mldsa_gh 0.1.0

data/ext/mldsa_gh_native/vendor/mldsa-native/mldsa/src/native/aarch64/meta.h

@@ -0,0 +1,314 @@
+/*
+ * Copyright (c) The mlkem-native project authors
+ * Copyright (c) The mldsa-native project authors
+ * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
+ */
+
+#ifndef MLD_NATIVE_AARCH64_META_H
+#define MLD_NATIVE_AARCH64_META_H
+
+/* Set of primitives that this backend replaces */
+#define MLD_USE_NATIVE_NTT
+#define MLD_USE_NATIVE_INTT
+#define MLD_USE_NATIVE_REJ_UNIFORM
+#define MLD_USE_NATIVE_REJ_UNIFORM_ETA2
+#define MLD_USE_NATIVE_REJ_UNIFORM_ETA4
+#define MLD_USE_NATIVE_POLY_DECOMPOSE_32
+#define MLD_USE_NATIVE_POLY_DECOMPOSE_88
+#define MLD_USE_NATIVE_POLY_CADDQ
+#define MLD_USE_NATIVE_POLY_USE_HINT_32
+#define MLD_USE_NATIVE_POLY_USE_HINT_88
+#define MLD_USE_NATIVE_POLY_CHKNORM
+#define MLD_USE_NATIVE_POLYZ_UNPACK_17
+#define MLD_USE_NATIVE_POLYZ_UNPACK_19
+#define MLD_USE_NATIVE_POINTWISE_MONTGOMERY
+#define MLD_USE_NATIVE_POLYVECL_POINTWISE_ACC_MONTGOMERY_L4
+#define MLD_USE_NATIVE_POLYVECL_POINTWISE_ACC_MONTGOMERY_L5
+#define MLD_USE_NATIVE_POLYVECL_POINTWISE_ACC_MONTGOMERY_L7
+
+/* Identifier for this backend so that source and assembly files
+ * in the build can be appropriately guarded. */
+#define MLD_ARITH_BACKEND_AARCH64
+
+
+#if !defined(__ASSEMBLER__)
+#include "../api.h"
+#include "src/arith_native_aarch64.h"
+
+MLD_MUST_CHECK_RETURN_VALUE
+static MLD_INLINE int mld_ntt_native(int32_t data[MLDSA_N])
+{
+  if (!mld_sys_check_capability(MLD_SYS_CAP_AARCH64_NEON))
+  {
+    return MLD_NATIVE_FUNC_FALLBACK;
+  }
+  mld_ntt_aarch64_asm(data, mld_aarch64_ntt_zetas_layer123456,
+                      mld_aarch64_ntt_zetas_layer78);
+  return MLD_NATIVE_FUNC_SUCCESS;
+}
+
+MLD_MUST_CHECK_RETURN_VALUE
+static MLD_INLINE int mld_intt_native(int32_t data[MLDSA_N])
+{
+  if (!mld_sys_check_capability(MLD_SYS_CAP_AARCH64_NEON))
+  {
+    return MLD_NATIVE_FUNC_FALLBACK;
+  }
+  mld_intt_aarch64_asm(data, mld_aarch64_intt_zetas_layer78,
+                       mld_aarch64_intt_zetas_layer123456);
+  return MLD_NATIVE_FUNC_SUCCESS;
+}
+
+MLD_MUST_CHECK_RETURN_VALUE
+static MLD_INLINE int mld_rej_uniform_native(int32_t *r, unsigned len,
+                                             const uint8_t *buf,
+                                             unsigned buflen)
+{
+  if (!mld_sys_check_capability(MLD_SYS_CAP_AARCH64_NEON) || len != MLDSA_N ||
+      buflen % 24 != 0)
+  {
+    return MLD_NATIVE_FUNC_FALLBACK;
+  }
+
+  /* Safety: outlen is at most MLDSA_N, hence, this cast is safe. */
+  return (int)mld_rej_uniform_aarch64_asm(r, buf, buflen,
+                                          mld_rej_uniform_table);
+}
+
+#if !defined(MLD_CONFIG_NO_KEYPAIR_API)
+#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLDSA_ETA == 2
+MLD_MUST_CHECK_RETURN_VALUE
+static MLD_INLINE int mld_rej_uniform_eta2_native(int32_t *r, unsigned len,
+                                                  const uint8_t *buf,
+                                                  unsigned buflen)
+{
+  uint64_t outlen;
+  /* AArch64 implementation assumes specific buffer lengths */
+  if (!mld_sys_check_capability(MLD_SYS_CAP_AARCH64_NEON) || len != MLDSA_N ||
+      buflen != MLD_AARCH64_REJ_UNIFORM_ETA2_BUFLEN)
+  {
+    return MLD_NATIVE_FUNC_FALLBACK;
+  }
+  /* Constant time: Inputs and outputs to this function are secret.
+   * It is safe to leak which coefficients are accepted/rejected.
+   * The assembly implementation must not leak any other information about the
+   * accepted coefficients. Constant-time testing cannot cover this, and we
+   * hence have to manually verify the assembly.
+   * We declassify prior the input data and mark the outputs as secret.
+   */
+  MLD_CT_TESTING_DECLASSIFY(buf, buflen);
+  outlen = mld_rej_uniform_eta2_aarch64_asm(r, buf, buflen,
+                                            mld_rej_uniform_eta_table);
+  MLD_CT_TESTING_SECRET(r, sizeof(int32_t) * outlen);
+  /* Safety: outlen is at most MLDSA_N and, hence, this cast is safe. */
+  return (int)outlen;
+}
+#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLDSA_ETA == 2 */
+
+#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLDSA_ETA == 4
+MLD_MUST_CHECK_RETURN_VALUE
+static MLD_INLINE int mld_rej_uniform_eta4_native(int32_t *r, unsigned len,
+                                                  const uint8_t *buf,
+                                                  unsigned buflen)
+{
+  uint64_t outlen;
+  /* AArch64 implementation assumes specific buffer lengths */
+  if (!mld_sys_check_capability(MLD_SYS_CAP_AARCH64_NEON) || len != MLDSA_N ||
+      buflen != MLD_AARCH64_REJ_UNIFORM_ETA4_BUFLEN)
+  {
+    return MLD_NATIVE_FUNC_FALLBACK;
+  }
+  /* Constant time: Inputs and outputs to this function are secret.
+   * It is safe to leak which coefficients are accepted/rejected.
+   * The assembly implementation must not leak any other information about the
+   * accepted coefficients. Constant-time testing cannot cover this, and we
+   * hence have to manually verify the assembly.
+   * We declassify prior the input data and mark the outputs as secret.
+   */
+  MLD_CT_TESTING_DECLASSIFY(buf, buflen);
+  outlen = mld_rej_uniform_eta4_aarch64_asm(r, buf, buflen,
+                                            mld_rej_uniform_eta_table);
+  MLD_CT_TESTING_SECRET(r, sizeof(int32_t) * outlen);
+  /* Safety: outlen is at most MLDSA_N and, hence, this cast is safe. */
+  return (int)outlen;
+}
+#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLDSA_ETA == 4 */
+#endif /* !MLD_CONFIG_NO_KEYPAIR_API */
+
+#if !defined(MLD_CONFIG_NO_SIGN_API)
+#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || \
+    (MLD_CONFIG_PARAMETER_SET == 65 || MLD_CONFIG_PARAMETER_SET == 87)
+MLD_MUST_CHECK_RETURN_VALUE
+static MLD_INLINE int mld_poly_decompose_32_native(int32_t *a1, int32_t *a0)
+{
+  if (!mld_sys_check_capability(MLD_SYS_CAP_AARCH64_NEON))
+  {
+    return MLD_NATIVE_FUNC_FALLBACK;
+  }
+  mld_poly_decompose_32_aarch64_asm(a1, a0);
+  return MLD_NATIVE_FUNC_SUCCESS;
+}
+#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLD_CONFIG_PARAMETER_SET == 65 \
+          || MLD_CONFIG_PARAMETER_SET == 87 */
+
+#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLD_CONFIG_PARAMETER_SET == 44
+MLD_MUST_CHECK_RETURN_VALUE
+static MLD_INLINE int mld_poly_decompose_88_native(int32_t *a1, int32_t *a0)
+{
+  if (!mld_sys_check_capability(MLD_SYS_CAP_AARCH64_NEON))
+  {
+    return MLD_NATIVE_FUNC_FALLBACK;
+  }
+  mld_poly_decompose_88_aarch64_asm(a1, a0);
+  return MLD_NATIVE_FUNC_SUCCESS;
+}
+#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLD_CONFIG_PARAMETER_SET == 44 \
+        */
+#endif /* !MLD_CONFIG_NO_SIGN_API */
+
+MLD_MUST_CHECK_RETURN_VALUE
+static MLD_INLINE int mld_poly_caddq_native(int32_t a[MLDSA_N])
+{
+  if (!mld_sys_check_capability(MLD_SYS_CAP_AARCH64_NEON))
+  {
+    return MLD_NATIVE_FUNC_FALLBACK;
+  }
+  mld_poly_caddq_aarch64_asm(a);
+  return MLD_NATIVE_FUNC_SUCCESS;
+}
+
+#if !defined(MLD_CONFIG_NO_VERIFY_API)
+#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || \
+    (MLD_CONFIG_PARAMETER_SET == 65 || MLD_CONFIG_PARAMETER_SET == 87)
+MLD_MUST_CHECK_RETURN_VALUE
+static MLD_INLINE int mld_poly_use_hint_32_native(int32_t *a, const int32_t *h)
+{
+  if (!mld_sys_check_capability(MLD_SYS_CAP_AARCH64_NEON))
+  {
+    return MLD_NATIVE_FUNC_FALLBACK;
+  }
+  mld_poly_use_hint_32_aarch64_asm(a, h);
+  return MLD_NATIVE_FUNC_SUCCESS;
+}
+#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLD_CONFIG_PARAMETER_SET == 65 \
+          || MLD_CONFIG_PARAMETER_SET == 87 */
+
+#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLD_CONFIG_PARAMETER_SET == 44
+MLD_MUST_CHECK_RETURN_VALUE
+static MLD_INLINE int mld_poly_use_hint_88_native(int32_t *a, const int32_t *h)
+{
+  if (!mld_sys_check_capability(MLD_SYS_CAP_AARCH64_NEON))
+  {
+    return MLD_NATIVE_FUNC_FALLBACK;
+  }
+  mld_poly_use_hint_88_aarch64_asm(a, h);
+  return MLD_NATIVE_FUNC_SUCCESS;
+}
+#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLD_CONFIG_PARAMETER_SET == 44 \
+        */
+#endif /* !MLD_CONFIG_NO_VERIFY_API */
+
+MLD_MUST_CHECK_RETURN_VALUE
+static MLD_INLINE int mld_poly_chknorm_native(const int32_t *a, int32_t B)
+{
+  if (!mld_sys_check_capability(MLD_SYS_CAP_AARCH64_NEON))
+  {
+    return MLD_NATIVE_FUNC_FALLBACK;
+  }
+  return mld_poly_chknorm_aarch64_asm(a, B);
+}
+
+#if !defined(MLD_CONFIG_NO_SIGN_API) || !defined(MLD_CONFIG_NO_VERIFY_API)
+#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLD_CONFIG_PARAMETER_SET == 44
+MLD_MUST_CHECK_RETURN_VALUE
+static MLD_INLINE int mld_polyz_unpack_17_native(int32_t *r, const uint8_t *buf)
+{
+  if (!mld_sys_check_capability(MLD_SYS_CAP_AARCH64_NEON))
+  {
+    return MLD_NATIVE_FUNC_FALLBACK;
+  }
+  mld_polyz_unpack_17_aarch64_asm(r, buf, mld_polyz_unpack_17_indices);
+  return MLD_NATIVE_FUNC_SUCCESS;
+}
+#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLD_CONFIG_PARAMETER_SET == 44 \
+        */
+
+#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || \
+    (MLD_CONFIG_PARAMETER_SET == 65 || MLD_CONFIG_PARAMETER_SET == 87)
+MLD_MUST_CHECK_RETURN_VALUE
+static MLD_INLINE int mld_polyz_unpack_19_native(int32_t *r, const uint8_t *buf)
+{
+  if (!mld_sys_check_capability(MLD_SYS_CAP_AARCH64_NEON))
+  {
+    return MLD_NATIVE_FUNC_FALLBACK;
+  }
+  mld_polyz_unpack_19_aarch64_asm(r, buf, mld_polyz_unpack_19_indices);
+  return MLD_NATIVE_FUNC_SUCCESS;
+}
+#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLD_CONFIG_PARAMETER_SET == 65 \
+          || MLD_CONFIG_PARAMETER_SET == 87 */
+#endif /* !MLD_CONFIG_NO_SIGN_API || !MLD_CONFIG_NO_VERIFY_API */
+
+#if !defined(MLD_CONFIG_NO_SIGN_API) || !defined(MLD_CONFIG_NO_VERIFY_API) || \
+    defined(MLD_CONFIG_REDUCE_RAM) || defined(MLD_UNIT_TEST)
+MLD_MUST_CHECK_RETURN_VALUE
+static MLD_INLINE int mld_poly_pointwise_montgomery_native(
+    int32_t a[MLDSA_N], const int32_t b[MLDSA_N])
+{
+  if (!mld_sys_check_capability(MLD_SYS_CAP_AARCH64_NEON))
+  {
+    return MLD_NATIVE_FUNC_FALLBACK;
+  }
+  mld_poly_pointwise_montgomery_aarch64_asm(a, b);
+  return MLD_NATIVE_FUNC_SUCCESS;
+}
+#endif /* !MLD_CONFIG_NO_SIGN_API || !MLD_CONFIG_NO_VERIFY_API || \
+          MLD_CONFIG_REDUCE_RAM || MLD_UNIT_TEST */
+
+#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLDSA_L == 4
+MLD_MUST_CHECK_RETURN_VALUE
+static MLD_INLINE int mld_polyvecl_pointwise_acc_montgomery_l4_native(
+    int32_t w[MLDSA_N], const int32_t u[4][MLDSA_N],
+    const int32_t v[4][MLDSA_N])
+{
+  if (!mld_sys_check_capability(MLD_SYS_CAP_AARCH64_NEON))
+  {
+    return MLD_NATIVE_FUNC_FALLBACK;
+  }
+  mld_polyvecl_pointwise_acc_montgomery_l4_aarch64_asm(w, u, v);
+  return MLD_NATIVE_FUNC_SUCCESS;
+}
+#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLDSA_L == 4 */
+
+#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLDSA_L == 5
+MLD_MUST_CHECK_RETURN_VALUE
+static MLD_INLINE int mld_polyvecl_pointwise_acc_montgomery_l5_native(
+    int32_t w[MLDSA_N], const int32_t u[5][MLDSA_N],
+    const int32_t v[5][MLDSA_N])
+{
+  if (!mld_sys_check_capability(MLD_SYS_CAP_AARCH64_NEON))
+  {
+    return MLD_NATIVE_FUNC_FALLBACK;
+  }
+  mld_polyvecl_pointwise_acc_montgomery_l5_aarch64_asm(w, u, v);
+  return MLD_NATIVE_FUNC_SUCCESS;
+}
+#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLDSA_L == 5 */
+
+#if defined(MLD_CONFIG_MULTILEVEL_WITH_SHARED) || MLDSA_L == 7
+MLD_MUST_CHECK_RETURN_VALUE
+static MLD_INLINE int mld_polyvecl_pointwise_acc_montgomery_l7_native(
+    int32_t w[MLDSA_N], const int32_t u[7][MLDSA_N],
+    const int32_t v[7][MLDSA_N])
+{
+  if (!mld_sys_check_capability(MLD_SYS_CAP_AARCH64_NEON))
+  {
+    return MLD_NATIVE_FUNC_FALLBACK;
+  }
+  mld_polyvecl_pointwise_acc_montgomery_l7_aarch64_asm(w, u, v);
+  return MLD_NATIVE_FUNC_SUCCESS;
+}
+#endif /* MLD_CONFIG_MULTILEVEL_WITH_SHARED || MLDSA_L == 7 */
+
+#endif /* !__ASSEMBLER__ */
+#endif /* !MLD_NATIVE_AARCH64_META_H */

data/ext/mldsa_gh_native/vendor/mldsa-native/mldsa/src/native/aarch64/src/aarch64_zetas.c

@@ -0,0 +1,248 @@
+/*
+ * Copyright (c) The mldsa-native project authors
+ * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
+ */
+
+/*
+ * WARNING: This file is auto-generated from scripts/autogen
+ *          in the mldsa-native repository.
+ *          Do not modify it directly.
+ */
+
+#include "../../../common.h"
+
+#if defined(MLD_ARITH_BACKEND_AARCH64) && \
+    !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED)
+
+#include "arith_native_aarch64.h"
+
+/*
+ * Table of zeta values used in the AArch64 forward NTT
+ * See autogen for details.
+ */
+MLD_ALIGN MLD_INTERNAL_DATA_DEFINITION const int32_t
+    mld_aarch64_ntt_zetas_layer123456[144] = {
+        -3572223, -915382907,  3765607,  964937599,   3761513,  963888510,
+        -3201494, -820383522,  -2883726, -738955404,  -3145678, -806080660,
+        -3201430, -820367122,  0,        0,           -601683,  -154181397,
+        -3370349, -863652652,  -4063053, -1041158200, 3602218,  923069133,
+        3182878,  815613168,   2740543,  702264730,   -3586446, -919027554,
+        0,        0,           3542485,  907762539,   2663378,  682491182,
+        -1674615, -429120452,  -3110818, -797147778,  2101410,  538486762,
+        3704823,  949361686,   1159875,  297218217,   0,        0,
+        2682288,  687336873,   -3524442, -903139016,  -434125,  -111244624,
+        394148,   101000509,   928749,   237992130,   1095468,  280713909,
+        -3506380, -898510625,  0,        0,           2129892,  545785280,
+        676590,   173376332,   -1335936, -342333886,  2071829,  530906624,
+        -4018989, -1029866791, 3241972,  830756018,   2156050,  552488273,
+        0,        0,           3764867,  964747974,   -3227876, -827143915,
+        1714295,  439288460,   3415069,  875112161,   1759347,  450833045,
+        -817536,  -209493775,  -3574466, -915957677,  0,        0,
+        -1005239, -257592709,  2453983,  628833668,   1460718,  374309300,
+        3756790,  962678241,   -1935799, -496048908,  -1716988, -439978542,
+        -3950053, -1012201926, 0,        0,           557458,   142848732,
+        -642628,  -164673562,  -3585098, -918682129,  -2897314, -742437332,
+        3192354,  818041395,   556856,   142694469,   3870317,  991769559,
+        0,        0,           -1221177, -312926867,  2815639,  721508096,
+        2283733,  585207070,   2917338,  747568486,   1853806,  475038184,
+        3345963,  857403734,   1858416,  476219497,   0,        0,
+};
+
+MLD_ALIGN MLD_INTERNAL_DATA_DEFINITION const int32_t
+    mld_aarch64_ntt_zetas_layer78[384] = {
+        3073009,     1277625,     -2635473,   3852015,     787459213,
+        327391679,   -675340520,  987079667,  1753,        -2659525,
+        2660408,     -59148,      449207,     -681503850,  681730119,
+        -15156688,   -1935420,    -1455890,   -1780227,    2772600,
+        -495951789,  -373072124,  -456183549, 710479343,   4183372,
+        -3222807,    -3121440,    -274060,    1071989969,  -825844983,
+        -799869667,  -70227934,   1182243,    636927,      -3956745,
+        -3284915,    302950022,   163212680,  -1013916752, -841760171,
+        87208,       -3965306,    -2296397,   -3716946,    22347069,
+        -1016110510, -588452222,  -952468207, 2508980,     2028118,
+        1937570,     -3815725,    642926661,  519705671,   496502727,
+        -977780347,  -27812,      1009365,    -1979497,    -3956944,
+        -7126831,    258649997,   -507246529, -1013967746, 822541,
+        -2454145,    1596822,     -3759465,   210776307,   -628875181,
+        409185979,   -963363710,  2811291,    -2983781,    -1109516,
+        4158088,     720393920,   -764594519, -284313712,  1065510939,
+        -1685153,    2678278,     -3551006,   -250446,     -431820817,
+        686309310,   -909946047,  -64176841,  -3410568,    -3768948,
+        635956,      -2455377,    -873958779, -965793731,  162963861,
+        -629190881,  1528066,     482649,     1148858,     -2962264,
+        391567239,   123678909,   294395108,  -759080783,  -4146264,
+        2192938,     2387513,     -268456,    -1062481036, 561940831,
+        611800717,   -68791907,   -1772588,   -1727088,    -3611750,
+        -3180456,    -454226054,  -442566669, -925511710,  -814992530,
+        -565603,     169688,      2462444,    -3334383,    -144935890,
+        43482586,    631001801,   -854436357, 3747250,     1239911,
+        3195676,     1254190,     960233614,  317727459,   818892658,
+        321386456,   2296099,     -3838479,   2642980,     -12417,
+        588375860,   -983611064,  677264190,  -3181859,    -4166425,
+        -3488383,    1987814,     -3197248,   -1067647297, -893898890,
+        509377762,   -819295484,  2998219,    -89301,      -1354892,
+        -1310261,    768294260,   -22883400,  -347191365,  -335754661,
+        141835,      2513018,     613238,     -2218467,    36345249,
+        643961400,   157142369,   -568482643, 1736313,     235407,
+        -3250154,    3258457,     444930577,  60323094,    -832852657,
+        834980303,   -458740,     4040196,    2039144,     -818761,
+        -117552223,  1035301089,  522531086,  -209807681,  -1921994,
+        -3472069,    -1879878,    -2178965,   -492511373,  -889718424,
+        -481719139,  -558360247,  -2579253,   1787943,     -2391089,
+        -2254727,    -660934133,  458160776,  -612717067,  -577774276,
+        -1623354,    -2374402,    586241,     527981,      -415984810,
+        -608441020,  150224382,   135295244,  2105286,     -2033807,
+        -1179613,    -2743411,    539479988,  -521163479,  -302276083,
+        -702999655,  3482206,     -4182915,   -1300016,    -2362063,
+        892316032,   -1071872863, -333129378, -605279149,  -1476985,
+        2491325,     507927,      -724804,    -378477722,  638402564,
+        130156402,   -185731180,  1994046,    -1393159,    -1187885,
+        -1834526,    510974714,   -356997292, -304395785,  -470097680,
+        -1317678,    2461387,     3035980,    621164,      -337655269,
+        630730945,   777970524,   159173408,  -3033742,    2647994,
+        -2612853,    749577,      -777397036, 678549029,   -669544140,
+        192079267,   -338420,     3009748,    4148469,     -4022750,
+        -86720197,   771248568,   1063046068, -1030830548, 3901472,
+        -1226661,    2925816,     3374250,    999753034,   -314332144,
+        749740976,   864652284,   3980599,    -1615530,    1665318,
+        1163598,     1020029345,  -413979908, 426738094,   298172236,
+        2569011,     1723229,     2028038,    -3369273,    658309618,
+        441577800,   519685171,   -863376927, 1356448,     -2775755,
+        2683270,     -2778788,    347590090,  -711287812,  687588511,
+        -712065019,  3994671,     -1370517,   3363542,     545376,
+        1023635298,  -351195274,  861908357,  139752717,   -11879,
+        3020393,     214880,      -770441,    -3043996,    773976352,
+        55063046,    -197425671,  -3467665,   2312838,     -653275,
+        -459163,     -888589898,  592665232,  -167401858,  -117660617,
+        3105558,     508145,      860144,     140244,      795799901,
+        130212265,   220412084,   35937555,   -1103344,    -553718,
+        3430436,     -1514152,    -282732136, -141890356,  879049958,
+        -388001774,  348812,      -327848,    1011223,     -2354215,
+        89383150,    -84011120,   259126110,  -603268097,  -2185084,
+        2358373,     -3014420,    2926054,    -559928242,  604333585,
+        -772445769,  749801963,   3123762,    -2193087,    -1716814,
+        -392707,     800464680,   -561979013, -439933955,  -100631253,
+        -3818627,    -1922253,    -2236726,   1744507,     -978523985,
+        -492577742,  -573161516,  447030292,  -303005,     -3974485,
+        1900052,     1054478,     -77645096,  -1018462631, 486888731,
+        270210213,   3531229,     -3773731,   -781875,     -731434,
+        904878186,   -967019376,  -200355636, -187430119,
+};
+
+MLD_ALIGN MLD_INTERNAL_DATA_DEFINITION const int32_t
+    mld_aarch64_intt_zetas_layer78[384] = {
+        -1744507,    2236726,    1922253,     3818627,     -447030292,
+        573161516,   492577742,  978523985,   731434,      781875,
+        3773731,     -3531229,   187430119,   200355636,   967019376,
+        -904878186,  -1054478,   -1900052,    3974485,     303005,
+        -270210213,  -486888731, 1018462631,  77645096,    2354215,
+        -1011223,    327848,     -348812,     603268097,   -259126110,
+        84011120,    -89383150,  392707,      1716814,     2193087,
+        -3123762,    100631253,  439933955,   561979013,   -800464680,
+        -2926054,    3014420,    -2358373,    2185084,     -749801963,
+        772445769,   -604333585, 559928242,   459163,      653275,
+        -2312838,    3467665,    117660617,   167401858,   -592665232,
+        888589898,   1514152,    -3430436,    553718,      1103344,
+        388001774,   -879049958, 141890356,   282732136,   -140244,
+        -860144,     -508145,    -3105558,    -35937555,   -220412084,
+        -130212265,  -795799901, 2778788,     -2683270,    2775755,
+        -1356448,    712065019,  -687588511,  711287812,   -347590090,
+        770441,      -214880,    -3020393,    11879,       197425671,
+        -55063046,   -773976352, 3043996,     -545376,     -3363542,
+        1370517,     -3994671,   -139752717,  -861908357,  351195274,
+        -1023635298, -3374250,   -2925816,    1226661,     -3901472,
+        -864652284,  -749740976, 314332144,   -999753034,  3369273,
+        -2028038,    -1723229,   -2569011,    863376927,   -519685171,
+        -441577800,  -658309618, -1163598,    -1665318,    1615530,
+        -3980599,    -298172236, -426738094,  413979908,   -1020029345,
+        -621164,     -3035980,   -2461387,    1317678,     -159173408,
+        -777970524,  -630730945, 337655269,   4022750,     -4148469,
+        -3009748,    338420,     1030830548,  -1063046068, -771248568,
+        86720197,    -749577,    2612853,     -2647994,    3033742,
+        -192079267,  669544140,  -678549029,  777397036,   2362063,
+        1300016,     4182915,    -3482206,    605279149,   333129378,
+        1071872863,  -892316032, 1834526,     1187885,     1393159,
+        -1994046,    470097680,  304395785,   356997292,   -510974714,
+        724804,      -507927,    -2491325,    1476985,     185731180,
+        -130156402,  -638402564, 378477722,   2254727,     2391089,
+        -1787943,    2579253,    577774276,   612717067,   -458160776,
+        660934133,   2743411,    1179613,     2033807,     -2105286,
+        702999655,   302276083,  521163479,   -539479988,  -527981,
+        -586241,     2374402,    1623354,     -135295244,  -150224382,
+        608441020,   415984810,  -3258457,    3250154,     -235407,
+        -1736313,    -834980303, 832852657,   -60323094,   -444930577,
+        2178965,     1879878,    3472069,     1921994,     558360247,
+        481719139,   889718424,  492511373,   818761,      -2039144,
+        -4040196,    458740,     209807681,   -522531086,  -1035301089,
+        117552223,   3197248,    -1987814,    3488383,     4166425,
+        819295484,   -509377762, 893898890,   1067647297,  2218467,
+        -613238,     -2513018,   -141835,     568482643,   -157142369,
+        -643961400,  -36345249,  1310261,     1354892,     89301,
+        -2998219,    335754661,  347191365,   22883400,    -768294260,
+        3334383,     -2462444,   -169688,     565603,      854436357,
+        -631001801,  -43482586,  144935890,   12417,       -2642980,
+        3838479,     -2296099,   3181859,     -677264190,  983611064,
+        -588375860,  -1254190,   -3195676,    -1239911,    -3747250,
+        -321386456,  -818892658, -317727459,  -960233614,  2962264,
+        -1148858,    -482649,    -1528066,    759080783,   -294395108,
+        -123678909,  -391567239, 3180456,     3611750,     1727088,
+        1772588,     814992530,  925511710,   442566669,   454226054,
+        268456,      -2387513,   -2192938,    4146264,     68791907,
+        -611800717,  -561940831, 1062481036,  -4158088,    1109516,
+        2983781,     -2811291,   -1065510939, 284313712,   764594519,
+        -720393920,  2455377,    -635956,     3768948,     3410568,
+        629190881,   -162963861, 965793731,   873958779,   250446,
+        3551006,     -2678278,   1685153,     64176841,    909946047,
+        -686309310,  431820817,  3815725,     -1937570,    -2028118,
+        -2508980,    977780347,  -496502727,  -519705671,  -642926661,
+        3759465,     -1596822,   2454145,     -822541,     963363710,
+        -409185979,  628875181,  -210776307,  3956944,     1979497,
+        -1009365,    27812,      1013967746,  507246529,   -258649997,
+        7126831,     274060,     3121440,     3222807,     -4183372,
+        70227934,    799869667,  825844983,   -1071989969, 3716946,
+        2296397,     3965306,    -87208,      952468207,   588452222,
+        1016110510,  -22347069,  3284915,     3956745,     -636927,
+        -1182243,    841760171,  1013916752,  -163212680,  -302950022,
+        -3852015,    2635473,    -1277625,    -3073009,    -987079667,
+        675340520,   -327391679, -787459213,  -2772600,    1780227,
+        1455890,     1935420,    -710479343,  456183549,   373072124,
+        495951789,   59148,      -2660408,    2659525,     -1753,
+        15156688,    -681730119, 681503850,   -449207,
+};
+
+MLD_ALIGN MLD_INTERNAL_DATA_DEFINITION const int32_t
+    mld_aarch64_intt_zetas_layer123456[160] = {
+        -2283733, -585207070, 0,        0,          -1858416, -476219497,
+        -3345963, -857403734, -2815639, -721508096, 0,        0,
+        -1853806, -475038184, -2917338, -747568486, 3585098,  918682129,
+        0,        0,          -3870317, -991769559, -556856,  -142694469,
+        642628,   164673562,  0,        0,          -3192354, -818041395,
+        2897314,  742437332,  -1460718, -374309300, 0,        0,
+        3950053,  1012201926, 1716988,  439978542,  -2453983, -628833668,
+        0,        0,          1935799,  496048908,  -3756790, -962678241,
+        -1714295, -439288460, 0,        0,          3574466,  915957677,
+        817536,   209493775,  3227876,  827143915,  0,        0,
+        -1759347, -450833045, -3415069, -875112161, 1335936,  342333886,
+        0,        0,          -2156050, -552488273, -3241972, -830756018,
+        -676590,  -173376332, 0,        0,          4018989,  1029866791,
+        -2071829, -530906624, 434125,   111244624,  0,        0,
+        3506380,  898510625,  -1095468, -280713909, 3524442,  903139016,
+        0,        0,          -928749,  -237992130, -394148,  -101000509,
+        1674615,  429120452,  0,        0,          -1159875, -297218217,
+        -3704823, -949361686, -2663378, -682491182, 0,        0,
+        -2101410, -538486762, 3110818,  797147778,  4063053,  1041158200,
+        0,        0,          3586446,  919027554,  -2740543, -702264730,
+        3370349,  863652652,  0,        0,          -3182878, -815613168,
+        -3602218, -923069133, -294725,  -75523344,  -3761513, -963888510,
+        -3765607, -964937599, 3201430,  820367122,  3145678,  806080660,
+        2883726,  738955404,  3201494,  820383522,  1221177,  312926867,
+        -557458,  -142848732, 1005239,  257592709,  -3764867, -964747974,
+        -2129892, -545785280, -2682288, -687336873, -3542485, -907762539,
+        601683,   154181397,  0,        0,
+};
+
+#else /* MLD_ARITH_BACKEND_AARCH64 && !MLD_CONFIG_MULTILEVEL_NO_SHARED */
+
+MLD_EMPTY_CU(aarch64_zetas)
+
+#endif /* !(MLD_ARITH_BACKEND_AARCH64 && !MLD_CONFIG_MULTILEVEL_NO_SHARED) */