pq_crypto 0.6.1 → 0.6.3

data/ext/pqcrypto/vendor/mldsa-native/mldsa/src/fips202/native/armv81m/src/keccak_f1600_x4_mve.c

@@ -12,114 +12,19 @@
 
 #include "fips202_native_armv81m.h"
 
-/*
- * TEMPORARY: Bit-interleaving using efficient shift-and-mask operations.
- * TODO: Replace with optimized MVE assembly implementations
- * (as a part of XORBytes and ExtractBytes)
- */
-
-/* Extract even-indexed bits from 64-bit value into lower 32 bits */
-static uint32_t bitinterleave_even(uint64_t x)
-{
-  uint64_t t;
-  t = x & 0x5555555555555555ULL;
-  t = (t | (t >> 1)) & 0x3333333333333333ULL;
-  t = (t | (t >> 2)) & 0x0f0f0f0f0f0f0f0fULL;
-  t = (t | (t >> 4)) & 0x00ff00ff00ff00ffULL;
-  t = (t | (t >> 8)) & 0x0000ffff0000ffffULL;
-  t = (t | (t >> 16)) & 0x00000000ffffffffULL;
-  return (uint32_t)t;
-}
-
-/* Extract odd-indexed bits from 64-bit value into lower 32 bits */
-static uint32_t bitinterleave_odd(uint64_t x)
-{
-  return bitinterleave_even(x >> 1);
-}
-
-/* Spread 32-bit value across even bit positions of 64-bit result */
-static uint64_t spread_even(uint32_t x)
-{
-  uint64_t t = x;
-  t = (t | (t << 16)) & 0x0000ffff0000ffffULL;
-  t = (t | (t << 8)) & 0x00ff00ff00ff00ffULL;
-  t = (t | (t << 4)) & 0x0f0f0f0f0f0f0f0fULL;
-  t = (t | (t << 2)) & 0x3333333333333333ULL;
-  t = (t | (t << 1)) & 0x5555555555555555ULL;
-  return t;
-}
-
-/* Combine even and odd 32-bit halves into interleaved 64-bit value */
-static uint64_t bitdeinterleave(uint32_t even, uint32_t odd)
-{
-  return spread_even(even) | (spread_even(odd) << 1);
-}
 
 /*
- * TEMPORARY: Naive C interleaving functions.
- * These will be replaced with optimized MVE assembly implementations.
+ * Keccak-f1600 x4 permutation (on bit-interleaved state)
+ * State is expected to already be in bit-interleaved format.
  */
-static void interleave_4fold(uint64_t *state_4x, const uint64_t *state0,
-                             const uint64_t *state1, const uint64_t *state2,
-                             const uint64_t *state3)
-{
-  uint32_t *state_4xl = (uint32_t *)state_4x;
-  uint32_t *state_4xh = (uint32_t *)state_4x + 100;
-
-  for (size_t i = 0; i < 25; i++)
-  {
-    state_4xl[i * 4 + 0] = bitinterleave_even(state0[i]);
-    state_4xl[i * 4 + 1] = bitinterleave_even(state1[i]);
-    state_4xl[i * 4 + 2] = bitinterleave_even(state2[i]);
-    state_4xl[i * 4 + 3] = bitinterleave_even(state3[i]);
-
-    state_4xh[i * 4 + 0] = bitinterleave_odd(state0[i]);
-    state_4xh[i * 4 + 1] = bitinterleave_odd(state1[i]);
-    state_4xh[i * 4 + 2] = bitinterleave_odd(state2[i]);
-    state_4xh[i * 4 + 3] = bitinterleave_odd(state3[i]);
-  }
-}
-
-static void deinterleave_4fold(uint64_t *state_4x, uint64_t *state0,
-                               uint64_t *state1, uint64_t *state2,
-                               uint64_t *state3)
-{
-  uint32_t *state_4xl = (uint32_t *)state_4x;
-  uint32_t *state_4xh = (uint32_t *)state_4x + 100;
-
-  for (size_t i = 0; i < 25; i++)
-  {
-    state0[i] = bitdeinterleave(state_4xl[i * 4 + 0], state_4xh[i * 4 + 0]);
-    state1[i] = bitdeinterleave(state_4xl[i * 4 + 1], state_4xh[i * 4 + 1]);
-    state2[i] = bitdeinterleave(state_4xl[i * 4 + 2], state_4xh[i * 4 + 2]);
-    state3[i] = bitdeinterleave(state_4xl[i * 4 + 3], state_4xh[i * 4 + 3]);
-  }
-}
-
 #define mld_keccak_f1600_x4_native_impl \
   MLD_NAMESPACE(keccak_f1600_x4_native_impl)
 int mld_keccak_f1600_x4_native_impl(uint64_t *state)
 {
-  /*
-   * TEMPORARY: Bit-interleaving using efficient shift-and-mask operations.
-   * TODO: Replace with optimized MVE assembly implementations
-   * (as a part of XORBytes and ExtractBytes)
-   */
-  MLD_ALIGN uint64_t state_4x[100];
-  MLD_ALIGN uint64_t state_4x_tmp[100];
-
-  /* Interleave the 4 states into bit-interleaved format */
-  interleave_4fold(state_4x, &state[0], &state[25], &state[50], &state[75]);
-
-  /* Run the permutation */
-  mld_keccak_f1600_x4_mve_asm(state_4x, state_4x_tmp,
+  MLD_ALIGN uint64_t state_tmp[100];
+  mld_keccak_f1600_x4_mve_asm(state, state_tmp,
                               mld_keccakf1600_round_constants);
-
-  /* Deinterleave back to 4 separate states */
-  deinterleave_4fold(state_4x, &state[0], &state[25], &state[50], &state[75]);
-
-  mld_zeroize(state_4x, sizeof(state_4x));
-  mld_zeroize(state_4x_tmp, sizeof(state_4x_tmp));
+  mld_zeroize(state_tmp, sizeof(state_tmp));
   return MLD_NATIVE_FUNC_SUCCESS;
 }
 

data/ext/pqcrypto/vendor/mldsa-native/mldsa/src/fips202/native/armv81m/src/keccakf1600_round_constants.c

@@ -17,31 +17,32 @@
  * - low word contains even-indexed bits
  * - high word contains odd-indexed bits
  */
-MLD_ALIGN const uint32_t mld_keccakf1600_round_constants[48] = {
-    0x00000001, 0x00000000, /* RC0 */
-    0x00000000, 0x00000089, /* RC1 */
-    0x00000000, 0x8000008b, /* RC2 */
-    0x00000000, 0x80008080, /* RC3 */
-    0x00000001, 0x0000008b, /* RC4 */
-    0x00000001, 0x00008000, /* RC5 */
-    0x00000001, 0x80008088, /* RC6 */
-    0x00000001, 0x80000082, /* RC7 */
-    0x00000000, 0x0000000b, /* RC8 */
-    0x00000000, 0x0000000a, /* RC9 */
-    0x00000001, 0x00008082, /* RC10 */
-    0x00000000, 0x00008003, /* RC11 */
-    0x00000001, 0x0000808b, /* RC12 */
-    0x00000001, 0x8000000b, /* RC13 */
-    0x00000001, 0x8000008a, /* RC14 */
-    0x00000001, 0x80000081, /* RC15 */
-    0x00000000, 0x80000081, /* RC16 */
-    0x00000000, 0x80000008, /* RC17 */
-    0x00000000, 0x00000083, /* RC18 */
-    0x00000000, 0x80008003, /* RC19 */
-    0x00000001, 0x80008088, /* RC20 */
-    0x00000000, 0x80000088, /* RC21 */
-    0x00000001, 0x00008000, /* RC22 */
-    0x00000000, 0x80008082, /* RC23 */
+MLD_ALIGN MLD_INTERNAL_DATA_DEFINITION const uint32_t
+    mld_keccakf1600_round_constants[48] = {
+        0x00000001, 0x00000000, /* RC0 */
+        0x00000000, 0x00000089, /* RC1 */
+        0x00000000, 0x8000008b, /* RC2 */
+        0x00000000, 0x80008080, /* RC3 */
+        0x00000001, 0x0000008b, /* RC4 */
+        0x00000001, 0x00008000, /* RC5 */
+        0x00000001, 0x80008088, /* RC6 */
+        0x00000001, 0x80000082, /* RC7 */
+        0x00000000, 0x0000000b, /* RC8 */
+        0x00000000, 0x0000000a, /* RC9 */
+        0x00000001, 0x00008082, /* RC10 */
+        0x00000000, 0x00008003, /* RC11 */
+        0x00000001, 0x0000808b, /* RC12 */
+        0x00000001, 0x8000000b, /* RC13 */
+        0x00000001, 0x8000008a, /* RC14 */
+        0x00000001, 0x80000081, /* RC15 */
+        0x00000000, 0x80000081, /* RC16 */
+        0x00000000, 0x80000008, /* RC17 */
+        0x00000000, 0x00000083, /* RC18 */
+        0x00000000, 0x80008003, /* RC19 */
+        0x00000001, 0x80008088, /* RC20 */
+        0x00000000, 0x80000088, /* RC21 */
+        0x00000001, 0x00008000, /* RC22 */
+        0x00000000, 0x80008082, /* RC23 */
 };
 
 #else /* MLD_FIPS202_ARMV81M_NEED_X4 && !MLD_CONFIG_MULTILEVEL_NO_SHARED */

data/ext/pqcrypto/vendor/mldsa-native/mldsa/src/fips202/native/armv81m/src/state_extract_bytes_x4_mve.S

@@ -0,0 +1,334 @@
+/*
+ * Copyright (c) The mlkem-native project authors
+ * Copyright (c) The mldsa-native project authors
+ * Copyright (c) 2026 Arm Limited
+ * SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
+ */
+
+// ---------------------------------------------------------------------------
+// Overview
+// ---------------------------------------------------------------------------
+// MVE/Helium implementation of KeccakF1600x4_StateExtractBytes
+// (inverse of state_xor_bytes_x4_mve.S).
+//
+// void KeccakF1600x4_StateExtractBytes(state, d0, d1, d2, d3, offset, length)
+//
+// Reads 'length' bytes from the bit-interleaved Keccak state starting at
+// byte 'offset', recombines the even and odd halves of each lane back
+// into plain bytes, and writes them to four output buffers (d0..d3).
+//
+// ---------------------------------------------------------------------------
+// Bit-interleaving background
+// ---------------------------------------------------------------------------
+// Each 64-bit Keccak lane is stored as two 32-bit words:
+//   even half -- bits 0, 2, 4, ..., 62 of the lane
+//   odd half  -- bits 1, 3, 5, ..., 63 of the lane
+// This representation allows 64-bit lane rotations (used in the Keccak
+// round function) to be implemented as pairs of 32-bit rotations.
+//
+// Batched (x4) processing:
+//   Four Keccak instances are processed as a batch.  Their states are
+//   stored interleaved in a single 800-byte buffer: first the even
+//   halves of all 25 lanes (400 bytes), then the odd halves (400 bytes).
+//   Within each 16-byte row, the four u32 words correspond to
+//   instances 0..3 of the same lane, enabling SIMD-parallel operations
+//   across all four instances.
+//
+// State memory layout (25 lanes x 4 instances x 2 halves):
+//   S[i][l]_even/odd = even/odd half of lane l, instance i  (u32)
+//   Each row is 16 bytes (one Q-register).
+//   Offset  Contents
+//     0     S[0][ 0]_even, S[1][ 0]_even, S[2][ 0]_even, S[3][ 0]_even
+//    16     S[0][ 1]_even, S[1][ 1]_even, S[2][ 1]_even, S[3][ 1]_even
+//    ...
+//   384     S[0][24]_even, S[1][24]_even, S[2][24]_even, S[3][24]_even
+//   400     S[0][ 0]_odd,  S[1][ 0]_odd,  S[2][ 0]_odd,  S[3][ 0]_odd
+//   416     S[0][ 1]_odd,  S[1][ 1]_odd,  S[2][ 1]_odd,  S[3][ 1]_odd
+//    ...
+//   784     S[0][24]_odd,  S[1][24]_odd,  S[2][24]_odd,  S[3][24]_odd
+//
+// ---------------------------------------------------------------------------
+// Three-phase structure
+// ---------------------------------------------------------------------------
+//   Prologue -- if offset is not 8-byte aligned, extract
+//               min(length, 8-(offset%8)) bytes via predicated byte stores.
+//   Main     -- process full 8-byte groups: load even/odd lane pair,
+//               de-interleave, scatter-store to output buffers.
+//   Tail     -- extract remaining <8 bytes via predicated byte stores.
+
+#include "../../../../common.h"
+#if defined(MLD_FIPS202_ARMV81M_NEED_X4) && \
+    !defined(MLD_CONFIG_MULTILEVEL_NO_SHARED)
+
+/*
+ * WARNING: This file is auto-derived from the mldsa-native source file
+ *   dev/fips202/armv81m/src/state_extract_bytes_x4_mve.S using scripts/simpasm. Do not modify it directly.
+ */
+
+.thumb
+.syntax unified
+
+.text
+.balign 4
+.global MLD_ASM_NAMESPACE(keccak_f1600_x4_state_extract_bytes_asm)
+MLD_ASM_FN_SYMBOL(keccak_f1600_x4_state_extract_bytes_asm)
+
+        .cfi_startproc
+        push.w {r4, r5, r6, r7, r8, r9, r10, r11, r12, lr}
+        .cfi_adjust_cfa_offset 0x28
+        .cfi_rel_offset r4, 0x0
+        .cfi_rel_offset r5, 0x4
+        .cfi_rel_offset r6, 0x8
+        .cfi_rel_offset r7, 0xc
+        .cfi_rel_offset r8, 0x10
+        .cfi_rel_offset r9, 0x14
+        .cfi_rel_offset r10, 0x18
+        .cfi_rel_offset r11, 0x1c
+        .cfi_rel_offset lr, 0x24
+        vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+        .cfi_adjust_cfa_offset 0x40
+        .cfi_rel_offset d8, 0x0
+        .cfi_rel_offset d9, 0x8
+        .cfi_rel_offset d10, 0x10
+        .cfi_rel_offset d11, 0x18
+        .cfi_rel_offset d12, 0x20
+        .cfi_rel_offset d13, 0x28
+        .cfi_rel_offset d14, 0x30
+        .cfi_rel_offset d15, 0x38
+        ldr r4, [sp, #0x68]
+        ldr.w r10, [sp, #0x6c]
+        ldr r6, [sp, #0x70]
+        cmp r6, #0x0
+        beq.w Lkeccak_f1600_x4_state_extract_bytes_asm_exit @ imm = #0x2ea
+        and r5, r10, #0x7
+        bic r9, r10, #0x7
+        add.w r8, r0, r9, lsl #1
+        add.w r7, r8, #0x190
+        cmp r5, #0x0
+        beq.w Lkeccak_f1600_x4_state_extract_bytes_asm_pre_main @ imm = #0x112
+        vldrw.u32 q0, [r8], #16
+        vldrw.u32 q1, [r7], #16
+        vrev32.16 q2, q0
+        vrev32.16 q3, q1
+        vsli.32 q0, q0, #0x8
+        vsli.16 q0, q0, #0x4
+        vsli.8 q0, q0, #0x1
+        vshr.u8 q4, q0, #0x3
+        vsli.8 q0, q4, #0x4
+        vshr.u8 q4, q0, #0x5
+        vsli.8 q0, q4, #0x6
+        vsli.32 q1, q1, #0x8
+        vsli.16 q1, q1, #0x4
+        vsli.8 q1, q1, #0x1
+        vshr.u8 q4, q1, #0x3
+        vsli.8 q1, q4, #0x4
+        vshr.u8 q4, q1, #0x5
+        vsli.8 q1, q4, #0x6
+        mov.w r0, #0x55
+        vdup.8 q4, r0
+        vand q0, q0, q4
+        vand q1, q1, q4
+        vshl.i32 q1, q1, #0x1
+        vorr q0, q0, q1
+        vsli.32 q2, q2, #0x8
+        vsli.16 q2, q2, #0x4
+        vsli.8 q2, q2, #0x1
+        vshr.u8 q1, q2, #0x3
+        vsli.8 q2, q1, #0x4
+        vshr.u8 q1, q2, #0x5
+        vsli.8 q2, q1, #0x6
+        vsli.32 q3, q3, #0x8
+        vsli.16 q3, q3, #0x4
+        vsli.8 q3, q3, #0x1
+        vshr.u8 q1, q3, #0x3
+        vsli.8 q3, q1, #0x4
+        vshr.u8 q1, q3, #0x5
+        vsli.8 q3, q1, #0x6
+        vand q1, q2, q4
+        vand q3, q3, q4
+        vshl.i32 q3, q3, #0x1
+        vorr q1, q1, q3
+        vrev64.32 q2, q0
+        vrev64.32 q3, q1
+        movw r0, #0xf0f
+        vmsr p0, r0
+        vpsel q0, q0, q3
+        vpsel q1, q2, q1
+        vmov.f64 d4, d1
+        vmov.f64 d6, d3
+        rsb.w lr, r5, #0x8
+        cmp r6, lr
+        it ls
+        movls lr, r6
+        vctp.8 lr
+        vmrs r11, p0
+        lsl.w r11, r11, r5
+        vmsr p0, r11
+        subs r1, r1, r5
+        subs r2, r2, r5
+        subs r3, r3, r5
+        subs r4, r4, r5
+        vpstttt
+        vstrbt.8 q0, [r1], #4
+        vstrbt.8 q1, [r2], #4
+        vstrbt.8 q2, [r3], #4
+        vstrbt.8 q3, [r4], #4
+        subs.w r6, r6, lr
+        cmp r6, #0x0
+        beq.w Lkeccak_f1600_x4_state_extract_bytes_asm_exit @ imm = #0x1cc
+        vmov q7[2], q7[0], r1, r3
+        vmov q7[3], q7[1], r2, r4
+        b Lkeccak_f1600_x4_state_extract_bytes_asm_main_body @ imm = #0xe
+
+Lkeccak_f1600_x4_state_extract_bytes_asm_pre_main:
+        vmov q7[2], q7[0], r1, r3
+        vmov q7[3], q7[1], r2, r4
+        mov.w r12, #0x4
+        vsub.i32 q7, q7, r12
+
+Lkeccak_f1600_x4_state_extract_bytes_asm_main_body:
+        lsr.w lr, r6, #0x3
+        wls lr, lr, Lkeccak_f1600_x4_state_extract_bytes_asm_main_loop_end @ imm = #0xb4
+
+Lkeccak_f1600_x4_state_extract_bytes_asm_main_loop_start:
+        vldrw.u32 q0, [r8], #16
+        vldrw.u32 q1, [r7], #16
+        vrev32.16 q2, q0
+        vrev32.16 q3, q1
+        vsli.32 q0, q0, #0x8
+        vsli.16 q0, q0, #0x4
+        vsli.8 q0, q0, #0x1
+        vshr.u8 q4, q0, #0x3
+        vsli.8 q0, q4, #0x4
+        vshr.u8 q4, q0, #0x5
+        vsli.8 q0, q4, #0x6
+        vsli.32 q1, q1, #0x8
+        vsli.16 q1, q1, #0x4
+        vsli.8 q1, q1, #0x1
+        vshr.u8 q4, q1, #0x3
+        vsli.8 q1, q4, #0x4
+        vshr.u8 q4, q1, #0x5
+        vsli.8 q1, q4, #0x6
+        mov.w r0, #0x55
+        vdup.8 q4, r0
+        vand q0, q0, q4
+        vand q1, q1, q4
+        vshl.i32 q1, q1, #0x1
+        vorr q0, q0, q1
+        vsli.32 q2, q2, #0x8
+        vsli.16 q2, q2, #0x4
+        vsli.8 q2, q2, #0x1
+        vshr.u8 q1, q2, #0x3
+        vsli.8 q2, q1, #0x4
+        vshr.u8 q1, q2, #0x5
+        vsli.8 q2, q1, #0x6
+        vsli.32 q3, q3, #0x8
+        vsli.16 q3, q3, #0x4
+        vsli.8 q3, q3, #0x1
+        vshr.u8 q1, q3, #0x3
+        vsli.8 q3, q1, #0x4
+        vshr.u8 q1, q3, #0x5
+        vsli.8 q3, q1, #0x6
+        vand q1, q2, q4
+        vand q3, q3, q4
+        vshl.i32 q3, q3, #0x1
+        vorr q1, q1, q3
+        vstrw.32 q0, [q7, #4]!
+        vstrw.32 q1, [q7, #4]!
+        le lr, Lkeccak_f1600_x4_state_extract_bytes_asm_main_loop_start @ imm = #-0xb4
+
+Lkeccak_f1600_x4_state_extract_bytes_asm_main_loop_end:
+        ands r6, r6, #0x7
+        beq Lkeccak_f1600_x4_state_extract_bytes_asm_exit @ imm = #0xee
+        mov.w r12, #0x4
+        vadd.i32 q7, q7, r12
+        vmov r1, r3, q7[2], q7[0]
+        vmov r2, r4, q7[3], q7[1]
+        vldrw.u32 q0, [r8], #16
+        vldrw.u32 q1, [r7], #16
+        vrev32.16 q2, q0
+        vrev32.16 q3, q1
+        vsli.32 q0, q0, #0x8
+        vsli.16 q0, q0, #0x4
+        vsli.8 q0, q0, #0x1
+        vshr.u8 q4, q0, #0x3
+        vsli.8 q0, q4, #0x4
+        vshr.u8 q4, q0, #0x5
+        vsli.8 q0, q4, #0x6
+        vsli.32 q1, q1, #0x8
+        vsli.16 q1, q1, #0x4
+        vsli.8 q1, q1, #0x1
+        vshr.u8 q4, q1, #0x3
+        vsli.8 q1, q4, #0x4
+        vshr.u8 q4, q1, #0x5
+        vsli.8 q1, q4, #0x6
+        mov.w r0, #0x55
+        vdup.8 q4, r0
+        vand q0, q0, q4
+        vand q1, q1, q4
+        vshl.i32 q1, q1, #0x1
+        vorr q0, q0, q1
+        vsli.32 q2, q2, #0x8
+        vsli.16 q2, q2, #0x4
+        vsli.8 q2, q2, #0x1
+        vshr.u8 q1, q2, #0x3
+        vsli.8 q2, q1, #0x4
+        vshr.u8 q1, q2, #0x5
+        vsli.8 q2, q1, #0x6
+        vsli.32 q3, q3, #0x8
+        vsli.16 q3, q3, #0x4
+        vsli.8 q3, q3, #0x1
+        vshr.u8 q1, q3, #0x3
+        vsli.8 q3, q1, #0x4
+        vshr.u8 q1, q3, #0x5
+        vsli.8 q3, q1, #0x6
+        vand q1, q2, q4
+        vand q3, q3, q4
+        vshl.i32 q3, q3, #0x1
+        vorr q1, q1, q3
+        vrev64.32 q2, q0
+        vrev64.32 q3, q1
+        movw r0, #0xf0f
+        vmsr p0, r0
+        vpsel q0, q0, q3
+        vpsel q1, q2, q1
+        vmov.f64 d4, d1
+        vmov.f64 d6, d3
+        vctp.8 r6
+        vpstttt
+        vstrbt.8 q0, [r1], #4
+        vstrbt.8 q1, [r2], #4
+        vstrbt.8 q2, [r3], #4
+        vstrbt.8 q3, [r4], #4
+
+Lkeccak_f1600_x4_state_extract_bytes_asm_exit:
+        vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+        .cfi_restore d8
+        .cfi_restore d9
+        .cfi_restore d10
+        .cfi_restore d11
+        .cfi_restore d12
+        .cfi_restore d13
+        .cfi_restore d14
+        .cfi_restore d15
+        .cfi_adjust_cfa_offset -0x40
+        pop.w {r4, r5, r6, r7, r8, r9, r10, r11, r12, pc}
+        .cfi_restore r4
+        .cfi_restore r5
+        .cfi_restore r6
+        .cfi_restore r7
+        .cfi_restore r8
+        .cfi_restore r9
+        .cfi_restore r10
+        .cfi_restore r11
+        .cfi_restore lr
+        .cfi_adjust_cfa_offset -0x28
+        .cfi_endproc
+
+MLD_ASM_FN_SIZE(keccak_f1600_x4_state_extract_bytes_asm)
+
+#endif /* MLD_FIPS202_ARMV81M_NEED_X4 && !MLD_CONFIG_MULTILEVEL_NO_SHARED */
+
+#if defined(__ELF__)
+.section .note.GNU-stack,"",%progbits
+#endif