@luma.gl/shaders 0.0.0-stage → 10.0.0-alpha.11

package/src/modules/math/fp64/fp64-arithmetic-wgsl.ts

@@ -0,0 +1,924 @@
+// luma.gl
+// SPDX-License-Identifier: MIT
+// SPDX-FileCopyrightText: Copyright (c) vis.gl contributors
+
+import {fp64arithmeticWGSLInteger} from './fp64-arithmetic-wgsl-integer';
+import {fp64arithmeticWGSLHybrid} from './fp64-arithmetic-wgsl-hybrid';
+
+/** WGSL source for fp64 arithmetic helpers and raw binary64 subtraction. */
+export const fp64arithmeticWGSL = /* wgsl */ `\
+struct Fp64ArithmeticUniforms {
+  ONE: f32,
+  SPLIT: f32,
+};
+
+@group(0) @binding(auto) var<uniform> fp64arithmetic : Fp64ArithmeticUniforms;
+
+#ifndef LUMA_FP64_F32_INPUT_ONLY
+struct Fp64Bits {
+  sign: u32,
+  exponent: i32,
+  significand: vec2u,
+  isZero: bool,
+  isInf: bool,
+  isNan: bool,
+};
+#endif
+
+#ifndef LUMA_FP64_PREDICATE_ONLY
+fn fp64_nan(seed: f32) -> f32 {
+  let nanBits = 0x7fc00000u | select(0u, 1u, seed < 0.0);
+  return bitcast<f32>(nanBits);
+}
+#endif
+
+fn fp64_u64_is_zero(value: vec2u) -> bool {
+  return value.x == 0u && value.y == 0u;
+}
+
+fn fp64_u64_compare(a: vec2u, b: vec2u) -> i32 {
+  if (a.x != b.x) {
+    return select(-1, 1, a.x > b.x);
+  }
+  if (a.y != b.y) {
+    return select(-1, 1, a.y > b.y);
+  }
+  return 0;
+}
+
+fn fp64_u64_add(a: vec2u, b: vec2u) -> vec2u {
+  let low = a.y + b.y;
+  let carry = select(0u, 1u, low < a.y);
+  return vec2u(a.x + b.x + carry, low);
+}
+
+fn fp64_u64_sub(a: vec2u, b: vec2u) -> vec2u {
+  let borrow = select(0u, 1u, a.y < b.y);
+  return vec2u(a.x - b.x - borrow, a.y - b.y);
+}
+
+fn fp64_u64_shift_left(value: vec2u, shift: u32) -> vec2u {
+  if (shift == 0u) {
+    return value;
+  }
+  if (shift < 32u) {
+    return vec2u((value.x << shift) | (value.y >> (32u - shift)), value.y << shift);
+  }
+  if (shift == 32u) {
+    return vec2u(value.y, 0u);
+  }
+  if (shift < 64u) {
+    return vec2u(value.y << (shift - 32u), 0u);
+  }
+  return vec2u(0u);
+}
+
+fn fp64_u64_shift_right(value: vec2u, shift: u32) -> vec2u {
+  if (shift == 0u) {
+    return value;
+  }
+  if (shift < 32u) {
+    return vec2u(value.x >> shift, (value.y >> shift) | (value.x << (32u - shift)));
+  }
+  if (shift == 32u) {
+    return vec2u(0u, value.x);
+  }
+  if (shift < 64u) {
+    return vec2u(0u, value.x >> (shift - 32u));
+  }
+  return vec2u(0u);
+}
+
+fn fp64_u64_get_bit(value: vec2u, bitIndex: u32) -> bool {
+  if (bitIndex >= 64u) {
+    return false;
+  }
+  if (bitIndex >= 32u) {
+    return ((value.x >> (bitIndex - 32u)) & 1u) != 0u;
+  }
+  return ((value.y >> bitIndex) & 1u) != 0u;
+}
+
+fn fp64_u64_has_bits_below(value: vec2u, bitCount: u32) -> bool {
+  if (bitCount == 0u) {
+    return false;
+  }
+  if (bitCount >= 64u) {
+    return !fp64_u64_is_zero(value);
+  }
+  if (bitCount > 32u) {
+    let highBitCount = bitCount - 32u;
+    let highMask = (1u << highBitCount) - 1u;
+    return value.y != 0u || (value.x & highMask) != 0u;
+  }
+  if (bitCount == 32u) {
+    return value.y != 0u;
+  }
+  let lowMask = (1u << bitCount) - 1u;
+  return (value.y & lowMask) != 0u;
+}
+
+#ifndef LUMA_FP64_F32_INPUT_ONLY
+fn fp64_u64_shift_right_sticky(value: vec2u, shift: u32) -> vec2u {
+  var shifted = fp64_u64_shift_right(value, shift);
+  if (fp64_u64_has_bits_below(value, shift)) {
+    shifted.y = shifted.y | 1u;
+  }
+  return shifted;
+}
+#endif
+
+fn fp64_u64_count_leading_zeros(value: vec2u) -> u32 {
+  if (value.x != 0u) {
+    return countLeadingZeros(value.x);
+  }
+  return 32u + countLeadingZeros(value.y);
+}
+
+fn fp64_round_shift_right_to_u32(value: vec2u, shift: u32) -> u32 {
+  if (shift == 0u) {
+    return value.y;
+  }
+
+  let truncated = fp64_u64_shift_right(value, shift);
+  var rounded = truncated.y;
+  let guard = fp64_u64_get_bit(value, shift - 1u);
+  let hasTrailingBits = fp64_u64_has_bits_below(value, shift - 1u);
+  if (guard && (hasTrailingBits || (rounded & 1u) == 1u)) {
+    rounded = rounded + 1u;
+  }
+  return rounded;
+}
+
+#ifndef LUMA_FP64_F32_INPUT_ONLY
+fn fp64_round_shift_right(value: vec2u, shift: u32) -> vec2u {
+  if (shift == 0u) {
+    return value;
+  }
+
+  var rounded = fp64_u64_shift_right(value, shift);
+  let guard = fp64_u64_get_bit(value, shift - 1u);
+  let hasTrailingBits = fp64_u64_has_bits_below(value, shift - 1u);
+  if (guard && (hasTrailingBits || (rounded.y & 1u) == 1u)) {
+    rounded = fp64_u64_add(rounded, vec2u(0u, 1u));
+  }
+  return rounded;
+}
+#endif
+
+fn fp64_make_f32_bits_from_u64(sign: u32, significand: vec2u, baseExponent: i32) -> u32 {
+  if (fp64_u64_is_zero(significand)) {
+    return sign << 31u;
+  }
+
+  let leadingZeros = fp64_u64_count_leading_zeros(significand);
+  let mostSignificantBit = 63u - leadingZeros;
+  var exponent = baseExponent + i32(mostSignificantBit);
+
+  if (exponent > 127) {
+    return (sign << 31u) | 0x7f800000u;
+  }
+
+  if (exponent >= -126) {
+    let shift = i32(mostSignificantBit) - 23;
+    var significand24: u32;
+    if (shift > 0) {
+      significand24 = fp64_round_shift_right_to_u32(significand, u32(shift));
+    } else {
+      significand24 = fp64_u64_shift_left(significand, u32(-shift)).y;
+    }
+
+    if (significand24 >= 0x1000000u) {
+      significand24 = significand24 >> 1u;
+      exponent = exponent + 1;
+      if (exponent > 127) {
+        return (sign << 31u) | 0x7f800000u;
+      }
+    }
+
+    return (sign << 31u) | (u32(exponent + 127) << 23u) | (significand24 & 0x7fffffu);
+  }
+
+  let scaleExponent = baseExponent + 149;
+  var mantissa: u32;
+  if (scaleExponent >= 0) {
+    mantissa = fp64_u64_shift_left(significand, u32(scaleExponent)).y;
+  } else {
+    mantissa = fp64_round_shift_right_to_u32(significand, u32(-scaleExponent));
+  }
+
+  if (mantissa >= 0x800000u) {
+    return (sign << 31u) | 0x00800000u;
+  }
+  return (sign << 31u) | mantissa;
+}
+
+#ifndef LUMA_FP64_F32_INPUT_ONLY
+fn fp64_decode_bits(bits: vec2u) -> Fp64Bits {
+  let sign = bits.x >> 31u;
+  let exponentBits = (bits.x >> 20u) & 0x7ffu;
+  let fractionHigh = bits.x & 0xfffffu;
+  let fractionLow = bits.y;
+  let fraction = vec2u(fractionHigh, fractionLow);
+
+  if (exponentBits == 0x7ffu) {
+    let isInf = fp64_u64_is_zero(fraction);
+    return Fp64Bits(sign, 0, vec2u(0u), false, isInf, !isInf);
+  }
+
+  if (exponentBits == 0u) {
+    let isZero = fp64_u64_is_zero(fraction);
+    return Fp64Bits(sign, -1022, fraction, isZero, false, false);
+  }
+
+  return Fp64Bits(sign, i32(exponentBits) - 1023, vec2u((1u << 20u) | fractionHigh, fractionLow), false, false, false);
+}
+
+fn fp64_finite_magnitude_compare(a: Fp64Bits, b: Fp64Bits) -> i32 {
+  if (a.exponent != b.exponent) {
+    return select(-1, 1, a.exponent > b.exponent);
+  }
+  return fp64_u64_compare(a.significand, b.significand);
+}
+#endif
+
+#ifndef LUMA_FP64_F32_INPUT_ONLY
+struct Fp64RawF32Bits {
+  sign: u32,
+  baseExponent: i32,
+  significand: u32,
+  isZero: bool,
+  isInf: bool,
+  isNan: bool,
+};
+
+// Decode an f32 as (-1)^sign * significand * 2^baseExponent. This shared
+// integer representation lets normalization remain independent of the
+// selected double-single arithmetic implementation.
+fn fp64_decode_raw_f32_bits(bits: u32) -> Fp64RawF32Bits {
+  let sign = bits >> 31u;
+  let exponentBits = (bits >> 23u) & 0xffu;
+  let fraction = bits & 0x7fffffu;
+
+  if (exponentBits == 0xffu) {
+    return Fp64RawF32Bits(sign, 0, 0u, false, fraction == 0u, fraction != 0u);
+  }
+  if (exponentBits == 0u) {
+    return Fp64RawF32Bits(sign, -149, fraction, fraction == 0u, false, false);
+  }
+  return Fp64RawF32Bits(
+    sign,
+    i32(exponentBits) - 150,
+    0x800000u | fraction,
+    false,
+    false,
+    false
+  );
+}
+
+fn fp64_raw_f32_magnitude_compare(aBits: u32, bBits: u32) -> i32 {
+  let aMagnitude = aBits & 0x7fffffffu;
+  let bMagnitude = bBits & 0x7fffffffu;
+  if (aMagnitude == bMagnitude) {
+    return 0;
+  }
+  return select(-1, 1, aMagnitude > bMagnitude);
+}
+
+fn fp64_make_raw_residual_f32_bits(
+  exactSign: u32,
+  exactMagnitude: vec2u,
+  exactBaseExponent: i32,
+  highBits: u32
+) -> u32 {
+  if (fp64_u64_is_zero(exactMagnitude)) {
+    return 0u;
+  }
+
+  let high = fp64_decode_raw_f32_bits(highBits);
+  if (high.isInf || high.isNan) {
+    return 0u;
+  }
+  if (high.isZero) {
+    return fp64_make_f32_bits_from_u64(exactSign, exactMagnitude, exactBaseExponent);
+  }
+
+  let commonBaseExponent = min(exactBaseExponent, high.baseExponent);
+  let exactShift = exactBaseExponent - commonBaseExponent;
+  let highShift = high.baseExponent - commonBaseExponent;
+  if (exactShift >= 64 || highShift >= 64) {
+    return 0u;
+  }
+
+  let exactAligned = fp64_u64_shift_left(exactMagnitude, u32(exactShift));
+  let highAligned = fp64_u64_shift_left(vec2u(0u, high.significand), u32(highShift));
+  let comparison = fp64_u64_compare(exactAligned, highAligned);
+  if (comparison == 0) {
+    return 0u;
+  }
+
+  var residualSign = exactSign;
+  var residualMagnitude: vec2u;
+  if (comparison > 0) {
+    residualMagnitude = fp64_u64_sub(exactAligned, highAligned);
+  } else {
+    residualSign = exactSign ^ 1u;
+    residualMagnitude = fp64_u64_sub(highAligned, exactAligned);
+  }
+  return fp64_make_f32_bits_from_u64(
+    residualSign,
+    residualMagnitude,
+    commonBaseExponent
+  );
+}
+
+fn fp64_split_raw_accumulator_bits(
+  sign: u32,
+  magnitude: vec2u,
+  baseExponent: i32
+) -> vec2u {
+  if (fp64_u64_is_zero(magnitude)) {
+    return vec2u(0u);
+  }
+  let highBits = fp64_make_f32_bits_from_u64(sign, magnitude, baseExponent);
+  let rawLowBits = fp64_make_raw_residual_f32_bits(sign, magnitude, baseExponent, highBits);
+  let lowBits = select(rawLowBits, 0u, (rawLowBits & 0x7fffffffu) == 0u);
+  if ((highBits & 0x7fffffffu) == 0u && (lowBits & 0x7fffffffu) == 0u) {
+    return vec2u(0u);
+  }
+  return vec2u(highBits, lowBits);
+}
+#endif
+
+#ifndef LUMA_FP64_F32_INPUT_ONLY
+// Round an arithmetic accumulator to binary64 before splitting it. The
+// aligned add/subtract paths retain three guard bits plus a sticky bit, which
+// is sufficient for round-to-nearest-even at the binary64 boundary.
+fn fp64_split_binary64_accumulator_bits(
+  sign: u32,
+  magnitude: vec2u,
+  baseExponent: i32
+) -> vec2u {
+  if (fp64_u64_is_zero(magnitude)) {
+    return vec2u(0u);
+  }
+
+  let mostSignificantBit = 63u - fp64_u64_count_leading_zeros(magnitude);
+  let exponent = baseExponent + i32(mostSignificantBit);
+  if (exponent > 1023) {
+    return vec2u((sign << 31u) | 0x7f800000u, 0u);
+  }
+
+  var roundedMagnitude = magnitude;
+  var roundedBaseExponent = baseExponent;
+  if (exponent >= -1022) {
+    if (mostSignificantBit > 52u) {
+      let shift = mostSignificantBit - 52u;
+      roundedMagnitude = fp64_round_shift_right(magnitude, shift);
+      roundedBaseExponent = baseExponent + i32(shift);
+    }
+  } else {
+    let shift = -1074 - baseExponent;
+    if (shift > 0) {
+      roundedMagnitude = fp64_round_shift_right(magnitude, u32(shift));
+      roundedBaseExponent = -1074;
+    }
+  }
+
+  if (fp64_u64_is_zero(roundedMagnitude)) {
+    return vec2u(0u);
+  }
+  return fp64_split_raw_accumulator_bits(sign, roundedMagnitude, roundedBaseExponent);
+}
+#endif
+
+#ifndef LUMA_FP64_PREDICATE_ONLY
+fn fp64_add_raw_f32_bits(aBits: u32, bBits: u32) -> vec2u {
+  let a = fp64_decode_raw_f32_bits(aBits);
+  let b = fp64_decode_raw_f32_bits(bBits);
+
+  if (a.isNan || b.isNan) {
+    return vec2u(0x7fc00000u, 0u);
+  }
+  if (a.isInf || b.isInf) {
+    if (a.isInf && b.isInf && a.sign != b.sign) {
+      return vec2u(0x7fc00000u, 0u);
+    }
+    return select(vec2u(bBits, 0u), vec2u(aBits, 0u), a.isInf);
+  }
+  if (a.isZero && b.isZero) {
+    return vec2u(0u);
+  }
+  if (a.isZero) {
+    return vec2u(bBits, 0u);
+  }
+  if (b.isZero) {
+    return vec2u(aBits, 0u);
+  }
+
+  let exponentDifference = abs(a.baseExponent - b.baseExponent);
+  if (exponentDifference > 25) {
+    if (fp64_raw_f32_magnitude_compare(aBits, bBits) >= 0) {
+      return vec2u(aBits, bBits);
+    }
+    return vec2u(bBits, aBits);
+  }
+
+  let commonBaseExponent = min(a.baseExponent, b.baseExponent);
+  let aMagnitude = fp64_u64_shift_left(
+    vec2u(0u, a.significand),
+    u32(a.baseExponent - commonBaseExponent)
+  );
+  let bMagnitude = fp64_u64_shift_left(
+    vec2u(0u, b.significand),
+    u32(b.baseExponent - commonBaseExponent)
+  );
+
+  var resultSign = a.sign;
+  var resultMagnitude: vec2u;
+  if (a.sign == b.sign) {
+    resultMagnitude = fp64_u64_add(aMagnitude, bMagnitude);
+  } else {
+    let comparison = fp64_u64_compare(aMagnitude, bMagnitude);
+    if (comparison == 0) {
+      return vec2u(0u);
+    }
+    if (comparison > 0) {
+      resultMagnitude = fp64_u64_sub(aMagnitude, bMagnitude);
+    } else {
+      resultSign = b.sign;
+      resultMagnitude = fp64_u64_sub(bMagnitude, aMagnitude);
+    }
+  }
+
+  return fp64_split_raw_accumulator_bits(
+    resultSign,
+    resultMagnitude,
+    commonBaseExponent
+  );
+}
+#endif
+
+#ifndef LUMA_FP64_F32_INPUT_ONLY
+fn fp64_add_aligned_magnitudes_to_fp64_bits(
+  sign: u32,
+  larger: Fp64Bits,
+  smaller: Fp64Bits
+) -> vec2u {
+  let largeSignificand = fp64_u64_shift_left(larger.significand, 3u);
+  let smallSignificand = fp64_u64_shift_right_sticky(
+    fp64_u64_shift_left(smaller.significand, 3u),
+    u32(larger.exponent - smaller.exponent)
+  );
+  let resultSignificand = fp64_u64_add(largeSignificand, smallSignificand);
+  return fp64_split_binary64_accumulator_bits(
+    sign,
+    resultSignificand,
+    larger.exponent - 55
+  );
+}
+
+fn fp64_sub_aligned_magnitudes_to_fp64_bits(
+  sign: u32,
+  larger: Fp64Bits,
+  smaller: Fp64Bits
+) -> vec2u {
+  let largeSignificand = fp64_u64_shift_left(larger.significand, 3u);
+  let smallSignificand = fp64_u64_shift_right_sticky(
+    fp64_u64_shift_left(smaller.significand, 3u),
+    u32(larger.exponent - smaller.exponent)
+  );
+  let resultSignificand = fp64_u64_sub(largeSignificand, smallSignificand);
+  return fp64_split_binary64_accumulator_bits(
+    sign,
+    resultSignificand,
+    larger.exponent - 55
+  );
+}
+
+fn fp64_add_aligned_magnitudes_to_f32_bits(sign: u32, larger: Fp64Bits, smaller: Fp64Bits) -> u32 {
+  let largeSignificand = fp64_u64_shift_left(larger.significand, 3u);
+  let smallSignificand = fp64_u64_shift_right_sticky(
+    fp64_u64_shift_left(smaller.significand, 3u),
+    u32(larger.exponent - smaller.exponent)
+  );
+  let resultSignificand = fp64_u64_add(largeSignificand, smallSignificand);
+  return fp64_make_f32_bits_from_u64(sign, resultSignificand, larger.exponent - 55);
+}
+
+fn fp64_sub_aligned_magnitudes_to_f32_bits(sign: u32, larger: Fp64Bits, smaller: Fp64Bits) -> u32 {
+  let largeSignificand = fp64_u64_shift_left(larger.significand, 3u);
+  let smallSignificand = fp64_u64_shift_right_sticky(
+    fp64_u64_shift_left(smaller.significand, 3u),
+    u32(larger.exponent - smaller.exponent)
+  );
+  let resultSignificand = fp64_u64_sub(largeSignificand, smallSignificand);
+  return fp64_make_f32_bits_from_u64(sign, resultSignificand, larger.exponent - 55);
+}
+
+// Subtract two raw binary64 values and round the exact result once to f32.
+// The input words are canonical high/low words: .x contains sign/exponent/high
+// fraction bits, and .y contains the low 32 fraction bits.
+fn sub_fp64u32_to_f32_bits(aBits: vec2u, bBits: vec2u) -> u32 {
+  let a = fp64_decode_bits(aBits);
+  let b = fp64_decode_bits(bBits);
+  let bSubtractionSign = b.sign ^ 1u;
+
+  if (a.isNan || b.isNan) {
+    return 0x7fc00000u;
+  }
+  if (a.isInf && b.isInf) {
+    if (a.sign == bSubtractionSign) {
+      return (a.sign << 31u) | 0x7f800000u;
+    }
+    return 0x7fc00000u;
+  }
+  if (a.isInf) {
+    return (a.sign << 31u) | 0x7f800000u;
+  }
+  if (b.isInf) {
+    return (bSubtractionSign << 31u) | 0x7f800000u;
+  }
+  if (a.isZero && b.isZero) {
+    return select(0u, 0x80000000u, a.sign == 1u && b.sign == 0u);
+  }
+
+  let magnitudeComparison = fp64_finite_magnitude_compare(a, b);
+  if (a.sign == bSubtractionSign) {
+    if (magnitudeComparison >= 0) {
+      return fp64_add_aligned_magnitudes_to_f32_bits(a.sign, a, b);
+    }
+    return fp64_add_aligned_magnitudes_to_f32_bits(a.sign, b, a);
+  }
+
+  if (magnitudeComparison == 0) {
+    return 0u;
+  }
+  if (magnitudeComparison > 0) {
+    return fp64_sub_aligned_magnitudes_to_f32_bits(a.sign, a, b);
+  }
+  return fp64_sub_aligned_magnitudes_to_f32_bits(bSubtractionSign, b, a);
+}
+
+fn sub_fp64u32_to_f32(aBits: vec2u, bBits: vec2u) -> f32 {
+  return bitcast<f32>(sub_fp64u32_to_f32_bits(aBits, bBits));
+}
+
+// Subtract two raw binary64 values, round once to binary64, then split the
+// result into normalized f32 limbs. Finite results must fit within the f32
+// exponent range; larger magnitudes map to infinity and smaller magnitudes
+// map to zero. The input words use canonical high/low word order.
+fn sub_fp64u32_to_fp64_bits(aBits: vec2u, bBits: vec2u) -> vec2u {
+  let a = fp64_decode_bits(aBits);
+  let b = fp64_decode_bits(bBits);
+  let bSubtractionSign = b.sign ^ 1u;
+
+  if (a.isNan || b.isNan) {
+    return vec2u(0x7fc00000u, 0u);
+  }
+  if (a.isInf && b.isInf) {
+    if (a.sign == bSubtractionSign) {
+      return vec2u((a.sign << 31u) | 0x7f800000u, 0u);
+    }
+    return vec2u(0x7fc00000u, 0u);
+  }
+  if (a.isInf) {
+    return vec2u((a.sign << 31u) | 0x7f800000u, 0u);
+  }
+  if (b.isInf) {
+    return vec2u((bSubtractionSign << 31u) | 0x7f800000u, 0u);
+  }
+  if (a.isZero && b.isZero) {
+    return vec2u(0u);
+  }
+
+  let magnitudeComparison = fp64_finite_magnitude_compare(a, b);
+  if (a.sign == bSubtractionSign) {
+    if (magnitudeComparison >= 0) {
+      return fp64_add_aligned_magnitudes_to_fp64_bits(a.sign, a, b);
+    }
+    return fp64_add_aligned_magnitudes_to_fp64_bits(a.sign, b, a);
+  }
+
+  if (magnitudeComparison == 0) {
+    return vec2u(0u);
+  }
+  if (magnitudeComparison > 0) {
+    return fp64_sub_aligned_magnitudes_to_fp64_bits(a.sign, a, b);
+  }
+  return fp64_sub_aligned_magnitudes_to_fp64_bits(bSubtractionSign, b, a);
+}
+
+fn sub_fp64u32_to_fp64(aBits: vec2u, bBits: vec2u) -> vec2f {
+  let resultBits = sub_fp64u32_to_fp64_bits(aBits, bBits);
+  return vec2f(bitcast<f32>(resultBits.x), bitcast<f32>(resultBits.y));
+}
+#endif
+
+#ifndef LUMA_FP64_PREDICATE_ONLY
+// Zero bits the compiler cannot prove are zero: the uniform ONE is 1.0 at runtime.
+// Some WGSL backends compile with fast-math flags (e.g. Firefox WebGPU on Windows, where DXC
+// runs without IEEE strictness). There, a float barrier such as \`x + ONE * 0.0\` folds away,
+// and the error terms of the double-single transforms are re-associated and lost. An integer
+// OR is opaque to floating-point simplification, and unlike XOR two nested barriers do not
+// cancel. Each rounding step the transforms rely on gets its own barrier.
+fn fp64_runtime_zero_bits() -> u32 {
+  return bitcast<u32>(fp64arithmetic.ONE) ^ 0x3f800000u;
+}
+
+fn fp64_runtime_zero() -> f32 {
+  return bitcast<f32>(fp64_runtime_zero_bits());
+}
+
+fn prevent_fp64_optimization(value: f32) -> f32 {
+#ifdef LUMA_FP64_CODE_ELIMINATION_WORKAROUND
+  return bitcast<f32>(bitcast<u32>(value) | fp64_runtime_zero_bits());
+#else
+  return value;
+#endif
+}
+#endif
+
+#ifdef LUMA_FP64_INTEGER_ARITHMETIC
+${fp64arithmeticWGSLInteger}
+#else
+#ifdef LUMA_FP64_HYBRID_ARITHMETIC
+${fp64arithmeticWGSLHybrid}
+#else
+fn split(a: f32) -> vec2f {
+  let splitValue = prevent_fp64_optimization(fp64arithmetic.SPLIT + fp64_runtime_zero());
+  let t = prevent_fp64_optimization(a * splitValue);
+  let temp = prevent_fp64_optimization(t - a);
+  let aHi = prevent_fp64_optimization(t - temp);
+  let aLo = prevent_fp64_optimization(a - aHi);
+  return vec2f(aHi, aLo);
+}
+
+fn split2(a: vec2f) -> vec2f {
+  var b = split(a.x);
+  b.y = prevent_fp64_optimization(b.y + a.y);
+  return b;
+}
+
+fn quickTwoSum(a: f32, b: f32) -> vec2f {
+  let sum = prevent_fp64_optimization(a + b);
+  let err = prevent_fp64_optimization(b - prevent_fp64_optimization(sum - a));
+  return vec2f(sum, err);
+}
+
+fn twoSum(a: f32, b: f32) -> vec2f {
+  let s = prevent_fp64_optimization(a + b);
+  let v = prevent_fp64_optimization(s - a);
+  let aErr = prevent_fp64_optimization(a - prevent_fp64_optimization(s - v));
+  let bErr = prevent_fp64_optimization(b - v);
+  let err = prevent_fp64_optimization(aErr + bErr);
+  return vec2f(s, err);
+}
+
+fn twoSub(a: f32, b: f32) -> vec2f {
+  let s = prevent_fp64_optimization(a - b);
+  let v = prevent_fp64_optimization(s - a);
+  let aErr = prevent_fp64_optimization(a - prevent_fp64_optimization(s - v));
+  let bErr = prevent_fp64_optimization(b + v);
+  let err = prevent_fp64_optimization(aErr - bErr);
+  return vec2f(s, err);
+}
+
+fn twoSqr(a: f32) -> vec2f {
+  let prod = prevent_fp64_optimization(a * a);
+  let aFp64 = split(a);
+  let highProduct = prevent_fp64_optimization(aFp64.x * aFp64.x);
+  let crossProduct = prevent_fp64_optimization(2.0 * aFp64.x * aFp64.y);
+  let lowProduct = prevent_fp64_optimization(aFp64.y * aFp64.y);
+  let highErr = prevent_fp64_optimization(highProduct - prod);
+  let err = prevent_fp64_optimization(
+    prevent_fp64_optimization(highErr + crossProduct) + lowProduct
+  );
+  return vec2f(prod, err);
+}
+
+fn twoProd(a: f32, b: f32) -> vec2f {
+  let prod = prevent_fp64_optimization(a * b);
+  let aFp64 = split(a);
+  let bFp64 = split(b);
+  let highProduct = prevent_fp64_optimization(aFp64.x * bFp64.x);
+  let crossProduct1 = prevent_fp64_optimization(aFp64.x * bFp64.y);
+  let crossProduct2 = prevent_fp64_optimization(aFp64.y * bFp64.x);
+  let lowProduct = prevent_fp64_optimization(aFp64.y * bFp64.y);
+#ifdef LUMA_FP64_CODE_ELIMINATION_WORKAROUND
+  let err1 = (highProduct - prod) * fp64arithmetic.ONE;
+  let err2 = crossProduct1 * fp64arithmetic.ONE * fp64arithmetic.ONE;
+  let err3 = crossProduct2 * fp64arithmetic.ONE * fp64arithmetic.ONE * fp64arithmetic.ONE;
+  let err4 =
+    lowProduct *
+    fp64arithmetic.ONE *
+    fp64arithmetic.ONE *
+    fp64arithmetic.ONE *
+    fp64arithmetic.ONE;
+#else
+  let err1 = highProduct - prod;
+  let err2 = crossProduct1;
+  let err3 = crossProduct2;
+  let err4 = lowProduct;
+#endif
+  let err12InputA = prevent_fp64_optimization(err1);
+  let err12InputB = prevent_fp64_optimization(err2);
+  let err12 = prevent_fp64_optimization(err12InputA + err12InputB);
+  let err123InputA = prevent_fp64_optimization(err12);
+  let err123InputB = prevent_fp64_optimization(err3);
+  let err123 = prevent_fp64_optimization(err123InputA + err123InputB);
+  let err1234InputA = prevent_fp64_optimization(err123);
+  let err1234InputB = prevent_fp64_optimization(err4);
+  let err = prevent_fp64_optimization(err1234InputA + err1234InputB);
+  return vec2f(prod, err);
+}
+
+fn sum_fp64(a: vec2f, b: vec2f) -> vec2f {
+  var s = twoSum(a.x, b.x);
+  let t = twoSum(a.y, b.y);
+  s.y = prevent_fp64_optimization(s.y + t.x);
+  s = quickTwoSum(s.x, s.y);
+  s.y = prevent_fp64_optimization(s.y + t.y);
+  s = quickTwoSum(s.x, s.y);
+  return s;
+}
+
+fn sub_fp64(a: vec2f, b: vec2f) -> vec2f {
+  var s = twoSub(a.x, b.x);
+  let t = twoSub(a.y, b.y);
+  s.y = prevent_fp64_optimization(s.y + t.x);
+  s = quickTwoSum(s.x, s.y);
+  s.y = prevent_fp64_optimization(s.y + t.y);
+  s = quickTwoSum(s.x, s.y);
+  return s;
+}
+
+fn mul_fp64(a: vec2f, b: vec2f) -> vec2f {
+  var prod = twoProd(a.x, b.x);
+  let crossProduct1 = prevent_fp64_optimization(a.x * b.y);
+  prod.y = prevent_fp64_optimization(prod.y + crossProduct1);
+#ifdef LUMA_FP64_HIGH_BITS_OVERFLOW_WORKAROUND
+  prod = split2(prod);
+#endif
+  prod = quickTwoSum(prod.x, prod.y);
+  let crossProduct2 = prevent_fp64_optimization(a.y * b.x);
+  prod.y = prevent_fp64_optimization(prod.y + crossProduct2);
+#ifdef LUMA_FP64_HIGH_BITS_OVERFLOW_WORKAROUND
+  prod = split2(prod);
+#endif
+  prod = quickTwoSum(prod.x, prod.y);
+  return prod;
+}
+
+#ifndef LUMA_FP64_PREDICATE_ONLY
+fn div_fp64(a: vec2f, b: vec2f) -> vec2f {
+  let xn = prevent_fp64_optimization(1.0 / b.x);
+  let yn = mul_fp64(a, vec2f(xn, fp64_runtime_zero()));
+  let diff = prevent_fp64_optimization(sub_fp64(a, mul_fp64(b, yn)).x);
+  let prod = twoProd(xn, diff);
+  return sum_fp64(yn, prod);
+}
+
+fn sqrt_fp64(a: vec2f) -> vec2f {
+  if (a.x == 0.0 && a.y == 0.0) {
+    return vec2f(0.0, 0.0);
+  }
+  if (a.x < 0.0) {
+    let nanValue = fp64_nan(a.x);
+    return vec2f(nanValue, nanValue);
+  }
+
+  let x = prevent_fp64_optimization(1.0 / sqrt(a.x));
+  let yn = prevent_fp64_optimization(a.x * x);
+#ifdef LUMA_FP64_CODE_ELIMINATION_WORKAROUND
+  let ynSqr = twoSqr(yn) * fp64arithmetic.ONE;
+#else
+  let ynSqr = twoSqr(yn);
+#endif
+  let diff = prevent_fp64_optimization(sub_fp64(a, ynSqr).x);
+  let prod = twoProd(prevent_fp64_optimization(x * 0.5), diff);
+#ifdef LUMA_FP64_HIGH_BITS_OVERFLOW_WORKAROUND
+  return sum_fp64(split(yn), prod);
+#else
+  return sum_fp64(vec2f(yn, 0.0), prod);
+#endif
+}
+#endif
+#endif
+#endif
+
+#ifndef LUMA_FP64_PREDICATE_ONLY
+fn fp64_f32_bits_is_nan(bits: u32) -> bool {
+  return (bits & 0x7fffffffu) > 0x7f800000u;
+}
+
+fn fp64_f32_bits_is_inf(bits: u32) -> bool {
+  return (bits & 0x7fffffffu) == 0x7f800000u;
+}
+
+fn fp64_compare_f32_bits(aBits: u32, bBits: u32) -> i32 {
+  let aMagnitude = aBits & 0x7fffffffu;
+  let bMagnitude = bBits & 0x7fffffffu;
+  if (aMagnitude == 0u && bMagnitude == 0u) {
+    return 0;
+  }
+  let aSign = aBits >> 31u;
+  let bSign = bBits >> 31u;
+  if (aSign != bSign) {
+    return select(1, -1, aSign == 1u);
+  }
+  if (aMagnitude == bMagnitude) {
+    return 0;
+  }
+  let magnitudeComparison = select(-1, 1, aMagnitude > bMagnitude);
+  return select(magnitudeComparison, -magnitudeComparison, aSign == 1u);
+}
+
+// Normalize an arbitrary pair of finite f32 limbs with integer accumulation.
+// This is independent of LUMA_FP64_INTEGER_ARITHMETIC and canonicalizes every
+// representation of zero to vec2f(+0.0, +0.0).
+fn normalize_fp64(value: vec2f) -> vec2f {
+  let resultBits = fp64_add_raw_f32_bits(bitcast<u32>(value.x), bitcast<u32>(value.y));
+  return vec2f(bitcast<f32>(resultBits.x), bitcast<f32>(resultBits.y));
+}
+
+fn is_nan_fp64(value: vec2f) -> bool {
+  let normalized = normalize_fp64(value);
+  return fp64_f32_bits_is_nan(bitcast<u32>(normalized.x)) ||
+    fp64_f32_bits_is_nan(bitcast<u32>(normalized.y));
+}
+
+fn is_finite_fp64(value: vec2f) -> bool {
+  let normalized = normalize_fp64(value);
+  let highBits = bitcast<u32>(normalized.x);
+  let lowBits = bitcast<u32>(normalized.y);
+  return !fp64_f32_bits_is_nan(highBits) && !fp64_f32_bits_is_nan(lowBits) &&
+    !fp64_f32_bits_is_inf(highBits) && !fp64_f32_bits_is_inf(lowBits);
+}
+
+// Returns -1, 0, or 1. NaN is unordered and returns 0; call is_nan_fp64 or
+// is_finite_fp64 first when 0 must mean a finite zero.
+fn sign_fp64(value: vec2f) -> i32 {
+  let normalized = normalize_fp64(value);
+  let highBits = bitcast<u32>(normalized.x);
+  let lowBits = bitcast<u32>(normalized.y);
+  if (fp64_f32_bits_is_nan(highBits) || fp64_f32_bits_is_nan(lowBits)) {
+    return 0;
+  }
+  if ((highBits & 0x7fffffffu) != 0u) {
+    return select(1, -1, (highBits >> 31u) == 1u);
+  }
+  if ((lowBits & 0x7fffffffu) != 0u) {
+    return select(1, -1, (lowBits >> 31u) == 1u);
+  }
+  return 0;
+}
+
+// Compares double-single values and returns -1, 0, or 1. NaN is unordered
+// and returns 0; callers that require equality semantics must first check
+// is_nan_fp64 or is_finite_fp64.
+fn compare_fp64(a: vec2f, b: vec2f) -> i32 {
+  let normalizedA = normalize_fp64(a);
+  let normalizedB = normalize_fp64(b);
+  let aHighBits = bitcast<u32>(normalizedA.x);
+  let aLowBits = bitcast<u32>(normalizedA.y);
+  let bHighBits = bitcast<u32>(normalizedB.x);
+  let bLowBits = bitcast<u32>(normalizedB.y);
+  if (fp64_f32_bits_is_nan(aHighBits) || fp64_f32_bits_is_nan(aLowBits) ||
+      fp64_f32_bits_is_nan(bHighBits) || fp64_f32_bits_is_nan(bLowBits)) {
+    return 0;
+  }
+  let highComparison = fp64_compare_f32_bits(aHighBits, bHighBits);
+  if (highComparison != 0) {
+    return highComparison;
+  }
+  return fp64_compare_f32_bits(aLowBits, bLowBits);
+}
+
+// Scale by 2^exponent under either arithmetic. The classic path repacks each limb with
+// integer operations and no floating-point arithmetic, so hardware that flushes subnormal
+// results cannot zero a subnormal limb. Its result has the value of the integer path's,
+// which also renormalizes the limbs; that renormalization would flush them here.
+fn fp64_scale_fp64(value: vec2f, exponent: i32) -> vec2f {
+#ifdef LUMA_FP64_INTEGER_ARITHMETIC
+  return fp64_scale_fp64_integer(value, exponent);
+#else
+  let high = fp64_decode_raw_f32_bits(bitcast<u32>(value.x));
+  let low = fp64_decode_raw_f32_bits(bitcast<u32>(value.y));
+  var scaledHigh = value.x;
+  if (!(high.isZero || high.isInf || high.isNan)) {
+    scaledHigh = bitcast<f32>(
+      fp64_make_f32_bits_from_u64(high.sign, vec2u(0u, high.significand), high.baseExponent + exponent)
+    );
+  }
+  var scaledLow = value.y;
+  if (!(low.isZero || low.isInf || low.isNan)) {
+    scaledLow = bitcast<f32>(
+      fp64_make_f32_bits_from_u64(low.sign, vec2u(0u, low.significand), low.baseExponent + exponent)
+    );
+  }
+  return vec2f(scaledHigh, scaledLow);
+#endif
+}
+#endif
+`;