@photivra/engine 1.5.0 → 2.1.0

package/dist/gpu/wgsl-sensor-kernel.js

@@ -0,0 +1,409 @@
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * WGSL kernel for the `webgpu-f32` sensor stage (#277), one invocation per native site. It evaluates the
+ * reference `simulatePackedSensorSite` (packed-sensor-raw.ts) with the same seeded noise model
+ * (`photivra-native-raw-noise` 0.4.0):
+ * - the MurmurHash3_x86_32 site seeds, fmix32 scrambling and xorshift32 streams are 32-bit integer
+ *   arithmetic, so every random state is bit-identical to the reference;
+ * - every comparison against a uniform value is made exactly on the integer state where the reference
+ *   value is an exact dyadic rational;
+ * - every other decision (Poisson multiplication-method threshold, PTRS floor and acceptance tests,
+ *   physical and pre-ADC limits, ADC and digital-gain rounding) is evaluated in float32 together with an
+ *   a-priori bound on how far the float32 value can be from the float64 reference value. A site where any
+ *   decision falls inside its bound is flagged. An unflagged site therefore makes every decision exactly
+ *   as the reference does, and its code and flags equal the reference.
+ * - Fixed-pattern noise and additional stored-charge components (2.0): the host evaluates the
+ *   fixed-pattern values in float64 with the model's own functions and packs, per site, the Poisson means
+ *   already multiplied by the PRNU and dark factors (exactly the reference's products), the pre-ADC limit
+ *   minus the readout offset and the offset in codes. Additional Poisson components are drawn from the
+ *   charge stream after photo and dark charge, in declaration order; deterministic components enter as an
+ *   exact integer part plus a float32 fraction, with their own bounds.
+ *
+ * The bounds use u = 2^-24 and the WGSL accuracy table (W3C WGSL, "Floating Point Accuracy"): correctly
+ * rounded + − ×, division 2.5 ULP, sqrt inherited from 1/inverseSqrt (≤ 4.5 ULP), exp 3 + 2|x| ULP,
+ * log 3 ULP or 2^-21 absolute on [0.5, 2]. WGSL's built-in cos has only a 2^-11 absolute bound, so
+ * the kernel uses its own range-reduced Taylor polynomials. Every margin is doubled (`MARGIN`).
+ * See docs/WEBGPU.md.
+ */
+/** Output word: code in bits 0–15, saturation flags in bits 16–19, the flagged (uncertain) bit 20. */
+export const SENSOR_OUTPUT = Object.freeze({ codeMask: 0xffff, flagShift: 16, uncertainBit: 1 << 20 });
+/** Auxiliary table layout (float32 words). */
+export const SENSOR_AUX = Object.freeze({ stirlingRemainder: 0, sigma: 256, inversePieces: 272, digitalPieces: 276, knots: 280 });
+export const SENSOR_WORKGROUP_SIZE = 256;
+/**
+ * Per-site packed words: photo word, photo fraction bits, dark word, dark fraction bits; then 2 words per
+ * additional Poisson component; then, with deterministic components, their sum (integer part, fraction
+ * bits); then, with fixed-pattern noise, (threshold − offset) and offset/gain as (floor, fraction bits).
+ */
+export const SENSOR_SITE_WORDS = 4;
+/** Bits of `Params.extras`. */
+export const SENSOR_EXTRAS = Object.freeze({ deterministic: 1, fixedPattern: 2 });
+/** Partial charge sums saturate here: above every supported capacity (2^24), so no decision changes. */
+export const SENSOR_REALIZED_CLAMP = 2 ** 30;
+/** Mean categories in bits 30–31 of a mean word, mirroring `samplePoisson`'s branches. */
+export const SENSOR_MEAN_CATEGORY = Object.freeze({ zero: 0, multiplication: 1, transformedRejection: 2, normal: 3 });
+/** Smallest integer m with (m + 0.5)/2^32 ≥ t, so `uS ≥ t` ⇔ `m ≥ result` exactly. */
+export function uniformThresholdInteger(t) {
+    return Math.ceil(t * 2 ** 32 - 0.5);
+}
+const SIN_COEFFICIENTS = [-1 / 6, 1 / 120, -1 / 5040, 1 / 362880, -1 / 39916800];
+const COS_COEFFICIENTS = [-1 / 2, 1 / 24, -1 / 720, 1 / 40320, -1 / 3628800, 1 / 479001600];
+const literal = (value) => {
+    const text = String(Math.fround(value));
+    return /[.e]/u.test(text) ? text : `${text}.0`;
+};
+export const SENSOR_RAW_WGSL = /* wgsl */ `
+const U: f32 = 5.9604644775390625e-8;
+const INV32: f32 = 2.3283064365386963e-10;
+const TWO_PI_INV32: f32 = ${literal(2 * Math.PI / 2 ** 32)};
+const MARGIN: f32 = 2.0;
+const CPU_REL: f32 = 7.105427357601002e-15;
+struct Params {
+  count: u32, row0: u32, region_w: u32, native_w: u32,
+  region_x: u32, region_y: u32, seed: u32, components: u32,
+  knots: u32, digital: u32, saturation: u32, adc_max: u32,
+  capacity_floor: u32, threshold_floor: u32, black: u32, sat_quantized: i32,
+  capacity_frac: f32, threshold_frac: f32, inv_gain: f32, inv_residual: f32,
+  digital_gain: f32, digital_residual: f32, inl_slope: f32, inl_offset: f32,
+  site_words: u32, poisson_count: u32, extras: u32, pad0: u32,
+  fpn_e_slack: f32, fpn_c_slack: f32, pad1: f32, pad2: f32,
+};
+@group(0) @binding(0) var<uniform> params: Params;
+@group(0) @binding(1) var<storage, read> means: array<u32>;
+@group(0) @binding(2) var<storage, read> aux: array<f32>;
+@group(0) @binding(3) var<storage, read_write> out: array<u32>;
+
+var<private> state: u32;
+var<private> uncertain: bool;
+var<private> radius: f32;
+var<private> ip: i32;
+var<private> fp: f32;
+
+fn fmix(value: u32) -> u32 {
+  var h = value;
+  h = h ^ (h >> 16u); h = h * 0x85ebca6bu; h = h ^ (h >> 13u); h = h * 0xc2b2ae35u; h = h ^ (h >> 16u);
+  return h;
+}
+fn mix_block(hash: u32, block: u32) -> u32 {
+  var k = block * 0xcc9e2d51u;
+  k = (k << 15u) | (k >> 17u);
+  k = k * 0x1b873593u;
+  var h = hash ^ k;
+  h = (h << 13u) | (h >> 19u);
+  return h * 5u + 0xe6546b64u;
+}
+fn site_seed(seed: u32, index: u32, stream: u32) -> u32 {
+  return fmix(mix_block(mix_block(mix_block(0u, seed), index), stream) ^ 12u);
+}
+fn start(seed: u32) {
+  state = fmix(seed ^ 0x9e3779b9u);
+  if (state == 0u) { state = 0x6d2b79f5u; }
+}
+fn next_bits() -> u32 {
+  state = state ^ (state << 13u); state = state ^ (state >> 17u); state = state ^ (state << 5u);
+  return state;
+}
+fn unit(bits: u32) -> f32 { return (f32(bits) + 0.5) * INV32; }
+fn check(distance: f32, margin: f32) { if (abs(distance) <= margin) { uncertain = true; } }
+
+// ln((bits + 0.5)/2^32), relative error <= 12u: WGSL log below 1/2, else 2·atanh(r) on the exact 1 - u.
+fn ln_unit(bits: u32) -> f32 {
+  if (bits < 0x80000000u) { return log(unit(bits)); }
+  let w = (f32(0xffffffffu - bits) + 0.5) * INV32;
+  let r = -w / (2.0 - w);
+  let r2 = r * r;
+  var p = ${literal(1 / 23)};
+  ${[21, 19, 17, 15, 13, 11, 9, 7, 5, 3].map(n => `p = p * r2 + ${literal(1 / n)};`).join(" ")}
+  p = p * r2 + 1.0;
+  return 2.0 * r * p;
+}
+// cos(2π (bits + 0.5)/2^32): exact octant reduction on the integer, Taylor polynomials on [0, π/4].
+fn cos_unit(bits: u32) -> f32 {
+  let octant = bits >> 29u;
+  let low = bits & 0x1fffffffu;
+  let even = (octant & 1u) == 0u;
+  var g = (f32(0x1fffffffu - low) + 0.5) * TWO_PI_INV32;
+  if (even) { g = (f32(low) + 0.5) * TWO_PI_INV32; }
+  let g2 = g * g;
+  var c = ${literal(COS_COEFFICIENTS[5])};
+  ${[4, 3, 2, 1, 0].map(j => `c = c * g2 + ${literal(COS_COEFFICIENTS[j])};`).join(" ")}
+  c = c * g2 + 1.0;
+  var s = ${literal(SIN_COEFFICIENTS[4])};
+  ${[3, 2, 1, 0].map(j => `s = s * g2 + ${literal(SIN_COEFFICIENTS[j])};`).join(" ")}
+  s = g * (s * g2 + 1.0);
+  let cb = select(s, c, even);
+  let sb = select(c, s, even);
+  let q = octant >> 1u;
+  if (q == 0u) { return cb; }
+  if (q == 1u) { return -sb; }
+  if (q == 2u) { return -cb; }
+  return sb;
+}
+// Box–Muller, one value per two uniforms; |z_f32 − z_f64| <= 24u·radius.
+fn std_normal() -> f32 {
+  let b1 = next_bits();
+  let b2 = next_bits();
+  radius = sqrt(-2.0 * ln_unit(b1));
+  return radius * cos_unit(b2);
+}
+fn poisson_small(mean: f32) -> u32 {
+  let threshold = exp(-mean);
+  var product = 1.0;
+  var count = 0u;
+  loop {
+    count = count + 1u;
+    product = product * unit(next_bits());
+    check(product - threshold, MARGIN * (3.0 * f32(count) + 130.0) * U * max(product, threshold));
+    if (!(product > threshold)) { break; }
+    if (count >= 1000u) { uncertain = true; break; }
+  }
+  return count - 1u;
+}
+// g(t) = ln(1 + t) − t for |t| <= 1/2 without cancellation: −t²/(2+t) + 2(s³/3 + s⁵/5 + …), s = t/(2+t).
+fn log1p_minus(t: f32) -> f32 {
+  let d = 2.0 + t;
+  let s = t / d;
+  let s2 = s * s;
+  var p = ${literal(1 / 25)};
+  ${[23, 21, 19, 17, 15, 13, 11, 9, 7, 5].map(n => `p = p * s2 + ${literal(1 / n)};`).join(" ")}
+  p = p * s2 + ${literal(1 / 3)};
+  return -(t * t) / d + 2.0 * s * s2 * p;
+}
+fn poisson_ptrs(mi: u32, mf: f32) -> u32 {
+  let mean = f32(mi) + mf;
+  let sq = sqrt(mean);
+  let b = 0.931 + 2.53 * sq;
+  let a = -0.059 + 0.02483 * b;
+  let inv_alpha = 1.1239 + 1.1328 / (b - 3.4);
+  let v_r = 0.9277 - 3.6224 / (b - 2.0);
+  let ln_mean = log(mean);
+  for (var iteration = 0u; iteration < 64u; iteration = iteration + 1u) {
+    let su = next_bits();
+    let sv = next_bits();
+    var m = su;
+    var sign = -1.0;
+    if (su >= 0x80000000u) { m = 0xffffffffu - su; sign = 1.0; }
+    let us = unit(m);
+    let u = sign * unit(0x7fffffffu - m);
+    let v = unit(sv);
+    let scale = 2.0 * a / us + b;
+    let t = scale * u;
+    let x = (t + mf) + 0.43;
+    let fl = floor(x);
+    let ex = MARGIN * (abs(t) * 40.0 * U + 2.0 * U * (abs(t) + 1.0) + 4.5e-16 * (abs(t) + mean + 1.0));
+    if (x - fl <= ex || fl + 1.0 - x <= ex) { uncertain = true; }
+    if (fl <= -1073741824.0 || fl >= 1073741824.0) { continue; }
+    let k = i32(mi) + i32(fl);
+    if (m >= ${uniformThresholdInteger(0.07)}u) {
+      if (k >= 0) { check(v - v_r, MARGIN * (16.0 * U + 2.0 * U * v)); }
+      if (v <= v_r && k >= 0) { return u32(k); }
+    }
+    if (k < 0 || (m < ${uniformThresholdInteger(0.013)}u && sv > m)) { continue; }
+    let lhs = log(v * inv_alpha / (a / (us * us) + b));
+    let dl = 60.0 * U + 4.76837158203125e-7 + 4.0 * U * abs(lhs);
+    var rhs: f32;
+    var dr: f32;
+    if (k == 0) {
+      rhs = -mean;
+      dr = 2.0 * U * mean + CPU_REL * mean;
+    } else {
+      let kf = f32(k);
+      let dd = f32(i32(mi) - k) + mf;
+      let t2 = dd / kf;
+      var kg: f32;
+      var dk: f32;
+      if (abs(t2) <= 0.5) {
+        kg = kf * log1p_minus(t2);
+        dk = 20.0 * U * abs(kg);
+      } else {
+        let lr = log(mean / kf);
+        kg = kf * lr - dd;
+        dk = kf * (8.0 * U + 4.76837158203125e-7 + 5.0 * U * abs(lr)) + 3.0 * U * abs(dd) + U * abs(kg);
+      }
+      let lnk = log(kf);
+      var corr: f32;
+      if (k < 256) { corr = aux[u32(k)]; } else { corr = 1.0 / (12.0 * kf) - 1.0 / (360.0 * kf * kf * kf); }
+      rhs = ((kg - 0.5 * lnk) - 0.91893853320467274) - corr;
+      dr = dk + 2.384185791015625e-7 + 2.0 * U * lnk + U + 0.6 * U + 3.0 * U * (abs(kg) + 0.5 * lnk + 1.0) +
+        CPU_REL * (mean + kf * (abs(ln_mean) + lnk + 1.0));
+    }
+    check(lhs - rhs, MARGIN * (dl + dr));
+    if (lhs <= rhs) { return u32(k); }
+  }
+  uncertain = true;
+  return mi;
+}
+fn poisson(word: u32, frac: f32) -> u32 {
+  let category = word >> 30u;
+  let mi = word & 0x3fffffffu;
+  if (category == 0u) { return 0u; }
+  if (category == 1u) { return poisson_small(f32(mi) + frac); }
+  if (category == 2u) { return poisson_ptrs(mi, frac); }
+  // Means of 1e9 and above: the draw always exceeds the capacity (<= 2^24), so the code does not depend on it.
+  let z = std_normal();
+  let y = sqrt(f32(mi) + frac) * z + frac + 0.5;
+  return u32(max(0, i32(mi) + i32(floor(y))));
+}
+// Adds an exact non-negative float32 value split into integer and fractional parts.
+fn accumulate(value: f32) {
+  let f = floor(value);
+  ip = ip + i32(f);
+  fp = fp + (value - f);
+}
+// n·(p0 + p1 + p2 + p3) for an integer 0 <= n < 2^24 and 12-bit pieces: every product is exact.
+fn scaled(n: u32, base: u32) {
+  let hi = f32(n >> 12u);
+  let lo = f32(n & 0xfffu);
+  for (var j = 0u; j < 4u; j = j + 1u) {
+    let piece = aux[base + j];
+    accumulate(hi * piece * 4096.0);
+    accumulate(lo * piece);
+  }
+}
+fn normalize() {
+  let f = floor(fp);
+  ip = ip + i32(f);
+  fp = fp - f;
+}
+fn inl_offset(code: f32) -> f32 {
+  let n = params.knots;
+  let base = ${"280u"};
+  if (code <= aux[base]) { return aux[base + n]; }
+  if (code >= aux[base + n - 1u]) { return aux[base + 2u * n - 1u]; }
+  var low = 0u;
+  var high = n - 1u;
+  loop {
+    if (high - low <= 1u) { break; }
+    let middle = (low + high) >> 1u;
+    if (aux[base + middle] <= code) { low = middle; } else { high = middle; }
+  }
+  let ac = aux[base + low];
+  let ao = aux[base + n + low];
+  return ao + ((code - ac) / (aux[base + high] - ac)) * (aux[base + n + high] - ao);
+}
+@compute @workgroup_size(${SENSOR_WORKGROUP_SIZE})
+fn main(@builtin(global_invocation_id) gid: vec3<u32>, @builtin(num_workgroups) groups: vec3<u32>) {
+  let i = gid.x + gid.y * groups.x * ${SENSOR_WORKGROUP_SIZE}u;
+  if (i >= params.count) { return; }
+  let row = params.row0 + i / params.region_w;
+  let index = (params.region_y + row) * params.native_w + params.region_x + i % params.region_w;
+  uncertain = false;
+  let base = i * params.site_words;
+  start(site_seed(params.seed, index, 0u));
+  let photo = poisson(means[base], bitcast<f32>(means[base + 1u]));
+  let dark = poisson(means[base + 2u], bitcast<f32>(means[base + 3u]));
+  // Saturating sums: any total above the clamp is above every supported capacity, as in the reference.
+  var realized = min(photo + dark, ${SENSOR_REALIZED_CLAMP}u);
+  for (var k = 0u; k < params.poisson_count; k = k + 1u) {
+    realized = min(realized + poisson(means[base + 4u + 2u * k], bitcast<f32>(means[base + 5u + 2u * k])), ${SENSOR_REALIZED_CLAMP}u);
+  }
+  var cursor = base + 4u + 2u * params.poisson_count;
+  let deterministic = (params.extras & ${SENSOR_EXTRAS.deterministic}u) != 0u;
+  let pattern = (params.extras & ${SENSOR_EXTRAS.fixedPattern}u) != 0u;
+  var r_frac = 0.0;
+  var det_slack = 0.0;
+  if (deterministic) {
+    realized = min(realized + means[cursor], ${SENSOR_REALIZED_CLAMP}u);
+    r_frac = bitcast<f32>(means[cursor + 1u]);
+    det_slack = U;
+    cursor = cursor + 2u;
+  }
+  var t_floor = i32(params.threshold_floor);
+  var t_frac = params.threshold_frac;
+  var fpn_e_slack = 0.0;
+  if (pattern) {
+    t_floor = bitcast<i32>(means[cursor]);
+    t_frac = bitcast<f32>(means[cursor + 1u]);
+    fpn_e_slack = params.fpn_e_slack;
+  }
+  start(site_seed(params.seed, index, 1u));
+  var noise = 0.0;
+  var noise_bound = 0.0;
+  for (var c = 0u; c < params.components; c = c + 1u) {
+    let sigma = aux[256u + c];
+    if (sigma != 0.0) {
+      let z = std_normal();
+      noise = noise + z * sigma;
+      noise_bound = noise_bound + sigma * radius * (28.0 + f32(params.components)) * U;
+    }
+  }
+  var flags = 0u;
+  var a_int = realized;
+  var a_frac = r_frac;
+  if (!deterministic) {
+    if (realized > params.capacity_floor) { flags = 1u; a_int = params.capacity_floor; a_frac = params.capacity_frac; }
+  } else {
+    // realized > capacity with a non-integer total: float32 difference with its bound.
+    let w = f32(i32(realized) - i32(params.capacity_floor));
+    let dcap = w + (r_frac - params.capacity_frac);
+    check(dcap, MARGIN * (5.0 * U + U * abs(w) + U * abs(dcap) + CPU_REL * (f32(realized) + 1.0)));
+    if (dcap > 0.0) { flags = 1u; a_int = params.capacity_floor; a_frac = params.capacity_frac; }
+  }
+  // Pre-ADC limit: afterPhysical + noise + offset > threshold, as afterPhysical − (threshold − offset) + noise > 0.
+  let whole = f32(i32(a_int) - t_floor);
+  let diff = (whole + (a_frac - t_frac)) + noise;
+  check(diff, MARGIN * (noise_bound + 3.0 * U + 2.0 * U * (abs(whole) + abs(noise) + 2.0) + CPU_REL * (f32(a_int) + abs(noise)) +
+    fpn_e_slack + det_slack));
+  var q: i32;
+  if (diff > 0.0) {
+    flags = flags | 2u;
+    q = params.sat_quantized;
+  } else {
+    // ideal = (afterPhysical + noise + offset)/gain + black, as an exact integer part plus a float32 fraction.
+    ip = i32(params.black);
+    fp = 0.0;
+    var fpn_c_slack = 0.0;
+    if (pattern) {
+      ip = ip + bitcast<i32>(means[cursor + 2u]);
+      fp = bitcast<f32>(means[cursor + 3u]);
+      fpn_c_slack = params.fpn_c_slack;
+    }
+    scaled(a_int, 272u);
+    let noise_codes = noise * params.inv_gain;
+    fp = fp + a_frac * params.inv_gain + noise_codes;
+    let a_codes = f32(a_int) * params.inv_gain;
+    var margin = MARGIN * (10.0 * U * (10.0 + abs(noise_codes)) + 4.0 * U * a_frac * params.inv_gain + 3.0 * U * abs(noise_codes) +
+      noise_bound * params.inv_gain + f32(a_int) * params.inv_residual + 2.0 * U +
+      CPU_REL * (a_codes + abs(noise_codes) + f32(params.black)) + fpn_c_slack + det_slack * params.inv_gain);
+    normalize();
+    if (params.knots > 0u) {
+      let ideal = f32(ip) + fp;
+      let offset = inl_offset(ideal);
+      fp = fp + offset;
+      margin = margin * (1.0 + params.inl_slope) + MARGIN * (params.inl_slope * 2.0 * U * (abs(ideal) + 1.0) +
+        8.0 * U * (params.inl_offset + 1.0) + CPU_REL * (abs(ideal) + params.inl_offset) * (1.0 + params.inl_slope));
+      normalize();
+    }
+    if ((params.digital & 4u) == 0u) { check(fp - 0.5, margin); }
+    q = ip + select(0, 1, fp >= 0.5);
+  }
+  if ((params.digital & 1u) != 0u) {
+    let adc = clamp(q, 0, i32(params.adc_max));
+    let delta = adc - i32(params.black);
+    let estimate = f32(delta) * params.digital_gain;
+    if (estimate >= 16777216.0) {
+      q = 16777216;
+    } else if (estimate <= -16777216.0) {
+      q = -16777216;
+    } else {
+      ip = 0;
+      fp = 0.0;
+      scaled(u32(abs(delta)), 276u);
+      if ((params.digital & 2u) == 0u) { check(fp - floor(fp) - 0.5, MARGIN * (40.0 * U * 9.0 + f32(abs(delta)) * params.digital_residual + CPU_REL * (abs(estimate) + f32(params.black)))); }
+      normalize();
+      var whole_part = ip;
+      var fraction = fp;
+      if (delta < 0) {
+        whole_part = -ip;
+        if (fp > 0.0) { whole_part = -ip - 1; fraction = 1.0 - fp; }
+      }
+      q = whole_part + i32(params.black) + select(0, 1, fraction >= 0.5);
+    }
+  }
+  if (q >= i32(params.saturation)) { flags = flags | 4u; }
+  if (q < 0) { flags = flags | 8u; }
+  let code = u32(clamp(q, 0, i32(params.saturation)));
+  out[i] = code | (flags << 16u) | select(0u, ${1 << 20}u, uncertain);
+}
+`;
+//# sourceMappingURL=wgsl-sensor-kernel.js.map

package/dist/gpu/wgsl-sensor-kernel.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"wgsl-sensor-kernel.js","sourceRoot":"","sources":["../../src/gpu/wgsl-sensor-kernel.ts"],"names":[],"mappings":"AAAA,sCAAsC;AAEtC;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH,sGAAsG;AACtG,MAAM,CAAC,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,EAAE,EAAE,YAAY,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;AACvG,8CAA8C;AAC9C,MAAM,CAAC,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,iBAAiB,EAAE,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,aAAa,EAAE,GAAG,EAAE,aAAa,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;AAClI,MAAM,CAAC,MAAM,qBAAqB,GAAG,GAAG,CAAC;AACzC;;;;GAIG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC;AACnC,+BAA+B;AAC/B,MAAM,CAAC,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC,CAAC;AAClF,wGAAwG;AACxG,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,IAAI,EAAE,CAAC;AAC7C,0FAA0F;AAC1F,MAAM,CAAC,MAAM,oBAAoB,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,oBAAoB,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC;AAEtH,sFAAsF;AACtF,MAAM,UAAU,uBAAuB,CAAC,CAAS;IAC/C,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,GAAG,CAAC,CAAC;AACtC,CAAC;AAED,MAAM,gBAAgB,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,CAAC,GAAG,IAAI,EAAE,CAAC,GAAG,MAAM,EAAE,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC;AACjF,MAAM,gBAAgB,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,GAAG,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,CAAC,GAAG,OAAO,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;AAC5F,MAAM,OAAO,GAAG,CAAC,KAAa,EAAU,EAAE;IACxC,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IACxC,OAAO,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,CAAC;AACjD,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,eAAe,GAAG,UAAU,CAAC;;;4BAGd,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;YAyD9C,OAAO,CAAC,CAAC,GAAG,EAAE,CAAC;IACvB,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,gBAAgB,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;;;;;;;;;;;;YAYlF,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAE,CAAC;IACrC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,gBAAgB,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;;YAE5E,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAE,CAAC;IACrC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,gBAAgB,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;YAmCzE,OAAO,CAAC,CAAC,GAAG,EAAE,CAAC;IACvB,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,gBAAgB,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;iBAC9E,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;eA4BhB,uBAAuB,CAAC,IAAI,CAAC;;;;wBAIpB,uBAAuB,CAAC,KAAK,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;eAqEvC,MAAM;;;;;;;;;;;;;;2BAcM,qBAAqB;;uCAET,qBAAqB;;;;;;;;;;qCAUvB,qBAAqB;;6GAEmD,qBAAqB;;;yCAGzF,aAAa,CAAC,aAAa;mCACjC,aAAa,CAAC,YAAY;;;;+CAId,qBAAqB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;gDAoGpB,CAAC,IAAI,EAAE;;CAEtD,CAAC","sourcesContent":["// SPDX-License-Identifier: Apache-2.0\n\n/**\n * WGSL kernel for the `webgpu-f32` sensor stage (#277), one invocation per native site. It evaluates the\n * reference `simulatePackedSensorSite` (packed-sensor-raw.ts) with the same seeded noise model\n * (`photivra-native-raw-noise` 0.4.0):\n * - the MurmurHash3_x86_32 site seeds, fmix32 scrambling and xorshift32 streams are 32-bit integer\n *   arithmetic, so every random state is bit-identical to the reference;\n * - every comparison against a uniform value is made exactly on the integer state where the reference\n *   value is an exact dyadic rational;\n * - every other decision (Poisson multiplication-method threshold, PTRS floor and acceptance tests,\n *   physical and pre-ADC limits, ADC and digital-gain rounding) is evaluated in float32 together with an\n *   a-priori bound on how far the float32 value can be from the float64 reference value. A site where any\n *   decision falls inside its bound is flagged. An unflagged site therefore makes every decision exactly\n *   as the reference does, and its code and flags equal the reference.\n * - Fixed-pattern noise and additional stored-charge components (2.0): the host evaluates the\n *   fixed-pattern values in float64 with the model's own functions and packs, per site, the Poisson means\n *   already multiplied by the PRNU and dark factors (exactly the reference's products), the pre-ADC limit\n *   minus the readout offset and the offset in codes. Additional Poisson components are drawn from the\n *   charge stream after photo and dark charge, in declaration order; deterministic components enter as an\n *   exact integer part plus a float32 fraction, with their own bounds.\n *\n * The bounds use u = 2^-24 and the WGSL accuracy table (W3C WGSL, \"Floating Point Accuracy\"): correctly\n * rounded + − ×, division 2.5 ULP, sqrt inherited from 1/inverseSqrt (≤ 4.5 ULP), exp 3 + 2|x| ULP,\n * log 3 ULP or 2^-21 absolute on [0.5, 2]. WGSL's built-in cos has only a 2^-11 absolute bound, so\n * the kernel uses its own range-reduced Taylor polynomials. Every margin is doubled (`MARGIN`).\n * See docs/WEBGPU.md.\n */\n\n/** Output word: code in bits 0–15, saturation flags in bits 16–19, the flagged (uncertain) bit 20. */\nexport const SENSOR_OUTPUT = Object.freeze({ codeMask: 0xffff, flagShift: 16, uncertainBit: 1 << 20 });\n/** Auxiliary table layout (float32 words). */\nexport const SENSOR_AUX = Object.freeze({ stirlingRemainder: 0, sigma: 256, inversePieces: 272, digitalPieces: 276, knots: 280 });\nexport const SENSOR_WORKGROUP_SIZE = 256;\n/**\n * Per-site packed words: photo word, photo fraction bits, dark word, dark fraction bits; then 2 words per\n * additional Poisson component; then, with deterministic components, their sum (integer part, fraction\n * bits); then, with fixed-pattern noise, (threshold − offset) and offset/gain as (floor, fraction bits).\n */\nexport const SENSOR_SITE_WORDS = 4;\n/** Bits of `Params.extras`. */\nexport const SENSOR_EXTRAS = Object.freeze({ deterministic: 1, fixedPattern: 2 });\n/** Partial charge sums saturate here: above every supported capacity (2^24), so no decision changes. */\nexport const SENSOR_REALIZED_CLAMP = 2 ** 30;\n/** Mean categories in bits 30–31 of a mean word, mirroring `samplePoisson`'s branches. */\nexport const SENSOR_MEAN_CATEGORY = Object.freeze({ zero: 0, multiplication: 1, transformedRejection: 2, normal: 3 });\n\n/** Smallest integer m with (m + 0.5)/2^32 ≥ t, so `uS ≥ t` ⇔ `m ≥ result` exactly. */\nexport function uniformThresholdInteger(t: number): number {\n  return Math.ceil(t * 2 ** 32 - 0.5);\n}\n\nconst SIN_COEFFICIENTS = [-1 / 6, 1 / 120, -1 / 5040, 1 / 362880, -1 / 39916800];\nconst COS_COEFFICIENTS = [-1 / 2, 1 / 24, -1 / 720, 1 / 40320, -1 / 3628800, 1 / 479001600];\nconst literal = (value: number): string => {\n  const text = String(Math.fround(value));\n  return /[.e]/u.test(text) ? text : `${text}.0`;\n};\n\nexport const SENSOR_RAW_WGSL = /* wgsl */ `\nconst U: f32 = 5.9604644775390625e-8;\nconst INV32: f32 = 2.3283064365386963e-10;\nconst TWO_PI_INV32: f32 = ${literal(2 * Math.PI / 2 ** 32)};\nconst MARGIN: f32 = 2.0;\nconst CPU_REL: f32 = 7.105427357601002e-15;\nstruct Params {\n  count: u32, row0: u32, region_w: u32, native_w: u32,\n  region_x: u32, region_y: u32, seed: u32, components: u32,\n  knots: u32, digital: u32, saturation: u32, adc_max: u32,\n  capacity_floor: u32, threshold_floor: u32, black: u32, sat_quantized: i32,\n  capacity_frac: f32, threshold_frac: f32, inv_gain: f32, inv_residual: f32,\n  digital_gain: f32, digital_residual: f32, inl_slope: f32, inl_offset: f32,\n  site_words: u32, poisson_count: u32, extras: u32, pad0: u32,\n  fpn_e_slack: f32, fpn_c_slack: f32, pad1: f32, pad2: f32,\n};\n@group(0) @binding(0) var<uniform> params: Params;\n@group(0) @binding(1) var<storage, read> means: array<u32>;\n@group(0) @binding(2) var<storage, read> aux: array<f32>;\n@group(0) @binding(3) var<storage, read_write> out: array<u32>;\n\nvar<private> state: u32;\nvar<private> uncertain: bool;\nvar<private> radius: f32;\nvar<private> ip: i32;\nvar<private> fp: f32;\n\nfn fmix(value: u32) -> u32 {\n  var h = value;\n  h = h ^ (h >> 16u); h = h * 0x85ebca6bu; h = h ^ (h >> 13u); h = h * 0xc2b2ae35u; h = h ^ (h >> 16u);\n  return h;\n}\nfn mix_block(hash: u32, block: u32) -> u32 {\n  var k = block * 0xcc9e2d51u;\n  k = (k << 15u) | (k >> 17u);\n  k = k * 0x1b873593u;\n  var h = hash ^ k;\n  h = (h << 13u) | (h >> 19u);\n  return h * 5u + 0xe6546b64u;\n}\nfn site_seed(seed: u32, index: u32, stream: u32) -> u32 {\n  return fmix(mix_block(mix_block(mix_block(0u, seed), index), stream) ^ 12u);\n}\nfn start(seed: u32) {\n  state = fmix(seed ^ 0x9e3779b9u);\n  if (state == 0u) { state = 0x6d2b79f5u; }\n}\nfn next_bits() -> u32 {\n  state = state ^ (state << 13u); state = state ^ (state >> 17u); state = state ^ (state << 5u);\n  return state;\n}\nfn unit(bits: u32) -> f32 { return (f32(bits) + 0.5) * INV32; }\nfn check(distance: f32, margin: f32) { if (abs(distance) <= margin) { uncertain = true; } }\n\n// ln((bits + 0.5)/2^32), relative error <= 12u: WGSL log below 1/2, else 2·atanh(r) on the exact 1 - u.\nfn ln_unit(bits: u32) -> f32 {\n  if (bits < 0x80000000u) { return log(unit(bits)); }\n  let w = (f32(0xffffffffu - bits) + 0.5) * INV32;\n  let r = -w / (2.0 - w);\n  let r2 = r * r;\n  var p = ${literal(1 / 23)};\n  ${[21, 19, 17, 15, 13, 11, 9, 7, 5, 3].map(n => `p = p * r2 + ${literal(1 / n)};`).join(\" \")}\n  p = p * r2 + 1.0;\n  return 2.0 * r * p;\n}\n// cos(2π (bits + 0.5)/2^32): exact octant reduction on the integer, Taylor polynomials on [0, π/4].\nfn cos_unit(bits: u32) -> f32 {\n  let octant = bits >> 29u;\n  let low = bits & 0x1fffffffu;\n  let even = (octant & 1u) == 0u;\n  var g = (f32(0x1fffffffu - low) + 0.5) * TWO_PI_INV32;\n  if (even) { g = (f32(low) + 0.5) * TWO_PI_INV32; }\n  let g2 = g * g;\n  var c = ${literal(COS_COEFFICIENTS[5]!)};\n  ${[4, 3, 2, 1, 0].map(j => `c = c * g2 + ${literal(COS_COEFFICIENTS[j]!)};`).join(\" \")}\n  c = c * g2 + 1.0;\n  var s = ${literal(SIN_COEFFICIENTS[4]!)};\n  ${[3, 2, 1, 0].map(j => `s = s * g2 + ${literal(SIN_COEFFICIENTS[j]!)};`).join(\" \")}\n  s = g * (s * g2 + 1.0);\n  let cb = select(s, c, even);\n  let sb = select(c, s, even);\n  let q = octant >> 1u;\n  if (q == 0u) { return cb; }\n  if (q == 1u) { return -sb; }\n  if (q == 2u) { return -cb; }\n  return sb;\n}\n// Box–Muller, one value per two uniforms; |z_f32 − z_f64| <= 24u·radius.\nfn std_normal() -> f32 {\n  let b1 = next_bits();\n  let b2 = next_bits();\n  radius = sqrt(-2.0 * ln_unit(b1));\n  return radius * cos_unit(b2);\n}\nfn poisson_small(mean: f32) -> u32 {\n  let threshold = exp(-mean);\n  var product = 1.0;\n  var count = 0u;\n  loop {\n    count = count + 1u;\n    product = product * unit(next_bits());\n    check(product - threshold, MARGIN * (3.0 * f32(count) + 130.0) * U * max(product, threshold));\n    if (!(product > threshold)) { break; }\n    if (count >= 1000u) { uncertain = true; break; }\n  }\n  return count - 1u;\n}\n// g(t) = ln(1 + t) − t for |t| <= 1/2 without cancellation: −t²/(2+t) + 2(s³/3 + s⁵/5 + …), s = t/(2+t).\nfn log1p_minus(t: f32) -> f32 {\n  let d = 2.0 + t;\n  let s = t / d;\n  let s2 = s * s;\n  var p = ${literal(1 / 25)};\n  ${[23, 21, 19, 17, 15, 13, 11, 9, 7, 5].map(n => `p = p * s2 + ${literal(1 / n)};`).join(\" \")}\n  p = p * s2 + ${literal(1 / 3)};\n  return -(t * t) / d + 2.0 * s * s2 * p;\n}\nfn poisson_ptrs(mi: u32, mf: f32) -> u32 {\n  let mean = f32(mi) + mf;\n  let sq = sqrt(mean);\n  let b = 0.931 + 2.53 * sq;\n  let a = -0.059 + 0.02483 * b;\n  let inv_alpha = 1.1239 + 1.1328 / (b - 3.4);\n  let v_r = 0.9277 - 3.6224 / (b - 2.0);\n  let ln_mean = log(mean);\n  for (var iteration = 0u; iteration < 64u; iteration = iteration + 1u) {\n    let su = next_bits();\n    let sv = next_bits();\n    var m = su;\n    var sign = -1.0;\n    if (su >= 0x80000000u) { m = 0xffffffffu - su; sign = 1.0; }\n    let us = unit(m);\n    let u = sign * unit(0x7fffffffu - m);\n    let v = unit(sv);\n    let scale = 2.0 * a / us + b;\n    let t = scale * u;\n    let x = (t + mf) + 0.43;\n    let fl = floor(x);\n    let ex = MARGIN * (abs(t) * 40.0 * U + 2.0 * U * (abs(t) + 1.0) + 4.5e-16 * (abs(t) + mean + 1.0));\n    if (x - fl <= ex || fl + 1.0 - x <= ex) { uncertain = true; }\n    if (fl <= -1073741824.0 || fl >= 1073741824.0) { continue; }\n    let k = i32(mi) + i32(fl);\n    if (m >= ${uniformThresholdInteger(0.07)}u) {\n      if (k >= 0) { check(v - v_r, MARGIN * (16.0 * U + 2.0 * U * v)); }\n      if (v <= v_r && k >= 0) { return u32(k); }\n    }\n    if (k < 0 || (m < ${uniformThresholdInteger(0.013)}u && sv > m)) { continue; }\n    let lhs = log(v * inv_alpha / (a / (us * us) + b));\n    let dl = 60.0 * U + 4.76837158203125e-7 + 4.0 * U * abs(lhs);\n    var rhs: f32;\n    var dr: f32;\n    if (k == 0) {\n      rhs = -mean;\n      dr = 2.0 * U * mean + CPU_REL * mean;\n    } else {\n      let kf = f32(k);\n      let dd = f32(i32(mi) - k) + mf;\n      let t2 = dd / kf;\n      var kg: f32;\n      var dk: f32;\n      if (abs(t2) <= 0.5) {\n        kg = kf * log1p_minus(t2);\n        dk = 20.0 * U * abs(kg);\n      } else {\n        let lr = log(mean / kf);\n        kg = kf * lr - dd;\n        dk = kf * (8.0 * U + 4.76837158203125e-7 + 5.0 * U * abs(lr)) + 3.0 * U * abs(dd) + U * abs(kg);\n      }\n      let lnk = log(kf);\n      var corr: f32;\n      if (k < 256) { corr = aux[u32(k)]; } else { corr = 1.0 / (12.0 * kf) - 1.0 / (360.0 * kf * kf * kf); }\n      rhs = ((kg - 0.5 * lnk) - 0.91893853320467274) - corr;\n      dr = dk + 2.384185791015625e-7 + 2.0 * U * lnk + U + 0.6 * U + 3.0 * U * (abs(kg) + 0.5 * lnk + 1.0) +\n        CPU_REL * (mean + kf * (abs(ln_mean) + lnk + 1.0));\n    }\n    check(lhs - rhs, MARGIN * (dl + dr));\n    if (lhs <= rhs) { return u32(k); }\n  }\n  uncertain = true;\n  return mi;\n}\nfn poisson(word: u32, frac: f32) -> u32 {\n  let category = word >> 30u;\n  let mi = word & 0x3fffffffu;\n  if (category == 0u) { return 0u; }\n  if (category == 1u) { return poisson_small(f32(mi) + frac); }\n  if (category == 2u) { return poisson_ptrs(mi, frac); }\n  // Means of 1e9 and above: the draw always exceeds the capacity (<= 2^24), so the code does not depend on it.\n  let z = std_normal();\n  let y = sqrt(f32(mi) + frac) * z + frac + 0.5;\n  return u32(max(0, i32(mi) + i32(floor(y))));\n}\n// Adds an exact non-negative float32 value split into integer and fractional parts.\nfn accumulate(value: f32) {\n  let f = floor(value);\n  ip = ip + i32(f);\n  fp = fp + (value - f);\n}\n// n·(p0 + p1 + p2 + p3) for an integer 0 <= n < 2^24 and 12-bit pieces: every product is exact.\nfn scaled(n: u32, base: u32) {\n  let hi = f32(n >> 12u);\n  let lo = f32(n & 0xfffu);\n  for (var j = 0u; j < 4u; j = j + 1u) {\n    let piece = aux[base + j];\n    accumulate(hi * piece * 4096.0);\n    accumulate(lo * piece);\n  }\n}\nfn normalize() {\n  let f = floor(fp);\n  ip = ip + i32(f);\n  fp = fp - f;\n}\nfn inl_offset(code: f32) -> f32 {\n  let n = params.knots;\n  let base = ${\"280u\"};\n  if (code <= aux[base]) { return aux[base + n]; }\n  if (code >= aux[base + n - 1u]) { return aux[base + 2u * n - 1u]; }\n  var low = 0u;\n  var high = n - 1u;\n  loop {\n    if (high - low <= 1u) { break; }\n    let middle = (low + high) >> 1u;\n    if (aux[base + middle] <= code) { low = middle; } else { high = middle; }\n  }\n  let ac = aux[base + low];\n  let ao = aux[base + n + low];\n  return ao + ((code - ac) / (aux[base + high] - ac)) * (aux[base + n + high] - ao);\n}\n@compute @workgroup_size(${SENSOR_WORKGROUP_SIZE})\nfn main(@builtin(global_invocation_id) gid: vec3<u32>, @builtin(num_workgroups) groups: vec3<u32>) {\n  let i = gid.x + gid.y * groups.x * ${SENSOR_WORKGROUP_SIZE}u;\n  if (i >= params.count) { return; }\n  let row = params.row0 + i / params.region_w;\n  let index = (params.region_y + row) * params.native_w + params.region_x + i % params.region_w;\n  uncertain = false;\n  let base = i * params.site_words;\n  start(site_seed(params.seed, index, 0u));\n  let photo = poisson(means[base], bitcast<f32>(means[base + 1u]));\n  let dark = poisson(means[base + 2u], bitcast<f32>(means[base + 3u]));\n  // Saturating sums: any total above the clamp is above every supported capacity, as in the reference.\n  var realized = min(photo + dark, ${SENSOR_REALIZED_CLAMP}u);\n  for (var k = 0u; k < params.poisson_count; k = k + 1u) {\n    realized = min(realized + poisson(means[base + 4u + 2u * k], bitcast<f32>(means[base + 5u + 2u * k])), ${SENSOR_REALIZED_CLAMP}u);\n  }\n  var cursor = base + 4u + 2u * params.poisson_count;\n  let deterministic = (params.extras & ${SENSOR_EXTRAS.deterministic}u) != 0u;\n  let pattern = (params.extras & ${SENSOR_EXTRAS.fixedPattern}u) != 0u;\n  var r_frac = 0.0;\n  var det_slack = 0.0;\n  if (deterministic) {\n    realized = min(realized + means[cursor], ${SENSOR_REALIZED_CLAMP}u);\n    r_frac = bitcast<f32>(means[cursor + 1u]);\n    det_slack = U;\n    cursor = cursor + 2u;\n  }\n  var t_floor = i32(params.threshold_floor);\n  var t_frac = params.threshold_frac;\n  var fpn_e_slack = 0.0;\n  if (pattern) {\n    t_floor = bitcast<i32>(means[cursor]);\n    t_frac = bitcast<f32>(means[cursor + 1u]);\n    fpn_e_slack = params.fpn_e_slack;\n  }\n  start(site_seed(params.seed, index, 1u));\n  var noise = 0.0;\n  var noise_bound = 0.0;\n  for (var c = 0u; c < params.components; c = c + 1u) {\n    let sigma = aux[256u + c];\n    if (sigma != 0.0) {\n      let z = std_normal();\n      noise = noise + z * sigma;\n      noise_bound = noise_bound + sigma * radius * (28.0 + f32(params.components)) * U;\n    }\n  }\n  var flags = 0u;\n  var a_int = realized;\n  var a_frac = r_frac;\n  if (!deterministic) {\n    if (realized > params.capacity_floor) { flags = 1u; a_int = params.capacity_floor; a_frac = params.capacity_frac; }\n  } else {\n    // realized > capacity with a non-integer total: float32 difference with its bound.\n    let w = f32(i32(realized) - i32(params.capacity_floor));\n    let dcap = w + (r_frac - params.capacity_frac);\n    check(dcap, MARGIN * (5.0 * U + U * abs(w) + U * abs(dcap) + CPU_REL * (f32(realized) + 1.0)));\n    if (dcap > 0.0) { flags = 1u; a_int = params.capacity_floor; a_frac = params.capacity_frac; }\n  }\n  // Pre-ADC limit: afterPhysical + noise + offset > threshold, as afterPhysical − (threshold − offset) + noise > 0.\n  let whole = f32(i32(a_int) - t_floor);\n  let diff = (whole + (a_frac - t_frac)) + noise;\n  check(diff, MARGIN * (noise_bound + 3.0 * U + 2.0 * U * (abs(whole) + abs(noise) + 2.0) + CPU_REL * (f32(a_int) + abs(noise)) +\n    fpn_e_slack + det_slack));\n  var q: i32;\n  if (diff > 0.0) {\n    flags = flags | 2u;\n    q = params.sat_quantized;\n  } else {\n    // ideal = (afterPhysical + noise + offset)/gain + black, as an exact integer part plus a float32 fraction.\n    ip = i32(params.black);\n    fp = 0.0;\n    var fpn_c_slack = 0.0;\n    if (pattern) {\n      ip = ip + bitcast<i32>(means[cursor + 2u]);\n      fp = bitcast<f32>(means[cursor + 3u]);\n      fpn_c_slack = params.fpn_c_slack;\n    }\n    scaled(a_int, 272u);\n    let noise_codes = noise * params.inv_gain;\n    fp = fp + a_frac * params.inv_gain + noise_codes;\n    let a_codes = f32(a_int) * params.inv_gain;\n    var margin = MARGIN * (10.0 * U * (10.0 + abs(noise_codes)) + 4.0 * U * a_frac * params.inv_gain + 3.0 * U * abs(noise_codes) +\n      noise_bound * params.inv_gain + f32(a_int) * params.inv_residual + 2.0 * U +\n      CPU_REL * (a_codes + abs(noise_codes) + f32(params.black)) + fpn_c_slack + det_slack * params.inv_gain);\n    normalize();\n    if (params.knots > 0u) {\n      let ideal = f32(ip) + fp;\n      let offset = inl_offset(ideal);\n      fp = fp + offset;\n      margin = margin * (1.0 + params.inl_slope) + MARGIN * (params.inl_slope * 2.0 * U * (abs(ideal) + 1.0) +\n        8.0 * U * (params.inl_offset + 1.0) + CPU_REL * (abs(ideal) + params.inl_offset) * (1.0 + params.inl_slope));\n      normalize();\n    }\n    if ((params.digital & 4u) == 0u) { check(fp - 0.5, margin); }\n    q = ip + select(0, 1, fp >= 0.5);\n  }\n  if ((params.digital & 1u) != 0u) {\n    let adc = clamp(q, 0, i32(params.adc_max));\n    let delta = adc - i32(params.black);\n    let estimate = f32(delta) * params.digital_gain;\n    if (estimate >= 16777216.0) {\n      q = 16777216;\n    } else if (estimate <= -16777216.0) {\n      q = -16777216;\n    } else {\n      ip = 0;\n      fp = 0.0;\n      scaled(u32(abs(delta)), 276u);\n      if ((params.digital & 2u) == 0u) { check(fp - floor(fp) - 0.5, MARGIN * (40.0 * U * 9.0 + f32(abs(delta)) * params.digital_residual + CPU_REL * (abs(estimate) + f32(params.black)))); }\n      normalize();\n      var whole_part = ip;\n      var fraction = fp;\n      if (delta < 0) {\n        whole_part = -ip;\n        if (fp > 0.0) { whole_part = -ip - 1; fraction = 1.0 - fp; }\n      }\n      q = whole_part + i32(params.black) + select(0, 1, fraction >= 0.5);\n    }\n  }\n  if (q >= i32(params.saturation)) { flags = flags | 4u; }\n  if (q < 0) { flags = flags | 8u; }\n  let code = u32(clamp(q, 0, i32(params.saturation)));\n  out[i] = code | (flags << 16u) | select(0u, ${1 << 20}u, uncertain);\n}\n`;\n"]}