@graphty/webgpu-graph-algorithms 0.6.24 → 0.6.25

package/dist/webgpu-graph-algorithms.js

@@ -1,5 +1,5 @@
-import { W as WebGpuGraphError, U as UNIFORM_SLOT_BYTES, B as BufferUsage, M as MAX_WORKGROUPS_PER_DIM, a as WGSL_RESERVED_WORDS, S as STATE_HEADER_BYTES, d as deviceLostError, i as isWebGpuGraphError, b as U32_MAX$2, c as MAX_LEVELS_PER_SUBMIT, R as RADIX_BINS, F as FUSED_FRONTIER_MAX, e as BEAMER_BETA, f as SSSP_DELTA_FACTOR, g as F32_INF_BITS, h as BC_EDGE_PARALLEL_GAMMA, j as BC_BATCH_BUDGET_FRACTION, k as BC_MAX_BATCH, l as BC_BACKWARD_LEVELS_PER_SUBMIT, A as APSP_MAX_DISPATCHES_PER_SUBMIT, m as APSP_TILE, n as GROUP_ROW_THREAD_MAX, o as GROUP_ROW_THREAD_LIMIT, p as GROUP_HASH_LOAD_FACTOR, P as PARALLEL_MERGE_LIMIT, L as LABEL_PROP_PASSES_PER_SUBMIT, q as BORUVKA_ROUNDS_PER_SUBMIT, r as GRID_COARSEST_SIDE, s as GRID_MIN_SIDE, t as GRID_SORT_BITS, u as FA2_DEFAULTS, v as MAX_ITERATIONS_PER_STEP, w as MAX_1D_ITEMS, x as hasErrorCode, y as FA2_FLAG_FIRST, z as PARTIAL_BYTES, E as EXACT_TILES_PER_PASS, I as INDIRECT_ARGS_STRIDE, C as GRID_HUB_CELL, D as LAYOUT_TUNING_DEFAULTS, H as EXACT_MAX_NODES, J as SETTLE_FLOOR_UNBOUNDED, T as TRACE_RECORD_BYTES, K as GRID_BBOX_MARGIN, N as GRID_EXTENT_FLOOR, O as FR_ADAPTIVE_MAX_ITERATIONS, Q as FR_START_TEMPERATURE, V as FA2_FLAG_ADAPTIVE, X as SETTLE_FLOOR_FRACTION, Y as FR_REHEAT_FRACTION, Z as FR_DEFAULTS, _ as SE_DEFAULTS, $ as SETTLE_FLOOR_REFERENCE_NODES, a0 as SE_SCALE_REFERENCE_NODES } from "./chunks/context-B40Z6lV_.js";
-import { a1, G, a2, a3, a4, a5 } from "./chunks/context-B40Z6lV_.js";
+import { W as WebGpuGraphError, U as UNIFORM_SLOT_BYTES, B as BufferUsage, M as MAX_WORKGROUPS_PER_DIM, a as WGSL_RESERVED_WORDS, S as STATE_HEADER_BYTES, d as deviceLostError, i as isWebGpuGraphError, b as U32_MAX$2, c as MAX_LEVELS_PER_SUBMIT, R as RADIX_BINS, F as FUSED_FRONTIER_MAX, e as BEAMER_BETA, f as SSSP_DELTA_FACTOR, g as F32_INF_BITS, h as BC_EDGE_PARALLEL_GAMMA, j as BC_BATCH_BUDGET_FRACTION, k as BC_MAX_BATCH, l as BC_BACKWARD_LEVELS_PER_SUBMIT, m as BC_COUNT_MAX_GROUPS, A as APSP_MAX_DISPATCHES_PER_SUBMIT, n as APSP_TILE, o as GROUP_ROW_THREAD_MAX, p as GROUP_ROW_THREAD_LIMIT, q as GROUP_HASH_LOAD_FACTOR, P as PARALLEL_MERGE_LIMIT, L as LABEL_PROP_PASSES_PER_SUBMIT, r as BORUVKA_ROUNDS_PER_SUBMIT, s as GRID_COARSEST_SIDE, t as GRID_MIN_SIDE, u as GRID_SORT_BITS, v as FA2_DEFAULTS, w as MAX_ITERATIONS_PER_STEP, x as MAX_1D_ITEMS, y as hasErrorCode, z as FA2_FLAG_FIRST, C as PARTIAL_BYTES, E as EXACT_TILES_PER_PASS, I as INDIRECT_ARGS_STRIDE, D as GRID_HUB_CELL, H as LAYOUT_TUNING_DEFAULTS, J as EXACT_MAX_NODES, K as SETTLE_FLOOR_UNBOUNDED, T as TRACE_RECORD_BYTES, N as GRID_BBOX_MARGIN, O as GRID_EXTENT_FLOOR, Q as FR_ADAPTIVE_MAX_ITERATIONS, V as FR_START_TEMPERATURE, X as FA2_FLAG_ADAPTIVE, Y as SETTLE_FLOOR_FRACTION, Z as FR_REHEAT_FRACTION, _ as FR_DEFAULTS, $ as SE_DEFAULTS, a0 as SETTLE_FLOOR_REFERENCE_NODES, a1 as SE_SCALE_REFERENCE_NODES } from "./chunks/context-BZY6SMsM.js";
+import { a2, G, a3, a4, a5, a6 } from "./chunks/context-BZY6SMsM.js";
 import { renumberPartition, INVALID_INDEX, foldArcs, makeMask, maskTest, expandEdges, fromEdgeArrays } from "@graphty/graph-format";
 class UniformRing {
   /**
@@ -722,6 +722,11 @@ fn apsp_init(@builtin(workgroup_id) wid: vec3<u32>, @builtin(local_invocation_id
 const bcBackwardWgsl = (
   /* wgsl */
   `
+fn sigma_of(word: u32) -> f32 {                                     // a stored count: u32, or f32 bits when SCALED
+    if (SCALED) { return bitcast<f32>(word); }
+    return f32(word);
+}
+
 @compute @workgroup_size(WG)
 fn bc_backward(@builtin(workgroup_id) wid: vec3<u32>, @builtin(local_invocation_id) lid: vec3<u32>) {
     for (var i = linear_id(wid, lid.x); i < P.count; i = i + P.stride) {
@@ -729,12 +734,13 @@ fn bc_backward(@builtin(workgroup_id) wid: vec3<u32>, @builtin(local_invocation_
         let w = t % P.n;
         let base = t - w;                                            // s * n
         let succ = depthK[t] + 1u;
-        let sw = f32(sigmaK[t]);
+        let sw = sigma_of(sigmaK[t]);
+        let shift = select(0i, sigma_shift(levelMax[depthK[t]]), SCALED);   // the scale step to the successors' depth
         var acc = 0.0;
         for (var a = rowPtr[w]; a < rowPtr[w + 1u]; a = a + 1u) {
             let v = base + colIdx[a];
             if (depthK[v] == succ) {                                 // v is a successor of w for source s
-                acc = acc + (sw / f32(sigmaK[v])) * (1.0 + deltaK[v]);
+                acc = acc + ldexp(sw / sigma_of(sigmaK[v]), -shift) * (1.0 + deltaK[v]);
             }
         }
         deltaK[t] = acc;                                             // written once per (w, s)
@@ -742,9 +748,42 @@ fn bc_backward(@builtin(workgroup_id) wid: vec3<u32>, @builtin(local_invocation_
 }
 `
 );
+const bcCountWgsl = (
+  /* wgsl */
+  `
+@compute @workgroup_size(WG)
+fn bc_count(@builtin(workgroup_id) wid: vec3<u32>, @builtin(local_invocation_id) lid: vec3<u32>) {
+    let level = atomicLoad(&counters[11]);
+    let start = ends[level + 1u];                                    // the boundary closed the claim-from level here
+    let count = atomicLoad(&counters[26]) - start;                   // stackTop: what the forward dispatch appended
+    let shift = sigma_shift(atomicLoad(&levelMax[level]));
+    for (var i = linear_id(wid, lid.x); i < count; i = i + P.stride) {
+        let t = S[start + i];                                        // s * n + x, at depth level + 1
+        let x = t % P.n;
+        let base = t - x;                                            // s * n
+        var acc = 0.0;
+        for (var a = rowPtr[x]; a < rowPtr[x + 1u]; a = a + 1u) {    // the in-arcs (v, x)
+            let v = base + colIdx[a];
+            if (depthK[v] == level) { acc = acc + bitcast<f32>(sigmaK[v]); }   // every predecessor's paths, CSR order
+        }
+        let sigma = ldexp(acc, -shift);
+        let bits = bitcast<u32>(sigma);
+        sigmaK[t] = bits;
+        let exponent = (bits >> 23u) & 0xffu;
+        if (exponent == 0u || exponent == 0xffu) { atomicOr(&counters[27], 1u); }   // out of f32's normal range
+        atomicMax(&levelMax[level + 1u], bits);
+    }
+}
+`
+);
 const bcEdgeGatherWgsl = (
   /* wgsl */
   `
+fn sigma_of(word: u32) -> f32 {                                     // a stored count: u32, or f32 bits when SCALED
+    if (SCALED) { return bitcast<f32>(word); }
+    return f32(word);
+}
+
 @compute @workgroup_size(WG)
 fn bc_edge_gather(@builtin(workgroup_id) wid: vec3<u32>, @builtin(local_invocation_id) lid: vec3<u32>) {
     for (var a = linear_id(wid, lid.x); a < P.count; a = a + P.stride) {
@@ -762,7 +801,9 @@ fn bc_edge_gather(@builtin(workgroup_id) wid: vec3<u32>, @builtin(local_invocati
             let base = s * P.n;
             let dw = depthK[base + w];
             if (dw != INVALID_INDEX && depthK[base + nbr] == dw + 1u) {   // (w, nbr) is on a shortest path from s
-                acc = acc + (f32(sigmaK[base + w]) / f32(sigmaK[base + nbr])) * (1.0 + deltaK[base + nbr]);
+                let shift = select(0i, sigma_shift(levelMax[dw]), SCALED);
+                let ratio = ldexp(sigma_of(sigmaK[base + w]) / sigma_of(sigmaK[base + nbr]), -shift);
+                acc = acc + ratio * (1.0 + deltaK[base + nbr]);
             }
         }
         arcScores[a] = acc;
@@ -780,8 +821,9 @@ fn bc_finalize(@builtin(local_invocation_id) lid: vec3<u32>) {
         for (var i = 0u; i < P.k; i = i + 1u) {
             let t = S[i];
             depthK[t] = 0u;                                          // the source is at depth 0
-            sigmaK[t] = 1u;                                          // with one shortest path, itself
+            sigmaK[t] = select(1u, bitcast<u32>(1.0), SCALED);       // with one shortest path, itself (f32 bits when SCALED)
         }
+        levelMax[0] = bitcast<u32>(1.0);                             // the largest count at depth 0
         ends[0] = 0u;
         atomicStore(&counters[26], P.k);                             // stackTop: the seeds are the log's first k entries
         atomicStore(&counters[27], 0u);                              // sigmaOverflow
@@ -865,7 +907,7 @@ fn bc_forward(@builtin(workgroup_id) wid: vec3<u32>, @builtin(local_invocation_i
                 if (atomicLoad(&depthK[x]) == INVALID_INDEX) {       // the pre-check of design 16.1
                     won = atomicMin(&depthK[x], next) == INVALID_INDEX;   // the claim: the one winner appends
                 }
-                if (atomicLoad(&depthK[x]) == next) {                // the count: EVERY arc on a shortest path adds
+                if (!SCALED && atomicLoad(&depthK[x]) == next) {     // the count: EVERY arc on a shortest path adds
                     let add = atomicLoad(&sigmaK[origin]);
                     let old = atomicAdd(&sigmaK[x], add);
                     if (old + add < old) { atomicOr(&counters[27], 1u); }   // the u32 wrap, reported
@@ -900,7 +942,7 @@ fn claim(x: u32, next: u32) -> bool {
 }
 
 fn count_paths(origin: u32, x: u32, next: u32) {
-    if (atomicLoad(&depthK[x]) == next) {                            // every arc on a shortest path adds
+    if (!SCALED && atomicLoad(&depthK[x]) == next) {                 // every arc on a shortest path adds
         let add = atomicLoad(&sigmaK[origin]);
         let old = atomicAdd(&sigmaK[x], add);
         if (old + add < old) { atomicOr(&counters[27], 1u); }        // the u32 wrap, reported
@@ -4381,9 +4423,10 @@ const BC_FINALIZE = {
     decl(1, 2, "S", "storage-ro", "array<u32>"),
     decl(1, 3, "depthK", "storage", "array<u32>"),
     decl(1, 4, "sigmaK", "storage", "array<u32>"),
+    decl(1, 5, "levelMax", "storage", "array<u32>"),
     decl(2, 0, "P", "uniform", "BcParams")
   ],
-  overrideDecls: [],
+  overrideDecls: [{ name: "SCALED", type: "bool", default: false }],
   uniforms: [BC_PARAMS],
   needs: [],
   snippetSlots: [],
@@ -4403,7 +4446,7 @@ const BC_FORWARD = {
     decl(1, 6, "sigmaK", "storage", "array<atomic<u32>>"),
     decl(2, 0, "P", "uniform", "BcParams")
   ],
-  overrideDecls: [],
+  overrideDecls: [{ name: "SCALED", type: "bool", default: false }],
   uniforms: [BC_PARAMS],
   needs: [],
   snippetSlots: [],
@@ -4420,9 +4463,10 @@ const BC_BACKWARD = {
     decl(1, 3, "depthK", "storage-ro", "array<u32>"),
     decl(1, 4, "sigmaK", "storage-ro", "array<u32>"),
     decl(1, 5, "deltaK", "storage", "array<f32>"),
+    decl(1, 6, "levelMax", "storage-ro", "array<u32>"),
     decl(2, 0, "P", "uniform", "BcParams")
   ],
-  overrideDecls: [],
+  overrideDecls: [{ name: "SCALED", type: "bool", default: false }],
   uniforms: [BC_PARAMS],
   needs: [],
   snippetSlots: [],
@@ -4454,9 +4498,10 @@ const BC_EDGE_GATHER = {
     decl(1, 3, "sigmaK", "storage-ro", "array<u32>"),
     decl(1, 4, "deltaK", "storage-ro", "array<f32>"),
     decl(1, 5, "arcScores", "storage", "array<f32>"),
+    decl(1, 6, "levelMax", "storage-ro", "array<u32>"),
     decl(2, 0, "P", "uniform", "BcParams")
   ],
-  overrideDecls: [],
+  overrideDecls: [{ name: "SCALED", type: "bool", default: false }],
   uniforms: [BC_PARAMS],
   needs: [],
   snippetSlots: [],
@@ -4476,7 +4521,31 @@ const BC_FORWARD_EDGE = {
     decl(1, 6, "sigmaK", "storage", "array<atomic<u32>>"),
     decl(2, 0, "P", "uniform", "BcParams")
   ],
-  overrideDecls: [{ name: "UNDIRECTED", type: "bool", default: false }],
+  overrideDecls: [
+    { name: "UNDIRECTED", type: "bool", default: false },
+    { name: "SCALED", type: "bool", default: false }
+  ],
+  uniforms: [BC_PARAMS],
+  needs: [],
+  snippetSlots: [],
+  phase: "P9"
+};
+const BC_COUNT = {
+  id: "bc-count",
+  body: bcCountWgsl,
+  entryPoint: "bc_count",
+  bindings: [
+    decl(1, 0, "rowPtr", "storage-ro", "array<u32>"),
+    decl(1, 1, "colIdx", "storage-ro", "array<u32>"),
+    decl(1, 2, "S", "storage-ro", "array<u32>"),
+    decl(1, 3, "ends", "storage-ro", "array<u32>"),
+    decl(1, 4, "counters", "storage", "array<atomic<u32>>"),
+    decl(1, 5, "depthK", "storage-ro", "array<u32>"),
+    decl(1, 6, "sigmaK", "storage", "array<u32>"),
+    decl(1, 7, "levelMax", "storage", "array<atomic<u32>>"),
+    decl(2, 0, "P", "uniform", "BcParams")
+  ],
+  overrideDecls: [],
   uniforms: [BC_PARAMS],
   needs: [],
   snippetSlots: [],
@@ -4733,6 +4802,7 @@ const REGISTRY = Object.freeze({
   "bc-forward": BC_FORWARD,
   "bc-backward": BC_BACKWARD,
   "bc-gather": BC_GATHER,
+  "bc-count": BC_COUNT,
   "bc-edge-gather": BC_EDGE_GATHER,
   "bc-forward-edge": BC_FORWARD_EDGE,
   "apsp-init": APSP_INIT,
@@ -8300,7 +8370,7 @@ async function bellmanFord(ctx, s, source, options) {
 const ALGORITHM$5 = "betweennessCentrality";
 const BYTES_PER_NODE_SOURCE = 16;
 const SAMPLE_SEED = 2654435769;
-const RING_SLOTS$4 = BC_BACKWARD_LEVELS_PER_SUBMIT + 16;
+const RING_SLOTS$4 = Math.max(3 * MAX_LEVELS_PER_SUBMIT, BC_BACKWARD_LEVELS_PER_SUBMIT) + 16;
 function planBatchSize(n, remaining, limits) {
   const needed = 4 * (n + 2);
   if (needed > limits.maxStorageBufferBindingSize) {
@@ -8356,6 +8426,26 @@ function resolveSources(options, n) {
   }
   return Array.from({ length: n }, (_, i) => i);
 }
+async function modeKernels(ctx, s, core, withForwardEdge, withEdges, scaled) {
+  const SCALED = scaled;
+  const [finalize, forward, backward] = await Promise.all(
+    ["bc-finalize", "bc-forward", "bc-backward"].map(
+      (id) => ctx.pipelines.kernel(kernelSpec(id, { SCALED }))
+    )
+  );
+  const forwardEdge = withForwardEdge ? await ctx.pipelines.kernel(kernelSpec("bc-forward-edge", { UNDIRECTED: !s.directed, SCALED })) : null;
+  const edgeGather = withEdges ? await ctx.pipelines.kernel(kernelSpec("bc-edge-gather", { SCALED })) : null;
+  let count = null;
+  if (scaled) {
+    const reverse = s.directed ? reverseOf(ctx, s) : core;
+    count = {
+      kernel: await ctx.pipelines.kernel(kernelSpec("bc-count")),
+      rowPtr: reverse.rowPtr,
+      colIdx: reverse.colIdx ?? reverse.rowPtr
+    };
+  }
+  return { finalize, forward, forwardEdge, backward, edgeGather, count };
+}
 function recordFill(state, pass, dst, count, value) {
   const { scope, fill, ctx } = state;
   const params = scope.params(FILL_PARAMS, { count, value, mode: 0, pad0: 0 });
@@ -8363,9 +8453,9 @@ function recordFill(state, pass, dst, count, value) {
     params.offset
   ]);
 }
-function recordBc(state, pass, kernel, resources, fields, items) {
+function recordBc(state, pass, kernel, resources, fields, items, maxGroups) {
   const { scope, ctx } = state;
-  const plan = planGridStride(items, ctx.workgroupSize, ctx.caps);
+  const plan = planGridStride(items, ctx.workgroupSize, ctx.caps, maxGroups);
   const params = scope.params(BC_PARAMS, { ...fields, stride: plan.stride ?? 0 });
   let bound = state.bound.get(kernel);
   if (bound === void 0) {
@@ -8384,8 +8474,8 @@ async function submit(state, batch, signal) {
   }
   return back;
 }
-async function runBatch(state, sources, form, levelsPerSubmit, tuning, signal) {
-  const { ctx, n, S, ends, depthK, sigmaK, deltaK, counters } = state;
+async function runBatch(state, mode, sources, form, levelsPerSubmit, tuning, signal) {
+  const { ctx, n, S, ends, levelMax, depthK, sigmaK, deltaK, counters } = state;
   const k = sources.length;
   const words = n * k;
   const seeds = Uint32Array.from(sources, (v, s) => s * n + v);
@@ -8403,15 +8493,18 @@ async function runBatch(state, sources, form, levelsPerSubmit, tuning, signal) {
       recordFill(state, pass, depthK, words, 4294967295);
       recordFill(state, pass, sigmaK, words, 0);
       recordFill(state, pass, deltaK, words, 0);
-      recordBc(state, pass, state.finalize, { counters, ends, S, depthK, sigmaK }, { n, k, role: 1 }, 1);
+      if (mode.count !== null) {
+        recordFill(state, pass, levelMax, n + 2, 0);
+      }
+      recordBc(state, pass, mode.finalize, { counters, ends, S, depthK, sigmaK, levelMax }, { n, k, role: 1 }, 1);
     }
     for (let level = 0; level < levelsPerSubmit; level++) {
-      recordBc(state, pass, state.finalize, { counters, ends, S, depthK, sigmaK }, { n, k, role: 0 }, 1);
-      if (form === "edge" && state.forwardEdge !== null && state.edgeSrc !== null && state.edgeDst !== null) {
+      recordBc(state, pass, mode.finalize, { counters, ends, S, depthK, sigmaK, levelMax }, { n, k, role: 0 }, 1);
+      if (form === "edge" && mode.forwardEdge !== null && state.edgeSrc !== null && state.edgeDst !== null) {
         recordBc(
           state,
           pass,
-          state.forwardEdge,
+          mode.forwardEdge,
           { edgeSrc: state.edgeSrc, edgeDst: state.edgeDst, S, ends, counters, depthK, sigmaK },
           forwardFields,
           forwardItems
@@ -8420,12 +8513,24 @@ async function runBatch(state, sources, form, levelsPerSubmit, tuning, signal) {
         recordBc(
           state,
           pass,
-          state.forward,
+          mode.forward,
           { rowPtr: state.rowPtr, colIdx: state.colIdx, S, ends, counters, depthK, sigmaK },
           forwardFields,
           forwardItems
         );
       }
+      if (mode.count !== null) {
+        const { kernel, rowPtr, colIdx } = mode.count;
+        recordBc(
+          state,
+          pass,
+          kernel,
+          { rowPtr, colIdx, S, ends, counters, depthK, sigmaK, levelMax },
+          { n, k },
+          words,
+          BC_COUNT_MAX_GROUPS
+        );
+      }
     }
     recorded += levelsPerSubmit;
     batch.endPass();
@@ -8438,6 +8543,9 @@ async function runBatch(state, sources, form, levelsPerSubmit, tuning, signal) {
       levels = words32[W.level];
       overflow = words32[W.sigmaOverflow] !== 0;
       endsWords = new Uint32Array(back, endsRequest.offset, endsCount).slice(0, levels + 1);
+      if (overflow && mode.count === null) {
+        return { levels, wrapped: true, overflow: false };
+      }
     } else if (recorded > n + 2) {
       throw new WebGpuGraphError("E_VALIDATION", `${ALGORITHM$5}: the done flag never rose in ${recorded} levels`, {
         label: ALGORITHM$5,
@@ -8461,20 +8569,28 @@ async function runBatch(state, sources, form, levelsPerSubmit, tuning, signal) {
       recordBc(
         state,
         pass,
-        state.backward,
-        { rowPtr: state.rowPtr, colIdx: state.colIdx, S, depthK, sigmaK, deltaK },
+        mode.backward,
+        { rowPtr: state.rowPtr, colIdx: state.colIdx, S, depthK, sigmaK, deltaK, levelMax },
         { n, k, start, count },
         count
       );
     }
     if (last) {
       recordBc(state, pass, state.gather, { deltaK, bc: state.bc }, { n, k }, n);
-      if (state.edgeGather !== null && state.arcScores !== null) {
+      if (mode.edgeGather !== null && state.arcScores !== null) {
         recordBc(
           state,
           pass,
-          state.edgeGather,
-          { rowPtr: state.rowPtr, colIdx: state.colIdx, depthK, sigmaK, deltaK, arcScores: state.arcScores },
+          mode.edgeGather,
+          {
+            rowPtr: state.rowPtr,
+            colIdx: state.colIdx,
+            depthK,
+            sigmaK,
+            deltaK,
+            arcScores: state.arcScores,
+            levelMax
+          },
           { n, k, count: state.arcCount },
           state.arcCount
         );
@@ -8500,12 +8616,13 @@ async function runBatch(state, sources, form, levelsPerSubmit, tuning, signal) {
     forward: form,
     levels,
     ends: endsWords.slice(),
+    scaled: mode.count !== null,
     sigmaOverflow: overflow,
     depthK: arrays?.depthK ?? null,
     sigmaK: arrays?.sigmaK ?? null,
     deltaK: arrays?.deltaK ?? null
   });
-  return { levels, overflow };
+  return { levels, wrapped: false, overflow };
 }
 async function runRaw(ctx, s, sources, withEdges, tuning, options) {
   const n = s.nodeCount;
@@ -8526,19 +8643,17 @@ async function runRaw(ctx, s, sources, withEdges, tuning, options) {
     const arrayBytes = 4 * n * kMax;
     const lease = (bytes, label) => bindingOf(scope.scratch(bytes, label), bytes);
     const arcBytes = 4 * Math.max(1, s.arcCount);
-    const [fill, finalize, forward, backward, gather] = await Promise.all(
-      ["fill", "bc-finalize", "bc-forward", "bc-backward", "bc-gather"].map(
-        (id) => ctx.pipelines.kernel(kernelSpec(id))
-      )
+    const [fill, gather] = await Promise.all(
+      ["fill", "bc-gather"].map((id) => ctx.pipelines.kernel(kernelSpec(id)))
     );
-    const forwardEdge = edgeView === null ? null : await ctx.pipelines.kernel(kernelSpec("bc-forward-edge", { UNDIRECTED: !s.directed }));
-    const edgeGather = withEdges ? await ctx.pipelines.kernel(kernelSpec("bc-edge-gather")) : null;
+    const exact = await modeKernels(ctx, s, core, edgeView !== null, withEdges, false);
     const state = {
       ctx,
       scope,
       n,
       S: lease(arrayBytes, "S"),
       ends: lease(4 * (n + 2), "ends"),
+      levelMax: lease(4 * (n + 2), "level-max"),
       depthK: lease(arrayBytes, "depthK"),
       sigmaK: lease(arrayBytes, "sigmaK"),
       deltaK: lease(arrayBytes, "deltaK"),
@@ -8552,12 +8667,9 @@ async function runRaw(ctx, s, sources, withEdges, tuning, options) {
       edgeCount,
       arcCount: s.arcCount,
       fill,
-      finalize,
-      forward,
-      forwardEdge,
-      backward,
       gather,
-      edgeGather,
+      exact,
+      scaled: null,
       bound: /* @__PURE__ */ new Map()
     };
     await ctx.allocator.check();
@@ -8578,11 +8690,23 @@ async function runRaw(ctx, s, sources, withEdges, tuning, options) {
       if (pinned === "auto" && previousLevels >= 0) {
         form = previousLevels < BC_EDGE_PARALLEL_GAMMA * Math.log2(n) ? "edge" : "frontier";
       }
-      if (state.forwardEdge === null) {
+      if (state.exact.forwardEdge === null) {
         form = "frontier";
       }
       const batch = sources.slice(start, start + k);
-      const outcome = await runBatch(state, batch, form, levelsPerSubmit, tuning, options?.signal);
+      let outcome = await runBatch(
+        state,
+        state.scaled ?? state.exact,
+        batch,
+        form,
+        levelsPerSubmit,
+        tuning,
+        options?.signal
+      );
+      if (outcome.wrapped) {
+        state.scaled ??= await modeKernels(ctx, s, core, edgeView !== null, withEdges, true);
+        outcome = await runBatch(state, state.scaled, batch, form, levelsPerSubmit, tuning, options?.signal);
+      }
       overflow = overflow || outcome.overflow;
       previousLevels = outcome.levels;
       start += k;
@@ -15508,7 +15632,7 @@ async function calibrateLayout(ctx, options) {
   };
 }
 export {
-  a1 as ARC_WINDOW_ALIGN,
+  a2 as ARC_WINDOW_ALIGN,
   EXACT_MAX_NODES,
   FA2_DEFAULTS,
   FR_DEFAULTS,
@@ -15516,10 +15640,10 @@ export {
   LAYOUT_TUNING_DEFAULTS,
   MAX_1D_ITEMS,
   MAX_WORKGROUPS_PER_DIM,
-  a2 as PASSTHROUGH_FORMAT_CODES,
+  a3 as PASSTHROUGH_FORMAT_CODES,
   SE_DEFAULTS,
-  a3 as STORAGE_ALIGN,
-  a4 as WORKGROUP_SIZE,
+  a4 as STORAGE_ALIGN,
+  a5 as WORKGROUP_SIZE,
   WebGpuGraphError,
   allPairsShortestPath,
   bellmanFord,
@@ -15537,7 +15661,7 @@ export {
   eigenvectorCentrality,
   hasErrorCode,
   hits,
-  a5 as isSoftwareAdapter,
+  a6 as isSoftwareAdapter,
   isWebGpuGraphError,
   katzCentrality,
   labelPropagation,