@ryanstark24/sfgraph-core 1.2.0 → 1.4.0

package/dist/extractors/live-org/bulk-retrieve.js

@@ -1,6 +1,7 @@
 import { discoverMetadataTypes } from "./discovery.js";
 import { buildDispatchTable } from "./dispatch.js";
 import { iterApex } from "./extractors/apex.js";
+import { iterCustomSettings } from "./extractors/custom-settings.js";
 import { iterFlow } from "./extractors/flow.js";
 import { iterGenericMetadata } from "./extractors/generic-metadata.js";
 import { iterIntegration } from "./extractors/integration.js";
@@ -9,6 +10,28 @@ import { iterObject } from "./extractors/object.js";
 import { iterOmnistudio } from "./extractors/omnistudio.js";
 import { iterSecurity } from "./extractors/security.js";
 import { iterVlocity } from "./extractors/vlocity.js";
+import { StopWaitingError, stopSignalContext } from "./rate-limit.js";
+function createWedgeCoordinator(warnings = [], overrides) {
+    const capEnv = Number.parseInt(process.env.SFGRAPH_MAX_BACKGROUND_WEDGES ?? "", 10);
+    // Default 32 (was 4): a full --rebuild on a large org wedges many slow/empty
+    // generic-metadata sources at the 90s first-yield watchdog AND the heavy Apex
+    // source; a cap of 4 evicted most of them (their data lost). The drain loop
+    // skips already-resolved entries cheaply, so a high cap costs little.
+    const cap = Number.isFinite(capEnv) && capEnv > 0 ? capEnv : 32;
+    const drainEnv = Number.parseInt(process.env.SFGRAPH_LATE_DRAIN_BUDGET_MS ?? "", 10);
+    // Default 300s (was 60s): the per-entry budget is TOTAL drain wall-clock.
+    // Apex needs ~75s to stream all classes once the Tooling pool frees up after
+    // the main loop; 60s clipped it mid-drain. Empty/fast entries still return
+    // `done` immediately and cost nothing, so the larger budget only helps the
+    // genuinely-heavy entries finish.
+    const drain = Number.isFinite(drainEnv) && drainEnv >= 0 ? drainEnv : 300_000;
+    return {
+        warnings,
+        backgroundWedges: [],
+        maxBackgroundWedges: overrides?.maxBackgroundWedges ?? cap,
+        lateDrainBudgetMs: overrides?.lateDrainBudgetMs ?? drain,
+    };
+}
 /** Best-effort classification so the end-of-run recommendation is targeted. */
 function classifySkip(msg) {
     const m = msg.toUpperCase();
@@ -34,27 +57,27 @@ export async function* mergeAsyncIterables(...iters) {
             yield v;
     }
 }
-/**
- * Sliding-window parallel merge. Keeps exactly `concurrency` iterators
- * live at any moment — when one completes (done:true), the next queued
- * iterator is started in its slot. Within the window, iterators race via
- * `Promise.race` for max throughput.
- *
- * Replaces the previous wave-based merger, which had a synchronisation
- * barrier at each wave boundary: all 6 iterators in a wave had to finish
- * before any of wave 2 could start. One hung source (very common for
- * `generic:Layout` on managed-package-heavy orgs) parked the whole wave
- * indefinitely with zero observable progress.
- *
- * Default concurrency = 8: enough to saturate the three rate-limit pools
- * (Tooling 5 / Metadata 5 / Data 10) with headroom for one slot to be
- * blocked on a slow source without starving the others. Override via
- * `SFGRAPH_SOURCE_CONCURRENCY=<n>` (1 = strictly sequential).
- *
- * Output ordering is non-deterministic. `live-ingest`'s processOne is
- * order-independent (idempotent per-record upserts), so this is safe.
- */
-export async function* mergeAsyncIterablesParallel(...iters) {
+export function mergeAsyncIterablesParallel(...args) {
+    // Disambiguate the two overloads. The 2-arg array+opts form is used by
+    // tests; the rest-args form is the production fan-out path.
+    let iters;
+    let opts;
+    if (args.length === 2 &&
+        Array.isArray(args[0]) &&
+        args[1] != null &&
+        typeof args[1] === "object" &&
+        !(args[1] instanceof Promise) &&
+        !(Symbol.asyncIterator in args[1])) {
+        iters = args[0];
+        opts = args[1];
+    }
+    else {
+        iters = args;
+        opts = undefined;
+    }
+    return mergeAsyncIterablesParallelImpl(iters, opts);
+}
+async function* mergeAsyncIterablesParallelImpl(iters, opts) {
     const envConcurrency = Number.parseInt(process.env.SFGRAPH_SOURCE_CONCURRENCY ?? "", 10);
     // Raised 8 -> 12: a wider window lets Tooling-backed sources (apex, lwc)
     // start immediately rather than wait behind the cohort of metadata-pool
@@ -62,7 +85,11 @@ export async function* mergeAsyncIterablesParallel(...iters) {
     // which spend most of their time queued in Bottleneck. Per-pool concurrency
     // still caps total HTTP fan-out, so this only opens up parallelism that
     // was being throttled at the wrong layer.
-    const concurrency = Number.isFinite(envConcurrency) && envConcurrency > 0 ? envConcurrency : 12;
+    const concurrency = opts?.concurrency != null && opts.concurrency > 0
+        ? opts.concurrency
+        : Number.isFinite(envConcurrency) && envConcurrency > 0
+            ? envConcurrency
+            : 12;
     const iterators = iters.map((it) => it[Symbol.asyncIterator]());
     const pending = new Map();
     let nextIterIdx = 0;
@@ -97,60 +124,275 @@ export async function* mergeAsyncIterablesParallel(...iters) {
         advance(idx);
     }
 }
-/** Wrap an iterable so a thrown error is captured + the stream ends cleanly
- *  instead of aborting the whole ingest. The error is recorded into a
- *  shared skip report (consumed at end-of-run) and a compact ✗ line is
- *  printed so the user sees something happened without the full error
- *  message scrolling past during the run. */
-async function* failSoft(label, factory, onError) {
+/**
+ * Per-source watchdog budgets. Centralized here so the failSoft wrapper and
+ * tests share the same constants.
+ *
+ * Phase 1.5 W1.5-01 contract: these are the budgets the watchdog uses, but
+ * the clock against which they measure starts at slot-acquisition time
+ * (i.e., on the FIRST `it.next()` call inside `failSoft`'s body), NOT at
+ * source registration. Queued-but-not-yet-executing sources accumulate
+ * ZERO clock time, so a wedge in one source can no longer cascade-kill
+ * its queued neighbors.
+ *
+ * Anti-feature reminder: do NOT raise these to "fix" a real wedge. The
+ * correct fix is per-source override (W1.5-future) or fixing the underlying
+ * pre-yield setup call. See `.planning/phase-1.5/PLAN.md` anti-features.
+ */
+const WATCHDOG_INACTIVITY_MS_DEFAULT = 5 * 60_000;
+const WATCHDOG_FIRST_YIELD_MS_DEFAULT = 90_000;
+/**
+ * Per-source watchdog overrides (the "per-source override" the anti-feature
+ * note above points to). Some sources legitimately need far longer than the
+ * 90s default to produce their FIRST record — most notably the Vlocity runner,
+ * which executes a long sequence of `vlocity_cmt` SObject queries and assembles
+ * DataPacks before yielding member #1, and can have multi-minute gaps between
+ * DataPack types once streaming. Raising the GLOBAL budget to accommodate it is
+ * wrong: it makes every other slow/empty source hold its merger slot 10x longer
+ * and starves the pool (observed: a 600s global first-yield stalled the whole
+ * fan-out behind a few empty generic sources). So scope the larger budget to
+ * the specific label only. Explicit env vars still win (debug/tests).
+ */
+const SOURCE_WATCHDOG_OVERRIDES = {
+    // Vlocity must stay INLINE: the post-merger background-drain cannot reliably
+    // recover it (it needs minutes to first-yield, longer than any sane drain
+    // budget, and competes for drain slots). Giving it a long inline budget lets
+    // it hold its DATA-pool slot and stream once the pool frees up. NB: do NOT
+    // do this for Tooling-pool sources like Apex — holding a scarce Tooling slot
+    // that long starves every other Tooling source and stalls the whole fan-out.
+    // Apex instead wedges fast (releasing its slot) and is recovered by the
+    // background-drain pass, which is sized generously in createWedgeCoordinator.
+    vlocity: { firstYieldMs: 20 * 60_000, inactivityMs: 10 * 60_000 },
+};
+/** Read watchdog budgets per-call so unit tests can override via env vars
+ *  without restarting the process. Production env should leave these unset:
+ *  ordinary sources get the documented 90s first-yield / 5min inactivity;
+ *  a source in SOURCE_WATCHDOG_OVERRIDES gets its scoped budget instead. */
+function readWatchdogBudgets(label) {
+    const fy = Number.parseInt(process.env.SFGRAPH_WATCHDOG_FIRST_YIELD_MS ?? "", 10);
+    const inact = Number.parseInt(process.env.SFGRAPH_WATCHDOG_INACTIVITY_MS ?? "", 10);
+    const override = label ? SOURCE_WATCHDOG_OVERRIDES[label] : undefined;
+    return {
+        firstYieldMs: Number.isFinite(fy) && fy > 0
+            ? fy
+            : (override?.firstYieldMs ?? WATCHDOG_FIRST_YIELD_MS_DEFAULT),
+        inactivityMs: Number.isFinite(inact) && inact > 0
+            ? inact
+            : (override?.inactivityMs ?? WATCHDOG_INACTIVITY_MS_DEFAULT),
+    };
+}
+/** Build the namespaced wedge warning string per the schema documented in
+ *  PLAN.md (`wedge:<source>:<stage>:<detail>`). Centralized here so the
+ *  format is stable across emit sites + grep-friendly. */
+function fmtWedgeWarning(source, stage, detail) {
+    const parts = [`wedge:${source}:${stage}`];
+    for (const [k, v] of Object.entries(detail)) {
+        parts.push(`${k}=${v}`);
+    }
+    return parts.join(":");
+}
+/**
+ * Enforce the background-wedge cap. When admitting a new entry would push
+ * the count above `maxBackgroundWedges`, mark the OLDEST entries' stop-
+ * waiting flags so their drain loops bail at the next iteration. Emits a
+ * `wedge:cap:backgroundWedgeAborted:...` warning per evicted entry. The
+ * iterator object itself is left in the set; the late-drain loop is what
+ * sees the flag and skips it. (We do NOT remove from `backgroundWedges`
+ * here so post-merger inspection still sees every wedge that ever fired
+ * during the run.)
+ */
+function enforceBackgroundWedgeCap(coordinator) {
+    const active = coordinator.backgroundWedges.filter((e) => !e.stopWaitingRef.value);
+    const overage = active.length - coordinator.maxBackgroundWedges;
+    if (overage <= 0)
+        return;
+    // Sort by registeredAt ascending; oldest first.
+    active.sort((a, b) => a.registeredAt - b.registeredAt);
+    for (let i = 0; i < overage; i += 1) {
+        const victim = active[i];
+        if (!victim)
+            continue;
+        victim.stopWaitingRef.value = true;
+        const ageMs = Date.now() - victim.registeredAt;
+        coordinator.warnings.push(fmtWedgeWarning("cap", "backgroundWedgeAborted", {
+            source: victim.label,
+            ageMs,
+            reason: "backgroundWedgeCapExceeded",
+        }));
+    }
+}
+/**
+ * Drain a single background-wedge iterator. Called once per wedged source,
+ * post-merger, with a wall-clock budget. Each iteration races the iterator's
+ * `it.next()` against the remaining budget. Records arriving inside the
+ * budget are yielded with `attributes.lateYield = true`. The pending HTTP
+ * request (if the wedge is on a network call) is NOT actively cancelled
+ * (W1.5-03's job); we just stop awaiting it.
+ *
+ * Bails immediately if `stopWaitingRef.value === true` (cap eviction).
+ */
+async function* drainBackgroundWedge(entry, budgetMs, warnings) {
+    const debug = process.env.SFGRAPH_DEBUG_INGEST === "1";
+    const startedAt = Date.now();
+    let lateCount = 0;
+    while (!entry.stopWaitingRef.value) {
+        const remaining = startedAt + budgetMs - Date.now();
+        if (remaining <= 0)
+            break;
+        // First iteration: consume the abandoned-but-still-pending `it.next()`
+        // promise from when the watchdog fired. Subsequent iterations call
+        // `iterator.next()` normally. Async-generator semantics queue
+        // concurrent `.next()` calls, so we MUST consume the pending one
+        // before issuing a new one — otherwise the late-yielded value goes
+        // to the abandoned promise and the drain sees the NEXT yield (or
+        // `done:true` if the generator is single-yield).
+        const nextPromise = entry.pendingFirstNext !== null
+            ? entry.pendingFirstNext
+            : stopSignalContext.run(entry.stopWaitingRef, () => entry.iterator.next());
+        if (entry.pendingFirstNext !== null)
+            entry.pendingFirstNext = null;
+        let next;
+        try {
+            next = await Promise.race([
+                nextPromise,
+                new Promise((resolve) => {
+                    setTimeout(
+                    // Resolve (not reject) with a synthetic done sentinel — bailing
+                    // is the normal exit path from a budget-exceeded drain, not an
+                    // error condition.
+                    () => resolve({ value: undefined, done: true }), remaining);
+                }),
+            ]);
+        }
+        catch (e) {
+            // Late-arriving error from the wedged iterator. Don't crash the drain;
+            // surface as a warning and stop.
+            const err = e?.message ?? String(e);
+            warnings.push(fmtWedgeWarning(entry.label, "lateError", { error: err.slice(0, 120) }));
+            break;
+        }
+        if (next.done)
+            break;
+        lateCount += 1;
+        const rec = next.value;
+        // Tag with lateYield so downstream consumers / tests can identify the
+        // record as having arrived after slot release.
+        const tagged = {
+            ...rec,
+            attributes: { ...(rec.attributes ?? {}), lateYield: true },
+        };
+        yield tagged;
+    }
+    if (lateCount > 0) {
+        const totalMs = Date.now() - entry.registeredAt;
+        warnings.push(fmtWedgeWarning(entry.label, "resolvedLate", {
+            totalMs,
+            records: lateCount,
+        }));
+    }
+    if (debug) {
+        console.log(`ingest: [debug] ${entry.label} background-wedge drain finished (${lateCount} late records, stopWaiting=${entry.stopWaitingRef.value})`);
+    }
+}
+/**
+ * Wrap an iterable so a thrown error is captured + the stream ends cleanly
+ * instead of aborting the whole ingest. The error is recorded into a
+ * shared skip report (consumed at end-of-run) and a compact ✗ line is
+ * printed so the user sees something happened without the full error
+ * message scrolling past during the run.
+ *
+ * Phase 1.5 W1.5-01 + W1.5-02 contract:
+ *
+ *  - The watchdog clock starts when this generator's BODY first executes
+ *    (i.e., when the merger calls `.next()` on us for the first time —
+ *    that's slot-acquisition time). Queued sources still inside
+ *    `mergeAsyncIterablesParallel`'s waiting list accumulate ZERO clock
+ *    time. Implemented via an internal `startedAtRef` that's written once
+ *    on body entry; if the caller passes a `startedAtRef` we update it in
+ *    place so the merger can observe the slot-acquisition moment.
+ *
+ *  - On watchdog fire, we DO NOT throw (which would abort the merger's
+ *    Promise.race for our slot AND lose the still-pending iterator).
+ *    Instead we:
+ *       1. Emit a `wedge:<label>:firstYield:90s:...` (or `:inactivity:300s:`)
+ *          warning into coordinator.warnings.
+ *       2. Register the still-pending iterator into coordinator.background
+ *          Wedges with a stopWaitingRef. Enforce the cap (evict oldest).
+ *       3. RETURN from the generator. This cleanly releases the merger's
+ *          slot — `pending.delete(idx)` runs in mergeAsyncIterablesParallel
+ *          and the next queued source acquires the slot.
+ *    Post-merger, the bulkRetrieve top-level fan-out drains the background
+ *    wedge set; records arriving inside the late-drain budget are yielded
+ *    with `attributes.lateYield = true`.
+ */
+async function* failSoft(label, factory, onError, coordinator, 
+/** Optional outer-visible clock cell. The merger does not currently read
+ *  this, but the contract is "set by failSoft on body entry"; tests use
+ *  it to assert slot-acquisition timing. */
+startedAtRef) {
     const debug = process.env.SFGRAPH_DEBUG_INGEST === "1";
+    // W1.5-01: the clock starts HERE — body-entry, which is slot-acquisition
+    // time. NOT at failSoft() invocation (that's source registration, which
+    // can be hundreds of seconds before the merger picks us out of the queue).
     const startedAt = Date.now();
+    if (startedAtRef)
+        startedAtRef.value = startedAt;
     let count = 0;
     let started = false;
-    // Watchdog at 5 minutes. Per-call withTimeouts (120s) catch hung HTTP;
-    // this catches the case where every call ALSO times out and the source
-    // never yields a single record. 5 min = ~2.5× the per-call budget,
-    // generous for queued-but-healthy, tight enough that a genuinely
-    // wedged source surfaces fast and doesn't park its sliding-window slot
-    // for half an hour. With the no-retry-on-timeout fix in
-    // readMetadataBatchAdaptive, the inner loop also can't balloon, so 5
-    // minutes is a real ceiling now.
-    const inactivityMs = 5 * 60_000;
-    // Tighter deadline for the FIRST record. The 5-min inactivity watchdog
-    // is for healthy-but-slow sources that have already started yielding;
-    // a source that hasn't yielded a single record in 90s is almost
-    // certainly parked on a pre-yield setup call (metadata.list, describe,
-    // EntityDefinition pagination) and won't recover. Killing it fast frees
-    // the sliding-window slot for queued peers.
-    const firstYieldMs = 90_000;
+    // Track last-yielded qualifiedName for diagnostic warnings — the smoking-
+    // gun field that names WHICH record was in flight when the wedge fired.
+    let lastYieldedQName = "<none>";
+    const { firstYieldMs, inactivityMs } = readWatchdogBudgets(label);
+    // W1.5-03: pre-allocate the per-source stop-signal at body entry and
+    // publish it into AsyncLocalStorage so any downstream rate-limit wrapper
+    // (`withTimeout`, `soqlWithTimeout`, `scheduleQuery`) inside the wedged
+    // iterator can observe `value = true` after cap-eviction flips it. Same
+    // ref is later attached to the BackgroundWedgeEntry so the keystone's
+    // `enforceBackgroundWedgeCap` flip propagates to in-flight calls without
+    // any per-extractor signature change. AsyncLocalStorage propagates across
+    // awaits within an async function started inside .run() — we factor the
+    // iterator into a helper started inside the .run() callback.
+    const stopWaitingRef = { value: false };
+    const it = stopSignalContext.run(stopWaitingRef, () => factory()[Symbol.asyncIterator]());
+    if (debug)
+        console.log(`ingest: [debug] ${label} ← invoked at ${startedAt}`);
     try {
-        if (debug)
-            console.log(`ingest: [debug] ${label} ← invoked at ${startedAt}`);
-        const it = factory()[Symbol.asyncIterator]();
         while (true) {
             const remainingFirstYield = started
                 ? Number.POSITIVE_INFINITY
                 : startedAt + firstYieldMs - Date.now();
             const watchdogMs = Math.min(inactivityMs, remainingFirstYield);
-            if (watchdogMs <= 0) {
-                throw new Error(`source watchdog (first-yield ${firstYieldMs / 1000}s): no record yielded — pre-yield setup wedged`);
-            }
-            const watchdogLabel = started
-                ? `${inactivityMs / 60_000}m inactivity`
-                : `first-yield ${firstYieldMs / 1000}s`;
-            const next = await Promise.race([
-                it.next(),
-                new Promise((_resolve, reject) => {
-                    setTimeout(() => reject(new Error(`source watchdog (${watchdogLabel}): no record yielded — pool jammed or call wedged`)), watchdogMs);
+            const stage = started ? "inactivity" : "firstYield";
+            const stageDetail = started ? `${inactivityMs / 1000}s` : `${firstYieldMs / 1000}s`;
+            // Run each `.next()` synchronously inside the ALS context so awaits
+            // inside the underlying generator's body that call into rate-limit
+            // wrappers can resolve the stop-signal via `getStore()`.
+            const pending = stopSignalContext.run(stopWaitingRef, () => it.next());
+            let timer;
+            const wedged = await Promise.race([
+                pending.then((result) => {
+                    if (timer)
+                        clearTimeout(timer);
+                    return { wedge: false, result };
+                }),
+                new Promise((resolve) => {
+                    // watchdogMs may be <=0 if the budget is already burned (defensive
+                    // — should be rare since the clock starts at body-entry). setTimeout
+                    // with non-positive delay fires on the next event-loop tick, which
+                    // is the correct wedge-immediately behavior.
+                    timer = setTimeout(() => resolve({ wedge: true }), Math.max(0, watchdogMs));
                 }),
             ]);
+            if (wedged.wedge) {
+                // Hand off the still-pending it.next() so the drain can consume
+                // it directly — see BackgroundWedgeEntry.pendingFirstNext.
+                emitWedge(stage, stageDetail, pending);
+                return;
+            }
+            const next = wedged.result;
             if (next.done)
                 break;
-            if (!started) {
-                started = true;
-                console.log(`ingest:   ${label} → starting…`);
-            }
-            count += 1;
+            recordYield(next.value);
             yield next.value;
         }
         if (started) {
@@ -175,6 +417,64 @@ async function* failSoft(label, factory, onError) {
             console.log(`ingest: [debug] ${label} → finalised (${count} records, ${Date.now() - startedAt}ms)`);
         }
     }
+    /** Inline helper: extracts qualifiedName-ish text from a RawMember for the
+     *  `lastYielded=...` diagnostic field. Safe against undefined. */
+    function recordYield(v) {
+        if (!started) {
+            started = true;
+            console.log(`ingest:   ${label} → starting…`);
+        }
+        count += 1;
+        const ref = v.ref;
+        if (ref) {
+            lastYieldedQName = `${ref.memberType}/${ref.memberName}`;
+        }
+    }
+    /**
+     * Slot-release + soft-isolate on watchdog fire. Two side effects:
+     *   1. Push a `wedge:<label>:<stage>:<detail>` warning into the shared sink.
+     *   2. Park the still-pending iterator into the background-wedge set so
+     *      the post-merger late-drain pass can capture any records that
+     *      eventually arrive. Enforce the cap (oldest evicted with a
+     *      `backgroundWedgeAborted` warning).
+     * The generator then RETURNS (caller is `for await` of the merger; the
+     * merger sees `done:true` and releases the slot).
+     */
+    function emitWedge(stage, stageDetail, pendingFirstNext) {
+        const wedgedFor = Date.now() - startedAt;
+        if (coordinator) {
+            // Schema (PLAN.md): `wedge:<label>:<stage>:<stageDetail>:<k=v>:<k=v>`.
+            // <stageDetail> is a positional value (e.g. "90s"), not a key=value.
+            const w = `wedge:${label}:${stage}:${stageDetail}` +
+                `:lastYielded=${lastYieldedQName}` +
+                `:wedgedForMs=${wedgedFor}`;
+            coordinator.warnings.push(w);
+            console.log(`ingest:   ${label} ⚠ wedged (${stage} ${stageDetail}); slot released, draining in background`);
+            // W1.5-03: reuse the SAME stopWaitingRef that was published into ALS
+            // at body entry. When enforceBackgroundWedgeCap later flips this ref's
+            // `.value = true` on cap eviction, any in-flight rate-limit wrapper
+            // inside the wedged iterator's continuing background pumps will see
+            // the flag through `stopSignalContext.getStore()` and reject with
+            // `StopWaitingError` quickly. The underlying jsforce HTTP call still
+            // leaks until socket idle timeout — documented in rate-limit.ts.
+            coordinator.backgroundWedges.push({
+                label,
+                iterator: it,
+                pendingFirstNext,
+                registeredAt: Date.now(),
+                stopWaitingRef,
+            });
+            enforceBackgroundWedgeCap(coordinator);
+        }
+        else {
+            // No coordinator (e.g. unit-test path that didn't wire one up, or
+            // back-compat caller): fall back to the old "log+skip" behavior so
+            // the caller still gets observable feedback.
+            const msg = `source watchdog (${stage} ${stageDetail}): no record yielded — pool jammed or call wedged`;
+            onError?.(label, new Error(msg));
+            console.log(`ingest:   ${label} ✗ skipped (${msg.slice(0, 80)})`);
+        }
+    }
 }
 /** Typed-extractor ownership map: which XML type names a dedicated extractor covers. */
 const APEX_TYPES = new Set(["ApexClass", "ApexTrigger"]);
@@ -291,10 +591,29 @@ const GENERIC_TYPE_WHITELIST = new Set([
     "OmniUiCard",
     "OmniProcess",
 ]);
+/**
+ * Types whose data is fully captured by a CONTAINER type, so routing them
+ * individually only produces value-less opaque duplicates that clobber the
+ * container's richer nodes. `CustomLabel` (singular) is covered by the
+ * `CustomLabels` container — whose rule extracts each label's value/language;
+ * the singular path would emit `CustomLabel:Name` nodes with no value and,
+ * depending on merge order, overwrite the real ones. Skip them.
+ */
+const CONTAINER_COVERED_TYPES = new Set(["CustomLabel"]);
 function shouldRouteGeneric(type) {
-    if (process.env.SFGRAPH_INCLUDE_ALL_GENERIC === "1")
-        return true;
-    return GENERIC_TYPE_WHITELIST.has(type);
+    // Never route a type a container already covers with richer data.
+    if (CONTAINER_COVERED_TYPES.has(type))
+        return false;
+    // Default: route EVERY long-tail metadata type through the generic
+    // extractor so the graph — and therefore semantic search — covers the org
+    // completely. Anything without a dedicated extractor still becomes a typed
+    // node (the rule parsers key off the type name) or an opaque node, which is
+    // what search needs. The old curated whitelist silently dropped ~250
+    // discovered types (search returned nothing for them). Set
+    // SFGRAPH_GENERIC_WHITELIST_ONLY=1 to restore the restricted subset.
+    if (process.env.SFGRAPH_GENERIC_WHITELIST_ONLY === "1")
+        return GENERIC_TYPE_WHITELIST.has(type);
+    return true;
 }
 export async function* bulkRetrieve(conn, caps, orgId, opts = {}) {
     // Back-compat: callers used to pass IngestSkipReport directly as the 4th arg.
@@ -304,6 +623,13 @@ export async function* bulkRetrieve(conn, caps, orgId, opts = {}) {
         : opts;
     const skipReport = normalized.skipReport;
     const onlyLabels = normalized.onlyLabels;
+    // Phase 1.5 W1.5-02 coordinator. Built only when a warnings sink was
+    // supplied — without one, no place to push wedge strings, so the legacy
+    // "watchdog fires → skip entry" behavior is preserved. The coordinator
+    // owns the background-wedge cap + late-drain budget.
+    const coordinator = normalized.warnings
+        ? createWedgeCoordinator(normalized.warnings)
+        : undefined;
     // Discover the type list this org actually supports. If discovery fails or
     // returns nothing usable, fall back to invoking every known extractor —
     // preserves Commit-A behavior for mocks that don't implement describe.
@@ -336,7 +662,7 @@ export async function* bulkRetrieve(conn, caps, orgId, opts = {}) {
         // ingest. The wrapper records the source label + error into skipReport
         // (consumed at end of run for a consolidated summary) and ends the
         // stream cleanly.
-        sources.push(failSoft(key, factory, onSkip));
+        sources.push(failSoft(key, factory, onSkip, coordinator));
     };
     if (types.length === 0) {
         // Discovery unavailable: invoke every dedicated extractor once.
@@ -394,12 +720,54 @@ export async function* bulkRetrieve(conn, caps, orgId, opts = {}) {
                     break;
             }
         }
+        // Child metadata types (ValidationRule, ListView, CompactLayout, FieldSet,
+        // WebLink, BusinessProcess, ...) are returned by describe() only as
+        // `childXmlNames` of a parent (mostly CustomObject), never as top-level
+        // types — so the dispatch loop above never fetches them. Most ARE
+        // independently listable via metadata.list, so route each through the
+        // generic extractor. This is how validation rules / list views on STANDARD
+        // objects get captured — iterObject only sees children embedded in the
+        // custom objects it reads, missing the bulk that live on Account/Case/etc.
+        // Children that aren't independently listable fail-soft to nothing. Skip
+        // children already owned by a dedicated extractor or dispatched above.
+        if (process.env.SFGRAPH_GENERIC_WHITELIST_ONLY !== "1") {
+            const typedOwned = new Set([
+                ...APEX_TYPES,
+                ...LWC_TYPES,
+                ...FLOW_TYPES,
+                ...OBJECT_TYPES,
+                ...SECURITY_TYPES,
+                ...INTEGRATION_TYPES,
+                "CustomField", // fetched in bulk by iterObject; re-listing is wasteful
+            ]);
+            const childTypes = new Set();
+            for (const t of types)
+                for (const c of t.childXmlNames)
+                    childTypes.add(c);
+            for (const child of childTypes) {
+                if (dispatch.has(child) || typedOwned.has(child))
+                    continue;
+                if (CONTAINER_COVERED_TYPES.has(child))
+                    continue; // CustomLabel ⊂ CustomLabels
+                invoke(`generic:${child}`, () => iterGenericMetadata(conn, String(orgId), child));
+            }
+        }
     }
     if (caps.vlocityLegacy) {
         invoke("vlocity", () => iterVlocity(conn, caps, String(orgId), onSkip));
     }
+    // Custom Setting ROW capture — self-guards (yields nothing if the org has no
+    // custom settings or EntityDefinition is unavailable), so invoke always.
+    invoke("custom-settings", () => iterCustomSettings(conn, String(orgId)));
     if (caps.omnistudioOncore) {
-        invoke("omnistudio", () => iterOmnistudio(conn));
+        // When the Metadata-API retrieve path is enabled it covers
+        // OmniDataTransform and OmniUiCard with higher fidelity — skip them on
+        // the SOQL path so the same component is not ingested twice (the
+        // retrieve XML would otherwise clobber the SOQL-enriched node, and
+        // vice versa, nondeterministically by merge order).
+        invoke("omnistudio", () => iterOmnistudio(conn, normalized.enableOmnistudioRetrieve
+            ? { skipTypes: new Set(["OmniDataTransform", "OmniUiCard"]) }
+            : {}));
         if (normalized.enableOmnistudioRetrieve) {
             invoke("omnistudio-retrieve", async function* () {
                 const { iterOmnistudioRetrieve } = await import("./extractors/omnistudio-retrieve.js");
@@ -410,10 +778,16 @@ export async function* bulkRetrieve(conn, caps, orgId, opts = {}) {
             });
         }
     }
-    // Log the generic-type filter summary once at fan-out start — makes the
-    // skip-vs-route decision visible without --debug.
+    // Log the generic-type routing summary once at fan-out start — makes the
+    // skip-vs-route decision visible without --debug. The default is FULL
+    // coverage (every discovered type routed); the opt-out is
+    // SFGRAPH_GENERIC_WHITELIST_ONLY=1 (restricts to the curated subset).
     if (types.length > 0) {
-        console.log(`ingest: dispatch routed=${sources.length} sources (${types.length} discovered metadata types; generic-type whitelist active — set SFGRAPH_INCLUDE_ALL_GENERIC=1 to invoke all)`);
+        const whitelistOnly = process.env.SFGRAPH_GENERIC_WHITELIST_ONLY === "1";
+        const mode = whitelistOnly
+            ? "curated whitelist only (SFGRAPH_GENERIC_WHITELIST_ONLY=1) — unset to route all"
+            : "full coverage (all discovered types) — set SFGRAPH_GENERIC_WHITELIST_ONLY=1 to restrict";
+        console.log(`ingest: dispatch routed=${sources.length} sources (${types.length} discovered metadata types; ${mode})`);
     }
     // Parallel by default: fan out across all source iterators so different
     // pools (Tooling/Metadata/Data) saturate simultaneously. Escape hatch:
@@ -421,5 +795,60 @@ export async function* bulkRetrieve(conn, caps, orgId, opts = {}) {
     // anyone who hits an ordering bug or wants the old log layout.
     const sequential = process.env.SFGRAPH_SEQUENTIAL_SOURCES === "1";
     yield* sequential ? mergeAsyncIterables(...sources) : mergeAsyncIterablesParallel(...sources);
+    // Phase 1.5 W1.5-02 late-yield drain. When the merger has finished its
+    // main loop (all live slots cleared), drain any background-wedge iterators
+    // for the configured budget. Records arriving inside the budget are
+    // yielded with `attributes.lateYield = true`. Iterators whose stopWaiting
+    // flag was set by the cap-eviction path skip immediately. Anything still
+    // unresolved after the budget is dropped on the floor — its underlying
+    // jsforce HTTP request continues running until libuv tears it down at
+    // process exit (W1.5-03 will surface a "stop-waiting" semantic for this).
+    if (coordinator && coordinator.backgroundWedges.length > 0 && coordinator.lateDrainBudgetMs > 0) {
+        if (process.env.SFGRAPH_DEBUG_INGEST === "1") {
+            console.log(`ingest: [debug] late-drain pass: ${coordinator.backgroundWedges.length} background wedge(s), budget=${coordinator.lateDrainBudgetMs}ms`);
+        }
+        for (const entry of coordinator.backgroundWedges) {
+            yield* drainBackgroundWedge(entry, coordinator.lateDrainBudgetMs, coordinator.warnings);
+        }
+    }
 }
+/**
+ * Internal-surface exports for unit tests in `__tests__/`. Not part of the
+ * public package API — these are intentionally namespaced under `__testing`
+ * so they don't appear in the package's exports surface to downstream
+ * consumers. (They're still importable, but the underscore convention
+ * signals "test only".)
+ *
+ * Exposed:
+ *   - `failSoft`: wrap an iterable with the W1.5-01/02 watchdog + soft-isolate.
+ *   - `createWedgeCoordinator`: build a coordinator + warnings sink with
+ *      optional cap/drain overrides for deterministic tests.
+ *   - `drainBackgroundWedge`: run the post-merger late-drain pass against
+ *      a single background-wedge entry.
+ *   - `mergeAsyncIterablesParallel`: re-export for symmetry; the (array,
+ *      opts) overload is what tests use to fix concurrency.
+ *   - `setTestWatchdogBudgets` / `resetTestWatchdogBudgets`: env-var-backed
+ *      knobs to squeeze the watchdog budget for fast tests. Always paired
+ *      via try/finally in tests so the environment is left clean.
+ */
+export const __testing = {
+    failSoft,
+    createWedgeCoordinator,
+    drainBackgroundWedge,
+    mergeAsyncIterablesParallel,
+    readWatchdogBudgets,
+    setTestWatchdogBudgets(opts) {
+        process.env.SFGRAPH_WATCHDOG_FIRST_YIELD_MS = String(opts.firstYieldMs);
+        process.env.SFGRAPH_WATCHDOG_INACTIVITY_MS = String(opts.inactivityMs);
+    },
+    resetTestWatchdogBudgets() {
+        // Note: assigning `undefined` to a `process.env.X` key in Node coerces to
+        // the string "undefined". That's fine here because readWatchdogBudgets()
+        // runs parseInt() on the value, parseInt("undefined") === NaN, and NaN
+        // falls through to the *_MS_DEFAULT branch. So semantically equivalent
+        // to `delete`, but satisfies biome's lint/performance/noDelete.
+        process.env.SFGRAPH_WATCHDOG_FIRST_YIELD_MS = undefined;
+        process.env.SFGRAPH_WATCHDOG_INACTIVITY_MS = undefined;
+    },
+};
 //# sourceMappingURL=bulk-retrieve.js.map