@ryanstark24/sfgraph-core 1.2.0 → 1.6.0

package/dist/extractors/live-org/bulk-retrieve.js

@@ -1,6 +1,7 @@
 import { discoverMetadataTypes } from "./discovery.js";
 import { buildDispatchTable } from "./dispatch.js";
 import { iterApex } from "./extractors/apex.js";
+import { iterCustomSettings } from "./extractors/custom-settings.js";
 import { iterFlow } from "./extractors/flow.js";
 import { iterGenericMetadata } from "./extractors/generic-metadata.js";
 import { iterIntegration } from "./extractors/integration.js";
@@ -9,6 +10,98 @@ import { iterObject } from "./extractors/object.js";
 import { iterOmnistudio } from "./extractors/omnistudio.js";
 import { iterSecurity } from "./extractors/security.js";
 import { iterVlocity } from "./extractors/vlocity.js";
+import { StopWaitingError, stopSignalContext } from "./rate-limit.js";
+import { IncompleteSourceError, TypeUnreadableError, UNREADABLE_EARLY_STOP, isPlatformRefusal, withRetry, } from "./retry.js";
+/** Report a parked wedge as a skip, at most once per entry. */
+function reportWedgeSkip(entry, reason) {
+    if (entry.skipReported)
+        return;
+    entry.skipReported = true;
+    entry.onError?.(entry.label, new Error(reason));
+    console.log(`ingest:   ${entry.label} ✗ skipped (${reason.slice(0, 80)})`);
+}
+function createWedgeCoordinator(warnings = [], overrides) {
+    const capEnv = Number.parseInt(process.env.SFGRAPH_MAX_BACKGROUND_WEDGES ?? "", 10);
+    // Default 32 (was 4): a full --rebuild on a large org wedges many slow/empty
+    // generic-metadata sources at the 90s first-yield watchdog AND the heavy Apex
+    // source; a cap of 4 evicted most of them (their data lost). The drain loop
+    // skips already-resolved entries cheaply, so a high cap costs little.
+    const cap = Number.isFinite(capEnv) && capEnv > 0 ? capEnv : 32;
+    const drainEnv = Number.parseInt(process.env.SFGRAPH_LATE_DRAIN_BUDGET_MS ?? "", 10);
+    // Default 300s (was 60s): the per-entry budget is TOTAL drain wall-clock.
+    // Apex needs ~75s to stream all classes once the Tooling pool frees up after
+    // the main loop; 60s clipped it mid-drain. Empty/fast entries still return
+    // `done` immediately and cost nothing, so the larger budget only helps the
+    // genuinely-heavy entries finish.
+    const drain = Number.isFinite(drainEnv) && drainEnv >= 0 ? drainEnv : 300_000;
+    return {
+        warnings,
+        backgroundWedges: [],
+        maxBackgroundWedges: overrides?.maxBackgroundWedges ?? cap,
+        lateDrainBudgetMs: overrides?.lateDrainBudgetMs ?? drain,
+    };
+}
+/**
+ * Turn a source failure into skip-report entries. An IncompleteSourceError
+ * is split: members the org refused deterministically become one
+ * `unreadable` entry, the rest one entry classified as before — so a source
+ * with both still gets re-run for the recoverable part, and an all-refused
+ * source is not re-run at all.
+ */
+export function skipEntriesFor(label, err) {
+    if (err instanceof IncompleteSourceError && err.failures.length > 0) {
+        const out = [];
+        const refused = err.failures.filter((f) => isPlatformRefusal(f.error));
+        if (refused.length > 0) {
+            out.push({ label, reason: describeUnreadable(err.source, refused), category: "unreadable" });
+        }
+        // Classify the recoverable rest per failure (review M5): the combined
+        // message holds member names and only the FIRST error, so one
+        // INSUFFICIENT_ACCESS member (or a name containing NOT_FOUND) mislabelled
+        // every other loss. One entry per category, each naming its members.
+        const byCategory = new Map();
+        for (const f of err.failures) {
+            if (isPlatformRefusal(f.error))
+                continue;
+            const msg = f.error instanceof Error ? f.error.message : String(f.error);
+            const cat = classifySkip(msg);
+            byCategory.set(cat, [...(byCategory.get(cat) ?? []), f]);
+        }
+        for (const [category, group] of byCategory) {
+            out.push({ label, reason: new IncompleteSourceError(err.source, group).message, category });
+        }
+        return out;
+    }
+    const reason = err?.message ?? String(err);
+    return [{ label, reason, category: classifySkip(reason) }];
+}
+/** Counts behind a source's unreadable failures: members refused, members
+ *  never requested after an early stop, and whether the listing was refused. */
+function unreadableCounts(refused) {
+    const members = refused.filter((f) => !(f.error instanceof TypeUnreadableError) && !f.name.startsWith("metadata.list "));
+    const unread = refused.reduce((n, f) => n + (f.error instanceof TypeUnreadableError ? f.error.unreadCount : 0), 0);
+    return { members, unread, listing: refused.some((f) => f.name.startsWith("metadata.list ")) };
+}
+function describeUnreadable(source, refused) {
+    const MAX_NAMES = 25;
+    const { members, unread, listing } = unreadableCounts(refused);
+    // The shared refusal of an early stop is its first member's (review N2);
+    // otherwise the first refusal recorded.
+    const stop = refused.find((f) => f.error instanceof TypeUnreadableError)?.error;
+    const first = stop?.cause ?? refused[0]?.error;
+    const firstMsg = first instanceof Error ? first.message : String(first ?? "");
+    if (listing && members.length === 0) {
+        // The type's listing itself was refused (e.g. ManagedTopic).
+        return `${source}: listing unreadable (org refused metadata.list); error: ${firstMsg}`;
+    }
+    const names = members.map((f) => f.name);
+    const shown = names.slice(0, MAX_NAMES).join(", ");
+    const more = names.length > MAX_NAMES ? ` (+${names.length - MAX_NAMES} more)` : "";
+    const stopped = unread > 0
+        ? `; stopped reading the type after ${UNREADABLE_EARLY_STOP} identical refusals — ${unread} more member(s) not read`
+        : "";
+    return `${source}: ${members.length} member(s) unreadable (org refused): ${shown}${more}${stopped}; first error: ${firstMsg}`;
+}
 /** Best-effort classification so the end-of-run recommendation is targeted. */
 function classifySkip(msg) {
     const m = msg.toUpperCase();
@@ -34,27 +127,27 @@ export async function* mergeAsyncIterables(...iters) {
             yield v;
     }
 }
-/**
- * Sliding-window parallel merge. Keeps exactly `concurrency` iterators
- * live at any moment — when one completes (done:true), the next queued
- * iterator is started in its slot. Within the window, iterators race via
- * `Promise.race` for max throughput.
- *
- * Replaces the previous wave-based merger, which had a synchronisation
- * barrier at each wave boundary: all 6 iterators in a wave had to finish
- * before any of wave 2 could start. One hung source (very common for
- * `generic:Layout` on managed-package-heavy orgs) parked the whole wave
- * indefinitely with zero observable progress.
- *
- * Default concurrency = 8: enough to saturate the three rate-limit pools
- * (Tooling 5 / Metadata 5 / Data 10) with headroom for one slot to be
- * blocked on a slow source without starving the others. Override via
- * `SFGRAPH_SOURCE_CONCURRENCY=<n>` (1 = strictly sequential).
- *
- * Output ordering is non-deterministic. `live-ingest`'s processOne is
- * order-independent (idempotent per-record upserts), so this is safe.
- */
-export async function* mergeAsyncIterablesParallel(...iters) {
+export function mergeAsyncIterablesParallel(...args) {
+    // Disambiguate the two overloads. The 2-arg array+opts form is used by
+    // tests; the rest-args form is the production fan-out path.
+    let iters;
+    let opts;
+    if (args.length === 2 &&
+        Array.isArray(args[0]) &&
+        args[1] != null &&
+        typeof args[1] === "object" &&
+        !(args[1] instanceof Promise) &&
+        !(Symbol.asyncIterator in args[1])) {
+        iters = args[0];
+        opts = args[1];
+    }
+    else {
+        iters = args;
+        opts = undefined;
+    }
+    return mergeAsyncIterablesParallelImpl(iters, opts);
+}
+async function* mergeAsyncIterablesParallelImpl(iters, opts) {
     const envConcurrency = Number.parseInt(process.env.SFGRAPH_SOURCE_CONCURRENCY ?? "", 10);
     // Raised 8 -> 12: a wider window lets Tooling-backed sources (apex, lwc)
     // start immediately rather than wait behind the cohort of metadata-pool
@@ -62,22 +155,32 @@ export async function* mergeAsyncIterablesParallel(...iters) {
     // which spend most of their time queued in Bottleneck. Per-pool concurrency
     // still caps total HTTP fan-out, so this only opens up parallelism that
     // was being throttled at the wrong layer.
-    const concurrency = Number.isFinite(envConcurrency) && envConcurrency > 0 ? envConcurrency : 12;
+    const concurrency = opts?.concurrency != null && opts.concurrency > 0
+        ? opts.concurrency
+        : Number.isFinite(envConcurrency) && envConcurrency > 0
+            ? envConcurrency
+            : 12;
     const iterators = iters.map((it) => it[Symbol.asyncIterator]());
     const pending = new Map();
     let nextIterIdx = 0;
+    // Sub-stream failures, in the order they happened. A failed stream frees
+    // its slot like a finished one so its peers (and queued streams) still
+    // drain, but the failure is re-thrown once everything else is done — it is
+    // never mapped to a silent `done` (review M7; the swallowed-error shape
+    // behind the original ApexClass truncation). bulkRetrieve's sources are
+    // failSoft-wrapped and never reject, so this only affects direct callers.
+    const errors = [];
     const advance = (idx) => {
         const it = iterators[idx];
         if (!it)
             return;
-        pending.set(idx, it.next().then((result) => ({ idx, result }), 
-        // Per-iter rejections are handled by the failSoft() wrapper one
-        // level up. Anything that reaches here is unexpected; mark the
-        // iter done so the outer race makes forward progress.
-        () => ({
-            idx,
-            result: { value: undefined, done: true },
-        })));
+        pending.set(idx, it.next().then((result) => ({ idx, result }), (e) => {
+            errors.push(e);
+            return {
+                idx,
+                result: { value: undefined, done: true },
+            };
+        }));
     };
     // Prime the window with the first `concurrency` iterators.
     while (pending.size < concurrency && nextIterIdx < iterators.length) {
@@ -96,61 +199,318 @@ export async function* mergeAsyncIterablesParallel(...iters) {
         yield result.value;
         advance(idx);
     }
+    if (errors.length === 1)
+        throw errors[0];
+    if (errors.length > 1) {
+        throw new AggregateError(errors, `${errors.length} merged sources failed`);
+    }
+}
+/**
+ * Per-source watchdog budgets. Centralized here so the failSoft wrapper and
+ * tests share the same constants.
+ *
+ * Phase 1.5 W1.5-01 contract: these are the budgets the watchdog uses, but
+ * the clock against which they measure starts at slot-acquisition time
+ * (i.e., on the FIRST `it.next()` call inside `failSoft`'s body), NOT at
+ * source registration. Queued-but-not-yet-executing sources accumulate
+ * ZERO clock time, so a wedge in one source can no longer cascade-kill
+ * its queued neighbors.
+ *
+ * Anti-feature reminder: do NOT raise these to "fix" a real wedge. The
+ * correct fix is per-source override (W1.5-future) or fixing the underlying
+ * pre-yield setup call. See `.planning/phase-1.5/PLAN.md` anti-features.
+ */
+const WATCHDOG_INACTIVITY_MS_DEFAULT = 5 * 60_000;
+const WATCHDOG_FIRST_YIELD_MS_DEFAULT = 90_000;
+/**
+ * Per-source watchdog overrides (the "per-source override" the anti-feature
+ * note above points to). Some sources legitimately need far longer than the
+ * 90s default to produce their FIRST record — most notably the Vlocity runner,
+ * which executes a long sequence of `vlocity_cmt` SObject queries and assembles
+ * DataPacks before yielding member #1, and can have multi-minute gaps between
+ * DataPack types once streaming. Raising the GLOBAL budget to accommodate it is
+ * wrong: it makes every other slow/empty source hold its merger slot 10x longer
+ * and starves the pool (observed: a 600s global first-yield stalled the whole
+ * fan-out behind a few empty generic sources). So scope the larger budget to
+ * the specific label only. Explicit env vars still win (debug/tests).
+ */
+const SOURCE_WATCHDOG_OVERRIDES = {
+    // Vlocity must stay INLINE: the post-merger background-drain cannot reliably
+    // recover it (it needs minutes to first-yield, longer than any sane drain
+    // budget, and competes for drain slots). Giving it a long inline budget lets
+    // it hold its DATA-pool slot and stream once the pool frees up. NB: do NOT
+    // do this for Tooling-pool sources like Apex — holding a scarce Tooling slot
+    // that long starves every other Tooling source and stalls the whole fan-out.
+    // Apex instead wedges fast (releasing its slot) and is recovered by the
+    // background-drain pass, which is sized generously in createWedgeCoordinator.
+    vlocity: { firstYieldMs: 20 * 60_000, inactivityMs: 10 * 60_000 },
+};
+/** Read watchdog budgets per-call so unit tests can override via env vars
+ *  without restarting the process. Production env should leave these unset:
+ *  ordinary sources get the documented 90s first-yield / 5min inactivity;
+ *  a source in SOURCE_WATCHDOG_OVERRIDES gets its scoped budget instead. */
+function readWatchdogBudgets(label) {
+    const fy = Number.parseInt(process.env.SFGRAPH_WATCHDOG_FIRST_YIELD_MS ?? "", 10);
+    const inact = Number.parseInt(process.env.SFGRAPH_WATCHDOG_INACTIVITY_MS ?? "", 10);
+    const override = label ? SOURCE_WATCHDOG_OVERRIDES[label] : undefined;
+    return {
+        firstYieldMs: Number.isFinite(fy) && fy > 0
+            ? fy
+            : (override?.firstYieldMs ?? WATCHDOG_FIRST_YIELD_MS_DEFAULT),
+        inactivityMs: Number.isFinite(inact) && inact > 0
+            ? inact
+            : (override?.inactivityMs ?? WATCHDOG_INACTIVITY_MS_DEFAULT),
+    };
+}
+/** Build the namespaced wedge warning string per the schema documented in
+ *  PLAN.md (`wedge:<source>:<stage>:<detail>`). Centralized here so the
+ *  format is stable across emit sites + grep-friendly. */
+function fmtWedgeWarning(source, stage, detail) {
+    const parts = [`wedge:${source}:${stage}`];
+    for (const [k, v] of Object.entries(detail)) {
+        parts.push(`${k}=${v}`);
+    }
+    return parts.join(":");
+}
+/**
+ * Enforce the background-wedge cap. When admitting a new entry would push
+ * the count above `maxBackgroundWedges`, mark the OLDEST entries' stop-
+ * waiting flags so their drain loops bail at the next iteration. Emits a
+ * `wedge:cap:backgroundWedgeAborted:...` warning per evicted entry. The
+ * iterator object itself is left in the set; the late-drain loop is what
+ * sees the flag and skips it. (We do NOT remove from `backgroundWedges`
+ * here so post-merger inspection still sees every wedge that ever fired
+ * during the run.)
+ */
+function enforceBackgroundWedgeCap(coordinator) {
+    const active = coordinator.backgroundWedges.filter((e) => !e.stopWaitingRef.value);
+    const overage = active.length - coordinator.maxBackgroundWedges;
+    if (overage <= 0)
+        return;
+    // Sort by registeredAt ascending; oldest first.
+    active.sort((a, b) => a.registeredAt - b.registeredAt);
+    for (let i = 0; i < overage; i += 1) {
+        const victim = active[i];
+        if (!victim)
+            continue;
+        victim.stopWaitingRef.value = true;
+        const ageMs = Date.now() - victim.registeredAt;
+        reportWedgeSkip(victim, `background wedge evicted (cap ${coordinator.maxBackgroundWedges}) after ${ageMs}ms — remaining rows not fetched`);
+        coordinator.warnings.push(fmtWedgeWarning("cap", "backgroundWedgeAborted", {
+            source: victim.label,
+            ageMs,
+            reason: "backgroundWedgeCapExceeded",
+        }));
+    }
+}
+/**
+ * Drain a single background-wedge iterator. Called once per wedged source,
+ * post-merger, with a wall-clock budget. Each iteration races the iterator's
+ * `it.next()` against the remaining budget. Records arriving inside the
+ * budget are yielded with `attributes.lateYield = true`. The pending HTTP
+ * request (if the wedge is on a network call) is NOT actively cancelled
+ * (W1.5-03's job); we just stop awaiting it.
+ *
+ * Bails immediately if `stopWaitingRef.value === true` (cap eviction).
+ */
+async function* drainBackgroundWedge(entry, budgetMs, warnings) {
+    const debug = process.env.SFGRAPH_DEBUG_INGEST === "1";
+    const startedAt = Date.now();
+    let lateCount = 0;
+    // Why the loop ended. Only "done" means the source finished; every other
+    // outcome loses rows and must become a skip (review I1).
+    let outcome = "evicted";
+    const BUDGET_SENTINEL = Symbol("late-drain budget");
+    while (!entry.stopWaitingRef.value) {
+        const remaining = startedAt + budgetMs - Date.now();
+        if (remaining <= 0) {
+            outcome = "budget";
+            break;
+        }
+        // First iteration: consume the abandoned-but-still-pending `it.next()`
+        // promise from when the watchdog fired. Subsequent iterations call
+        // `iterator.next()` normally. Async-generator semantics queue
+        // concurrent `.next()` calls, so we MUST consume the pending one
+        // before issuing a new one — otherwise the late-yielded value goes
+        // to the abandoned promise and the drain sees the NEXT yield (or
+        // `done:true` if the generator is single-yield).
+        const nextPromise = entry.pendingFirstNext !== null
+            ? entry.pendingFirstNext
+            : stopSignalContext.run(entry.stopWaitingRef, () => entry.iterator.next());
+        if (entry.pendingFirstNext !== null)
+            entry.pendingFirstNext = null;
+        let next;
+        let timer;
+        try {
+            next = await Promise.race([
+                nextPromise,
+                new Promise((resolve) => {
+                    // A distinct sentinel (not a synthetic `done`) so budget exhaustion
+                    // is never mistaken for the source finishing.
+                    timer = setTimeout(() => resolve(BUDGET_SENTINEL), remaining);
+                }),
+            ]);
+        }
+        catch (e) {
+            // Late-arriving error from the wedged iterator (e.g. an
+            // IncompleteSourceError after a truncated drain). Don't crash the
+            // drain; keep the warning AND record the skip.
+            const err = e;
+            const msg = err?.message ?? String(e);
+            warnings.push(fmtWedgeWarning(entry.label, "lateError", { error: msg.slice(0, 120) }));
+            outcome = "error";
+            if (!entry.skipReported) {
+                entry.skipReported = true;
+                entry.onError?.(entry.label, err instanceof Error ? err : new Error(msg));
+                console.log(`ingest:   ${entry.label} ✗ skipped (${msg.slice(0, 80)})`);
+            }
+            break;
+        }
+        finally {
+            if (timer)
+                clearTimeout(timer);
+        }
+        if (next === BUDGET_SENTINEL) {
+            outcome = "budget";
+            break;
+        }
+        if (next.done) {
+            outcome = "done";
+            break;
+        }
+        lateCount += 1;
+        const rec = next.value;
+        // Tag with lateYield so downstream consumers / tests can identify the
+        // record as having arrived after slot release.
+        const tagged = {
+            ...rec,
+            attributes: { ...(rec.attributes ?? {}), lateYield: true },
+        };
+        yield tagged;
+    }
+    if (outcome === "budget") {
+        reportWedgeSkip(entry, `late-drain budget exhausted (${budgetMs}ms) after ${lateCount} late records — source did not finish`);
+    }
+    else if (outcome === "evicted") {
+        reportWedgeSkip(entry, "background wedge evicted before the late drain finished it");
+    }
+    if (lateCount > 0) {
+        const totalMs = Date.now() - entry.registeredAt;
+        warnings.push(fmtWedgeWarning(entry.label, "resolvedLate", {
+            totalMs,
+            records: lateCount,
+        }));
+    }
+    if (debug) {
+        console.log(`ingest: [debug] ${entry.label} background-wedge drain finished (${lateCount} late records, stopWaiting=${entry.stopWaitingRef.value})`);
+    }
 }
-/** Wrap an iterable so a thrown error is captured + the stream ends cleanly
- *  instead of aborting the whole ingest. The error is recorded into a
- *  shared skip report (consumed at end-of-run) and a compact ✗ line is
- *  printed so the user sees something happened without the full error
- *  message scrolling past during the run. */
-async function* failSoft(label, factory, onError) {
+/**
+ * Wrap an iterable so a thrown error is captured + the stream ends cleanly
+ * instead of aborting the whole ingest. The error is recorded into a
+ * shared skip report (consumed at end-of-run) and a compact ✗ line is
+ * printed so the user sees something happened without the full error
+ * message scrolling past during the run.
+ *
+ * Phase 1.5 W1.5-01 + W1.5-02 contract:
+ *
+ *  - The watchdog clock starts when this generator's BODY first executes
+ *    (i.e., when the merger calls `.next()` on us for the first time —
+ *    that's slot-acquisition time). Queued sources still inside
+ *    `mergeAsyncIterablesParallel`'s waiting list accumulate ZERO clock
+ *    time. Implemented via an internal `startedAtRef` that's written once
+ *    on body entry; if the caller passes a `startedAtRef` we update it in
+ *    place so the merger can observe the slot-acquisition moment.
+ *
+ *  - On watchdog fire, we DO NOT throw (which would abort the merger's
+ *    Promise.race for our slot AND lose the still-pending iterator).
+ *    Instead we:
+ *       1. Emit a `wedge:<label>:firstYield:90s:...` (or `:inactivity:300s:`)
+ *          warning into coordinator.warnings.
+ *       2. Register the still-pending iterator into coordinator.background
+ *          Wedges with a stopWaitingRef. Enforce the cap (evict oldest).
+ *       3. RETURN from the generator. This cleanly releases the merger's
+ *          slot — `pending.delete(idx)` runs in mergeAsyncIterablesParallel
+ *          and the next queued source acquires the slot.
+ *    Post-merger, the bulkRetrieve top-level fan-out drains the background
+ *    wedge set; records arriving inside the late-drain budget are yielded
+ *    with `attributes.lateYield = true`.
+ */
+async function* failSoft(label, factory, onError, coordinator, 
+/** Optional outer-visible clock cell. The merger does not currently read
+ *  this, but the contract is "set by failSoft on body entry"; tests use
+ *  it to assert slot-acquisition timing. */
+startedAtRef) {
     const debug = process.env.SFGRAPH_DEBUG_INGEST === "1";
+    // W1.5-01: the clock starts HERE — body-entry, which is slot-acquisition
+    // time. NOT at failSoft() invocation (that's source registration, which
+    // can be hundreds of seconds before the merger picks us out of the queue).
     const startedAt = Date.now();
+    if (startedAtRef)
+        startedAtRef.value = startedAt;
     let count = 0;
     let started = false;
-    // Watchdog at 5 minutes. Per-call withTimeouts (120s) catch hung HTTP;
-    // this catches the case where every call ALSO times out and the source
-    // never yields a single record. 5 min = ~2.5× the per-call budget,
-    // generous for queued-but-healthy, tight enough that a genuinely
-    // wedged source surfaces fast and doesn't park its sliding-window slot
-    // for half an hour. With the no-retry-on-timeout fix in
-    // readMetadataBatchAdaptive, the inner loop also can't balloon, so 5
-    // minutes is a real ceiling now.
-    const inactivityMs = 5 * 60_000;
-    // Tighter deadline for the FIRST record. The 5-min inactivity watchdog
-    // is for healthy-but-slow sources that have already started yielding;
-    // a source that hasn't yielded a single record in 90s is almost
-    // certainly parked on a pre-yield setup call (metadata.list, describe,
-    // EntityDefinition pagination) and won't recover. Killing it fast frees
-    // the sliding-window slot for queued peers.
-    const firstYieldMs = 90_000;
+    // Track last-yielded qualifiedName for diagnostic warnings — the smoking-
+    // gun field that names WHICH record was in flight when the wedge fired.
+    let lastYieldedQName = "<none>";
+    const { firstYieldMs, inactivityMs } = readWatchdogBudgets(label);
+    // W1.5-03: pre-allocate the per-source stop-signal at body entry and
+    // publish it into AsyncLocalStorage so any downstream rate-limit wrapper
+    // (`withTimeout`, `soqlWithTimeout`, `scheduleQuery`) inside the wedged
+    // iterator can observe `value = true` after cap-eviction flips it. Same
+    // ref is later attached to the BackgroundWedgeEntry so the keystone's
+    // `enforceBackgroundWedgeCap` flip propagates to in-flight calls without
+    // any per-extractor signature change. AsyncLocalStorage propagates across
+    // awaits within an async function started inside .run() — we factor the
+    // iterator into a helper started inside the .run() callback.
+    const stopWaitingRef = { value: false };
+    const it = stopSignalContext.run(stopWaitingRef, () => factory()[Symbol.asyncIterator]());
+    if (debug)
+        console.log(`ingest: [debug] ${label} ← invoked at ${startedAt}`);
     try {
-        if (debug)
-            console.log(`ingest: [debug] ${label} ← invoked at ${startedAt}`);
-        const it = factory()[Symbol.asyncIterator]();
         while (true) {
             const remainingFirstYield = started
                 ? Number.POSITIVE_INFINITY
                 : startedAt + firstYieldMs - Date.now();
             const watchdogMs = Math.min(inactivityMs, remainingFirstYield);
-            if (watchdogMs <= 0) {
-                throw new Error(`source watchdog (first-yield ${firstYieldMs / 1000}s): no record yielded — pre-yield setup wedged`);
+            const stage = started ? "inactivity" : "firstYield";
+            const stageDetail = started ? `${inactivityMs / 1000}s` : `${firstYieldMs / 1000}s`;
+            // Run each `.next()` synchronously inside the ALS context so awaits
+            // inside the underlying generator's body that call into rate-limit
+            // wrappers can resolve the stop-signal via `getStore()`.
+            const pending = stopSignalContext.run(stopWaitingRef, () => it.next());
+            let timer;
+            let wedged;
+            try {
+                wedged = await Promise.race([
+                    pending.then((result) => ({ wedge: false, result })),
+                    new Promise((resolve) => {
+                        // watchdogMs may be <=0 if the budget is already burned (defensive
+                        // — should be rare since the clock starts at body-entry). setTimeout
+                        // with non-positive delay fires on the next event-loop tick, which
+                        // is the correct wedge-immediately behavior.
+                        timer = setTimeout(() => resolve({ wedge: true }), Math.max(0, watchdogMs));
+                    }),
+                ]);
             }
-            const watchdogLabel = started
-                ? `${inactivityMs / 60_000}m inactivity`
-                : `first-yield ${firstYieldMs / 1000}s`;
-            const next = await Promise.race([
-                it.next(),
-                new Promise((_resolve, reject) => {
-                    setTimeout(() => reject(new Error(`source watchdog (${watchdogLabel}): no record yielded — pool jammed or call wedged`)), watchdogMs);
-                }),
-            ]);
+            finally {
+                // Clear on EVERY outcome. Clearing only when `pending` resolved left
+                // the ref'd timer (up to the 5 min inactivity budget) armed whenever
+                // the source threw, so a run whose last sources skipped held the
+                // process open after its final line (exit hang, 2026-10-09).
+                if (timer)
+                    clearTimeout(timer);
+            }
+            if (wedged.wedge) {
+                // Hand off the still-pending it.next() so the drain can consume
+                // it directly — see BackgroundWedgeEntry.pendingFirstNext.
+                emitWedge(stage, stageDetail, pending);
+                return;
+            }
+            const next = wedged.result;
             if (next.done)
                 break;
-            if (!started) {
-                started = true;
-                console.log(`ingest:   ${label} → starting…`);
-            }
-            count += 1;
+            recordYield(next.value);
             yield next.value;
         }
         if (started) {
@@ -163,7 +523,30 @@ async function* failSoft(label, factory, onError) {
     catch (e) {
         const err = e;
         onError?.(label, err);
-        console.log(`ingest:   ${label} ✗ skipped (${err?.message?.slice(0, 80) ?? "unknown"})`);
+        // Print the unreadable and recoverable counts separately (review M6):
+        // the combined message counts both as "not fetched after retries".
+        let detail = err?.message?.slice(0, 80) ?? "unknown";
+        let verdict = "skipped";
+        if (err instanceof IncompleteSourceError && err.failures.length > 0) {
+            const refused = err.failures.filter((f) => isPlatformRefusal(f.error));
+            const lost = err.failures.length - refused.length;
+            if (refused.length > 0) {
+                const { members, unread, listing } = unreadableCounts(refused);
+                const parts = [];
+                if (lost > 0)
+                    parts.push(`${lost} member(s) not fetched`);
+                if (listing)
+                    parts.push("listing refused");
+                if (members.length > 0)
+                    parts.push(`${members.length} member(s) unreadable`);
+                if (unread > 0)
+                    parts.push(`${unread} not read after early stop`);
+                detail = parts.join(", ");
+                if (lost === 0)
+                    verdict = "unreadable";
+            }
+        }
+        console.log(`ingest:   ${label} ✗ ${verdict} (${detail})`);
         if (debug) {
             console.error(`ingest: [debug] ${label} failure detail: ${err?.message ?? String(err)}`);
             if (err?.stack)
@@ -175,6 +558,65 @@ async function* failSoft(label, factory, onError) {
             console.log(`ingest: [debug] ${label} → finalised (${count} records, ${Date.now() - startedAt}ms)`);
         }
     }
+    /** Inline helper: extracts qualifiedName-ish text from a RawMember for the
+     *  `lastYielded=...` diagnostic field. Safe against undefined. */
+    function recordYield(v) {
+        if (!started) {
+            started = true;
+            console.log(`ingest:   ${label} → starting…`);
+        }
+        count += 1;
+        const ref = v.ref;
+        if (ref) {
+            lastYieldedQName = `${ref.memberType}/${ref.memberName}`;
+        }
+    }
+    /**
+     * Slot-release + soft-isolate on watchdog fire. Two side effects:
+     *   1. Push a `wedge:<label>:<stage>:<detail>` warning into the shared sink.
+     *   2. Park the still-pending iterator into the background-wedge set so
+     *      the post-merger late-drain pass can capture any records that
+     *      eventually arrive. Enforce the cap (oldest evicted with a
+     *      `backgroundWedgeAborted` warning).
+     * The generator then RETURNS (caller is `for await` of the merger; the
+     * merger sees `done:true` and releases the slot).
+     */
+    function emitWedge(stage, stageDetail, pendingFirstNext) {
+        const wedgedFor = Date.now() - startedAt;
+        if (coordinator) {
+            // Schema (PLAN.md): `wedge:<label>:<stage>:<stageDetail>:<k=v>:<k=v>`.
+            // <stageDetail> is a positional value (e.g. "90s"), not a key=value.
+            const w = `wedge:${label}:${stage}:${stageDetail}` +
+                `:lastYielded=${lastYieldedQName}` +
+                `:wedgedForMs=${wedgedFor}`;
+            coordinator.warnings.push(w);
+            console.log(`ingest:   ${label} ⚠ wedged (${stage} ${stageDetail}); slot released, draining in background`);
+            // W1.5-03: reuse the SAME stopWaitingRef that was published into ALS
+            // at body entry. When enforceBackgroundWedgeCap later flips this ref's
+            // `.value = true` on cap eviction, any in-flight rate-limit wrapper
+            // inside the wedged iterator's continuing background pumps will see
+            // the flag through `stopSignalContext.getStore()` and reject with
+            // `StopWaitingError` quickly. The underlying jsforce HTTP call still
+            // leaks until socket idle timeout — documented in rate-limit.ts.
+            coordinator.backgroundWedges.push({
+                label,
+                iterator: it,
+                pendingFirstNext,
+                registeredAt: Date.now(),
+                stopWaitingRef,
+                ...(onError ? { onError } : {}),
+            });
+            enforceBackgroundWedgeCap(coordinator);
+        }
+        else {
+            // No coordinator (e.g. unit-test path that didn't wire one up, or
+            // back-compat caller): fall back to the old "log+skip" behavior so
+            // the caller still gets observable feedback.
+            const msg = `source watchdog (${stage} ${stageDetail}): no record yielded — pool jammed or call wedged`;
+            onError?.(label, new Error(msg));
+            console.log(`ingest:   ${label} ✗ skipped (${msg.slice(0, 80)})`);
+        }
+    }
 }
 /** Typed-extractor ownership map: which XML type names a dedicated extractor covers. */
 const APEX_TYPES = new Set(["ApexClass", "ApexTrigger"]);
@@ -291,10 +733,29 @@ const GENERIC_TYPE_WHITELIST = new Set([
     "OmniUiCard",
     "OmniProcess",
 ]);
+/**
+ * Types whose data is fully captured by a CONTAINER type, so routing them
+ * individually only produces value-less opaque duplicates that clobber the
+ * container's richer nodes. `CustomLabel` (singular) is covered by the
+ * `CustomLabels` container — whose rule extracts each label's value/language;
+ * the singular path would emit `CustomLabel:Name` nodes with no value and,
+ * depending on merge order, overwrite the real ones. Skip them.
+ */
+const CONTAINER_COVERED_TYPES = new Set(["CustomLabel"]);
 function shouldRouteGeneric(type) {
-    if (process.env.SFGRAPH_INCLUDE_ALL_GENERIC === "1")
-        return true;
-    return GENERIC_TYPE_WHITELIST.has(type);
+    // Never route a type a container already covers with richer data.
+    if (CONTAINER_COVERED_TYPES.has(type))
+        return false;
+    // Default: route EVERY long-tail metadata type through the generic
+    // extractor so the graph — and therefore semantic search — covers the org
+    // completely. Anything without a dedicated extractor still becomes a typed
+    // node (the rule parsers key off the type name) or an opaque node, which is
+    // what search needs. The old curated whitelist silently dropped ~250
+    // discovered types (search returned nothing for them). Set
+    // SFGRAPH_GENERIC_WHITELIST_ONLY=1 to restore the restricted subset.
+    if (process.env.SFGRAPH_GENERIC_WHITELIST_ONLY === "1")
+        return GENERIC_TYPE_WHITELIST.has(type);
+    return true;
 }
 export async function* bulkRetrieve(conn, caps, orgId, opts = {}) {
     // Back-compat: callers used to pass IngestSkipReport directly as the 4th arg.
@@ -304,24 +765,58 @@ export async function* bulkRetrieve(conn, caps, orgId, opts = {}) {
         : opts;
     const skipReport = normalized.skipReport;
     const onlyLabels = normalized.onlyLabels;
-    // Discover the type list this org actually supports. If discovery fails or
-    // returns nothing usable, fall back to invoking every known extractor —
-    // preserves Commit-A behavior for mocks that don't implement describe.
+    // An explicit re-run (--retry-skipped / --only) reads every member of the
+    // generic types it names: no early stop (review N1c). Every generic source
+    // such a run launches was named, directly or via `discovery`.
+    const genericOpts = { earlyStop: !(onlyLabels && normalized.explicitOnlyLabels) };
+    // Phase 1.5 W1.5-02 coordinator. Built only when a warnings sink was
+    // supplied — without one, no place to push wedge strings, so the legacy
+    // "watchdog fires → skip entry" behavior is preserved. The coordinator
+    // owns the background-wedge cap + late-drain budget.
+    const coordinator = normalized.warnings
+        ? createWedgeCoordinator(normalized.warnings)
+        : undefined;
+    const onSkip = skipReport
+        ? (label, err) => {
+            skipReport.skips.push(...skipEntriesFor(label, err));
+        }
+        : undefined;
+    /** Record a skip for a label the --only filter selects (a pre-fan-out
+     *  failure under an unrelated `--only` is not this run's loss). */
+    const skipIfSelected = (label, err, selected) => {
+        if (onlyLabels && !selected)
+            return;
+        onSkip?.(label, err);
+        console.log(`ingest:   ${label} ✗ skipped (${err.message.slice(0, 80)})`);
+    };
+    // Discover the type list this org actually supports. Retried like every
+    // other fetch. If it still fails, fall back to invoking every dedicated
+    // extractor AND record a `discovery` skip: without the type list every
+    // generic:<Type> source (and every child type) is never invoked, which
+    // used to be silent (review I2). `--retry-skipped` feeds `discovery` back
+    // as an only-label, which selects every generic source below.
     let types = [];
     try {
-        types = await discoverMetadataTypes(conn);
+        types = await withRetry(() => discoverMetadataTypes(conn));
     }
-    catch {
+    catch (e) {
         types = [];
+        const err = e instanceof Error ? e : new Error(String(e));
+        skipIfSelected("discovery", new Error(`metadata discovery failed — generic metadata types not fetched: ${err.message}`, {
+            cause: err,
+        }), !onlyLabels ||
+            onlyLabels.has("discovery") ||
+            [...onlyLabels].some((l) => l.startsWith("generic:")));
+    }
+    // Capability probes that failed transiently (not "absent"): the gated
+    // source was not invoked, so it is a skip under that source's label.
+    for (const pe of caps.probeErrors ?? []) {
+        if (!pe.source)
+            continue;
+        skipIfSelected(pe.source, new Error(`capability probe ${pe.probe} failed — source not run: ${pe.error}`), onlyLabels?.has(pe.source) ?? true);
     }
     const sources = [];
     const invoked = new Set(); // source-key dedup
-    const onSkip = skipReport
-        ? (label, err) => {
-            const reason = err?.message ?? String(err);
-            skipReport.skips.push({ label, reason, category: classifySkip(reason) });
-        }
-        : undefined;
     const invoke = (key, factory) => {
         if (invoked.has(key))
             return;
@@ -329,14 +824,19 @@ export async function* bulkRetrieve(conn, caps, orgId, opts = {}) {
         // --only filter: when onlyLabels is set, skip any source not in the set.
         // Filter applies to the exact source key ('apex', 'generic:Profile', etc.)
         // for precise targeting of retry/partial-refresh flows.
-        if (onlyLabels && !onlyLabels.has(key))
+        // A `discovery` only-label (from a skipped discovery) selects every
+        // generic source — those are what a failed discovery lost.
+        if (onlyLabels &&
+            !onlyLabels.has(key) &&
+            !(onlyLabels.has("discovery") && key.startsWith("generic:"))) {
             return;
+        }
         // Each source is wrapped fail-soft so one failing type (e.g. a metadata
         // category the user's profile lacks access to) doesn't abort the whole
         // ingest. The wrapper records the source label + error into skipReport
         // (consumed at end of run for a consolidated summary) and ends the
         // stream cleanly.
-        sources.push(failSoft(key, factory, onSkip));
+        sources.push(failSoft(key, factory, onSkip, coordinator));
     };
     if (types.length === 0) {
         // Discovery unavailable: invoke every dedicated extractor once.
@@ -360,7 +860,7 @@ export async function* bulkRetrieve(conn, caps, orgId, opts = {}) {
                         invoke("lwc", () => iterLwc(conn));
                     else if (shouldRouteGeneric(type)) {
                         routedGeneric += 1;
-                        invoke(`generic:${type}`, () => iterGenericMetadata(conn, String(orgId), type));
+                        invoke(`generic:${type}`, () => iterGenericMetadata(conn, String(orgId), type, {}, genericOpts));
                     }
                     else {
                         skippedGeneric += 1;
@@ -377,7 +877,7 @@ export async function* bulkRetrieve(conn, caps, orgId, opts = {}) {
                         invoke("integration", () => iterIntegration(conn));
                     else if (shouldRouteGeneric(type)) {
                         routedGeneric += 1;
-                        invoke(`generic:${type}`, () => iterGenericMetadata(conn, String(orgId), type));
+                        invoke(`generic:${type}`, () => iterGenericMetadata(conn, String(orgId), type, {}, genericOpts));
                     }
                     else {
                         skippedGeneric += 1;
@@ -394,12 +894,54 @@ export async function* bulkRetrieve(conn, caps, orgId, opts = {}) {
                     break;
             }
         }
+        // Child metadata types (ValidationRule, ListView, CompactLayout, FieldSet,
+        // WebLink, BusinessProcess, ...) are returned by describe() only as
+        // `childXmlNames` of a parent (mostly CustomObject), never as top-level
+        // types — so the dispatch loop above never fetches them. Most ARE
+        // independently listable via metadata.list, so route each through the
+        // generic extractor. This is how validation rules / list views on STANDARD
+        // objects get captured — iterObject only sees children embedded in the
+        // custom objects it reads, missing the bulk that live on Account/Case/etc.
+        // Children that aren't independently listable fail-soft to nothing. Skip
+        // children already owned by a dedicated extractor or dispatched above.
+        if (process.env.SFGRAPH_GENERIC_WHITELIST_ONLY !== "1") {
+            const typedOwned = new Set([
+                ...APEX_TYPES,
+                ...LWC_TYPES,
+                ...FLOW_TYPES,
+                ...OBJECT_TYPES,
+                ...SECURITY_TYPES,
+                ...INTEGRATION_TYPES,
+                "CustomField", // fetched in bulk by iterObject; re-listing is wasteful
+            ]);
+            const childTypes = new Set();
+            for (const t of types)
+                for (const c of t.childXmlNames)
+                    childTypes.add(c);
+            for (const child of childTypes) {
+                if (dispatch.has(child) || typedOwned.has(child))
+                    continue;
+                if (CONTAINER_COVERED_TYPES.has(child))
+                    continue; // CustomLabel ⊂ CustomLabels
+                invoke(`generic:${child}`, () => iterGenericMetadata(conn, String(orgId), child, {}, genericOpts));
+            }
+        }
     }
     if (caps.vlocityLegacy) {
         invoke("vlocity", () => iterVlocity(conn, caps, String(orgId), onSkip));
     }
+    // Custom Setting ROW capture — self-guards (yields nothing if the org has no
+    // custom settings or EntityDefinition is unavailable), so invoke always.
+    invoke("custom-settings", () => iterCustomSettings(conn, String(orgId)));
     if (caps.omnistudioOncore) {
-        invoke("omnistudio", () => iterOmnistudio(conn));
+        // When the Metadata-API retrieve path is enabled it covers
+        // OmniDataTransform and OmniUiCard with higher fidelity — skip them on
+        // the SOQL path so the same component is not ingested twice (the
+        // retrieve XML would otherwise clobber the SOQL-enriched node, and
+        // vice versa, nondeterministically by merge order).
+        invoke("omnistudio", () => iterOmnistudio(conn, normalized.enableOmnistudioRetrieve
+            ? { skipTypes: new Set(["OmniDataTransform", "OmniUiCard"]) }
+            : {}));
         if (normalized.enableOmnistudioRetrieve) {
             invoke("omnistudio-retrieve", async function* () {
                 const { iterOmnistudioRetrieve } = await import("./extractors/omnistudio-retrieve.js");
@@ -410,10 +952,16 @@ export async function* bulkRetrieve(conn, caps, orgId, opts = {}) {
             });
         }
     }
-    // Log the generic-type filter summary once at fan-out start — makes the
-    // skip-vs-route decision visible without --debug.
+    // Log the generic-type routing summary once at fan-out start — makes the
+    // skip-vs-route decision visible without --debug. The default is FULL
+    // coverage (every discovered type routed); the opt-out is
+    // SFGRAPH_GENERIC_WHITELIST_ONLY=1 (restricts to the curated subset).
     if (types.length > 0) {
-        console.log(`ingest: dispatch routed=${sources.length} sources (${types.length} discovered metadata types; generic-type whitelist active — set SFGRAPH_INCLUDE_ALL_GENERIC=1 to invoke all)`);
+        const whitelistOnly = process.env.SFGRAPH_GENERIC_WHITELIST_ONLY === "1";
+        const mode = whitelistOnly
+            ? "curated whitelist only (SFGRAPH_GENERIC_WHITELIST_ONLY=1) — unset to route all"
+            : "full coverage (all discovered types) — set SFGRAPH_GENERIC_WHITELIST_ONLY=1 to restrict";
+        console.log(`ingest: dispatch routed=${sources.length} sources (${types.length} discovered metadata types; ${mode})`);
     }
     // Parallel by default: fan out across all source iterators so different
     // pools (Tooling/Metadata/Data) saturate simultaneously. Escape hatch:
@@ -421,5 +969,69 @@ export async function* bulkRetrieve(conn, caps, orgId, opts = {}) {
     // anyone who hits an ordering bug or wants the old log layout.
     const sequential = process.env.SFGRAPH_SEQUENTIAL_SOURCES === "1";
     yield* sequential ? mergeAsyncIterables(...sources) : mergeAsyncIterablesParallel(...sources);
+    // Phase 1.5 W1.5-02 late-yield drain. When the merger has finished its
+    // main loop (all live slots cleared), drain any background-wedge iterators
+    // for the configured budget. Records arriving inside the budget are
+    // yielded with `attributes.lateYield = true`. Iterators whose stopWaiting
+    // flag was set by the cap-eviction path skip immediately. Anything still
+    // unresolved after the budget is dropped on the floor — its underlying
+    // jsforce HTTP request continues running until libuv tears it down at
+    // process exit (W1.5-03 will surface a "stop-waiting" semantic for this).
+    if (coordinator &&
+        coordinator.backgroundWedges.length > 0 &&
+        coordinator.lateDrainBudgetMs <= 0) {
+        // Late drain disabled (SFGRAPH_LATE_DRAIN_BUDGET_MS=0): every parked
+        // wedge is abandoned, so each one is a skip, not a silent loss.
+        for (const entry of coordinator.backgroundWedges) {
+            reportWedgeSkip(entry, "wedged and late drain disabled — source did not finish");
+        }
+    }
+    else if (coordinator && coordinator.backgroundWedges.length > 0) {
+        if (process.env.SFGRAPH_DEBUG_INGEST === "1") {
+            console.log(`ingest: [debug] late-drain pass: ${coordinator.backgroundWedges.length} background wedge(s), budget=${coordinator.lateDrainBudgetMs}ms`);
+        }
+        for (const entry of coordinator.backgroundWedges) {
+            yield* drainBackgroundWedge(entry, coordinator.lateDrainBudgetMs, coordinator.warnings);
+        }
+    }
 }
+/**
+ * Internal-surface exports for unit tests in `__tests__/`. Not part of the
+ * public package API — these are intentionally namespaced under `__testing`
+ * so they don't appear in the package's exports surface to downstream
+ * consumers. (They're still importable, but the underscore convention
+ * signals "test only".)
+ *
+ * Exposed:
+ *   - `failSoft`: wrap an iterable with the W1.5-01/02 watchdog + soft-isolate.
+ *   - `createWedgeCoordinator`: build a coordinator + warnings sink with
+ *      optional cap/drain overrides for deterministic tests.
+ *   - `drainBackgroundWedge`: run the post-merger late-drain pass against
+ *      a single background-wedge entry.
+ *   - `mergeAsyncIterablesParallel`: re-export for symmetry; the (array,
+ *      opts) overload is what tests use to fix concurrency.
+ *   - `setTestWatchdogBudgets` / `resetTestWatchdogBudgets`: env-var-backed
+ *      knobs to squeeze the watchdog budget for fast tests. Always paired
+ *      via try/finally in tests so the environment is left clean.
+ */
+export const __testing = {
+    failSoft,
+    createWedgeCoordinator,
+    drainBackgroundWedge,
+    mergeAsyncIterablesParallel,
+    readWatchdogBudgets,
+    setTestWatchdogBudgets(opts) {
+        process.env.SFGRAPH_WATCHDOG_FIRST_YIELD_MS = String(opts.firstYieldMs);
+        process.env.SFGRAPH_WATCHDOG_INACTIVITY_MS = String(opts.inactivityMs);
+    },
+    resetTestWatchdogBudgets() {
+        // Note: assigning `undefined` to a `process.env.X` key in Node coerces to
+        // the string "undefined". That's fine here because readWatchdogBudgets()
+        // runs parseInt() on the value, parseInt("undefined") === NaN, and NaN
+        // falls through to the *_MS_DEFAULT branch. So semantically equivalent
+        // to `delete`, but satisfies biome's lint/performance/noDelete.
+        process.env.SFGRAPH_WATCHDOG_FIRST_YIELD_MS = undefined;
+        process.env.SFGRAPH_WATCHDOG_INACTIVITY_MS = undefined;
+    },
+};
 //# sourceMappingURL=bulk-retrieve.js.map