@ryanstark24/sfgraph-core 1.2.0 → 1.6.0

package/dist/ingest/parse-pool.d.ts

@@ -0,0 +1,139 @@
+import type { Logger } from "@ryanstark24/sfgraph-shared";
+import type { RawMember } from "../extractors/interfaces/metadata-source.js";
+import { type MemberParse, type ParseCtxBase } from "./parse-member.js";
+/**
+ * Hard cap on the default worker count. Each worker loads every parser. On a
+ * large corpus (a large Vlocity repo) the memory diagnosis estimated ~200 MB of heap per
+ * worker once the Apex DFA cache is released (APEX_DFA_CLEAR_INTERVAL in
+ * parsers/apex/class.ts) and up to ~450 MB without it; measured whole-process
+ * RSS for the repo ingest at 4 workers fell from ~3.0 GB to ~1.95 GB. The ONNX embedding worker needs CPU during fan-out
+ * too; 4 workers gave most of the 8-worker speed-up (3,000 members: 10.3 s vs
+ * 11.2 s at 8, 17.9-21.4 s serial). Set SFGRAPH_PARSE_WORKERS to go higher.
+ */
+export declare const DEFAULT_MAX_PARSE_WORKERS = 4;
+/**
+ * Below this many members the pool is not started and the input is parsed on
+ * the main thread. Workers cost ~0.3 s (1) to ~0.8-1.0 s (8) to load every
+ * parser, plus each worker's first Apex parse loads apex-parser. Measured on
+ * synthetic Apex + LWC members (8 workers, compiled build, shared 10-core
+ * box): serial was faster at 200 (1.2 s vs 2.8 s) and 400 (2.5 s vs 3.5 s),
+ * about even at 1,000 (5.9 s vs 5.8 s), and the pool won above that (3,000: 21.4 s
+ * vs 11.2 s). 500 keeps small projects and deltas serial. Override with
+ * SFGRAPH_PARSE_POOL_MIN. With the default 4 workers: 200 members 1.3 s
+ * serial vs 2.1 s pool, 400 even (2.8 s), 1,000 6.7 s vs 4.4 s.
+ */
+export declare const DEFAULT_PARSE_POOL_MIN_MEMBERS = 500;
+/**
+ * Heap-cap OOMs before the pool gives up on workers: max(10, workers * 4). An
+ * OOM is a large member, not a broken worker, so it gets a looser bound than
+ * crashes; past it, members are evidently too big for the cap.
+ */
+export declare function maxOomFallbacks(workers: number): number;
+/**
+ * Old-generation heap cap per parse worker, in MB (T2 cycle 2). A worker's
+ * live heap is small (about 40 MB on a large Vlocity repo), but without a cap V8
+ * lets old space grow to about 180 MB per worker before a major GC; with 4
+ * workers that slack put the repo rebuild at 1.9-2.06 GB RSS. A cap puts V8
+ * in its memory-constrained mode, which collects sooner. A worker that still
+ * runs out (ERR_WORKER_OUT_OF_MEMORY) has its member re-parsed on the main
+ * thread, so the output does not change. SFGRAPH_PARSE_WORKER_HEAP_MB
+ * overrides it (0 = no cap).
+ */
+export declare const DEFAULT_PARSE_WORKER_HEAP_MB = 256;
+/**
+ * Worker count. `SFGRAPH_PARSE_WORKERS` set to an integer >= 0 is used as is
+ * (`0` disables the pool). Otherwise `Math.max(1, cpus - 1)`, capped at 4,
+ * where cpus is `os.availableParallelism()` (honours affinity, unlike
+ * `os.cpus().length`).
+ */
+export declare function resolveParseWorkers(env?: string | undefined, cpus?: number): number;
+/**
+ * The compiled worker next to this module, or null when it is not there
+ * (running from TypeScript sources, e.g. under vitest). Callers then parse on
+ * the main thread.
+ */
+export declare function defaultParseWorkerUrl(): URL | null;
+/** One recorded parser log call. */
+export interface ParseLogEntry {
+    level: "debug" | "info" | "warn" | "error";
+    msg: string;
+    meta?: Record<string, unknown>;
+}
+/** Main → worker. */
+export interface ParseJobMsg {
+    seq: number;
+    member: RawMember;
+}
+/** Worker → main. */
+export type ParseWorkerMsg = {
+    type: "ready";
+} | {
+    type: "done";
+    seq: number;
+    outcome: MemberParse;
+    logs: ParseLogEntry[];
+}
+/** The result is not plain data or could not be cloned: re-parse on main. */
+ | {
+    type: "fallback";
+    seq: number;
+    reason: string;
+};
+/** Data handed to each worker. */
+export interface ParseWorkerData {
+    base: ParseCtxBase;
+}
+export interface ParsePoolStats {
+    /** Workers actually used (0 = the whole input was parsed on the main thread). */
+    workers: number;
+    /** Members parsed in a worker. */
+    pooled: number;
+    /** Members parsed on the main thread (serial mode, fallbacks, broken pool). */
+    inProcess: number;
+    /** Members a worker could not ship back as plain data (re-parsed on main). */
+    fallbacks: number;
+    /** Worker crashes or stalls (each fails one member). */
+    crashes: number;
+    /**
+     * Members whose worker hit its heap cap (ERR_WORKER_OUT_OF_MEMORY); each is
+     * re-parsed on main. Not counted in `crashes`; bounded by MAX_OOM_FALLBACKS.
+     */
+    oomFallbacks?: number;
+}
+export interface ParsePoolOpts extends ParseCtxBase {
+    /** Worker threads; 0 parses on the main thread. Default resolveParseWorkers(). */
+    workers?: number;
+    /** Logger for parser log calls (replayed in member order). */
+    logger?: Logger;
+    /** Worker script; default the compiled `./parse-worker.js` (null → main thread). */
+    workerUrl?: URL | string | null;
+    /** Inputs shorter than this stay on the main thread. Default 500 (SFGRAPH_PARSE_POOL_MIN). */
+    minMembers?: number;
+    /** Members queued per worker. Default 4. */
+    perWorker?: number;
+    /** A ready worker with queued members and no reply for this long is treated as hung. Default 10 min. */
+    stallTimeoutMs?: number;
+    /** A worker not ready this long after it was spawned counts as a start-up failure. Default 2 min. */
+    startupTimeoutMs?: number;
+    /** Old-generation heap cap per worker in MB; 0 = none. Default 256 (SFGRAPH_PARSE_WORKER_HEAP_MB). */
+    workerHeapMb?: number;
+    /** Filled in as the pool runs. */
+    stats?: ParsePoolStats;
+    /**
+     * An outcome to use instead of parsing (T15 N5-1: a quarantined member is
+     * never dispatched to a worker); undefined parses as usual.
+     */
+    preset?: (member: RawMember) => MemberParse | undefined;
+}
+export interface ParsedMember<M extends RawMember = RawMember> {
+    member: M;
+    outcome: MemberParse;
+}
+/**
+ * Parse `members` (in workers when worthwhile) and yield `{member, outcome}`
+ * in input order. Obsolete members (deletions) are yielded with a `skip`
+ * outcome; the caller applies them. If the source throws, every member read
+ * before the throw is yielded first, then the error is rethrown.
+ */
+export declare function parsePool<M extends RawMember>(members: AsyncIterable<M> | Iterable<M>, opts: ParsePoolOpts): AsyncGenerator<ParsedMember<M>, void, undefined>;
+//# sourceMappingURL=parse-pool.d.ts.map

package/dist/ingest/parse-pool.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parse-pool.d.ts","sourceRoot":"","sources":["../../src/ingest/parse-pool.ts"],"names":[],"mappings":"AAwCA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,6BAA6B,CAAC;AAC1D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,6CAA6C,CAAC;AAE7E,OAAO,EAAE,KAAK,WAAW,EAAE,KAAK,YAAY,EAAe,MAAM,mBAAmB,CAAC;AAErF;;;;;;;;GAQG;AACH,eAAO,MAAM,yBAAyB,IAAI,CAAC;AAE3C;;;;;;;;;;GAUG;AACH,eAAO,MAAM,8BAA8B,MAAM,CAAC;AASlD;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAEvD;AAED;;;;;;;;;GASG;AACH,eAAO,MAAM,4BAA4B,MAAM,CAAC;AAOhD;;;;;GAKG;AACH,wBAAgB,mBAAmB,CACjC,GAAG,GAAE,MAAM,GAAG,SAA6C,EAC3D,IAAI,GAAE,MAEc,GACnB,MAAM,CAMR;AAED;;;;GAIG;AACH,wBAAgB,qBAAqB,IAAI,GAAG,GAAG,IAAI,CAOlD;AAED,oCAAoC;AACpC,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;IAC3C,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAChC;AAED,qBAAqB;AACrB,MAAM,WAAW,WAAW;IAC1B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,SAAS,CAAC;CACnB;AAED,qBAAqB;AACrB,MAAM,MAAM,cAAc,GACtB;IAAE,IAAI,EAAE,OAAO,CAAA;CAAE,GACjB;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,WAAW,CAAC;IAAC,IAAI,EAAE,aAAa,EAAE,CAAA;CAAE;AAC5E,6EAA6E;GAC3E;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC;AAEtD,kCAAkC;AAClC,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,YAAY,CAAC;CACpB;AAED,MAAM,WAAW,cAAc;IAC7B,iFAAiF;IACjF,OAAO,EAAE,MAAM,CAAC;IAChB,kCAAkC;IAClC,MAAM,EAAE,MAAM,CAAC;IACf,+EAA+E;IAC/E,SAAS,EAAE,MAAM,CAAC;IAClB,8EAA8E;IAC9E,SAAS,EAAE,MAAM,CAAC;IAClB,wDAAwD;IACxD,OAAO,EAAE,MAAM,CAAC;IAChB;;;OAGG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,aAAc,SAAQ,YAAY;IACjD,kFAAkF;IAClF,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,8DAA8D;IAC9D,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,oFAAoF;IACpF,SAAS,CAAC,EAAE,GAAG,GAAG,MAAM,GAAG,IAAI,CAAC;IAChC,8FAA8F;IAC9F,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,4CAA4C;IAC5C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,wGAAwG;IACxG,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,qGAAqG;IACrG,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,sGAAsG;IACtG,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,kCAAkC;IAClC,KAAK,CAAC,EAAE,cAAc,CAAC;IACvB;;;OAGG;IACH,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,SAAS,KAAK,WAAW,GAAG,SAAS,CAAC;CACzD;AAED,MAAM,WAAW,YAAY,CAAC,CAAC,SAAS,SAAS,GAAG,SAAS;IAC3D,MAAM,EAAE,CAAC,CAAC;IACV,OAAO,EAAE,WAAW,CAAC;CACtB;AAqDD;;;;;GAKG;AACH,wBAAuB,SAAS,CAAC,CAAC,SAAS,SAAS,EAClD,OAAO,EAAE,aAAa,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,EACvC,IAAI,EAAE,aAAa,GAClB,cAAc,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,SAAS,CAAC,CAwblD"}