@ryanstark24/sfgraph-core 1.1.4 → 1.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/analyze/test-coverage.d.ts +0 -2
- package/dist/analyze/test-coverage.d.ts.map +1 -1
- package/dist/analyze/test-coverage.js +0 -10
- package/dist/analyze/test-coverage.js.map +1 -1
- package/dist/embedding/embed.d.ts +31 -0
- package/dist/embedding/embed.d.ts.map +1 -0
- package/dist/embedding/embed.js +69 -0
- package/dist/embedding/embed.js.map +1 -0
- package/dist/embedding/index.d.ts +2 -0
- package/dist/embedding/index.d.ts.map +1 -1
- package/dist/embedding/index.js +1 -0
- package/dist/embedding/index.js.map +1 -1
- package/dist/embedding/queue.js +7 -49
- package/dist/embedding/queue.js.map +1 -1
- package/dist/extractors/live-org/index.d.ts +2 -0
- package/dist/extractors/live-org/index.d.ts.map +1 -1
- package/dist/extractors/live-org/index.js +1 -0
- package/dist/extractors/live-org/index.js.map +1 -1
- package/dist/extractors/live-org/liveness.d.ts +42 -0
- package/dist/extractors/live-org/liveness.d.ts.map +1 -0
- package/dist/extractors/live-org/liveness.js +76 -0
- package/dist/extractors/live-org/liveness.js.map +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/ingest/live-ingest.d.ts +0 -1
- package/dist/ingest/live-ingest.d.ts.map +1 -1
- package/dist/ingest/live-ingest.js +17 -6
- package/dist/ingest/live-ingest.js.map +1 -1
- package/dist/parsers/__tests__/omnistudio.golden.test.js +51 -3
- package/dist/parsers/__tests__/omnistudio.golden.test.js.map +1 -1
- package/dist/parsers/__tests__/vlocity.golden.test.js +106 -5
- package/dist/parsers/__tests__/vlocity.golden.test.js.map +1 -1
- package/dist/parsers/contract.d.ts +0 -2
- package/dist/parsers/contract.d.ts.map +1 -1
- package/dist/parsers/contract.js +0 -7
- package/dist/parsers/contract.js.map +1 -1
- package/dist/parsers/omnistudio/integration-procedure.d.ts.map +1 -1
- package/dist/parsers/omnistudio/integration-procedure.js +6 -3
- package/dist/parsers/omnistudio/integration-procedure.js.map +1 -1
- package/dist/parsers/omnistudio/process.d.ts.map +1 -1
- package/dist/parsers/omnistudio/process.js +17 -4
- package/dist/parsers/omnistudio/process.js.map +1 -1
- package/dist/parsers/omnistudio/ui-card.d.ts.map +1 -1
- package/dist/parsers/omnistudio/ui-card.js +6 -3
- package/dist/parsers/omnistudio/ui-card.js.map +1 -1
- package/dist/parsers/vlocity/common.d.ts +1 -2
- package/dist/parsers/vlocity/common.d.ts.map +1 -1
- package/dist/parsers/vlocity/common.js +0 -3
- package/dist/parsers/vlocity/common.js.map +1 -1
- package/dist/parsers/vlocity/integration-procedure.d.ts.map +1 -1
- package/dist/parsers/vlocity/integration-procedure.js +19 -3
- package/dist/parsers/vlocity/integration-procedure.js.map +1 -1
- package/dist/parsers/vlocity/omni-script.d.ts.map +1 -1
- package/dist/parsers/vlocity/omni-script.js +5 -2
- package/dist/parsers/vlocity/omni-script.js.map +1 -1
- package/dist/parsers/vlocity/vlocity-card.d.ts.map +1 -1
- package/dist/parsers/vlocity/vlocity-card.js +5 -2
- package/dist/parsers/vlocity/vlocity-card.js.map +1 -1
- package/dist/storage/interfaces.d.ts +4 -0
- package/dist/storage/interfaces.d.ts.map +1 -1
- package/dist/storage/sqlite/vector-store.d.ts +4 -0
- package/dist/storage/sqlite/vector-store.d.ts.map +1 -1
- package/dist/storage/sqlite/vector-store.js +18 -0
- package/dist/storage/sqlite/vector-store.js.map +1 -1
- package/package.json +1 -1
|
@@ -1,6 +1,4 @@
|
|
|
1
1
|
import type { OrgId, QualifiedName } from "@ryanstark24/sfgraph-shared";
|
|
2
|
-
import type { NodeFact } from "../domain/index.js";
|
|
3
2
|
import type { GraphStore } from "../storage/interfaces.js";
|
|
4
|
-
export declare function findTestsFor(store: GraphStore, orgId: OrgId, qname: QualifiedName): NodeFact[];
|
|
5
3
|
export declare function hasTestCoverage(store: GraphStore, orgId: OrgId, qname: QualifiedName): boolean;
|
|
6
4
|
//# sourceMappingURL=test-coverage.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"test-coverage.d.ts","sourceRoot":"","sources":["../../src/analyze/test-coverage.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;
|
|
1
|
+
{"version":3,"file":"test-coverage.d.ts","sourceRoot":"","sources":["../../src/analyze/test-coverage.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAExE,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AAE3D,wBAAgB,eAAe,CAAC,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,aAAa,GAAG,OAAO,CAE9F"}
|
|
@@ -1,14 +1,4 @@
|
|
|
1
1
|
import { REL_TYPES } from "../domain/rel-types.js";
|
|
2
|
-
export function findTestsFor(store, orgId, qname) {
|
|
3
|
-
const edges = store.listEdgesTo(orgId, qname, REL_TYPES.IS_TEST_FOR);
|
|
4
|
-
const out = [];
|
|
5
|
-
for (const e of edges) {
|
|
6
|
-
const n = store.getNode(orgId, e.srcQualifiedName);
|
|
7
|
-
if (n)
|
|
8
|
-
out.push(n);
|
|
9
|
-
}
|
|
10
|
-
return out;
|
|
11
|
-
}
|
|
12
2
|
export function hasTestCoverage(store, orgId, qname) {
|
|
13
3
|
return store.listEdgesTo(orgId, qname, REL_TYPES.IS_TEST_FOR).length > 0;
|
|
14
4
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"test-coverage.js","sourceRoot":"","sources":["../../src/analyze/test-coverage.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"test-coverage.js","sourceRoot":"","sources":["../../src/analyze/test-coverage.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AAGnD,MAAM,UAAU,eAAe,CAAC,KAAiB,EAAE,KAAY,EAAE,KAAoB;IACnF,OAAO,KAAK,CAAC,WAAW,CAAC,KAAK,EAAE,KAAK,EAAE,SAAS,CAAC,WAAW,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;AAC3E,CAAC"}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Standalone embedder. Lazy-imports @xenova/transformers (optionalDependency
|
|
3
|
+
* on @ryanstark24/sfgraph-core), uses the vendored MiniLM-L6 weights from
|
|
4
|
+
* @ryanstark24/sfgraph-models when no override is set, and gracefully falls
|
|
5
|
+
* back to zero-vectors when either is unavailable.
|
|
6
|
+
*
|
|
7
|
+
* Env overrides (also accepted by the ingest CLI):
|
|
8
|
+
* SFGRAPH_EMBED_MODEL_PATH absolute path to a model directory
|
|
9
|
+
* SFGRAPH_EMBED_MODEL_ID model id inside that dir (default: Xenova/all-MiniLM-L6-v2)
|
|
10
|
+
* SFGRAPH_EMBED_MODEL_DIM embedding dim (default: 384)
|
|
11
|
+
*
|
|
12
|
+
* Exported because callers outside the ingest pipeline (e.g. the MCP
|
|
13
|
+
* `find_similar` tool's free-text mode) need to embed ad-hoc strings.
|
|
14
|
+
* Until this lived as a module-private closure inside queue.ts, ad-hoc
|
|
15
|
+
* embeddings weren't possible.
|
|
16
|
+
*/
|
|
17
|
+
export interface EmbedOptions {
|
|
18
|
+
/** Override the default model path (otherwise @ryanstark24/sfgraph-models). */
|
|
19
|
+
modelPath?: string;
|
|
20
|
+
/** Override model id. */
|
|
21
|
+
modelId?: string;
|
|
22
|
+
/** Override embedding dimension. */
|
|
23
|
+
dim?: number;
|
|
24
|
+
}
|
|
25
|
+
export declare function embedTexts(texts: string[], opts?: EmbedOptions): Promise<Float32Array[]>;
|
|
26
|
+
/** Convenience: embed a single string. Returns null when transformers/model
|
|
27
|
+
* unavailable — distinct from a zero-vector so callers can distinguish
|
|
28
|
+
* "the runtime can't embed" from "I embedded but the result happens to be
|
|
29
|
+
* all zeros." */
|
|
30
|
+
export declare function embedSingle(text: string, opts?: EmbedOptions): Promise<Float32Array | null>;
|
|
31
|
+
//# sourceMappingURL=embed.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"embed.d.ts","sourceRoot":"","sources":["../../src/embedding/embed.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AACH,MAAM,WAAW,YAAY;IAC3B,+EAA+E;IAC/E,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,yBAAyB;IACzB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,oCAAoC;IACpC,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AAKD,wBAAsB,UAAU,CAC9B,KAAK,EAAE,MAAM,EAAE,EACf,IAAI,GAAE,YAAiB,GACtB,OAAO,CAAC,YAAY,EAAE,CAAC,CA8CzB;AAED;;;kBAGkB;AAClB,wBAAsB,WAAW,CAC/B,IAAI,EAAE,MAAM,EACZ,IAAI,GAAE,YAAiB,GACtB,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,CAoB9B"}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
const DEFAULT_MODEL_ID = "Xenova/all-MiniLM-L6-v2";
|
|
2
|
+
const DEFAULT_DIM = 384;
|
|
3
|
+
export async function embedTexts(texts, opts = {}) {
|
|
4
|
+
try {
|
|
5
|
+
const mod = (await import("@xenova/transformers"));
|
|
6
|
+
const { pipeline, env } = mod;
|
|
7
|
+
env.allowRemoteModels = false;
|
|
8
|
+
const envPath = opts.modelPath ?? process.env.SFGRAPH_EMBED_MODEL_PATH;
|
|
9
|
+
const modelId = opts.modelId ?? process.env.SFGRAPH_EMBED_MODEL_ID ?? DEFAULT_MODEL_ID;
|
|
10
|
+
const dim = opts.dim ??
|
|
11
|
+
(process.env.SFGRAPH_EMBED_MODEL_DIM
|
|
12
|
+
? Number.parseInt(process.env.SFGRAPH_EMBED_MODEL_DIM, 10)
|
|
13
|
+
: DEFAULT_DIM);
|
|
14
|
+
if (envPath) {
|
|
15
|
+
env.localModelPath = envPath;
|
|
16
|
+
}
|
|
17
|
+
else {
|
|
18
|
+
try {
|
|
19
|
+
const models = (await import("@ryanstark24/sfgraph-models"));
|
|
20
|
+
if (models?.MODEL_DATA_DIR)
|
|
21
|
+
env.localModelPath = models.MODEL_DATA_DIR;
|
|
22
|
+
}
|
|
23
|
+
catch {
|
|
24
|
+
/* models package optional */
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
const pipe = await pipeline("feature-extraction", modelId, {
|
|
28
|
+
quantized: true,
|
|
29
|
+
local_files_only: true,
|
|
30
|
+
});
|
|
31
|
+
const out = await pipe(texts, { pooling: "mean", normalize: true });
|
|
32
|
+
const results = [];
|
|
33
|
+
for (let i = 0; i < texts.length; i++) {
|
|
34
|
+
results.push(new Float32Array(out.data.slice(i * dim, (i + 1) * dim)));
|
|
35
|
+
}
|
|
36
|
+
return results;
|
|
37
|
+
}
|
|
38
|
+
catch {
|
|
39
|
+
return texts.map(() => new Float32Array(DEFAULT_DIM));
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
/** Convenience: embed a single string. Returns null when transformers/model
|
|
43
|
+
* unavailable — distinct from a zero-vector so callers can distinguish
|
|
44
|
+
* "the runtime can't embed" from "I embedded but the result happens to be
|
|
45
|
+
* all zeros." */
|
|
46
|
+
export async function embedSingle(text, opts = {}) {
|
|
47
|
+
try {
|
|
48
|
+
// Probe import directly so we can return null on missing runtime,
|
|
49
|
+
// without conflating that with "zero-vector fallback."
|
|
50
|
+
await import("@xenova/transformers");
|
|
51
|
+
}
|
|
52
|
+
catch {
|
|
53
|
+
return null;
|
|
54
|
+
}
|
|
55
|
+
const [vec] = await embedTexts([text], opts);
|
|
56
|
+
if (!vec)
|
|
57
|
+
return null;
|
|
58
|
+
// Heuristic: if every element is exactly 0, the model load failed silently
|
|
59
|
+
// inside embedTexts and we got the fallback. Surface that to the caller.
|
|
60
|
+
let allZero = true;
|
|
61
|
+
for (let i = 0; i < vec.length; i++) {
|
|
62
|
+
if (vec[i] !== 0) {
|
|
63
|
+
allZero = false;
|
|
64
|
+
break;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
return allZero ? null : vec;
|
|
68
|
+
}
|
|
69
|
+
//# sourceMappingURL=embed.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"embed.js","sourceRoot":"","sources":["../../src/embedding/embed.ts"],"names":[],"mappings":"AAyBA,MAAM,gBAAgB,GAAG,yBAAyB,CAAC;AACnD,MAAM,WAAW,GAAG,GAAG,CAAC;AAExB,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,KAAe,EACf,OAAqB,EAAE;IAEvB,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,CAAC,MAAM,MAAM,CAAC,sBAAsB,CAAC,CAKhD,CAAC;QACF,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,GAAG,CAAC;QAC9B,GAAG,CAAC,iBAAiB,GAAG,KAAK,CAAC;QAE9B,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,IAAI,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC;QACvE,MAAM,OAAO,GACX,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,GAAG,CAAC,sBAAsB,IAAI,gBAAgB,CAAC;QACzE,MAAM,GAAG,GACP,IAAI,CAAC,GAAG;YACR,CAAC,OAAO,CAAC,GAAG,CAAC,uBAAuB;gBAClC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,uBAAuB,EAAE,EAAE,CAAC;gBAC1D,CAAC,CAAC,WAAW,CAAC,CAAC;QAEnB,IAAI,OAAO,EAAE,CAAC;YACZ,GAAG,CAAC,cAAc,GAAG,OAAO,CAAC;QAC/B,CAAC;aAAM,CAAC;YACN,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,CAAC,MAAM,MAAM,CAC1B,6BAAkD,CACnD,CAAgC,CAAC;gBAClC,IAAI,MAAM,EAAE,cAAc;oBAAE,GAAG,CAAC,cAAc,GAAG,MAAM,CAAC,cAAc,CAAC;YACzE,CAAC;YAAC,MAAM,CAAC;gBACP,6BAA6B;YAC/B,CAAC;QACH,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,oBAAoB,EAAE,OAAO,EAAE;YACzD,SAAS,EAAE,IAAI;YACf,gBAAgB,EAAE,IAAI;SACvB,CAAC,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,KAAK,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACpE,MAAM,OAAO,GAAmB,EAAE,CAAC;QACnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,OAAO,CAAC,IAAI,CAAC,IAAI,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;QACzE,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,IAAI,YAAY,CAAC,WAAW,CAAC,CAAC,CAAC;IACxD,CAAC;AACH,CAAC;AAED;;;kBAGkB;AAClB,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,IAAY,EACZ,OAAqB,EAAE;IAEvB,IAAI,CAAC;QACH,kEAAkE;QAClE,uDAAuD;QACvD,MAAM,MAAM,CAAC,sBAAsB,CAAC,CAAC;IACvC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,UAAU,CAAC,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,CAAC;IAC7C,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IACtB,2EAA2E;IAC3E,yEAAyE;IACzE,IAAI,OAAO,GAAG,IAAI,CAAC;IACnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACpC,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC;YACjB,OAAO,GAAG,KAAK,CAAC;YAChB,MAAM;QACR,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC;AAC9B,CAAC"}
|
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
export { EmbeddingQueue } from "./queue.js";
|
|
2
2
|
export type { EmbeddingItem, EmbeddingQueueOpts, VectorSink } from "./queue.js";
|
|
3
3
|
export { EMBED_DIM } from "./pool.js";
|
|
4
|
+
export { embedTexts, embedSingle } from "./embed.js";
|
|
5
|
+
export type { EmbedOptions } from "./embed.js";
|
|
4
6
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/embedding/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAC5C,YAAY,EAAE,aAAa,EAAE,kBAAkB,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAChF,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/embedding/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAC5C,YAAY,EAAE,aAAa,EAAE,kBAAkB,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAChF,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACrD,YAAY,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC"}
|
package/dist/embedding/index.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/embedding/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAE5C,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/embedding/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAE5C,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC"}
|
package/dist/embedding/queue.js
CHANGED
|
@@ -68,55 +68,13 @@ export class EmbeddingQueue {
|
|
|
68
68
|
}
|
|
69
69
|
}
|
|
70
70
|
}
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
*/
|
|
71
|
+
// Ingest's batched embedder is now a thin wrapper around the shared
|
|
72
|
+
// `embedTexts` function exported from ./embed.ts. The function used to be a
|
|
73
|
+
// closure here; pulling it out means the MCP `find_similar` tool (and any
|
|
74
|
+
// future ad-hoc embedding caller) gets the same pipeline + same env-var
|
|
75
|
+
// overrides, with no risk of drift.
|
|
77
76
|
async function defaultEmbed(texts) {
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
// package so consumers who don't need embeddings can skip the WASM
|
|
81
|
-
// runtime. Use a plain dynamic import (no string cast) — when the
|
|
82
|
-
// optional dep is missing, the import rejects and we fall through to
|
|
83
|
-
// the zero-vector fallback below.
|
|
84
|
-
const mod = (await import("@xenova/transformers"));
|
|
85
|
-
const { pipeline, env } = mod;
|
|
86
|
-
env.allowRemoteModels = false;
|
|
87
|
-
const envPath = process.env.SFGRAPH_EMBED_MODEL_PATH;
|
|
88
|
-
const modelId = process.env.SFGRAPH_EMBED_MODEL_ID ?? "Xenova/all-MiniLM-L6-v2";
|
|
89
|
-
const dim = process.env.SFGRAPH_EMBED_MODEL_DIM
|
|
90
|
-
? Number.parseInt(process.env.SFGRAPH_EMBED_MODEL_DIM, 10)
|
|
91
|
-
: 384;
|
|
92
|
-
if (envPath) {
|
|
93
|
-
env.localModelPath = envPath;
|
|
94
|
-
}
|
|
95
|
-
else {
|
|
96
|
-
try {
|
|
97
|
-
const models = (await import("@ryanstark24/sfgraph-models"));
|
|
98
|
-
if (models?.MODEL_DATA_DIR)
|
|
99
|
-
env.localModelPath = models.MODEL_DATA_DIR;
|
|
100
|
-
}
|
|
101
|
-
catch {
|
|
102
|
-
/* models package optional */
|
|
103
|
-
}
|
|
104
|
-
}
|
|
105
|
-
const pipe = await pipeline("feature-extraction", modelId, {
|
|
106
|
-
quantized: true,
|
|
107
|
-
local_files_only: true,
|
|
108
|
-
});
|
|
109
|
-
const out = await pipe(texts, { pooling: "mean", normalize: true });
|
|
110
|
-
const results = [];
|
|
111
|
-
for (let i = 0; i < texts.length; i++) {
|
|
112
|
-
results.push(new Float32Array(out.data.slice(i * dim, (i + 1) * dim)));
|
|
113
|
-
}
|
|
114
|
-
return results;
|
|
115
|
-
}
|
|
116
|
-
catch {
|
|
117
|
-
// Transformers (or model) not present — return zero vectors so caller can
|
|
118
|
-
// still upsert placeholder embeddings if desired. Ingest must not crash.
|
|
119
|
-
return texts.map(() => new Float32Array(384));
|
|
120
|
-
}
|
|
77
|
+
const { embedTexts } = await import("./embed.js");
|
|
78
|
+
return embedTexts(texts);
|
|
121
79
|
}
|
|
122
80
|
//# sourceMappingURL=queue.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"queue.js","sourceRoot":"","sources":["../../src/embedding/queue.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,6BAA6B,CAAC;AA2BxE;;;;;;;GAOG;AACH,MAAM,OAAO,cAAc;IAMI;IALrB,MAAM,GAAoB,EAAE,CAAC;IAC7B,QAAQ,GAAyB,IAAI,CAAC;IAC7B,SAAS,CAAS;IAClB,OAAO,CAA+C;IAEvE,YAA6B,IAAwB;QAAxB,SAAI,GAAJ,IAAI,CAAoB;QACnD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC;QACtC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,KAAK,IAAI,YAAY,CAAC;IAC5C,CAAC;IAED,IAAI,CAAC,IAAmB;QACtB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvB,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC3D,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE;gBAC7C,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;YACvB,CAAC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK;QACT,wEAAwE;QACxE,oEAAoE;QACpE,qEAAqE;QACrE,sEAAsE;QACtE,4CAA4C;QAC5C,OAAO,IAAI,EAAE,CAAC;YACZ,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAClB,MAAM,IAAI,CAAC,QAAQ,CAAC;gBACpB,SAAS;YACX,CAAC;YACD,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO;YACrC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE;gBAC7C,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;YACvB,CAAC,CAAC,CAAC;YACH,MAAM,IAAI,CAAC,QAAQ,CAAC;QACtB,CAAC;IACH,CAAC;IAED,0BAA0B;IAC1B,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;IAC5B,CAAC;IAEO,KAAK,CAAC,UAAU;QACtB,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;QACpD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAC/B,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;YAC7D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACtC,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;gBACrB,IAAI,CAAC,CAAC;oBAAE,SAAS;gBACjB,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC,CAAkB,CAAC;gBACpC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,gBAAgB,CACpC,CAAC,CAAC,KAAyB,EAC3B,eAAe,CAAC,CAAC,CAAC,KAAK,CAAC,EACxB,CAAC,CAAC,KAAK,EACP,CAAC,EACD,QAAQ,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,CAAC,CAC5B,CAAC;YACJ,CAAC;QACH,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAU,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;CACF;AAED
|
|
1
|
+
{"version":3,"file":"queue.js","sourceRoot":"","sources":["../../src/embedding/queue.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,6BAA6B,CAAC;AA2BxE;;;;;;;GAOG;AACH,MAAM,OAAO,cAAc;IAMI;IALrB,MAAM,GAAoB,EAAE,CAAC;IAC7B,QAAQ,GAAyB,IAAI,CAAC;IAC7B,SAAS,CAAS;IAClB,OAAO,CAA+C;IAEvE,YAA6B,IAAwB;QAAxB,SAAI,GAAJ,IAAI,CAAoB;QACnD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC;QACtC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,KAAK,IAAI,YAAY,CAAC;IAC5C,CAAC;IAED,IAAI,CAAC,IAAmB;QACtB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvB,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC3D,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE;gBAC7C,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;YACvB,CAAC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK;QACT,wEAAwE;QACxE,oEAAoE;QACpE,qEAAqE;QACrE,sEAAsE;QACtE,4CAA4C;QAC5C,OAAO,IAAI,EAAE,CAAC;YACZ,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;gBAClB,MAAM,IAAI,CAAC,QAAQ,CAAC;gBACpB,SAAS;YACX,CAAC;YACD,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO;YACrC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE;gBAC7C,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;YACvB,CAAC,CAAC,CAAC;YACH,MAAM,IAAI,CAAC,QAAQ,CAAC;QACtB,CAAC;IACH,CAAC;IAED,0BAA0B;IAC1B,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;IAC5B,CAAC;IAEO,KAAK,CAAC,UAAU;QACtB,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;QACpD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAC/B,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;YAC7D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACtC,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;gBACrB,IAAI,CAAC,CAAC;oBAAE,SAAS;gBACjB,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC,CAAkB,CAAC;gBACpC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,gBAAgB,CACpC,CAAC,CAAC,KAAyB,EAC3B,eAAe,CAAC,CAAC,CAAC,KAAK,CAAC,EACxB,CAAC,CAAC,KAAK,EACP,CAAC,EACD,QAAQ,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,CAAC,CAC5B,CAAC;YACJ,CAAC;QACH,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAU,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;CACF;AAED,oEAAoE;AACpE,4EAA4E;AAC5E,0EAA0E;AAC1E,wEAAwE;AACxE,oCAAoC;AACpC,KAAK,UAAU,YAAY,CAAC,KAAe;IACzC,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,MAAM,CAAC,YAAY,CAAC,CAAC;IAClD,OAAO,UAAU,CAAC,KAAK,CAAC,CAAC;AAC3B,CAAC"}
|
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
export { wrapConnectionReadOnly } from "./read-only-proxy.js";
|
|
2
|
+
export { startLivenessProbe } from "./liveness.js";
|
|
3
|
+
export type { LivenessProbeHandle, LivenessProbeOptions } from "./liveness.js";
|
|
2
4
|
export { ReadOnlyViolationError } from "./errors.js";
|
|
3
5
|
export { resolveOrg, resolveDefaultOrgAlias, safeOrgInfo } from "./auth.js";
|
|
4
6
|
export type { ResolvedOrg, ResolveOrgDeps } from "./auth.js";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/extractors/live-org/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAC9D,OAAO,EAAE,sBAAsB,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,sBAAsB,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAC5E,YAAY,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAC7D,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AACtD,YAAY,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACzD,OAAO,EACL,UAAU,EACV,OAAO,EACP,aAAa,EACb,gBAAgB,EAChB,YAAY,EACZ,WAAW,EACX,YAAY,EACZ,QAAQ,EACR,oBAAoB,EACpB,qBAAqB,EACrB,wBAAwB,GACzB,MAAM,iBAAiB,CAAC;AACzB,YAAY,EAAE,cAAc,EAAE,wBAAwB,EAAE,MAAM,iBAAiB,CAAC;AAChF,OAAO,EACL,YAAY,EACZ,mBAAmB,EACnB,2BAA2B,GAC5B,MAAM,oBAAoB,CAAC;AAC5B,YAAY,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,EAAE,QAAQ,EAAE,OAAO,IAAI,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACxE,OAAO,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAC;AAC9C,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAChD,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;AAC9D,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAC5D,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/extractors/live-org/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAC9D,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACnD,YAAY,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,eAAe,CAAC;AAC/E,OAAO,EAAE,sBAAsB,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,sBAAsB,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAC5E,YAAY,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAC7D,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AACtD,YAAY,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACzD,OAAO,EACL,UAAU,EACV,OAAO,EACP,aAAa,EACb,gBAAgB,EAChB,YAAY,EACZ,WAAW,EACX,YAAY,EACZ,QAAQ,EACR,oBAAoB,EACpB,qBAAqB,EACrB,wBAAwB,GACzB,MAAM,iBAAiB,CAAC;AACzB,YAAY,EAAE,cAAc,EAAE,wBAAwB,EAAE,MAAM,iBAAiB,CAAC;AAChF,OAAO,EACL,YAAY,EACZ,mBAAmB,EACnB,2BAA2B,GAC5B,MAAM,oBAAoB,CAAC;AAC5B,YAAY,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACzE,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,EAAE,QAAQ,EAAE,OAAO,IAAI,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACxE,OAAO,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAC;AAC9C,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAChD,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;AAC9D,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAC5D,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC"}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
export { wrapConnectionReadOnly } from "./read-only-proxy.js";
|
|
2
|
+
export { startLivenessProbe } from "./liveness.js";
|
|
2
3
|
export { ReadOnlyViolationError } from "./errors.js";
|
|
3
4
|
export { resolveOrg, resolveDefaultOrgAlias, safeOrgInfo } from "./auth.js";
|
|
4
5
|
export { probeCapabilities } from "./capabilities.js";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/extractors/live-org/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAC9D,OAAO,EAAE,sBAAsB,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,sBAAsB,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAE5E,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAEtD,OAAO,EACL,UAAU,EACV,OAAO,EACP,aAAa,EACb,gBAAgB,EAChB,YAAY,EACZ,WAAW,EACX,YAAY,EACZ,QAAQ,EACR,oBAAoB,EACpB,qBAAqB,EACrB,wBAAwB,GACzB,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EACL,YAAY,EACZ,mBAAmB,EACnB,2BAA2B,GAC5B,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,EAAE,QAAQ,EAAE,OAAO,IAAI,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACxE,OAAO,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAC;AAC9C,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAChD,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;AAC9D,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAC5D,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/extractors/live-org/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAC9D,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAEnD,OAAO,EAAE,sBAAsB,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,sBAAsB,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAE5E,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAEtD,OAAO,EACL,UAAU,EACV,OAAO,EACP,aAAa,EACb,gBAAgB,EAChB,YAAY,EACZ,WAAW,EACX,YAAY,EACZ,QAAQ,EACR,oBAAoB,EACpB,qBAAqB,EACrB,wBAAwB,GACzB,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EACL,YAAY,EACZ,mBAAmB,EACnB,2BAA2B,GAC5B,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,EAAE,QAAQ,EAAE,OAAO,IAAI,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACxE,OAAO,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAC;AAC9C,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAChD,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;AAC9D,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAC5D,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC"}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Background connection-liveness probe.
|
|
3
|
+
*
|
|
4
|
+
* The dead-socket failure mode reproduces cleanly on corporate VPNs / NATs
|
|
5
|
+
* with idle eviction: jsforce's HTTP socket goes half-open, every
|
|
6
|
+
* outstanding request hangs forever, and the only escape is the per-call
|
|
7
|
+
* 60s `withTimeout` wrap inside each extractor. That works — the run
|
|
8
|
+
* doesn't wedge — but the user-visible experience is "10 sources each
|
|
9
|
+
* fail with timeout errors, ingest finishes with a giant skip list, no
|
|
10
|
+
* obvious cause." This probe makes the cause obvious.
|
|
11
|
+
*
|
|
12
|
+
* Every `intervalMs` we call `conn.identity()` (the cheapest jsforce
|
|
13
|
+
* request: a single REST hit to `/services/oauth2/userinfo`). If it
|
|
14
|
+
* succeeds, the connection is alive; we reset the failure counter and
|
|
15
|
+
* keep going. If it times out or rejects, we increment a counter; once
|
|
16
|
+
* `consecutiveFailures >= maxFailures` we declare the connection dead,
|
|
17
|
+
* log a prominent warning, and fire `onDead` once.
|
|
18
|
+
*
|
|
19
|
+
* We do NOT abort the ingest. Aborting mid-flight would need an AbortSignal
|
|
20
|
+
* threaded through every extractor — too invasive for marginal benefit
|
|
21
|
+
* given the per-call timeouts already cap exposure. The probe's value is
|
|
22
|
+
* SIGNALLING: the user sees "ingest: ⚠ connection lost — surviving
|
|
23
|
+
* extractors will time out within ~60s" and knows to re-run from a fresh
|
|
24
|
+
* terminal, instead of guessing at pool tuning.
|
|
25
|
+
*/
|
|
26
|
+
export interface LivenessProbeOptions {
|
|
27
|
+
intervalMs?: number;
|
|
28
|
+
identityTimeoutMs?: number;
|
|
29
|
+
maxFailures?: number;
|
|
30
|
+
onDead?: (failureReason: string) => void;
|
|
31
|
+
/** Override the logger; falls back to console. Tests inject silent. */
|
|
32
|
+
log?: (line: string) => void;
|
|
33
|
+
}
|
|
34
|
+
export interface LivenessProbeHandle {
|
|
35
|
+
stop(): void;
|
|
36
|
+
isDead(): boolean;
|
|
37
|
+
failureCount(): number;
|
|
38
|
+
}
|
|
39
|
+
export declare function startLivenessProbe(conn: {
|
|
40
|
+
identity: () => Promise<unknown>;
|
|
41
|
+
}, options?: LivenessProbeOptions): LivenessProbeHandle;
|
|
42
|
+
//# sourceMappingURL=liveness.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"liveness.d.ts","sourceRoot":"","sources":["../../../src/extractors/live-org/liveness.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,MAAM,WAAW,oBAAoB;IACnC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,CAAC,aAAa,EAAE,MAAM,KAAK,IAAI,CAAC;IACzC,uEAAuE;IACvE,GAAG,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;CAC9B;AAED,MAAM,WAAW,mBAAmB;IAClC,IAAI,IAAI,IAAI,CAAC;IACb,MAAM,IAAI,OAAO,CAAC;IAClB,YAAY,IAAI,MAAM,CAAC;CACxB;AA2BD,wBAAgB,kBAAkB,CAChC,IAAI,EAAE;IAAE,QAAQ,EAAE,MAAM,OAAO,CAAC,OAAO,CAAC,CAAA;CAAE,EAC1C,OAAO,GAAE,oBAAyB,GACjC,mBAAmB,CA2DrB"}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
const DEFAULTS = {
|
|
2
|
+
intervalMs: 30_000,
|
|
3
|
+
identityTimeoutMs: 10_000,
|
|
4
|
+
maxFailures: 2,
|
|
5
|
+
};
|
|
6
|
+
/** Race a promise against a setTimeout; reject if the promise doesn't
|
|
7
|
+
* settle within `ms`. Kept local so this module has no dependency on
|
|
8
|
+
* rate-limit.ts's withTimeout (which would create a circular ref). */
|
|
9
|
+
function withDeadline(p, ms, label) {
|
|
10
|
+
return new Promise((resolve, reject) => {
|
|
11
|
+
const t = setTimeout(() => reject(new Error(`${label} deadline (${ms}ms)`)), ms);
|
|
12
|
+
p.then((v) => {
|
|
13
|
+
clearTimeout(t);
|
|
14
|
+
resolve(v);
|
|
15
|
+
}, (e) => {
|
|
16
|
+
clearTimeout(t);
|
|
17
|
+
reject(e);
|
|
18
|
+
});
|
|
19
|
+
});
|
|
20
|
+
}
|
|
21
|
+
export function startLivenessProbe(conn, options = {}) {
|
|
22
|
+
const intervalMs = options.intervalMs ?? DEFAULTS.intervalMs;
|
|
23
|
+
const identityTimeoutMs = options.identityTimeoutMs ?? DEFAULTS.identityTimeoutMs;
|
|
24
|
+
const maxFailures = options.maxFailures ?? DEFAULTS.maxFailures;
|
|
25
|
+
const log = options.log ?? ((line) => console.log(line));
|
|
26
|
+
let consecutiveFailures = 0;
|
|
27
|
+
let dead = false;
|
|
28
|
+
let firedDead = false;
|
|
29
|
+
let stopped = false;
|
|
30
|
+
const tick = async () => {
|
|
31
|
+
if (stopped || dead)
|
|
32
|
+
return;
|
|
33
|
+
try {
|
|
34
|
+
await withDeadline(conn.identity(), identityTimeoutMs, "liveness identity()");
|
|
35
|
+
// Success — reset counter. Don't log on success or we'd flood.
|
|
36
|
+
consecutiveFailures = 0;
|
|
37
|
+
}
|
|
38
|
+
catch (e) {
|
|
39
|
+
consecutiveFailures += 1;
|
|
40
|
+
const reason = e?.message ?? String(e);
|
|
41
|
+
// Soft warning at the first failure, escalation at the threshold.
|
|
42
|
+
// The intermediate state ("1 of 2 failures") is real signal that
|
|
43
|
+
// something's wrong even before we're sure.
|
|
44
|
+
if (consecutiveFailures < maxFailures) {
|
|
45
|
+
log(`ingest: [liveness] probe failed (${consecutiveFailures}/${maxFailures}): ${reason.slice(0, 120)}`);
|
|
46
|
+
}
|
|
47
|
+
else if (!firedDead) {
|
|
48
|
+
dead = true;
|
|
49
|
+
firedDead = true;
|
|
50
|
+
log("");
|
|
51
|
+
log(`ingest: ⚠ CONNECTION LOST — ${consecutiveFailures} consecutive identity() probes failed. Surviving extractors will fail within ~60s as their per-call timeouts trip. The ingest will finish with a skip list. Re-run \`sfgraph ingest --org <alias> --retry-skipped\` from a fresh terminal once connectivity is restored.`);
|
|
52
|
+
log("");
|
|
53
|
+
options.onDead?.(reason);
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
};
|
|
57
|
+
// Fire-and-forget interval. The first tick fires after intervalMs (not
|
|
58
|
+
// immediately) — startup is when the connection is freshest and we
|
|
59
|
+
// don't want to add latency to ingest's first API calls by racing
|
|
60
|
+
// identity() against probeCapabilities().
|
|
61
|
+
const timer = setInterval(() => {
|
|
62
|
+
void tick();
|
|
63
|
+
}, intervalMs);
|
|
64
|
+
// Don't keep the event loop alive on this timer alone; the ingest's
|
|
65
|
+
// own keep-alive sentinel handles that.
|
|
66
|
+
timer.unref?.();
|
|
67
|
+
return {
|
|
68
|
+
stop: () => {
|
|
69
|
+
stopped = true;
|
|
70
|
+
clearInterval(timer);
|
|
71
|
+
},
|
|
72
|
+
isDead: () => dead,
|
|
73
|
+
failureCount: () => consecutiveFailures,
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
//# sourceMappingURL=liveness.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"liveness.js","sourceRoot":"","sources":["../../../src/extractors/live-org/liveness.ts"],"names":[],"mappings":"AAwCA,MAAM,QAAQ,GAAG;IACf,UAAU,EAAE,MAAM;IAClB,iBAAiB,EAAE,MAAM;IACzB,WAAW,EAAE,CAAC;CACf,CAAC;AAEF;;uEAEuE;AACvE,SAAS,YAAY,CAAI,CAAa,EAAE,EAAU,EAAE,KAAa;IAC/D,OAAO,IAAI,OAAO,CAAI,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACxC,MAAM,CAAC,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,GAAG,KAAK,cAAc,EAAE,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACjF,CAAC,CAAC,IAAI,CACJ,CAAC,CAAC,EAAE,EAAE;YACJ,YAAY,CAAC,CAAC,CAAC,CAAC;YAChB,OAAO,CAAC,CAAC,CAAC,CAAC;QACb,CAAC,EACD,CAAC,CAAC,EAAE,EAAE;YACJ,YAAY,CAAC,CAAC,CAAC,CAAC;YAChB,MAAM,CAAC,CAAC,CAAC,CAAC;QACZ,CAAC,CACF,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,kBAAkB,CAChC,IAA0C,EAC1C,UAAgC,EAAE;IAElC,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,QAAQ,CAAC,UAAU,CAAC;IAC7D,MAAM,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,IAAI,QAAQ,CAAC,iBAAiB,CAAC;IAClF,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,QAAQ,CAAC,WAAW,CAAC;IAChE,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,CAAC,CAAC,IAAY,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;IAEjE,IAAI,mBAAmB,GAAG,CAAC,CAAC;IAC5B,IAAI,IAAI,GAAG,KAAK,CAAC;IACjB,IAAI,SAAS,GAAG,KAAK,CAAC;IACtB,IAAI,OAAO,GAAG,KAAK,CAAC;IAEpB,MAAM,IAAI,GAAG,KAAK,IAAmB,EAAE;QACrC,IAAI,OAAO,IAAI,IAAI;YAAE,OAAO;QAC5B,IAAI,CAAC;YACH,MAAM,YAAY,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,iBAAiB,EAAE,qBAAqB,CAAC,CAAC;YAC9E,+DAA+D;YAC/D,mBAAmB,GAAG,CAAC,CAAC;QAC1B,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,mBAAmB,IAAI,CAAC,CAAC;YACzB,MAAM,MAAM,GAAI,CAAW,EAAE,OAAO,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC;YAClD,kEAAkE;YAClE,iEAAiE;YACjE,4CAA4C;YAC5C,IAAI,mBAAmB,GAAG,WAAW,EAAE,CAAC;gBACtC,GAAG,CACD,oCAAoC,mBAAmB,IAAI,WAAW,MAAM,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CACnG,CAAC;YACJ,CAAC;iBAAM,IAAI,CAAC,SAAS,EAAE,CAAC;gBACtB,IAAI,GAAG,IAAI,CAAC;gBACZ,SAAS,GAAG,IAAI,CAAC;gBACjB,GAAG,CAAC,EAAE,CAAC,CAAC;gBACR,GAAG,CACD,+BAA+B,mBAAmB,0QAA0Q,CAC7T,CAAC;gBACF,GAAG,CAAC,EAAE,CAAC,CAAC;gBACR,OAAO,CAAC,MAAM,EAAE,CAAC,MAAM,CAAC,CAAC;YAC3B,CAAC;QACH,CAAC;IACH,CAAC,CAAC;IAEF,uEAAuE;IACvE,mEAAmE;IACnE,kEAAkE;IAClE,0CAA0C;IAC1C,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE;QAC7B,KAAK,IAAI,EAAE,CAAC;IACd,CAAC,EAAE,UAAU,CAAC,CAAC;IACf,oEAAoE;IACpE,wCAAwC;IACxC,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC;IAEhB,OAAO;QACL,IAAI,EAAE,GAAG,EAAE;YACT,OAAO,GAAG,IAAI,CAAC;YACf,aAAa,CAAC,KAAK,CAAC,CAAC;QACvB,CAAC;QACD,MAAM,EAAE,GAAG,EAAE,CAAC,IAAI;QAClB,YAAY,EAAE,GAAG,EAAE,CAAC,mBAAmB;KACxC,CAAC;AACJ,CAAC"}
|
package/dist/index.d.ts
CHANGED
|
@@ -3,6 +3,7 @@ export * from "./telemetry/index.js";
|
|
|
3
3
|
export * from "./extractors/live-org/index.js";
|
|
4
4
|
export * from "./storage/index.js";
|
|
5
5
|
export * from "./ingest/index.js";
|
|
6
|
+
export * from "./embedding/index.js";
|
|
6
7
|
export type { MetadataSource, MemberRef, RawMember, } from "./extractors/interfaces/metadata-source.js";
|
|
7
8
|
export { FilesystemMetadataSource } from "./extractors/filesystem/index.js";
|
|
8
9
|
export * as render from "./render/mermaid/index.js";
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,mBAAmB,CAAC;AAClC,cAAc,sBAAsB,CAAC;AACrC,cAAc,gCAAgC,CAAC;AAC/C,cAAc,oBAAoB,CAAC;AACnC,cAAc,mBAAmB,CAAC;AAClC,YAAY,EACV,cAAc,EACd,SAAS,EACT,SAAS,GACV,MAAM,4CAA4C,CAAC;AACpD,OAAO,EAAE,wBAAwB,EAAE,MAAM,kCAAkC,CAAC;AAC5E,OAAO,KAAK,MAAM,MAAM,2BAA2B,CAAC;AACpD,OAAO,KAAK,OAAO,MAAM,oBAAoB,CAAC;AAE9C,OAAO,EACL,kBAAkB,EAClB,mBAAmB,EACnB,KAAK,WAAW,EAChB,KAAK,SAAS,EACd,KAAK,kBAAkB,GACxB,MAAM,0BAA0B,CAAC;AAClC,YAAY,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,mBAAmB,CAAC;AAClC,cAAc,sBAAsB,CAAC;AACrC,cAAc,gCAAgC,CAAC;AAC/C,cAAc,oBAAoB,CAAC;AACnC,cAAc,mBAAmB,CAAC;AAClC,cAAc,sBAAsB,CAAC;AACrC,YAAY,EACV,cAAc,EACd,SAAS,EACT,SAAS,GACV,MAAM,4CAA4C,CAAC;AACpD,OAAO,EAAE,wBAAwB,EAAE,MAAM,kCAAkC,CAAC;AAC5E,OAAO,KAAK,MAAM,MAAM,2BAA2B,CAAC;AACpD,OAAO,KAAK,OAAO,MAAM,oBAAoB,CAAC;AAE9C,OAAO,EACL,kBAAkB,EAClB,mBAAmB,EACnB,KAAK,WAAW,EAChB,KAAK,SAAS,EACd,KAAK,kBAAkB,GACxB,MAAM,0BAA0B,CAAC;AAClC,YAAY,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -3,6 +3,7 @@ export * from "./telemetry/index.js";
|
|
|
3
3
|
export * from "./extractors/live-org/index.js";
|
|
4
4
|
export * from "./storage/index.js";
|
|
5
5
|
export * from "./ingest/index.js";
|
|
6
|
+
export * from "./embedding/index.js";
|
|
6
7
|
export { FilesystemMetadataSource } from "./extractors/filesystem/index.js";
|
|
7
8
|
export * as render from "./render/mermaid/index.js";
|
|
8
9
|
export * as analyze from "./analyze/index.js";
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,mBAAmB,CAAC;AAClC,cAAc,sBAAsB,CAAC;AACrC,cAAc,gCAAgC,CAAC;AAC/C,cAAc,oBAAoB,CAAC;AACnC,cAAc,mBAAmB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,mBAAmB,CAAC;AAClC,cAAc,sBAAsB,CAAC;AACrC,cAAc,gCAAgC,CAAC;AAC/C,cAAc,oBAAoB,CAAC;AACnC,cAAc,mBAAmB,CAAC;AAClC,cAAc,sBAAsB,CAAC;AAMrC,OAAO,EAAE,wBAAwB,EAAE,MAAM,kCAAkC,CAAC;AAC5E,OAAO,KAAK,MAAM,MAAM,2BAA2B,CAAC;AACpD,OAAO,KAAK,OAAO,MAAM,oBAAoB,CAAC;AAC9C,sEAAsE;AACtE,OAAO,EACL,kBAAkB,EAClB,mBAAmB,GAIpB,MAAM,0BAA0B,CAAC"}
|
|
@@ -71,5 +71,4 @@ export interface LiveIngestResult {
|
|
|
71
71
|
danglingEdges: number;
|
|
72
72
|
}
|
|
73
73
|
export declare function liveIngest(opts: LiveIngestOpts): Promise<LiveIngestResult>;
|
|
74
|
-
export declare function rethrowAsIngestError(e: unknown, ctx: string): never;
|
|
75
74
|
//# sourceMappingURL=live-ingest.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"live-ingest.d.ts","sourceRoot":"","sources":["../../src/ingest/live-ingest.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"live-ingest.d.ts","sourceRoot":"","sources":["../../src/ingest/live-ingest.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,KAAK,EAAmB,MAAM,6BAA6B,CAAC;AAC1E,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,6BAA6B,CAAC;AAO1D,OAAO,qBAAqB,CAAC;AAG7B,OAAO,EAAkB,KAAK,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAExE,OAAO,EAAE,KAAK,cAAc,EAAE,KAAK,WAAW,EAAc,MAAM,gCAAgC,CAAC;AAEnG,OAAO,EAAE,KAAK,eAAe,EAAqB,MAAM,wCAAwC,CAAC;AAQjG,OAAO,KAAK,EAAE,qBAAqB,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AAClF,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAE9D,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,aAAa,GAAG,MAAM,CAAC;AAEzD,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,UAAU,CAAC;IAClB,UAAU,EAAE,UAAU,CAAC;IACvB,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,6CAA6C;IAC7C,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,8CAA8C;IAC9C,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,2CAA2C;IAC3C,WAAW,CAAC,EAAE,cAAc,CAAC;IAC7B,oFAAoF;IACpF,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,gEAAgE;IAChE,UAAU,CAAC,EAAE,qBAAqB,CAAC;IACnC,yEAAyE;IACzE,WAAW,CAAC,EAAE,UAAU,CAAC;IACzB;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B;;;;OAIG;IACH,UAAU,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IACzB;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,qEAAqE;IACrE,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,2DAA2D;IAC3D,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,gEAAgE;IAChE,2BAA2B,CAAC,EAAE,OAAO,CAAC;IACtC,oDAAoD;IACpD,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,KAAK,CAAC;IACb,YAAY,EAAE,eAAe,CAAC;IAC9B,IAAI,EAAE,UAAU,CAAC;IACjB,gBAAgB,EAAE,MAAM,CAAC;IACzB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,yEAAyE;IACzE,gBAAgB,EAAE,MAAM,CAAC;IACzB,2EAA2E;IAC3E,aAAa,EAAE,MAAM,CAAC;IACtB,gEAAgE;IAChE,mBAAmB,EAAE,MAAM,CAAC;IAC5B,2EAA2E;IAC3E,aAAa,EAAE,MAAM,CAAC;CACvB;AAmLD,wBAAsB,UAAU,CAAC,IAAI,EAAE,cAAc,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAwiBhF"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { asQualifiedName } from "@ryanstark24/sfgraph-shared";
|
|
2
2
|
import { resolveApexMethodArity } from "../parsers/apex/arity-resolver.js";
|
|
3
3
|
import { resolveCrossFlavor } from "../parsers/cross-flavor-resolver.js";
|
|
4
4
|
import { resolveFlowApexMethods } from "../parsers/flow/invocable-resolver.js";
|
|
@@ -12,6 +12,7 @@ import { resolveOrg } from "../extractors/live-org/auth.js";
|
|
|
12
12
|
import { bulkRetrieve } from "../extractors/live-org/bulk-retrieve.js";
|
|
13
13
|
import { probeCapabilities } from "../extractors/live-org/capabilities.js";
|
|
14
14
|
import { iterChanges } from "../extractors/live-org/source-member.js";
|
|
15
|
+
import { startLivenessProbe, } from "../extractors/live-org/liveness.js";
|
|
15
16
|
import { dataPool, metadataPool, toolingPool } from "../extractors/live-org/rate-limit.js";
|
|
16
17
|
import { loadAllRules } from "../parsers/rules/_loader.js";
|
|
17
18
|
const NOOP_LOG = {
|
|
@@ -374,6 +375,7 @@ export async function liveIngest(opts) {
|
|
|
374
375
|
const debug = process.env.SFGRAPH_DEBUG_INGEST === "1";
|
|
375
376
|
let heartbeatTimer = null;
|
|
376
377
|
let keepAlive = null;
|
|
378
|
+
let livenessProbe = null;
|
|
377
379
|
// Track signal handlers we register so we can remove them in finally.
|
|
378
380
|
// Without this, every invocation of liveIngest in debug mode leaked one
|
|
379
381
|
// listener per signal (SIGTERM/SIGINT/SIGUSR2), eventually triggering
|
|
@@ -394,6 +396,19 @@ export async function liveIngest(opts) {
|
|
|
394
396
|
// and Node will happily exit(0) on a perfectly healthy pending Promise
|
|
395
397
|
// if no ref'd handle is left.
|
|
396
398
|
keepAlive = setInterval(() => { }, 60_000);
|
|
399
|
+
// Background liveness probe. Polls conn.identity() every 30s with a
|
|
400
|
+
// 10s deadline; after two consecutive failures it logs a prominent
|
|
401
|
+
// "CONNECTION LOST" warning so the user knows why surviving
|
|
402
|
+
// extractors are about to time out one by one. We intentionally do
|
|
403
|
+
// NOT abort the ingest from here — per-call 60s timeouts (added in
|
|
404
|
+
// 1.1.3) already cap exposure, and aborting mid-flight would need
|
|
405
|
+
// an AbortSignal threaded through every extractor for marginal
|
|
406
|
+
// benefit. The probe's job is observability, not control flow.
|
|
407
|
+
// Disable via SFGRAPH_NO_LIVENESS_PROBE=1 for hermetic test runs
|
|
408
|
+
// where the mock connection has no identity() implementation.
|
|
409
|
+
if (process.env.SFGRAPH_NO_LIVENESS_PROBE !== "1") {
|
|
410
|
+
livenessProbe = startLivenessProbe(resolved.conn);
|
|
411
|
+
}
|
|
397
412
|
// Wall-clock heartbeat — always on. Without this, silent phases (where
|
|
398
413
|
// all in-window sources are awaiting Bottleneck-queued metadata reads
|
|
399
414
|
// without yielding) look indistinguishable from a hung process. The
|
|
@@ -506,6 +521,7 @@ export async function liveIngest(opts) {
|
|
|
506
521
|
clearInterval(heartbeatTimer);
|
|
507
522
|
if (keepAlive)
|
|
508
523
|
clearInterval(keepAlive);
|
|
524
|
+
livenessProbe?.stop();
|
|
509
525
|
for (const h of installedSignalHandlers)
|
|
510
526
|
process.off(h.sig, h.fn);
|
|
511
527
|
}
|
|
@@ -696,9 +712,4 @@ export async function liveIngest(opts) {
|
|
|
696
712
|
danglingEdges,
|
|
697
713
|
};
|
|
698
714
|
}
|
|
699
|
-
export function rethrowAsIngestError(e, ctx) {
|
|
700
|
-
if (e instanceof SfgraphError)
|
|
701
|
-
throw e;
|
|
702
|
-
throw new SfgraphError(ErrorCode.E_SF_INGEST, `${ctx}: ${e.message ?? String(e)}`);
|
|
703
|
-
}
|
|
704
715
|
//# sourceMappingURL=live-ingest.js.map
|