@seanhogg/builderforce-memory 2026.7.14 → 2026.9.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -0
- package/dist/bridges/AnthropicBridge.d.ts +9 -1
- package/dist/bridges/AnthropicBridge.d.ts.map +1 -1
- package/dist/bridges/AnthropicBridge.js +58 -2
- package/dist/bridges/AnthropicBridge.js.map +1 -1
- package/dist/bridges/CachingBridge.d.ts +9 -1
- package/dist/bridges/CachingBridge.d.ts.map +1 -1
- package/dist/bridges/CachingBridge.js +24 -1
- package/dist/bridges/CachingBridge.js.map +1 -1
- package/dist/bridges/OpenAIBridge.d.ts +4 -1
- package/dist/bridges/OpenAIBridge.d.ts.map +1 -1
- package/dist/bridges/OpenAIBridge.js +40 -2
- package/dist/bridges/OpenAIBridge.js.map +1 -1
- package/dist/bridges/SemanticCachingBridge.d.ts +8 -1
- package/dist/bridges/SemanticCachingBridge.d.ts.map +1 -1
- package/dist/bridges/SemanticCachingBridge.js +26 -1
- package/dist/bridges/SemanticCachingBridge.js.map +1 -1
- package/dist/bridges/TransformerBridge.d.ts +13 -0
- package/dist/bridges/TransformerBridge.d.ts.map +1 -1
- package/dist/bridges/VertexAIBridge.d.ts +102 -0
- package/dist/bridges/VertexAIBridge.d.ts.map +1 -0
- package/dist/bridges/VertexAIBridge.js +224 -0
- package/dist/bridges/VertexAIBridge.js.map +1 -0
- package/dist/bridges/index.d.ts +3 -1
- package/dist/bridges/index.d.ts.map +1 -1
- package/dist/bridges/index.js +1 -0
- package/dist/bridges/index.js.map +1 -1
- package/dist/eval/EvalHarness.d.ts +37 -0
- package/dist/eval/EvalHarness.d.ts.map +1 -0
- package/dist/eval/EvalHarness.js +178 -0
- package/dist/eval/EvalHarness.js.map +1 -0
- package/dist/eval/graders.d.ts +76 -0
- package/dist/eval/graders.d.ts.map +1 -0
- package/dist/eval/graders.js +168 -0
- package/dist/eval/graders.js.map +1 -0
- package/dist/eval/index.d.ts +26 -0
- package/dist/eval/index.d.ts.map +1 -0
- package/dist/eval/index.js +26 -0
- package/dist/eval/index.js.map +1 -0
- package/dist/eval/types.d.ts +97 -0
- package/dist/eval/types.d.ts.map +1 -0
- package/dist/eval/types.js +5 -0
- package/dist/eval/types.js.map +1 -0
- package/dist/evermind/contract.d.ts +40 -0
- package/dist/evermind/contract.d.ts.map +1 -0
- package/dist/evermind/contract.js +13 -0
- package/dist/evermind/contract.js.map +1 -0
- package/dist/evermind/index.d.ts +15 -0
- package/dist/evermind/index.d.ts.map +1 -0
- package/dist/evermind/index.js +4 -0
- package/dist/evermind/index.js.map +1 -0
- package/dist/evermind/memoryBlock.d.ts +7 -0
- package/dist/evermind/memoryBlock.d.ts.map +1 -0
- package/dist/evermind/memoryBlock.js +37 -0
- package/dist/evermind/memoryBlock.js.map +1 -0
- package/dist/evermind/recall.d.ts +55 -0
- package/dist/evermind/recall.d.ts.map +1 -0
- package/dist/evermind/recall.js +105 -0
- package/dist/evermind/recall.js.map +1 -0
- package/dist/index.d.ts +18 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +14 -1
- package/dist/index.js.map +1 -1
- package/dist/ingest/IngestionPipeline.d.ts +95 -0
- package/dist/ingest/IngestionPipeline.d.ts.map +1 -0
- package/dist/ingest/IngestionPipeline.js +295 -0
- package/dist/ingest/IngestionPipeline.js.map +1 -0
- package/dist/ingest/index.d.ts +10 -0
- package/dist/ingest/index.d.ts.map +1 -0
- package/dist/ingest/index.js +8 -0
- package/dist/ingest/index.js.map +1 -0
- package/dist/ingest/parsers.d.ts +53 -0
- package/dist/ingest/parsers.d.ts.map +1 -0
- package/dist/ingest/parsers.js +306 -0
- package/dist/ingest/parsers.js.map +1 -0
- package/dist/ingest/types.d.ts +114 -0
- package/dist/ingest/types.d.ts.map +1 -0
- package/dist/ingest/types.js +19 -0
- package/dist/ingest/types.js.map +1 -0
- package/dist/memory/MemoryStore.d.ts +33 -0
- package/dist/memory/MemoryStore.d.ts.map +1 -1
- package/dist/memory/MemoryStore.js +29 -3
- package/dist/memory/MemoryStore.js.map +1 -1
- package/dist/memory/index.d.ts +1 -1
- package/dist/memory/index.d.ts.map +1 -1
- package/dist/orchestration/AgentGraph.d.ts +104 -0
- package/dist/orchestration/AgentGraph.d.ts.map +1 -0
- package/dist/orchestration/AgentGraph.js +364 -0
- package/dist/orchestration/AgentGraph.js.map +1 -0
- package/dist/orchestration/checkpoint.d.ts +58 -0
- package/dist/orchestration/checkpoint.d.ts.map +1 -0
- package/dist/orchestration/checkpoint.js +92 -0
- package/dist/orchestration/checkpoint.js.map +1 -0
- package/dist/orchestration/index.d.ts +14 -0
- package/dist/orchestration/index.d.ts.map +1 -0
- package/dist/orchestration/index.js +10 -0
- package/dist/orchestration/index.js.map +1 -0
- package/dist/orchestration/patterns.d.ts +159 -0
- package/dist/orchestration/patterns.d.ts.map +1 -0
- package/dist/orchestration/patterns.js +371 -0
- package/dist/orchestration/patterns.js.map +1 -0
- package/dist/orchestration/types.d.ts +146 -0
- package/dist/orchestration/types.d.ts.map +1 -0
- package/dist/orchestration/types.js +20 -0
- package/dist/orchestration/types.js.map +1 -0
- package/dist/rag/EnterpriseRetriever.d.ts +114 -0
- package/dist/rag/EnterpriseRetriever.d.ts.map +1 -0
- package/dist/rag/EnterpriseRetriever.js +240 -0
- package/dist/rag/EnterpriseRetriever.js.map +1 -0
- package/dist/rag/index.d.ts +6 -0
- package/dist/rag/index.d.ts.map +1 -0
- package/dist/rag/index.js +5 -0
- package/dist/rag/index.js.map +1 -0
- package/dist/retrieval/bm25.d.ts +48 -1
- package/dist/retrieval/bm25.d.ts.map +1 -1
- package/dist/retrieval/bm25.js +59 -13
- package/dist/retrieval/bm25.js.map +1 -1
- package/dist/retrieval/index.d.ts +2 -2
- package/dist/retrieval/index.d.ts.map +1 -1
- package/dist/retrieval/index.js +1 -1
- package/dist/retrieval/index.js.map +1 -1
- package/dist/telemetry/InstrumentedBridge.d.ts +62 -0
- package/dist/telemetry/InstrumentedBridge.d.ts.map +1 -0
- package/dist/telemetry/InstrumentedBridge.js +120 -0
- package/dist/telemetry/InstrumentedBridge.js.map +1 -0
- package/dist/telemetry/MetricsRegistry.d.ts +88 -0
- package/dist/telemetry/MetricsRegistry.d.ts.map +1 -0
- package/dist/telemetry/MetricsRegistry.js +167 -0
- package/dist/telemetry/MetricsRegistry.js.map +1 -0
- package/dist/telemetry/Tracer.d.ts +137 -0
- package/dist/telemetry/Tracer.d.ts.map +1 -0
- package/dist/telemetry/Tracer.js +244 -0
- package/dist/telemetry/Tracer.js.map +1 -0
- package/dist/telemetry/exporters.d.ts +68 -0
- package/dist/telemetry/exporters.d.ts.map +1 -0
- package/dist/telemetry/exporters.js +145 -0
- package/dist/telemetry/exporters.js.map +1 -0
- package/dist/telemetry/index.d.ts +19 -0
- package/dist/telemetry/index.d.ts.map +1 -0
- package/dist/telemetry/index.js +12 -0
- package/dist/telemetry/index.js.map +1 -0
- package/dist/telemetry/pricing.d.ts +74 -0
- package/dist/telemetry/pricing.d.ts.map +1 -0
- package/dist/telemetry/pricing.js +98 -0
- package/dist/telemetry/pricing.js.map +1 -0
- package/dist/telemetry/traceparent.d.ts +50 -0
- package/dist/telemetry/traceparent.d.ts.map +1 -0
- package/dist/telemetry/traceparent.js +77 -0
- package/dist/telemetry/traceparent.js.map +1 -0
- package/dist/telemetry/types.d.ts +90 -0
- package/dist/telemetry/types.d.ts.map +1 -0
- package/dist/telemetry/types.js +17 -0
- package/dist/telemetry/types.js.map +1 -0
- package/dist/vectorstore/MemoryVectorStore.d.ts +72 -0
- package/dist/vectorstore/MemoryVectorStore.d.ts.map +1 -0
- package/dist/vectorstore/MemoryVectorStore.js +205 -0
- package/dist/vectorstore/MemoryVectorStore.js.map +1 -0
- package/dist/vectorstore/RestVectorStore.d.ts +79 -0
- package/dist/vectorstore/RestVectorStore.d.ts.map +1 -0
- package/dist/vectorstore/RestVectorStore.js +170 -0
- package/dist/vectorstore/RestVectorStore.js.map +1 -0
- package/dist/vectorstore/dialects.d.ts +101 -0
- package/dist/vectorstore/dialects.d.ts.map +1 -0
- package/dist/vectorstore/dialects.js +618 -0
- package/dist/vectorstore/dialects.js.map +1 -0
- package/dist/vectorstore/filter.d.ts +42 -0
- package/dist/vectorstore/filter.d.ts.map +1 -0
- package/dist/vectorstore/filter.js +132 -0
- package/dist/vectorstore/filter.js.map +1 -0
- package/dist/vectorstore/index.d.ts +15 -0
- package/dist/vectorstore/index.d.ts.map +1 -0
- package/dist/vectorstore/index.js +11 -0
- package/dist/vectorstore/index.js.map +1 -0
- package/dist/vectorstore/types.d.ts +151 -0
- package/dist/vectorstore/types.d.ts.map +1 -0
- package/dist/vectorstore/types.js +30 -0
- package/dist/vectorstore/types.js.map +1 -0
- package/package.json +39 -5
- package/src/bridges/AnthropicBridge.ts +68 -3
- package/src/bridges/CachingBridge.ts +28 -2
- package/src/bridges/OpenAIBridge.ts +48 -3
- package/src/bridges/SemanticCachingBridge.ts +29 -2
- package/src/bridges/TransformerBridge.ts +16 -0
- package/src/bridges/VertexAIBridge.ts +297 -0
- package/src/bridges/index.ts +3 -1
- package/src/eval/EvalHarness.ts +214 -0
- package/src/eval/graders.ts +218 -0
- package/src/eval/index.ts +26 -0
- package/src/eval/types.ts +107 -0
- package/src/evermind/contract.ts +39 -0
- package/src/evermind/index.ts +21 -0
- package/src/evermind/memoryBlock.ts +38 -0
- package/src/evermind/recall.ts +122 -0
- package/src/index.ts +232 -0
- package/src/ingest/IngestionPipeline.ts +378 -0
- package/src/ingest/index.ts +46 -0
- package/src/ingest/parsers.ts +318 -0
- package/src/ingest/types.ts +116 -0
- package/src/memory/MemoryStore.ts +55 -3
- package/src/memory/index.ts +1 -1
- package/src/orchestration/AgentGraph.ts +462 -0
- package/src/orchestration/checkpoint.ts +129 -0
- package/src/orchestration/index.ts +63 -0
- package/src/orchestration/patterns.ts +532 -0
- package/src/orchestration/types.ts +128 -0
- package/src/rag/EnterpriseRetriever.ts +348 -0
- package/src/rag/index.ts +18 -0
- package/src/retrieval/bm25.ts +71 -13
- package/src/retrieval/index.ts +2 -2
- package/src/telemetry/InstrumentedBridge.ts +152 -0
- package/src/telemetry/MetricsRegistry.ts +263 -0
- package/src/telemetry/Tracer.ts +310 -0
- package/src/telemetry/exporters.ts +187 -0
- package/src/telemetry/index.ts +56 -0
- package/src/telemetry/pricing.ts +129 -0
- package/src/telemetry/traceparent.ts +89 -0
- package/src/telemetry/types.ts +105 -0
- package/src/vectorstore/MemoryVectorStore.ts +250 -0
- package/src/vectorstore/RestVectorStore.ts +236 -0
- package/src/vectorstore/dialects.ts +702 -0
- package/src/vectorstore/filter.ts +148 -0
- package/src/vectorstore/index.ts +53 -0
- package/src/vectorstore/types.ts +149 -0
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* evermind/recall.ts — the lexical recall rules for a project Evermind's taught memories.
|
|
3
|
+
*
|
|
4
|
+
* The gateway ranks a project's retained contributions against a query ("which learned
|
|
5
|
+
* memory would answer this?"), and the chat client decides which recalled memories an
|
|
6
|
+
* answer RECONCILES (restates enough of to supersede). Both must key on the SAME
|
|
7
|
+
* meaningful terms, so the tokenizer, stopword set and both rules live here, once.
|
|
8
|
+
*
|
|
9
|
+
* Pure and dependency-free: safe in a Worker, in Node, and in a browser bundle.
|
|
10
|
+
*/
|
|
11
|
+
/** Tiny code+English stopword set — dropped so recall keys on meaningful terms. */
|
|
12
|
+
export declare const RECALL_STOPWORDS: ReadonlySet<string>;
|
|
13
|
+
/** Lowercase, split on non-word runs, drop stopwords and 1-char tokens. */
|
|
14
|
+
export declare function recallTokens(text: string): string[];
|
|
15
|
+
/** One contribution the ranker scores (a subset of a retained-contribution entry). */
|
|
16
|
+
export interface RecallScorable {
|
|
17
|
+
id: number;
|
|
18
|
+
kind: 'text' | 'delta';
|
|
19
|
+
version: number;
|
|
20
|
+
at: number;
|
|
21
|
+
weight: number;
|
|
22
|
+
prompt?: string;
|
|
23
|
+
text?: string;
|
|
24
|
+
/** The chat that contributed this memory, when one did. Absent means project-wide. */
|
|
25
|
+
chatId?: number;
|
|
26
|
+
}
|
|
27
|
+
/** A scored recall match — the entry plus its 0..1 lexical relevance to the query. */
|
|
28
|
+
export interface RankedEvermindRecall extends RecallScorable {
|
|
29
|
+
/** Lexical relevance to the query, 0..1 (rounded to 3 dp). */
|
|
30
|
+
score: number;
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Rank contributions by lexical relevance to `query`, best first. Only entries carrying
|
|
34
|
+
* text can match (a weight delta has none); scores at or below the noise floor are
|
|
35
|
+
* dropped, and the result is capped at `limit` (default 8).
|
|
36
|
+
*/
|
|
37
|
+
export declare function rankEvermindRecall(query: string, recent: readonly RecallScorable[], opts?: {
|
|
38
|
+
limit?: number;
|
|
39
|
+
}): RankedEvermindRecall[];
|
|
40
|
+
/** Stable, bounded (djb2, base36) hash of a recall query — keeps a cache key finite. */
|
|
41
|
+
export declare function hashRecallQuery(query: string): string;
|
|
42
|
+
/**
|
|
43
|
+
* Fraction of a recalled memory's meaningful tokens an answer must restate for the
|
|
44
|
+
* turn to count as RECONCILING (superseding) that memory.
|
|
45
|
+
*/
|
|
46
|
+
export declare const RECONCILE_OVERLAP = 0.6;
|
|
47
|
+
/**
|
|
48
|
+
* How many recalled memories `answer` reconciles — restates enough of that the
|
|
49
|
+
* contributed turn supersedes them. A heuristic over token overlap; it surfaces the
|
|
50
|
+
* reconcile step and never gates learning.
|
|
51
|
+
*/
|
|
52
|
+
export declare function countReconciledMemories(items: ReadonlyArray<{
|
|
53
|
+
text: string;
|
|
54
|
+
}>, answer: string): number;
|
|
55
|
+
//# sourceMappingURL=recall.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"recall.d.ts","sourceRoot":"","sources":["../../src/evermind/recall.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,mFAAmF;AACnF,eAAO,MAAM,gBAAgB,EAAE,WAAW,CAAC,MAAM,CAK/C,CAAC;AAEH,2EAA2E;AAC3E,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAEnD;AAED,sFAAsF;AACtF,MAAM,WAAW,cAAc;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,sFAAsF;IACtF,MAAM,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,sFAAsF;AACtF,MAAM,WAAW,oBAAqB,SAAQ,cAAc;IACxD,8DAA8D;IAC9D,KAAK,EAAE,MAAM,CAAC;CACjB;AA2BD;;;;GAIG;AACH,wBAAgB,kBAAkB,CAC9B,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,SAAS,cAAc,EAAE,EACjC,IAAI,CAAC,EAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,GAC1B,oBAAoB,EAAE,CAYxB;AAED,wFAAwF;AACxF,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAIrD;AAED;;;GAGG;AACH,eAAO,MAAM,iBAAiB,MAAM,CAAC;AAErC;;;;GAIG;AACH,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,aAAa,CAAC;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAYtG"}
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* evermind/recall.ts — the lexical recall rules for a project Evermind's taught memories.
|
|
3
|
+
*
|
|
4
|
+
* The gateway ranks a project's retained contributions against a query ("which learned
|
|
5
|
+
* memory would answer this?"), and the chat client decides which recalled memories an
|
|
6
|
+
* answer RECONCILES (restates enough of to supersede). Both must key on the SAME
|
|
7
|
+
* meaningful terms, so the tokenizer, stopword set and both rules live here, once.
|
|
8
|
+
*
|
|
9
|
+
* Pure and dependency-free: safe in a Worker, in Node, and in a browser bundle.
|
|
10
|
+
*/
|
|
11
|
+
/** Tiny code+English stopword set — dropped so recall keys on meaningful terms. */
|
|
12
|
+
export const RECALL_STOPWORDS = new Set([
|
|
13
|
+
'the', 'a', 'an', 'and', 'or', 'to', 'of', 'in', 'on', 'for', 'with', 'is', 'are',
|
|
14
|
+
'be', 'as', 'at', 'by', 'it', 'this', 'that', 'from', 'you', 'your', 'i', 'we',
|
|
15
|
+
'they', 'he', 'she', 'can', 'will', 'how', 'do', 'does', 'what', 'why', 'when',
|
|
16
|
+
'which', 'use', 'using', 'used', 'please', 'need', 'want', 'me', 'my', 'so', 'if',
|
|
17
|
+
]);
|
|
18
|
+
/** Lowercase, split on non-word runs, drop stopwords and 1-char tokens. */
|
|
19
|
+
export function recallTokens(text) {
|
|
20
|
+
return (text.toLowerCase().match(/[a-z0-9_]+/g) ?? []).filter((w) => w.length >= 2 && !RECALL_STOPWORDS.has(w));
|
|
21
|
+
}
|
|
22
|
+
function termFreq(tokens) {
|
|
23
|
+
const m = new Map();
|
|
24
|
+
for (const tok of tokens)
|
|
25
|
+
m.set(tok, (m.get(tok) ?? 0) + 1);
|
|
26
|
+
return m;
|
|
27
|
+
}
|
|
28
|
+
/** Cosine similarity between two TF maps (0..1). Iterates the smaller map. */
|
|
29
|
+
function tfCosine(a, b) {
|
|
30
|
+
let na = 0;
|
|
31
|
+
for (const v of a.values())
|
|
32
|
+
na += v * v;
|
|
33
|
+
let nb = 0;
|
|
34
|
+
for (const v of b.values())
|
|
35
|
+
nb += v * v;
|
|
36
|
+
if (na === 0 || nb === 0)
|
|
37
|
+
return 0;
|
|
38
|
+
const [small, big] = a.size <= b.size ? [a, b] : [b, a];
|
|
39
|
+
let dot = 0;
|
|
40
|
+
for (const [k, v] of small) {
|
|
41
|
+
const w = big.get(k);
|
|
42
|
+
if (w)
|
|
43
|
+
dot += v * w;
|
|
44
|
+
}
|
|
45
|
+
return dot / (Math.sqrt(na) * Math.sqrt(nb));
|
|
46
|
+
}
|
|
47
|
+
/** Below this cosine a match is noise, not recall. */
|
|
48
|
+
const MIN_RECALL_SCORE = 0.02;
|
|
49
|
+
/**
|
|
50
|
+
* Rank contributions by lexical relevance to `query`, best first. Only entries carrying
|
|
51
|
+
* text can match (a weight delta has none); scores at or below the noise floor are
|
|
52
|
+
* dropped, and the result is capped at `limit` (default 8).
|
|
53
|
+
*/
|
|
54
|
+
export function rankEvermindRecall(query, recent, opts) {
|
|
55
|
+
const q = termFreq(recallTokens(query));
|
|
56
|
+
if (q.size === 0)
|
|
57
|
+
return [];
|
|
58
|
+
const scored = [];
|
|
59
|
+
for (const e of recent) {
|
|
60
|
+
const hay = `${e.prompt ?? ''} ${e.text ?? ''}`.trim();
|
|
61
|
+
if (!hay)
|
|
62
|
+
continue;
|
|
63
|
+
const score = tfCosine(q, termFreq(recallTokens(hay)));
|
|
64
|
+
if (score > MIN_RECALL_SCORE)
|
|
65
|
+
scored.push({ ...e, score: Math.round(score * 1000) / 1000 });
|
|
66
|
+
}
|
|
67
|
+
scored.sort((a, b) => b.score - a.score || b.at - a.at);
|
|
68
|
+
return scored.slice(0, Math.max(1, opts?.limit ?? 8));
|
|
69
|
+
}
|
|
70
|
+
/** Stable, bounded (djb2, base36) hash of a recall query — keeps a cache key finite. */
|
|
71
|
+
export function hashRecallQuery(query) {
|
|
72
|
+
let h = 5381;
|
|
73
|
+
for (let i = 0; i < query.length; i++)
|
|
74
|
+
h = ((h << 5) + h + query.charCodeAt(i)) | 0;
|
|
75
|
+
return (h >>> 0).toString(36);
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Fraction of a recalled memory's meaningful tokens an answer must restate for the
|
|
79
|
+
* turn to count as RECONCILING (superseding) that memory.
|
|
80
|
+
*/
|
|
81
|
+
export const RECONCILE_OVERLAP = 0.6;
|
|
82
|
+
/**
|
|
83
|
+
* How many recalled memories `answer` reconciles — restates enough of that the
|
|
84
|
+
* contributed turn supersedes them. A heuristic over token overlap; it surfaces the
|
|
85
|
+
* reconcile step and never gates learning.
|
|
86
|
+
*/
|
|
87
|
+
export function countReconciledMemories(items, answer) {
|
|
88
|
+
const ans = new Set(recallTokens(answer));
|
|
89
|
+
if (ans.size === 0)
|
|
90
|
+
return 0;
|
|
91
|
+
let n = 0;
|
|
92
|
+
for (const it of items) {
|
|
93
|
+
const mem = new Set(recallTokens(it.text));
|
|
94
|
+
if (mem.size === 0)
|
|
95
|
+
continue;
|
|
96
|
+
let hit = 0;
|
|
97
|
+
for (const tok of mem)
|
|
98
|
+
if (ans.has(tok))
|
|
99
|
+
hit++;
|
|
100
|
+
if (hit / mem.size >= RECONCILE_OVERLAP)
|
|
101
|
+
n++;
|
|
102
|
+
}
|
|
103
|
+
return n;
|
|
104
|
+
}
|
|
105
|
+
//# sourceMappingURL=recall.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"recall.js","sourceRoot":"","sources":["../../src/evermind/recall.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,mFAAmF;AACnF,MAAM,CAAC,MAAM,gBAAgB,GAAwB,IAAI,GAAG,CAAC;IACzD,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK;IACjF,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI;IAC9E,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM;IAC9E,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI;CACpF,CAAC,CAAC;AAEH,2EAA2E;AAC3E,MAAM,UAAU,YAAY,CAAC,IAAY;IACrC,OAAO,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;AACpH,CAAC;AAqBD,SAAS,QAAQ,CAAC,MAAgB;IAC9B,MAAM,CAAC,GAAG,IAAI,GAAG,EAAkB,CAAC;IACpC,KAAK,MAAM,GAAG,IAAI,MAAM;QAAE,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC5D,OAAO,CAAC,CAAC;AACb,CAAC;AAED,8EAA8E;AAC9E,SAAS,QAAQ,CAAC,CAAsB,EAAE,CAAsB;IAC5D,IAAI,EAAE,GAAG,CAAC,CAAC;IACX,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE;QAAE,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;IACxC,IAAI,EAAE,GAAG,CAAC,CAAC;IACX,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE;QAAE,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;IACxC,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACnC,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACxD,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACrB,IAAI,CAAC;YAAE,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;IACxB,CAAC;IACD,OAAO,GAAG,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;AACjD,CAAC;AAED,sDAAsD;AACtD,MAAM,gBAAgB,GAAG,IAAI,CAAC;AAE9B;;;;GAIG;AACH,MAAM,UAAU,kBAAkB,CAC9B,KAAa,EACb,MAAiC,EACjC,IAAyB;IAEzB,MAAM,CAAC,GAAG,QAAQ,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC;IACxC,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAC5B,MAAM,MAAM,GAA2B,EAAE,CAAC;IAC1C,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACrB,MAAM,GAAG,GAAG,GAAG,CAAC,CAAC,MAAM,IAAI,EAAE,IAAI,CAAC,CAAC,IAAI,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,CAAC;QACvD,IAAI,CAAC,GAAG;YAAE,SAAS;QACnB,MAAM,KAAK,GAAG,QAAQ,CAAC,CAAC,EAAE,QAAQ,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACvD,IAAI,KAAK,GAAG,gBAAgB;YAAE,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC;IAChG,CAAC;IACD,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;IACxD,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC;AAC1D,CAAC;AAED,wFAAwF;AACxF,MAAM,UAAU,eAAe,CAAC,KAAa;IACzC,IAAI,CAAC,GAAG,IAAI,CAAC;IACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE;QAAE,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IACpF,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;AAClC,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAErC;;;;GAIG;AACH,MAAM,UAAU,uBAAuB,CAAC,KAAsC,EAAE,MAAc;IAC1F,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC;IAC1C,IAAI,GAAG,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAC7B,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,KAAK,MAAM,EAAE,IAAI,KAAK,EAAE,CAAC;QACrB,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC;QAC3C,IAAI,GAAG,CAAC,IAAI,KAAK,CAAC;YAAE,SAAS;QAC7B,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,KAAK,MAAM,GAAG,IAAI,GAAG;YAAE,IAAI,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,GAAG,EAAE,CAAC;QAC/C,IAAI,GAAG,GAAG,GAAG,CAAC,IAAI,IAAI,iBAAiB;YAAE,CAAC,EAAE,CAAC;IACjD,CAAC;IACD,OAAO,CAAC,CAAC;AACb,CAAC"}
|
package/dist/index.d.ts
CHANGED
|
@@ -34,12 +34,15 @@ export { SSMRuntime } from './runtime/SSMRuntime.js';
|
|
|
34
34
|
export type { SSMRuntimeOptions, GenerateOptions } from './runtime/SSMRuntime.js';
|
|
35
35
|
export type { TransformerBridge, BridgeGenerateOptions } from './bridges/TransformerBridge.js';
|
|
36
36
|
export { OpenAIBridge } from './bridges/OpenAIBridge.js';
|
|
37
|
+
export { VertexAIBridge, VertexAIEmbedder } from './bridges/VertexAIBridge.js';
|
|
37
38
|
export { AnthropicBridge } from './bridges/AnthropicBridge.js';
|
|
38
39
|
export { FetchBridge } from './bridges/FetchBridge.js';
|
|
39
40
|
export { CachingBridge } from './bridges/CachingBridge.js';
|
|
40
41
|
export { SemanticCachingBridge } from './bridges/SemanticCachingBridge.js';
|
|
41
42
|
export { ResponseCache, buildCacheKey } from './bridges/ResponseCache.js';
|
|
43
|
+
export type { BridgeCallInfo } from './bridges/TransformerBridge.js';
|
|
42
44
|
export type { OpenAIBridgeOptions } from './bridges/OpenAIBridge.js';
|
|
45
|
+
export type { VertexAIBridgeOptions, VertexAIEmbedderOptions } from './bridges/VertexAIBridge.js';
|
|
43
46
|
export type { AnthropicBridgeOptions } from './bridges/AnthropicBridge.js';
|
|
44
47
|
export type { FetchBridgeOptions } from './bridges/FetchBridge.js';
|
|
45
48
|
export type { CachingBridgeOptions } from './bridges/CachingBridge.js';
|
|
@@ -49,12 +52,24 @@ export { SemanticCache } from './cache/SemanticCache.js';
|
|
|
49
52
|
export { FetchSemanticCacheBackend } from './cache/FetchSemanticCacheBackend.js';
|
|
50
53
|
export type { Embedder, SemanticCacheBackend, SemanticCacheHit, SemanticCacheOptions, FetchSemanticCacheBackendOptions, } from './cache/index.js';
|
|
51
54
|
export { cosineSimilarity, jaccardSimilarity, tokenize } from './similarity/index.js';
|
|
52
|
-
export { chunkText, bm25Search, reciprocalRankFusion, maximalMarginalRelevance, hybridRetrieve, } from './retrieval/index.js';
|
|
53
|
-
export type { Chunk, ChunkOptions, Bm25Doc, Bm25Hit, Bm25Options, RankedList, FusedHit, MmrCandidate, RetrievalCandidate, HybridQuery, HybridRetrieveOptions, HybridHit, } from './retrieval/index.js';
|
|
55
|
+
export { chunkText, bm25Search, bm25Idf, bm25LengthNorm, bm25TermScore, BM25_DEFAULT_K1, BM25_DEFAULT_B, reciprocalRankFusion, maximalMarginalRelevance, hybridRetrieve, } from './retrieval/index.js';
|
|
56
|
+
export type { Chunk, ChunkOptions, Bm25Doc, Bm25Hit, Bm25Options, Bm25Tokenizer, RankedList, FusedHit, MmrCandidate, RetrievalCandidate, HybridQuery, HybridRetrieveOptions, HybridHit, } from './retrieval/index.js';
|
|
57
|
+
export { Tracer, Span, parseTraceparent, formatTraceparent, MetricsRegistry, InstrumentedBridge, InMemorySpanExporter, ConsoleSpanExporter, OtlpHttpSpanExporter, toOtlpPayload, toOtlpSpan, percentile, estimateCostUsd, estimateTokens, resolveRate, DEFAULT_PRICE_BOOK, ANTHROPIC_PRICE_BOOK, EVERMIND_PRICE_BOOK, DEFAULT_CACHE_READ_MULTIPLIER, DEFAULT_CACHE_WRITE_MULTIPLIER, } from './telemetry/index.js';
|
|
58
|
+
export type { TracerOptions, SpanOptions, TraceContext, SpanData, SpanKind, SpanStatus, SpanEvent, SpanExporter, AttributeValue, LlmUsage, LlmMetricSnapshot, KindMetricSnapshot, MetricsRegistryOptions, InstrumentedBridgeOptions, ConsoleSpanExporterOptions, OtlpHttpSpanExporterOptions, ModelRate, PriceBook, } from './telemetry/index.js';
|
|
59
|
+
export { MemoryVectorStore, RestVectorStore, getDialect, listDialects, registerDialect, evermindDialect, qdrantDialect, pineconeDialect, vertexAiDialect, accessFilter, combineFilters, matchesFilter, filterFields, TENANT_FIELD, ACL_FIELD, SENSITIVITY_FIELD, ACL_PUBLIC, } from './vectorstore/index.js';
|
|
60
|
+
export type { MemoryVectorStoreOptions, RestVectorStoreOptions, VectorDialect, DialectContext, TranslatedFilter, AccessScope, ComparisonOp, MetadataFilter, MetadataRecord, MetadataValue, UpsertResult, VectorMatch, VectorQuery, VectorRecord, VectorStore, } from './vectorstore/index.js';
|
|
61
|
+
export { IngestionPipeline, InMemoryIngestManifest, ParserRegistry, BUILTIN_PARSERS, rowsParser, csvParser, jsonParser, markdownParser, htmlParser, plainTextParser, parseCsv, serializeRow, hashText, mapWithConcurrency, SOURCE_ID_FIELD, CHUNK_INDEX_FIELD, CONTENT_HASH_FIELD, TITLE_FIELD, URI_FIELD, PARSER_FIELD, } from './ingest/index.js';
|
|
62
|
+
export type { IngestionPipelineOptions, BatchEmbedder, ChunkFingerprint, DocumentParser, IngestManifest, IngestionReport, ParsedSection, SourceContent, SourceDocument, StructuredRow, SyncStrategy, } from './ingest/index.js';
|
|
63
|
+
export { EnterpriseRetriever, buildGroundedPrompt, GROUNDED_SYSTEM_PROMPT, NO_EVIDENCE_ANSWER, } from './rag/index.js';
|
|
64
|
+
export type { EnterpriseRetrieverOptions, RetrieveOptions, RetrievalResult, RetrievalMode, RetrievedPassage, Reranker, } from './rag/index.js';
|
|
65
|
+
export { AgentGraph, appendReducer, mergeReducer, sumReducer, unionReducer, InMemoryCheckpointer, KeyValueCheckpointer, createReactAgent, createReflectionAgent, createSupervisor, asWorker, parseReactTurn, renderToolCatalog, renderTranscript, REACT_PROTOCOL, DEFAULT_CRITIC_PROMPT, FINISH, START, END, } from './orchestration/index.js';
|
|
66
|
+
export type { AgentGraphOptions, RunOptions, InMemoryCheckpointerOptions, KeyValueCheckpointerOptions, KeyValueLike, AgentTool, AgentTurn, AgentTurnRole, ParsedAction, ReactAgentOptions, ReactState, ReflectionAgentOptions, ReflectionState, SupervisorOptions, SupervisorState, Worker, ChannelReducer, ChannelSpec, ChannelSpecs, Checkpoint, Checkpointer, EdgeCondition, GraphEndReason, GraphEvent, GraphRunResult, GraphState, NodeContext, NodeFn, } from './orchestration/index.js';
|
|
67
|
+
export { EvalHarness, evaluateGate, formatReport, exactMatch, substringChecks, embeddingSimilarity, retrievalRecall, citationValidity, llmJudge, noError, costBudget, DEFAULT_RAG_GRADERS, DEFAULT_JUDGE_RUBRIC, } from './eval/index.js';
|
|
68
|
+
export type { EvalHarnessOptions, EvalCase, EvalDataset, EvalGate, EvalOutput, EvalReport, EvalTarget, CaseResult, GateVerdict, GradeResult, Grader, LlmJudgeOptions, SimilarityGraderOptions, } from './eval/index.js';
|
|
54
69
|
export { InferenceRouter } from './router/InferenceRouter.js';
|
|
55
70
|
export type { RoutingStrategy, RoutingDecision, RouterContext, InferenceRouterOptions, RoutingAuditEntry, } from './router/InferenceRouter.js';
|
|
56
71
|
export { MemoryStore } from './memory/MemoryStore.js';
|
|
57
|
-
export type { MemoryEntry, MemoryStoreOptions, RememberOptions, FactType, } from './memory/MemoryStore.js';
|
|
72
|
+
export type { MemoryEntry, MemoryStoreOptions, RememberOptions, FactType, RecallMethod, ScoredMemory, RankedRecall, } from './memory/MemoryStore.js';
|
|
58
73
|
export { EvermindCognition, workspacePresenceGatherer, canonicalizeSubjectKey, normalizeSubjectKey, buildAliasTable, sanitizeRecalledFact, buildRecallContext, trustScore, } from './cognition/index.js';
|
|
59
74
|
export type { EvermindCognitionOptions, WorkspacePresenceRule, Claim, CognitionFactStore, CommitResult, EvidenceContext, EvidenceGatherer, EvidenceResult, Verdict, AliasTable, RecalledFact, FactProvenance, } from './cognition/index.js';
|
|
60
75
|
export { runStackDiagnostic } from './diagnostics/index.js';
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAGH,OAAO,EAAE,YAAY,EAAE,MAAmB,oBAAoB,CAAC;AAC/D,OAAO,EAAE,YAAY,EAAE,MAAmB,oBAAoB,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAE7F,YAAY,EAAE,gBAAgB,EAAE,MAAU,oBAAoB,CAAC;AAC/D,YAAY,EAAE,mBAAmB,EAAE,MAAO,oBAAoB,CAAC;AAC/D,YAAY,EACR,mBAAmB,EACnB,eAAe,EACf,YAAY,EACZ,WAAW,EACX,WAAW,EACX,WAAW,EACX,aAAa,EACb,mBAAmB,EACnB,WAAW,EACX,eAAe,EACf,gBAAgB,EAChB,OAAO,EACP,SAAS,GACZ,MAAM,oBAAoB,CAAC;AAG5B,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAC1D,YAAY,EAAE,oBAAoB,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAGrF,OAAO,EACH,WAAW,EACX,aAAa,EACb,UAAU,EACV,gBAAgB,EAChB,gBAAgB,EAChB,aAAa,EACb,aAAa,EACb,MAAM,EACN,UAAU,EACV,YAAY,EACZ,aAAa,EACb,aAAa,GAChB,MAAM,sCAAsC,CAAC;AAC9C,YAAY,EACR,iBAAiB,EACjB,aAAa,EACb,YAAY,EACZ,kBAAkB,EAClB,aAAa,EACb,MAAM,GACT,MAAM,sCAAsC,CAAC;AAI9C,OAAO,EACH,eAAe,EACf,sBAAsB,EACtB,kBAAkB,EAClB,gBAAgB,EAChB,UAAU,EACV,oBAAoB,GACvB,MAAM,sCAAsC,CAAC;AAC9C,YAAY,EACR,SAAS,EACT,QAAQ,EACR,WAAW,EACX,SAAS,EACT,eAAe,EACf,cAAc,EACd,qBAAqB,EACrB,iBAAiB,EACjB,WAAW,EACX,gBAAgB,GACnB,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,iBAAiB,EAAE,MAAM,sCAAsC,CAAC;AACrF,YAAY,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,sCAAsC,CAAC;AAG3G,OAAO,EAAE,UAAU,EAAE,MAAS,yBAAyB,CAAC;AACxD,YAAY,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAGlF,YAAY,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,MAAM,gCAAgC,CAAC;AAC/F,OAAO,EAAE,YAAY,EAAE,MAAS,2BAA2B,CAAC;AAC5D,OAAO,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AAC/D,OAAO,EAAE,WAAW,EAAE,MAAU,0BAA0B,CAAC;AAC3D,OAAO,EAAE,aAAa,EAAE,MAAQ,4BAA4B,CAAC;AAC7D,OAAO,EAAE,qBAAqB,EAAE,MAAM,oCAAoC,CAAC;AAC3E,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC1E,YAAY,EAAE,mBAAmB,EAAE,MAAS,2BAA2B,CAAC;AACxE,YAAY,EAAE,sBAAsB,EAAE,MAAM,8BAA8B,CAAC;AAC3E,YAAY,EAAE,kBAAkB,EAAE,MAAU,0BAA0B,CAAC;AACvE,YAAY,EAAE,oBAAoB,EAAE,MAAQ,4BAA4B,CAAC;AACzE,YAAY,EAAE,4BAA4B,EAAE,MAAM,oCAAoC,CAAC;AACvF,YAAY,EAAE,oBAAoB,EAAE,MAAQ,4BAA4B,CAAC;AAGzE,OAAO,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AACzD,OAAO,EAAE,yBAAyB,EAAE,MAAM,sCAAsC,CAAC;AACjF,YAAY,EACR,QAAQ,EACR,oBAAoB,EACpB,gBAAgB,EAChB,oBAAoB,EACpB,gCAAgC,GACnC,MAAM,kBAAkB,CAAC;AAG1B,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAGtF,OAAO,EACH,SAAS,EACT,UAAU,EACV,oBAAoB,EACpB,wBAAwB,EACxB,cAAc,GACjB,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EACR,KAAK,EACL,YAAY,EACZ,OAAO,EACP,OAAO,EACP,WAAW,EACX,UAAU,EACV,QAAQ,EACR,YAAY,EACZ,kBAAkB,EAClB,WAAW,EACX,qBAAqB,EACrB,SAAS,GACZ,MAAM,sBAAsB,CAAC;AAG9B,OAAO,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;AAC9D,YAAY,EACR,eAAe,EACf,eAAe,EACf,aAAa,EACb,sBAAsB,EACtB,iBAAiB,GACpB,MAAM,6BAA6B,CAAC;AAGrC,OAAO,EAAE,WAAW,EAAE,MAAO,yBAAyB,CAAC;AACvD,YAAY,EACR,WAAW,EACX,kBAAkB,EAClB,eAAe,EACf,QAAQ,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAGH,OAAO,EAAE,YAAY,EAAE,MAAmB,oBAAoB,CAAC;AAC/D,OAAO,EAAE,YAAY,EAAE,MAAmB,oBAAoB,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAE7F,YAAY,EAAE,gBAAgB,EAAE,MAAU,oBAAoB,CAAC;AAC/D,YAAY,EAAE,mBAAmB,EAAE,MAAO,oBAAoB,CAAC;AAC/D,YAAY,EACR,mBAAmB,EACnB,eAAe,EACf,YAAY,EACZ,WAAW,EACX,WAAW,EACX,WAAW,EACX,aAAa,EACb,mBAAmB,EACnB,WAAW,EACX,eAAe,EACf,gBAAgB,EAChB,OAAO,EACP,SAAS,GACZ,MAAM,oBAAoB,CAAC;AAG5B,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAC1D,YAAY,EAAE,oBAAoB,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAGrF,OAAO,EACH,WAAW,EACX,aAAa,EACb,UAAU,EACV,gBAAgB,EAChB,gBAAgB,EAChB,aAAa,EACb,aAAa,EACb,MAAM,EACN,UAAU,EACV,YAAY,EACZ,aAAa,EACb,aAAa,GAChB,MAAM,sCAAsC,CAAC;AAC9C,YAAY,EACR,iBAAiB,EACjB,aAAa,EACb,YAAY,EACZ,kBAAkB,EAClB,aAAa,EACb,MAAM,GACT,MAAM,sCAAsC,CAAC;AAI9C,OAAO,EACH,eAAe,EACf,sBAAsB,EACtB,kBAAkB,EAClB,gBAAgB,EAChB,UAAU,EACV,oBAAoB,GACvB,MAAM,sCAAsC,CAAC;AAC9C,YAAY,EACR,SAAS,EACT,QAAQ,EACR,WAAW,EACX,SAAS,EACT,eAAe,EACf,cAAc,EACd,qBAAqB,EACrB,iBAAiB,EACjB,WAAW,EACX,gBAAgB,GACnB,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,iBAAiB,EAAE,MAAM,sCAAsC,CAAC;AACrF,YAAY,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,sCAAsC,CAAC;AAG3G,OAAO,EAAE,UAAU,EAAE,MAAS,yBAAyB,CAAC;AACxD,YAAY,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAGlF,YAAY,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,MAAM,gCAAgC,CAAC;AAC/F,OAAO,EAAE,YAAY,EAAE,MAAS,2BAA2B,CAAC;AAC5D,OAAO,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AAC/E,OAAO,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AAC/D,OAAO,EAAE,WAAW,EAAE,MAAU,0BAA0B,CAAC;AAC3D,OAAO,EAAE,aAAa,EAAE,MAAQ,4BAA4B,CAAC;AAC7D,OAAO,EAAE,qBAAqB,EAAE,MAAM,oCAAoC,CAAC;AAC3E,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC1E,YAAY,EAAE,cAAc,EAAE,MAAc,gCAAgC,CAAC;AAC7E,YAAY,EAAE,mBAAmB,EAAE,MAAS,2BAA2B,CAAC;AACxE,YAAY,EAAE,qBAAqB,EAAE,uBAAuB,EAAE,MAAM,6BAA6B,CAAC;AAClG,YAAY,EAAE,sBAAsB,EAAE,MAAM,8BAA8B,CAAC;AAC3E,YAAY,EAAE,kBAAkB,EAAE,MAAU,0BAA0B,CAAC;AACvE,YAAY,EAAE,oBAAoB,EAAE,MAAQ,4BAA4B,CAAC;AACzE,YAAY,EAAE,4BAA4B,EAAE,MAAM,oCAAoC,CAAC;AACvF,YAAY,EAAE,oBAAoB,EAAE,MAAQ,4BAA4B,CAAC;AAGzE,OAAO,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AACzD,OAAO,EAAE,yBAAyB,EAAE,MAAM,sCAAsC,CAAC;AACjF,YAAY,EACR,QAAQ,EACR,oBAAoB,EACpB,gBAAgB,EAChB,oBAAoB,EACpB,gCAAgC,GACnC,MAAM,kBAAkB,CAAC;AAG1B,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAGtF,OAAO,EACH,SAAS,EACT,UAAU,EACV,OAAO,EACP,cAAc,EACd,aAAa,EACb,eAAe,EACf,cAAc,EACd,oBAAoB,EACpB,wBAAwB,EACxB,cAAc,GACjB,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EACR,KAAK,EACL,YAAY,EACZ,OAAO,EACP,OAAO,EACP,WAAW,EACX,aAAa,EACb,UAAU,EACV,QAAQ,EACR,YAAY,EACZ,kBAAkB,EAClB,WAAW,EACX,qBAAqB,EACrB,SAAS,GACZ,MAAM,sBAAsB,CAAC;AAG9B,OAAO,EACH,MAAM,EACN,IAAI,EACJ,gBAAgB,EAChB,iBAAiB,EACjB,eAAe,EACf,kBAAkB,EAClB,oBAAoB,EACpB,mBAAmB,EACnB,oBAAoB,EACpB,aAAa,EACb,UAAU,EACV,UAAU,EACV,eAAe,EACf,cAAc,EACd,WAAW,EACX,kBAAkB,EAClB,oBAAoB,EACpB,mBAAmB,EACnB,6BAA6B,EAC7B,8BAA8B,GACjC,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EACR,aAAa,EACb,WAAW,EACX,YAAY,EACZ,QAAQ,EACR,QAAQ,EACR,UAAU,EACV,SAAS,EACT,YAAY,EACZ,cAAc,EACd,QAAQ,EACR,iBAAiB,EACjB,kBAAkB,EAClB,sBAAsB,EACtB,yBAAyB,EACzB,0BAA0B,EAC1B,2BAA2B,EAC3B,SAAS,EACT,SAAS,GACZ,MAAM,sBAAsB,CAAC;AAG9B,OAAO,EACH,iBAAiB,EACjB,eAAe,EACf,UAAU,EACV,YAAY,EACZ,eAAe,EACf,eAAe,EACf,aAAa,EACb,eAAe,EACf,eAAe,EACf,YAAY,EACZ,cAAc,EACd,aAAa,EACb,YAAY,EACZ,YAAY,EACZ,SAAS,EACT,iBAAiB,EACjB,UAAU,GACb,MAAM,wBAAwB,CAAC;AAChC,YAAY,EACR,wBAAwB,EACxB,sBAAsB,EACtB,aAAa,EACb,cAAc,EACd,gBAAgB,EAChB,WAAW,EACX,YAAY,EACZ,cAAc,EACd,cAAc,EACd,aAAa,EACb,YAAY,EACZ,WAAW,EACX,WAAW,EACX,YAAY,EACZ,WAAW,GACd,MAAM,wBAAwB,CAAC;AAGhC,OAAO,EACH,iBAAiB,EACjB,sBAAsB,EACtB,cAAc,EACd,eAAe,EACf,UAAU,EACV,SAAS,EACT,UAAU,EACV,cAAc,EACd,UAAU,EACV,eAAe,EACf,QAAQ,EACR,YAAY,EACZ,QAAQ,EACR,kBAAkB,EAClB,eAAe,EACf,iBAAiB,EACjB,kBAAkB,EAClB,WAAW,EACX,SAAS,EACT,YAAY,GACf,MAAM,mBAAmB,CAAC;AAC3B,YAAY,EACR,wBAAwB,EACxB,aAAa,EACb,gBAAgB,EAChB,cAAc,EACd,cAAc,EACd,eAAe,EACf,aAAa,EACb,aAAa,EACb,cAAc,EACd,aAAa,EACb,YAAY,GACf,MAAM,mBAAmB,CAAC;AAG3B,OAAO,EACH,mBAAmB,EACnB,mBAAmB,EACnB,sBAAsB,EACtB,kBAAkB,GACrB,MAAM,gBAAgB,CAAC;AACxB,YAAY,EACR,0BAA0B,EAC1B,eAAe,EACf,eAAe,EACf,aAAa,EACb,gBAAgB,EAChB,QAAQ,GACX,MAAM,gBAAgB,CAAC;AAGxB,OAAO,EACH,UAAU,EACV,aAAa,EACb,YAAY,EACZ,UAAU,EACV,YAAY,EACZ,oBAAoB,EACpB,oBAAoB,EACpB,gBAAgB,EAChB,qBAAqB,EACrB,gBAAgB,EAChB,QAAQ,EACR,cAAc,EACd,iBAAiB,EACjB,gBAAgB,EAChB,cAAc,EACd,qBAAqB,EACrB,MAAM,EACN,KAAK,EACL,GAAG,GACN,MAAM,0BAA0B,CAAC;AAClC,YAAY,EACR,iBAAiB,EACjB,UAAU,EACV,2BAA2B,EAC3B,2BAA2B,EAC3B,YAAY,EACZ,SAAS,EACT,SAAS,EACT,aAAa,EACb,YAAY,EACZ,iBAAiB,EACjB,UAAU,EACV,sBAAsB,EACtB,eAAe,EACf,iBAAiB,EACjB,eAAe,EACf,MAAM,EACN,cAAc,EACd,WAAW,EACX,YAAY,EACZ,UAAU,EACV,YAAY,EACZ,aAAa,EACb,cAAc,EACd,UAAU,EACV,cAAc,EACd,UAAU,EACV,WAAW,EACX,MAAM,GACT,MAAM,0BAA0B,CAAC;AAGlC,OAAO,EACH,WAAW,EACX,YAAY,EACZ,YAAY,EACZ,UAAU,EACV,eAAe,EACf,mBAAmB,EACnB,eAAe,EACf,gBAAgB,EAChB,QAAQ,EACR,OAAO,EACP,UAAU,EACV,mBAAmB,EACnB,oBAAoB,GACvB,MAAM,iBAAiB,CAAC;AACzB,YAAY,EACR,kBAAkB,EAClB,QAAQ,EACR,WAAW,EACX,QAAQ,EACR,UAAU,EACV,UAAU,EACV,UAAU,EACV,UAAU,EACV,WAAW,EACX,WAAW,EACX,MAAM,EACN,eAAe,EACf,uBAAuB,GAC1B,MAAM,iBAAiB,CAAC;AAGzB,OAAO,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;AAC9D,YAAY,EACR,eAAe,EACf,eAAe,EACf,aAAa,EACb,sBAAsB,EACtB,iBAAiB,GACpB,MAAM,6BAA6B,CAAC;AAGrC,OAAO,EAAE,WAAW,EAAE,MAAO,yBAAyB,CAAC;AACvD,YAAY,EACR,WAAW,EACX,kBAAkB,EAClB,eAAe,EACf,QAAQ,EACR,YAAY,EACZ,YAAY,EACZ,YAAY,GACf,MAAM,yBAAyB,CAAC;AAGjC,OAAO,EACH,iBAAiB,EACjB,yBAAyB,EACzB,sBAAsB,EACtB,mBAAmB,EACnB,eAAe,EACf,oBAAoB,EACpB,kBAAkB,EAClB,UAAU,GACb,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EACR,wBAAwB,EACxB,qBAAqB,EACrB,KAAK,EACL,kBAAkB,EAClB,YAAY,EACZ,eAAe,EACf,gBAAgB,EAChB,cAAc,EACd,OAAO,EACP,UAAU,EACV,YAAY,EACZ,cAAc,GACjB,MAAM,sBAAsB,CAAC;AAG9B,OAAO,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAC5D,YAAY,EACR,SAAS,EACT,eAAe,EACf,YAAY,EACZ,qBAAqB,EACrB,eAAe,EACf,UAAU,GACb,MAAM,wBAAwB,CAAC;AAGhC,OAAO,EACH,WAAW,EACX,eAAe,EACf,gBAAgB,EAChB,uBAAuB,EACvB,gBAAgB,EAChB,qBAAqB,EACrB,mBAAmB,EACnB,aAAa,EACb,mBAAmB,EACnB,SAAS,EACT,UAAU,EACV,kBAAkB,EAClB,WAAW,EACX,aAAa,EACb,WAAW,EACX,UAAU,GACb,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EACR,cAAc,EACd,kBAAkB,EAClB,YAAY,EACZ,WAAW,EACX,uBAAuB,EACvB,kBAAkB,EAClB,YAAY,EACZ,QAAQ,EACR,cAAc,GACjB,MAAM,qBAAqB,CAAC;AAG7B,OAAO,EAAE,oBAAoB,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAC5E,YAAY,EAAE,iBAAiB,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAG/F,OAAO,EAAE,kBAAkB,EAAE,MAAM,sCAAsC,CAAC;AAC1E,YAAY,EACR,cAAc,EACd,aAAa,EACb,kBAAkB,EAClB,eAAe,EACf,WAAW,EACX,gBAAgB,GACnB,MAAM,sCAAsC,CAAC;AAG9C,OAAO,EAAE,QAAQ,EAAE,MAAO,qBAAqB,CAAC;AAChD,YAAY,EAAE,eAAe,EAAE,YAAY,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAGpG,OAAO,EAAE,QAAQ,EAAE,MAAO,sBAAsB,CAAC;AACjD,YAAY,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAMzD,OAAO,EAAE,UAAU,EAAE,MAAe,yBAAyB,CAAC;AAC9D,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAEjE,eAAO,MAAM,GAAG;IACZ;;;;;OAKG;4BACY,iBAAiB;CAC1B,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -32,6 +32,7 @@ export { EvermindLM, EvermindLMTrainer } from '@seanhogg/builderforce-memory-eng
|
|
|
32
32
|
// ── Runtime ───────────────────────────────────────────────────────────────────
|
|
33
33
|
export { SSMRuntime } from './runtime/SSMRuntime.js';
|
|
34
34
|
export { OpenAIBridge } from './bridges/OpenAIBridge.js';
|
|
35
|
+
export { VertexAIBridge, VertexAIEmbedder } from './bridges/VertexAIBridge.js';
|
|
35
36
|
export { AnthropicBridge } from './bridges/AnthropicBridge.js';
|
|
36
37
|
export { FetchBridge } from './bridges/FetchBridge.js';
|
|
37
38
|
export { CachingBridge } from './bridges/CachingBridge.js';
|
|
@@ -43,7 +44,19 @@ export { FetchSemanticCacheBackend } from './cache/FetchSemanticCacheBackend.js'
|
|
|
43
44
|
// ── Similarity primitives ──────────────────────────────────────────────────────
|
|
44
45
|
export { cosineSimilarity, jaccardSimilarity, tokenize } from './similarity/index.js';
|
|
45
46
|
// ── Retrieval (chunking, BM25, rank fusion, hybrid RAG) ────────────────────────
|
|
46
|
-
export { chunkText, bm25Search, reciprocalRankFusion, maximalMarginalRelevance, hybridRetrieve, } from './retrieval/index.js';
|
|
47
|
+
export { chunkText, bm25Search, bm25Idf, bm25LengthNorm, bm25TermScore, BM25_DEFAULT_K1, BM25_DEFAULT_B, reciprocalRankFusion, maximalMarginalRelevance, hybridRetrieve, } from './retrieval/index.js';
|
|
48
|
+
// ── Telemetry (granular tracing + LLM-native metrics + cost-per-request) ──────
|
|
49
|
+
export { Tracer, Span, parseTraceparent, formatTraceparent, MetricsRegistry, InstrumentedBridge, InMemorySpanExporter, ConsoleSpanExporter, OtlpHttpSpanExporter, toOtlpPayload, toOtlpSpan, percentile, estimateCostUsd, estimateTokens, resolveRate, DEFAULT_PRICE_BOOK, ANTHROPIC_PRICE_BOOK, EVERMIND_PRICE_BOOK, DEFAULT_CACHE_READ_MULTIPLIER, DEFAULT_CACHE_WRITE_MULTIPLIER, } from './telemetry/index.js';
|
|
50
|
+
// ── Vector store (port + adapters + tenant/ACL scoping) ───────────────────────
|
|
51
|
+
export { MemoryVectorStore, RestVectorStore, getDialect, listDialects, registerDialect, evermindDialect, qdrantDialect, pineconeDialect, vertexAiDialect, accessFilter, combineFilters, matchesFilter, filterFields, TENANT_FIELD, ACL_FIELD, SENSITIVITY_FIELD, ACL_PUBLIC, } from './vectorstore/index.js';
|
|
52
|
+
// ── Ingestion (structured + unstructured → vector store, incremental sync) ────
|
|
53
|
+
export { IngestionPipeline, InMemoryIngestManifest, ParserRegistry, BUILTIN_PARSERS, rowsParser, csvParser, jsonParser, markdownParser, htmlParser, plainTextParser, parseCsv, serializeRow, hashText, mapWithConcurrency, SOURCE_ID_FIELD, CHUNK_INDEX_FIELD, CONTENT_HASH_FIELD, TITLE_FIELD, URI_FIELD, PARSER_FIELD, } from './ingest/index.js';
|
|
54
|
+
// ── Enterprise RAG (hybrid, scoped, cited) ────────────────────────────────────
|
|
55
|
+
export { EnterpriseRetriever, buildGroundedPrompt, GROUNDED_SYSTEM_PROMPT, NO_EVIDENCE_ANSWER, } from './rag/index.js';
|
|
56
|
+
// ── Multi-agent orchestration (graph + ReAct/reflection/delegation) ───────────
|
|
57
|
+
export { AgentGraph, appendReducer, mergeReducer, sumReducer, unionReducer, InMemoryCheckpointer, KeyValueCheckpointer, createReactAgent, createReflectionAgent, createSupervisor, asWorker, parseReactTurn, renderToolCatalog, renderTranscript, REACT_PROTOCOL, DEFAULT_CRITIC_PROMPT, FINISH, START, END, } from './orchestration/index.js';
|
|
58
|
+
// ── Evaluation (the conception → launch gate) ─────────────────────────────────
|
|
59
|
+
export { EvalHarness, evaluateGate, formatReport, exactMatch, substringChecks, embeddingSimilarity, retrievalRecall, citationValidity, llmJudge, noError, costBudget, DEFAULT_RAG_GRADERS, DEFAULT_JUDGE_RUBRIC, } from './eval/index.js';
|
|
47
60
|
// ── Router ────────────────────────────────────────────────────────────────────
|
|
48
61
|
export { InferenceRouter } from './router/InferenceRouter.js';
|
|
49
62
|
// ── Memory ────────────────────────────────────────────────────────────────────
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,iFAAiF;AACjF,OAAO,EAAE,YAAY,EAAE,MAAmB,oBAAoB,CAAC;AAC/D,OAAO,EAAE,YAAY,EAAE,MAAmB,oBAAoB,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAoB7F,iFAAiF;AACjF,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAE1D,gFAAgF;AAChF,qEAAqE;AACrE,OAAO,EACH,WAAW,EACX,aAAa,EACb,UAAU,EACV,gBAAgB,EAChB,gBAAgB,EAChB,aAAa,EACb,aAAa,EACb,MAAM,EACN,UAAU,EACV,YAAY,EACZ,aAAa,EACb,aAAa,GAChB,MAAM,sCAAsC,CAAC;AAU9C,sFAAsF;AACtF,wFAAwF;AACxF,OAAO,EACH,eAAe,EACf,sBAAsB,EACtB,kBAAkB,EAClB,gBAAgB,EAChB,UAAU,EACV,oBAAoB,GACvB,MAAM,sCAAsC,CAAC;AAa9C,OAAO,EAAE,UAAU,EAAE,iBAAiB,EAAE,MAAM,sCAAsC,CAAC;AAGrF,iFAAiF;AACjF,OAAO,EAAE,UAAU,EAAE,MAAS,yBAAyB,CAAC;AAKxD,OAAO,EAAE,YAAY,EAAE,MAAS,2BAA2B,CAAC;AAC5D,OAAO,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AAC/D,OAAO,EAAE,WAAW,EAAE,MAAU,0BAA0B,CAAC;AAC3D,OAAO,EAAE,aAAa,EAAE,MAAQ,4BAA4B,CAAC;AAC7D,OAAO,EAAE,qBAAqB,EAAE,MAAM,oCAAoC,CAAC;AAC3E,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,iFAAiF;AACjF,OAAO,EAAE,YAAY,EAAE,MAAmB,oBAAoB,CAAC;AAC/D,OAAO,EAAE,YAAY,EAAE,MAAmB,oBAAoB,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAoB7F,iFAAiF;AACjF,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAE1D,gFAAgF;AAChF,qEAAqE;AACrE,OAAO,EACH,WAAW,EACX,aAAa,EACb,UAAU,EACV,gBAAgB,EAChB,gBAAgB,EAChB,aAAa,EACb,aAAa,EACb,MAAM,EACN,UAAU,EACV,YAAY,EACZ,aAAa,EACb,aAAa,GAChB,MAAM,sCAAsC,CAAC;AAU9C,sFAAsF;AACtF,wFAAwF;AACxF,OAAO,EACH,eAAe,EACf,sBAAsB,EACtB,kBAAkB,EAClB,gBAAgB,EAChB,UAAU,EACV,oBAAoB,GACvB,MAAM,sCAAsC,CAAC;AAa9C,OAAO,EAAE,UAAU,EAAE,iBAAiB,EAAE,MAAM,sCAAsC,CAAC;AAGrF,iFAAiF;AACjF,OAAO,EAAE,UAAU,EAAE,MAAS,yBAAyB,CAAC;AAKxD,OAAO,EAAE,YAAY,EAAE,MAAS,2BAA2B,CAAC;AAC5D,OAAO,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AAC/E,OAAO,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAC;AAC/D,OAAO,EAAE,WAAW,EAAE,MAAU,0BAA0B,CAAC;AAC3D,OAAO,EAAE,aAAa,EAAE,MAAQ,4BAA4B,CAAC;AAC7D,OAAO,EAAE,qBAAqB,EAAE,MAAM,oCAAoC,CAAC;AAC3E,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAU1E,kFAAkF;AAClF,OAAO,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AACzD,OAAO,EAAE,yBAAyB,EAAE,MAAM,sCAAsC,CAAC;AASjF,kFAAkF;AAClF,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAEtF,kFAAkF;AAClF,OAAO,EACH,SAAS,EACT,UAAU,EACV,OAAO,EACP,cAAc,EACd,aAAa,EACb,eAAe,EACf,cAAc,EACd,oBAAoB,EACpB,wBAAwB,EACxB,cAAc,GACjB,MAAM,sBAAsB,CAAC;AAiB9B,iFAAiF;AACjF,OAAO,EACH,MAAM,EACN,IAAI,EACJ,gBAAgB,EAChB,iBAAiB,EACjB,eAAe,EACf,kBAAkB,EAClB,oBAAoB,EACpB,mBAAmB,EACnB,oBAAoB,EACpB,aAAa,EACb,UAAU,EACV,UAAU,EACV,eAAe,EACf,cAAc,EACd,WAAW,EACX,kBAAkB,EAClB,oBAAoB,EACpB,mBAAmB,EACnB,6BAA6B,EAC7B,8BAA8B,GACjC,MAAM,sBAAsB,CAAC;AAsB9B,iFAAiF;AACjF,OAAO,EACH,iBAAiB,EACjB,eAAe,EACf,UAAU,EACV,YAAY,EACZ,eAAe,EACf,eAAe,EACf,aAAa,EACb,eAAe,EACf,eAAe,EACf,YAAY,EACZ,cAAc,EACd,aAAa,EACb,YAAY,EACZ,YAAY,EACZ,SAAS,EACT,iBAAiB,EACjB,UAAU,GACb,MAAM,wBAAwB,CAAC;AAmBhC,iFAAiF;AACjF,OAAO,EACH,iBAAiB,EACjB,sBAAsB,EACtB,cAAc,EACd,eAAe,EACf,UAAU,EACV,SAAS,EACT,UAAU,EACV,cAAc,EACd,UAAU,EACV,eAAe,EACf,QAAQ,EACR,YAAY,EACZ,QAAQ,EACR,kBAAkB,EAClB,eAAe,EACf,iBAAiB,EACjB,kBAAkB,EAClB,WAAW,EACX,SAAS,EACT,YAAY,GACf,MAAM,mBAAmB,CAAC;AAe3B,iFAAiF;AACjF,OAAO,EACH,mBAAmB,EACnB,mBAAmB,EACnB,sBAAsB,EACtB,kBAAkB,GACrB,MAAM,gBAAgB,CAAC;AAUxB,iFAAiF;AACjF,OAAO,EACH,UAAU,EACV,aAAa,EACb,YAAY,EACZ,UAAU,EACV,YAAY,EACZ,oBAAoB,EACpB,oBAAoB,EACpB,gBAAgB,EAChB,qBAAqB,EACrB,gBAAgB,EAChB,QAAQ,EACR,cAAc,EACd,iBAAiB,EACjB,gBAAgB,EAChB,cAAc,EACd,qBAAqB,EACrB,MAAM,EACN,KAAK,EACL,GAAG,GACN,MAAM,0BAA0B,CAAC;AAgClC,iFAAiF;AACjF,OAAO,EACH,WAAW,EACX,YAAY,EACZ,YAAY,EACZ,UAAU,EACV,eAAe,EACf,mBAAmB,EACnB,eAAe,EACf,gBAAgB,EAChB,QAAQ,EACR,OAAO,EACP,UAAU,EACV,mBAAmB,EACnB,oBAAoB,GACvB,MAAM,iBAAiB,CAAC;AAiBzB,iFAAiF;AACjF,OAAO,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;AAS9D,iFAAiF;AACjF,OAAO,EAAE,WAAW,EAAE,MAAO,yBAAyB,CAAC;AAWvD,iFAAiF;AACjF,OAAO,EACH,iBAAiB,EACjB,yBAAyB,EACzB,sBAAsB,EACtB,mBAAmB,EACnB,eAAe,EACf,oBAAoB,EACpB,kBAAkB,EAClB,UAAU,GACb,MAAM,sBAAsB,CAAC;AAgB9B,kFAAkF;AAClF,OAAO,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAU5D,mFAAmF;AACnF,OAAO,EACH,WAAW,EACX,eAAe,EACf,gBAAgB,EAChB,uBAAuB,EACvB,gBAAgB,EAChB,qBAAqB,EACrB,mBAAmB,EACnB,aAAa,EACb,mBAAmB,EACnB,SAAS,EACT,UAAU,EACV,kBAAkB,EAClB,WAAW,EACX,aAAa,EACb,WAAW,EACX,UAAU,GACb,MAAM,qBAAqB,CAAC;AAa7B,mFAAmF;AACnF,OAAO,EAAE,oBAAoB,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAG5E,iFAAiF;AACjF,OAAO,EAAE,kBAAkB,EAAE,MAAM,sCAAsC,CAAC;AAU1E,iFAAiF;AACjF,OAAO,EAAE,QAAQ,EAAE,MAAO,qBAAqB,CAAC;AAGhD,iFAAiF;AACjF,OAAO,EAAE,QAAQ,EAAE,MAAO,sBAAsB,CAAC;AAGjD,iFAAiF;AACjF,mDAAmD;AACnD,qEAAqE;AAErE,OAAO,EAAE,UAAU,EAAE,MAAe,yBAAyB,CAAC;AAG9D,MAAM,CAAC,MAAM,GAAG,GAAG;IACf;;;;;OAKG;IACH,MAAM,EAAE,CAAC,IAAuB,EAAE,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC;CACtD,CAAC"}
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ingest/IngestionPipeline.ts — source → parse → chunk → embed → upsert.
|
|
3
|
+
*
|
|
4
|
+
* The one-off script that loads a corpus into a vector database is easy. What
|
|
5
|
+
* makes a pipeline PRODUCTION-grade is everything that happens on the second run:
|
|
6
|
+
*
|
|
7
|
+
* • **Incremental sync.** Re-ingesting a 200k-document corpus must not re-embed
|
|
8
|
+
* 200k documents because three of them changed. Chunks carry a content hash;
|
|
9
|
+
* with a manifest, unchanged hashes are skipped and vanished chunks deleted.
|
|
10
|
+
* Embedding is the cost line, so this is the difference between a pipeline that
|
|
11
|
+
* runs nightly and one that runs once.
|
|
12
|
+
* • **Deterministic ids.** A chunk's id is derived from its source id and its
|
|
13
|
+
* ordinal, so a re-run overwrites rather than duplicating. Random ids are why
|
|
14
|
+
* corpora silently triple in size.
|
|
15
|
+
* • **Batching and bounded concurrency.** Embedders and vector databases both
|
|
16
|
+
* rate-limit; an unbounded `Promise.all` over a corpus is a self-inflicted 429.
|
|
17
|
+
* • **Per-document failure isolation.** One malformed file fails that document
|
|
18
|
+
* and is reported, rather than aborting the run.
|
|
19
|
+
* • **Governance by construction.** Tenant, ACL and sensitivity are written onto
|
|
20
|
+
* every chunk during ingest, because a retrieval-time ACL that depends on
|
|
21
|
+
* ingest-time diligence is not an ACL.
|
|
22
|
+
*/
|
|
23
|
+
import { type ChunkOptions } from '../retrieval/chunk.js';
|
|
24
|
+
import type { Tracer } from '../telemetry/Tracer.js';
|
|
25
|
+
import { type VectorStore } from '../vectorstore/types.js';
|
|
26
|
+
import { ParserRegistry } from './parsers.js';
|
|
27
|
+
import { type BatchEmbedder, type ChunkFingerprint, type DocumentParser, type IngestManifest, type IngestionReport, type SourceDocument, type SyncStrategy } from './types.js';
|
|
28
|
+
export interface IngestionPipelineOptions {
|
|
29
|
+
store: VectorStore;
|
|
30
|
+
embed: BatchEmbedder;
|
|
31
|
+
/** Chunking parameters. Defaults to 1000 chars / 200 overlap. */
|
|
32
|
+
chunk?: ChunkOptions;
|
|
33
|
+
/** Custom parser set. Defaults to the builtin registry. */
|
|
34
|
+
parsers?: ParserRegistry | readonly DocumentParser[];
|
|
35
|
+
/** Enables `diff` sync — see {@link IngestManifest}. */
|
|
36
|
+
manifest?: IngestManifest;
|
|
37
|
+
/** Default `'diff'` when a manifest is supplied, `'replace'` otherwise. */
|
|
38
|
+
syncStrategy?: SyncStrategy;
|
|
39
|
+
/** Texts per embedder call. Default 64. */
|
|
40
|
+
embedBatchSize?: number;
|
|
41
|
+
/** Records per store upsert. Default 100. */
|
|
42
|
+
upsertBatchSize?: number;
|
|
43
|
+
/** Documents processed concurrently. Default 4. */
|
|
44
|
+
concurrency?: number;
|
|
45
|
+
/** Model label recorded on embedding spans, for cost attribution. */
|
|
46
|
+
embeddingModel?: string;
|
|
47
|
+
tracer?: Tracer;
|
|
48
|
+
}
|
|
49
|
+
export declare class IngestionPipeline {
|
|
50
|
+
private readonly _store;
|
|
51
|
+
private readonly _embed;
|
|
52
|
+
private readonly _chunkOpts;
|
|
53
|
+
private readonly _parsers;
|
|
54
|
+
private readonly _manifest;
|
|
55
|
+
private readonly _strategy;
|
|
56
|
+
private readonly _embedBatch;
|
|
57
|
+
private readonly _upsertBatch;
|
|
58
|
+
private readonly _concurrency;
|
|
59
|
+
private readonly _embeddingModel;
|
|
60
|
+
private readonly _tracer;
|
|
61
|
+
constructor(opts: IngestionPipelineOptions);
|
|
62
|
+
ingest(sources: readonly SourceDocument[]): Promise<IngestionReport>;
|
|
63
|
+
/** Removes every chunk a source produced — a GDPR/right-to-erasure primitive. */
|
|
64
|
+
forget(sourceId: string): Promise<number>;
|
|
65
|
+
private _ingestOne;
|
|
66
|
+
/**
|
|
67
|
+
* Decides what actually has to be written and deleted.
|
|
68
|
+
*
|
|
69
|
+
* `diff` compares content hashes against the manifest; `replace` clears the
|
|
70
|
+
* source's chunks first (correct without a manifest, at the cost of re-embedding);
|
|
71
|
+
* `append` writes everything and deletes nothing.
|
|
72
|
+
*/
|
|
73
|
+
private _planSync;
|
|
74
|
+
private _embedBatchTraced;
|
|
75
|
+
private _baseMetadata;
|
|
76
|
+
}
|
|
77
|
+
/** In-process manifest — the default for a single-process or test deployment. */
|
|
78
|
+
export declare class InMemoryIngestManifest implements IngestManifest {
|
|
79
|
+
private readonly _entries;
|
|
80
|
+
get(sourceId: string): Promise<ChunkFingerprint[] | undefined>;
|
|
81
|
+
set(sourceId: string, fingerprints: ChunkFingerprint[]): Promise<void>;
|
|
82
|
+
delete(sourceId: string): Promise<void>;
|
|
83
|
+
get size(): number;
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* FNV-1a, 32-bit, rendered as hex.
|
|
87
|
+
*
|
|
88
|
+
* A content hash here answers "did this chunk change", not "was this chunk
|
|
89
|
+
* tampered with", so a fast non-cryptographic hash is the right tool — and it is
|
|
90
|
+
* synchronous, unlike WebCrypto, which keeps chunking a pure function.
|
|
91
|
+
*/
|
|
92
|
+
export declare function hashText(text: string): string;
|
|
93
|
+
/** Bounded-concurrency map — the reason a large corpus does not become a 429. */
|
|
94
|
+
export declare function mapWithConcurrency<T>(items: readonly T[], limit: number, fn: (item: T, index: number) => Promise<void>): Promise<void>;
|
|
95
|
+
//# sourceMappingURL=IngestionPipeline.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"IngestionPipeline.d.ts","sourceRoot":"","sources":["../../src/ingest/IngestionPipeline.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,EAAa,KAAK,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACrE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAC;AAErD,OAAO,EAOH,KAAK,WAAW,EACnB,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,EAOH,KAAK,aAAa,EAClB,KAAK,gBAAgB,EACrB,KAAK,cAAc,EACnB,KAAK,cAAc,EACnB,KAAK,eAAe,EACpB,KAAK,cAAc,EACnB,KAAK,YAAY,EACpB,MAAM,YAAY,CAAC;AAEpB,MAAM,WAAW,wBAAwB;IACrC,KAAK,EAAE,WAAW,CAAC;IACnB,KAAK,EAAE,aAAa,CAAC;IACrB,iEAAiE;IACjE,KAAK,CAAC,EAAE,YAAY,CAAC;IACrB,2DAA2D;IAC3D,OAAO,CAAC,EAAE,cAAc,GAAG,SAAS,cAAc,EAAE,CAAC;IACrD,wDAAwD;IACxD,QAAQ,CAAC,EAAE,cAAc,CAAC;IAC1B,2EAA2E;IAC3E,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,2CAA2C;IAC3C,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,6CAA6C;IAC7C,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,mDAAmD;IACnD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,qEAAqE;IACrE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,MAAM,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,qBAAa,iBAAiB;IAC1B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAc;IACrC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgB;IACvC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAe;IAC1C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAiB;IAC1C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAA6B;IACvD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAe;IACzC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAS;IACzC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAqB;gBAEjC,IAAI,EAAE,wBAAwB;IAgBpC,MAAM,CAAC,OAAO,EAAE,SAAS,cAAc,EAAE,GAAG,OAAO,CAAC,eAAe,CAAC;IA6C1E,iFAAiF;IAC3E,MAAM,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;YAQjC,UAAU;IA8FxB;;;;;;OAMG;YACW,SAAS;YAyBT,iBAAiB;IA2B/B,OAAO,CAAC,aAAa;CAgBxB;AAED,iFAAiF;AACjF,qBAAa,sBAAuB,YAAW,cAAc;IACzD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAyC;IAE5D,GAAG,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,EAAE,GAAG,SAAS,CAAC;IAI9D,GAAG,CAAC,QAAQ,EAAE,MAAM,EAAE,YAAY,EAAE,gBAAgB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAItE,MAAM,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAI7C,IAAI,IAAI,IAAI,MAAM,CAA+B;CACpD;AAED;;;;;;GAMG;AACH,wBAAgB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAO7C;AAED,iFAAiF;AACjF,wBAAsB,kBAAkB,CAAC,CAAC,EACtC,KAAK,EAAE,SAAS,CAAC,EAAE,EACnB,KAAK,EAAE,MAAM,EACb,EAAE,EAAE,CAAC,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,GAC9C,OAAO,CAAC,IAAI,CAAC,CASf"}
|