@getplumb/core 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +48 -0
- package/dist/bm25.d.ts +23 -0
- package/dist/bm25.d.ts.map +1 -0
- package/dist/bm25.js +74 -0
- package/dist/bm25.js.map +1 -0
- package/dist/chunker.d.ts +35 -0
- package/dist/chunker.d.ts.map +1 -0
- package/dist/chunker.js +45 -0
- package/dist/chunker.js.map +1 -0
- package/dist/context-builder.d.ts +33 -0
- package/dist/context-builder.d.ts.map +1 -0
- package/dist/context-builder.js +101 -0
- package/dist/context-builder.js.map +1 -0
- package/dist/embedder.d.ts +34 -0
- package/dist/embedder.d.ts.map +1 -0
- package/dist/embedder.js +88 -0
- package/dist/embedder.js.map +1 -0
- package/dist/extractor.d.ts +21 -0
- package/dist/extractor.d.ts.map +1 -0
- package/dist/extractor.js +89 -0
- package/dist/extractor.js.map +1 -0
- package/dist/fact-search.d.ts +28 -0
- package/dist/fact-search.d.ts.map +1 -0
- package/dist/fact-search.js +155 -0
- package/dist/fact-search.js.map +1 -0
- package/dist/index.d.ts +18 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +12 -0
- package/dist/index.js.map +1 -0
- package/dist/llm-client.d.ts +28 -0
- package/dist/llm-client.d.ts.map +1 -0
- package/dist/llm-client.js +115 -0
- package/dist/llm-client.js.map +1 -0
- package/dist/local-store.d.ts +99 -0
- package/dist/local-store.d.ts.map +1 -0
- package/dist/local-store.js +292 -0
- package/dist/local-store.js.map +1 -0
- package/dist/raw-log-search.d.ts +33 -0
- package/dist/raw-log-search.d.ts.map +1 -0
- package/dist/raw-log-search.js +137 -0
- package/dist/raw-log-search.js.map +1 -0
- package/dist/read-path.d.ts +60 -0
- package/dist/read-path.d.ts.map +1 -0
- package/dist/read-path.js +76 -0
- package/dist/read-path.js.map +1 -0
- package/dist/schema.d.ts +34 -0
- package/dist/schema.d.ts.map +1 -0
- package/dist/schema.js +119 -0
- package/dist/schema.js.map +1 -0
- package/dist/scorer.d.ts +30 -0
- package/dist/scorer.d.ts.map +1 -0
- package/dist/scorer.js +50 -0
- package/dist/scorer.js.map +1 -0
- package/dist/store.d.ts +25 -0
- package/dist/store.d.ts.map +1 -0
- package/dist/store.js +2 -0
- package/dist/store.js.map +1 -0
- package/dist/types.d.ts +44 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +7 -0
- package/dist/types.js.map +1 -0
- package/package.json +43 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Plumb Contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
# @getplumb/core
|
|
2
|
+
|
|
3
|
+
> Cross-session AI memory — storage abstraction, types, and local SQLite driver
|
|
4
|
+
|
|
5
|
+
The core library for [Plumb](https://getplumb.dev) — a local-first, MCP-native memory layer for AI agents.
|
|
6
|
+
|
|
7
|
+
## What it does
|
|
8
|
+
|
|
9
|
+
- **Two-layer memory:** raw conversation log (Layer 1) + extracted fact graph (Layer 2)
|
|
10
|
+
- **Hybrid search:** BM25 + semantic vectors + RRF fusion + cross-encoder reranking
|
|
11
|
+
- **Local storage:** SQLite + sqlite-vec, zero external dependencies
|
|
12
|
+
- **Pluggable LLM:** bring your own OpenAI or Anthropic client for fact extraction
|
|
13
|
+
|
|
14
|
+
## Install
|
|
15
|
+
|
|
16
|
+
```bash
|
|
17
|
+
npm install @getplumb/core
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
## Quick start
|
|
21
|
+
|
|
22
|
+
```ts
|
|
23
|
+
import { LocalStore } from '@getplumb/core';
|
|
24
|
+
|
|
25
|
+
const store = new LocalStore({ dbPath: '~/.plumb/memory.db' });
|
|
26
|
+
await store.init();
|
|
27
|
+
|
|
28
|
+
// Ingest a conversation turn
|
|
29
|
+
await store.ingest({
|
|
30
|
+
sessionId: 'my-session',
|
|
31
|
+
userId: 'user-123',
|
|
32
|
+
userMessage: 'I prefer TypeScript over Python',
|
|
33
|
+
assistantMessage: 'Noted! TypeScript it is.'
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
// Retrieve relevant memory
|
|
37
|
+
const results = await store.searchRawLog('TypeScript preferences', 5);
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
## Links
|
|
41
|
+
|
|
42
|
+
- [Docs](https://docs.getplumb.dev)
|
|
43
|
+
- [GitHub](https://github.com/getplumb/plumb)
|
|
44
|
+
- [getplumb.dev](https://getplumb.dev)
|
|
45
|
+
|
|
46
|
+
## License
|
|
47
|
+
|
|
48
|
+
MIT
|
package/dist/bm25.d.ts
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* BM25 Okapi — pure TypeScript in-memory implementation.
|
|
3
|
+
*
|
|
4
|
+
* Parameters: k1 = 1.5, b = 0.75 (standard values from the original paper).
|
|
5
|
+
* IDF variant: Robertson-Walker smooth IDF to avoid negative values for
|
|
6
|
+
* very common terms: IDF(q) = log((N - df + 0.5) / (df + 0.5) + 1)
|
|
7
|
+
*
|
|
8
|
+
* Usage:
|
|
9
|
+
* const index = new Bm25(corpus); // corpus: string[]
|
|
10
|
+
* const scores = index.scores(query); // scores: number[], same order as corpus
|
|
11
|
+
*/
|
|
12
|
+
/** Tokenize text to lowercase alphanumeric tokens. */
|
|
13
|
+
export declare function tokenize(text: string): string[];
|
|
14
|
+
export declare class Bm25 {
|
|
15
|
+
#private;
|
|
16
|
+
constructor(corpus: readonly string[]);
|
|
17
|
+
/**
|
|
18
|
+
* Compute BM25 scores for all corpus documents against the query.
|
|
19
|
+
* Returns an array of scores in the same order as the constructor corpus.
|
|
20
|
+
*/
|
|
21
|
+
scores(query: string): number[];
|
|
22
|
+
}
|
|
23
|
+
//# sourceMappingURL=bm25.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"bm25.d.ts","sourceRoot":"","sources":["../src/bm25.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAKH,sDAAsD;AACtD,wBAAgB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAE/C;AAED,qBAAa,IAAI;;gBAOH,MAAM,EAAE,SAAS,MAAM,EAAE;IA4BrC;;;OAGG;IACH,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE;CAqBhC"}
|
package/dist/bm25.js
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* BM25 Okapi — pure TypeScript in-memory implementation.
|
|
3
|
+
*
|
|
4
|
+
* Parameters: k1 = 1.5, b = 0.75 (standard values from the original paper).
|
|
5
|
+
* IDF variant: Robertson-Walker smooth IDF to avoid negative values for
|
|
6
|
+
* very common terms: IDF(q) = log((N - df + 0.5) / (df + 0.5) + 1)
|
|
7
|
+
*
|
|
8
|
+
* Usage:
|
|
9
|
+
* const index = new Bm25(corpus); // corpus: string[]
|
|
10
|
+
* const scores = index.scores(query); // scores: number[], same order as corpus
|
|
11
|
+
*/
|
|
12
|
+
const K1 = 1.5;
|
|
13
|
+
const B = 0.75;
|
|
14
|
+
/** Tokenize text to lowercase alphanumeric tokens. */
|
|
15
|
+
export function tokenize(text) {
|
|
16
|
+
return text.toLowerCase().match(/\b[a-z0-9]+\b/g) ?? [];
|
|
17
|
+
}
|
|
18
|
+
export class Bm25 {
|
|
19
|
+
#n;
|
|
20
|
+
#avgdl;
|
|
21
|
+
#idf;
|
|
22
|
+
#tf;
|
|
23
|
+
#docLengths;
|
|
24
|
+
constructor(corpus) {
|
|
25
|
+
this.#n = corpus.length;
|
|
26
|
+
const tokenized = corpus.map(tokenize);
|
|
27
|
+
this.#docLengths = tokenized.map((t) => t.length);
|
|
28
|
+
const totalLen = this.#docLengths.reduce((a, b) => a + b, 0);
|
|
29
|
+
this.#avgdl = this.#n > 0 ? totalLen / this.#n : 1;
|
|
30
|
+
// Build term-frequency maps and document-frequency counts.
|
|
31
|
+
const df = new Map();
|
|
32
|
+
this.#tf = tokenized.map((tokens) => {
|
|
33
|
+
const freq = new Map();
|
|
34
|
+
for (const tok of tokens) {
|
|
35
|
+
freq.set(tok, (freq.get(tok) ?? 0) + 1);
|
|
36
|
+
}
|
|
37
|
+
for (const tok of freq.keys()) {
|
|
38
|
+
df.set(tok, (df.get(tok) ?? 0) + 1);
|
|
39
|
+
}
|
|
40
|
+
return freq;
|
|
41
|
+
});
|
|
42
|
+
// Precompute IDF for each unique term.
|
|
43
|
+
this.#idf = new Map();
|
|
44
|
+
const N = this.#n;
|
|
45
|
+
for (const [term, dfTerm] of df) {
|
|
46
|
+
this.#idf.set(term, Math.log((N - dfTerm + 0.5) / (dfTerm + 0.5) + 1));
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Compute BM25 scores for all corpus documents against the query.
|
|
51
|
+
* Returns an array of scores in the same order as the constructor corpus.
|
|
52
|
+
*/
|
|
53
|
+
scores(query) {
|
|
54
|
+
const queryTerms = tokenize(query);
|
|
55
|
+
const result = new Array(this.#n).fill(0);
|
|
56
|
+
if (queryTerms.length === 0 || this.#n === 0)
|
|
57
|
+
return result;
|
|
58
|
+
for (const term of queryTerms) {
|
|
59
|
+
const idf = this.#idf.get(term);
|
|
60
|
+
if (idf === undefined)
|
|
61
|
+
continue;
|
|
62
|
+
for (let i = 0; i < this.#n; i++) {
|
|
63
|
+
const tf = this.#tf[i]?.get(term) ?? 0;
|
|
64
|
+
if (tf === 0)
|
|
65
|
+
continue;
|
|
66
|
+
const dl = this.#docLengths[i] ?? 0;
|
|
67
|
+
const norm = tf * (K1 + 1) / (tf + K1 * (1 - B + B * dl / this.#avgdl));
|
|
68
|
+
result[i] = (result[i] ?? 0) + idf * norm;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
return result;
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
//# sourceMappingURL=bm25.js.map
|
package/dist/bm25.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"bm25.js","sourceRoot":"","sources":["../src/bm25.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,MAAM,EAAE,GAAG,GAAG,CAAC;AACf,MAAM,CAAC,GAAG,IAAI,CAAC;AAEf,sDAAsD;AACtD,MAAM,UAAU,QAAQ,CAAC,IAAY;IACnC,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,gBAAgB,CAAC,IAAI,EAAE,CAAC;AAC1D,CAAC;AAED,MAAM,OAAO,IAAI;IACN,EAAE,CAAS;IACX,MAAM,CAAS;IACf,IAAI,CAAsB;IAC1B,GAAG,CAA6B;IAChC,WAAW,CAAW;IAE/B,YAAY,MAAyB;QACnC,IAAI,CAAC,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC;QACxB,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACvC,IAAI,CAAC,WAAW,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QAClD,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;QAC7D,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAEnD,2DAA2D;QAC3D,MAAM,EAAE,GAAG,IAAI,GAAG,EAAkB,CAAC;QACrC,IAAI,CAAC,GAAG,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;YAClC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAkB,CAAC;YACvC,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;gBACzB,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YAC1C,CAAC;YACD,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;gBAC9B,EAAE,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YACtC,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC,CAAC,CAAC;QAEH,uCAAuC;QACvC,IAAI,CAAC,IAAI,GAAG,IAAI,GAAG,EAAE,CAAC;QACtB,MAAM,CAAC,GAAG,IAAI,CAAC,EAAE,CAAC;QAClB,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC;YAChC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACzE,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,MAAM,CAAC,KAAa;QAClB,MAAM,UAAU,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;QACnC,MAAM,MAAM,GAAG,IAAI,KAAK,CAAS,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAElD,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,EAAE,KAAK,CAAC;YAAE,OAAO,MAAM,CAAC;QAE5D,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;YAC9B,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAChC,IAAI,GAAG,KAAK,SAAS;gBAAE,SAAS;YAEhC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;gBACjC,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACvC,IAAI,EAAE,KAAK,CAAC;oBAAE,SAAS;gBACvB,MAAM,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;gBACpC,MAAM,IAAI,GAAG,EAAE,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;gBACxE,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,GAAG,GAAG,IAAI,CAAC;YAC5C,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;CACF"}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Chunker — splits a MessageExchange into overlapping text chunks for indexing.
|
|
3
|
+
*
|
|
4
|
+
* Design (T-004):
|
|
5
|
+
* - Each exchange is formatted as "User: {msg}\nAgent: {response}"
|
|
6
|
+
* - Short exchanges (≤ CHUNK_WORDS words) produce one chunk
|
|
7
|
+
* - Long exchanges are split with OVERLAP_WORDS word overlap to preserve
|
|
8
|
+
* context at chunk boundaries
|
|
9
|
+
*
|
|
10
|
+
* Word-based splitting is used (not token-based) to avoid a tokenizer
|
|
11
|
+
* dependency at ingest time. At 384-dim bge-small, 300 words ≈ 400 tokens
|
|
12
|
+
* which is within the model's 512-token limit.
|
|
13
|
+
*/
|
|
14
|
+
import type { MessageExchange } from './types.js';
|
|
15
|
+
export declare const CHUNK_WORDS = 300;
|
|
16
|
+
export declare const OVERLAP_WORDS = 50;
|
|
17
|
+
export interface Chunk {
|
|
18
|
+
/** Full formatted text of this chunk. */
|
|
19
|
+
readonly text: string;
|
|
20
|
+
/** 0-based index within this exchange's chunk sequence. */
|
|
21
|
+
readonly chunkIndex: number;
|
|
22
|
+
/** Total chunks produced from this exchange. */
|
|
23
|
+
readonly totalChunks: number;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Format a MessageExchange into its canonical chunk text.
|
|
27
|
+
* Used both by the chunker and by ingest() for the stored chunk_text column.
|
|
28
|
+
*/
|
|
29
|
+
export declare function formatExchange(exchange: MessageExchange): string;
|
|
30
|
+
/**
|
|
31
|
+
* Split a MessageExchange into one or more overlapping text chunks.
|
|
32
|
+
* Returns at least one chunk (even for empty exchanges).
|
|
33
|
+
*/
|
|
34
|
+
export declare function chunkExchange(exchange: MessageExchange): Chunk[];
|
|
35
|
+
//# sourceMappingURL=chunker.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"chunker.d.ts","sourceRoot":"","sources":["../src/chunker.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAElD,eAAO,MAAM,WAAW,MAAM,CAAC;AAC/B,eAAO,MAAM,aAAa,KAAK,CAAC;AAEhC,MAAM,WAAW,KAAK;IACpB,yCAAyC;IACzC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,2DAA2D;IAC3D,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,gDAAgD;IAChD,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAC9B;AAED;;;GAGG;AACH,wBAAgB,cAAc,CAAC,QAAQ,EAAE,eAAe,GAAG,MAAM,CAEhE;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,eAAe,GAAG,KAAK,EAAE,CAmBhE"}
|
package/dist/chunker.js
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Chunker — splits a MessageExchange into overlapping text chunks for indexing.
|
|
3
|
+
*
|
|
4
|
+
* Design (T-004):
|
|
5
|
+
* - Each exchange is formatted as "User: {msg}\nAgent: {response}"
|
|
6
|
+
* - Short exchanges (≤ CHUNK_WORDS words) produce one chunk
|
|
7
|
+
* - Long exchanges are split with OVERLAP_WORDS word overlap to preserve
|
|
8
|
+
* context at chunk boundaries
|
|
9
|
+
*
|
|
10
|
+
* Word-based splitting is used (not token-based) to avoid a tokenizer
|
|
11
|
+
* dependency at ingest time. At 384-dim bge-small, 300 words ≈ 400 tokens
|
|
12
|
+
* which is within the model's 512-token limit.
|
|
13
|
+
*/
|
|
14
|
+
export const CHUNK_WORDS = 300;
|
|
15
|
+
export const OVERLAP_WORDS = 50;
|
|
16
|
+
/**
|
|
17
|
+
* Format a MessageExchange into its canonical chunk text.
|
|
18
|
+
* Used both by the chunker and by ingest() for the stored chunk_text column.
|
|
19
|
+
*/
|
|
20
|
+
export function formatExchange(exchange) {
|
|
21
|
+
return `User: ${exchange.userMessage}\nAgent: ${exchange.agentResponse}`;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Split a MessageExchange into one or more overlapping text chunks.
|
|
25
|
+
* Returns at least one chunk (even for empty exchanges).
|
|
26
|
+
*/
|
|
27
|
+
export function chunkExchange(exchange) {
|
|
28
|
+
const text = formatExchange(exchange);
|
|
29
|
+
const words = text.split(/\s+/).filter((w) => w.length > 0);
|
|
30
|
+
if (words.length <= CHUNK_WORDS) {
|
|
31
|
+
return [{ text, chunkIndex: 0, totalChunks: 1 }];
|
|
32
|
+
}
|
|
33
|
+
const chunks = [];
|
|
34
|
+
let start = 0;
|
|
35
|
+
while (start < words.length) {
|
|
36
|
+
const end = Math.min(start + CHUNK_WORDS, words.length);
|
|
37
|
+
chunks.push(words.slice(start, end).join(' '));
|
|
38
|
+
if (end >= words.length)
|
|
39
|
+
break;
|
|
40
|
+
start += CHUNK_WORDS - OVERLAP_WORDS;
|
|
41
|
+
}
|
|
42
|
+
const totalChunks = chunks.length;
|
|
43
|
+
return chunks.map((chunkText, chunkIndex) => ({ text: chunkText, chunkIndex, totalChunks }));
|
|
44
|
+
}
|
|
45
|
+
//# sourceMappingURL=chunker.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"chunker.js","sourceRoot":"","sources":["../src/chunker.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAIH,MAAM,CAAC,MAAM,WAAW,GAAG,GAAG,CAAC;AAC/B,MAAM,CAAC,MAAM,aAAa,GAAG,EAAE,CAAC;AAWhC;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,QAAyB;IACtD,OAAO,SAAS,QAAQ,CAAC,WAAW,YAAY,QAAQ,CAAC,aAAa,EAAE,CAAC;AAC3E,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,QAAyB;IACrD,MAAM,IAAI,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;IACtC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAE5D,IAAI,KAAK,CAAC,MAAM,IAAI,WAAW,EAAE,CAAC;QAChC,OAAO,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC,CAAC;IACnD,CAAC;IAED,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,OAAO,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;QAC5B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;QACxD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QAC/C,IAAI,GAAG,IAAI,KAAK,CAAC,MAAM;YAAE,MAAM;QAC/B,KAAK,IAAI,WAAW,GAAG,aAAa,CAAC;IACvC,CAAC;IAED,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC;IAClC,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,UAAU,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,UAAU,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC;AAC/F,CAAC"}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Context builder — formats a MemoryContext into a [MEMORY CONTEXT] string
|
|
3
|
+
* suitable for injection into an agent's system prompt.
|
|
4
|
+
*
|
|
5
|
+
* Output format (example):
|
|
6
|
+
*
|
|
7
|
+
* [MEMORY CONTEXT]
|
|
8
|
+
*
|
|
9
|
+
* ## High confidence facts
|
|
10
|
+
* - user is building a product called Plumb (0.98, session: tech-planning, today)
|
|
11
|
+
*
|
|
12
|
+
* ## Medium confidence facts
|
|
13
|
+
* - user uses TypeScript (0.65, session: dev-chat, 2 days ago)
|
|
14
|
+
*
|
|
15
|
+
* ## Related conversations
|
|
16
|
+
* - [tech-planning] today: "Let me help you design the memory system..."
|
|
17
|
+
*
|
|
18
|
+
* Empty MemoryContext returns an empty string — no block is injected.
|
|
19
|
+
*/
|
|
20
|
+
import type { MemoryContext } from './read-path.js';
|
|
21
|
+
/**
|
|
22
|
+
* Converts an age in fractional days to a human-readable string.
|
|
23
|
+
* Examples: 'today', 'yesterday', '3 days ago', '2 weeks ago', '1 month ago'.
|
|
24
|
+
*/
|
|
25
|
+
export declare function formatAge(ageInDays: number): string;
|
|
26
|
+
/**
|
|
27
|
+
* Formats a MemoryContext into a [MEMORY CONTEXT] prompt block.
|
|
28
|
+
*
|
|
29
|
+
* Returns an empty string if the context has no facts and no raw chunks,
|
|
30
|
+
* so callers can skip injection without additional checks.
|
|
31
|
+
*/
|
|
32
|
+
export declare function formatContextBlock(context: MemoryContext): string;
|
|
33
|
+
//# sourceMappingURL=context-builder.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"context-builder.d.ts","sourceRoot":"","sources":["../src/context-builder.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAwB,MAAM,gBAAgB,CAAC;AAI1E;;;GAGG;AACH,wBAAgB,SAAS,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAUnD;AAsBD;;;;;GAKG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,aAAa,GAAG,MAAM,CAsCjE"}
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Context builder — formats a MemoryContext into a [MEMORY CONTEXT] string
|
|
3
|
+
* suitable for injection into an agent's system prompt.
|
|
4
|
+
*
|
|
5
|
+
* Output format (example):
|
|
6
|
+
*
|
|
7
|
+
* [MEMORY CONTEXT]
|
|
8
|
+
*
|
|
9
|
+
* ## High confidence facts
|
|
10
|
+
* - user is building a product called Plumb (0.98, session: tech-planning, today)
|
|
11
|
+
*
|
|
12
|
+
* ## Medium confidence facts
|
|
13
|
+
* - user uses TypeScript (0.65, session: dev-chat, 2 days ago)
|
|
14
|
+
*
|
|
15
|
+
* ## Related conversations
|
|
16
|
+
* - [tech-planning] today: "Let me help you design the memory system..."
|
|
17
|
+
*
|
|
18
|
+
* Empty MemoryContext returns an empty string — no block is injected.
|
|
19
|
+
*/
|
|
20
|
+
// ─── Age formatting ───────────────────────────────────────────────────────────
|
|
21
|
+
/**
|
|
22
|
+
* Converts an age in fractional days to a human-readable string.
|
|
23
|
+
* Examples: 'today', 'yesterday', '3 days ago', '2 weeks ago', '1 month ago'.
|
|
24
|
+
*/
|
|
25
|
+
export function formatAge(ageInDays) {
|
|
26
|
+
if (ageInDays < 1)
|
|
27
|
+
return 'today';
|
|
28
|
+
if (ageInDays < 2)
|
|
29
|
+
return 'yesterday';
|
|
30
|
+
if (ageInDays < 7)
|
|
31
|
+
return `${Math.floor(ageInDays)} days ago`;
|
|
32
|
+
if (ageInDays < 14)
|
|
33
|
+
return '1 week ago';
|
|
34
|
+
if (ageInDays < 30)
|
|
35
|
+
return `${Math.floor(ageInDays / 7)} weeks ago`;
|
|
36
|
+
if (ageInDays < 60)
|
|
37
|
+
return '1 month ago';
|
|
38
|
+
if (ageInDays < 365)
|
|
39
|
+
return `${Math.floor(ageInDays / 30)} months ago`;
|
|
40
|
+
if (ageInDays < 730)
|
|
41
|
+
return '1 year ago';
|
|
42
|
+
return `${Math.floor(ageInDays / 365)} years ago`;
|
|
43
|
+
}
|
|
44
|
+
// ─── Line formatters ──────────────────────────────────────────────────────────
|
|
45
|
+
function formatFactLine(sf) {
|
|
46
|
+
const { fact, score, ageInDays } = sf;
|
|
47
|
+
const description = `${fact.subject} ${fact.predicate} ${fact.object}`;
|
|
48
|
+
const sessionLabel = fact.sourceSessionLabel ?? fact.sourceSessionId;
|
|
49
|
+
const age = formatAge(ageInDays);
|
|
50
|
+
return `- ${description} (${score.toFixed(2)}, session: ${sessionLabel}, ${age})`;
|
|
51
|
+
}
|
|
52
|
+
function formatChunkLine(chunk) {
|
|
53
|
+
const excerpt = chunk.chunkText.slice(0, 200);
|
|
54
|
+
const sessionLabel = chunk.sessionLabel ?? chunk.sessionId;
|
|
55
|
+
const ageInDays = (Date.now() - chunk.timestamp.getTime()) / (1_000 * 60 * 60 * 24);
|
|
56
|
+
const age = formatAge(ageInDays);
|
|
57
|
+
return `- [${sessionLabel}] ${age}: "${excerpt}"`;
|
|
58
|
+
}
|
|
59
|
+
// ─── Public API ───────────────────────────────────────────────────────────────
|
|
60
|
+
/**
|
|
61
|
+
* Formats a MemoryContext into a [MEMORY CONTEXT] prompt block.
|
|
62
|
+
*
|
|
63
|
+
* Returns an empty string if the context has no facts and no raw chunks,
|
|
64
|
+
* so callers can skip injection without additional checks.
|
|
65
|
+
*/
|
|
66
|
+
export function formatContextBlock(context) {
|
|
67
|
+
const { highConfidence, mediumConfidence, lowConfidence, relatedConversations } = context;
|
|
68
|
+
const isEmpty = highConfidence.length === 0 &&
|
|
69
|
+
mediumConfidence.length === 0 &&
|
|
70
|
+
lowConfidence.length === 0 &&
|
|
71
|
+
relatedConversations.length === 0;
|
|
72
|
+
if (isEmpty)
|
|
73
|
+
return '';
|
|
74
|
+
const lines = ['[MEMORY CONTEXT]'];
|
|
75
|
+
if (highConfidence.length > 0) {
|
|
76
|
+
lines.push('');
|
|
77
|
+
lines.push('## High confidence facts');
|
|
78
|
+
for (const sf of highConfidence)
|
|
79
|
+
lines.push(formatFactLine(sf));
|
|
80
|
+
}
|
|
81
|
+
if (mediumConfidence.length > 0) {
|
|
82
|
+
lines.push('');
|
|
83
|
+
lines.push('## Medium confidence facts');
|
|
84
|
+
for (const sf of mediumConfidence)
|
|
85
|
+
lines.push(formatFactLine(sf));
|
|
86
|
+
}
|
|
87
|
+
if (lowConfidence.length > 0) {
|
|
88
|
+
lines.push('');
|
|
89
|
+
lines.push('## Low confidence facts');
|
|
90
|
+
for (const sf of lowConfidence)
|
|
91
|
+
lines.push(formatFactLine(sf));
|
|
92
|
+
}
|
|
93
|
+
if (relatedConversations.length > 0) {
|
|
94
|
+
lines.push('');
|
|
95
|
+
lines.push('## Related conversations');
|
|
96
|
+
for (const chunk of relatedConversations)
|
|
97
|
+
lines.push(formatChunkLine(chunk));
|
|
98
|
+
}
|
|
99
|
+
return lines.join('\n');
|
|
100
|
+
}
|
|
101
|
+
//# sourceMappingURL=context-builder.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"context-builder.js","sourceRoot":"","sources":["../src/context-builder.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAIH,iFAAiF;AAEjF;;;GAGG;AACH,MAAM,UAAU,SAAS,CAAC,SAAiB;IACzC,IAAI,SAAS,GAAG,CAAC;QAAE,OAAO,OAAO,CAAC;IAClC,IAAI,SAAS,GAAG,CAAC;QAAE,OAAO,WAAW,CAAC;IACtC,IAAI,SAAS,GAAG,CAAC;QAAE,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,WAAW,CAAC;IAC9D,IAAI,SAAS,GAAG,EAAE;QAAE,OAAO,YAAY,CAAC;IACxC,IAAI,SAAS,GAAG,EAAE;QAAE,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,YAAY,CAAC;IACpE,IAAI,SAAS,GAAG,EAAE;QAAE,OAAO,aAAa,CAAC;IACzC,IAAI,SAAS,GAAG,GAAG;QAAE,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,EAAE,CAAC,aAAa,CAAC;IACvE,IAAI,SAAS,GAAG,GAAG;QAAE,OAAO,YAAY,CAAC;IACzC,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,GAAG,CAAC,YAAY,CAAC;AACpD,CAAC;AAED,iFAAiF;AAEjF,SAAS,cAAc,CAAC,EAAc;IACpC,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC;IACtC,MAAM,WAAW,GAAG,GAAG,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;IACvE,MAAM,YAAY,GAAG,IAAI,CAAC,kBAAkB,IAAI,IAAI,CAAC,eAAe,CAAC;IACrE,MAAM,GAAG,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC;IACjC,OAAO,KAAK,WAAW,KAAK,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,cAAc,YAAY,KAAK,GAAG,GAAG,CAAC;AACpF,CAAC;AAED,SAAS,eAAe,CAAC,KAAe;IACtC,MAAM,OAAO,GAAG,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IAC9C,MAAM,YAAY,GAAG,KAAK,CAAC,YAAY,IAAI,KAAK,CAAC,SAAS,CAAC;IAC3D,MAAM,SAAS,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC,GAAG,CAAC,KAAK,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;IACpF,MAAM,GAAG,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC;IACjC,OAAO,MAAM,YAAY,KAAK,GAAG,MAAM,OAAO,GAAG,CAAC;AACpD,CAAC;AAED,iFAAiF;AAEjF;;;;;GAKG;AACH,MAAM,UAAU,kBAAkB,CAAC,OAAsB;IACvD,MAAM,EAAE,cAAc,EAAE,gBAAgB,EAAE,aAAa,EAAE,oBAAoB,EAAE,GAAG,OAAO,CAAC;IAE1F,MAAM,OAAO,GACX,cAAc,CAAC,MAAM,KAAK,CAAC;QAC3B,gBAAgB,CAAC,MAAM,KAAK,CAAC;QAC7B,aAAa,CAAC,MAAM,KAAK,CAAC;QAC1B,oBAAoB,CAAC,MAAM,KAAK,CAAC,CAAC;IAEpC,IAAI,OAAO;QAAE,OAAO,EAAE,CAAC;IAEvB,MAAM,KAAK,GAAa,CAAC,kBAAkB,CAAC,CAAC;IAE7C,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QACvC,KAAK,MAAM,EAAE,IAAI,cAAc;YAAE,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC;IAClE,CAAC;IAED,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAChC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC;QACzC,KAAK,MAAM,EAAE,IAAI,gBAAgB;YAAE,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC;IACpE,CAAC;IAED,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC;QACtC,KAAK,MAAM,EAAE,IAAI,aAAa;YAAE,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC;IACjE,CAAC;IAED,IAAI,oBAAoB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QACvC,KAAK,MAAM,KAAK,IAAI,oBAAoB;YAAE,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC,CAAC;IAC/E,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC"}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Embedder — wraps @xenova/transformers for local CPU inference.
|
|
3
|
+
*
|
|
4
|
+
* Models:
|
|
5
|
+
* Passage embedder: Xenova/bge-small-en-v1.5 (384-dim, normalized cosine)
|
|
6
|
+
* Cross-encoder: Xenova/ms-marco-MiniLM-L-6-v2 (relevance logit)
|
|
7
|
+
*
|
|
8
|
+
* BGE convention:
|
|
9
|
+
* - Index-time text: no prefix (raw passage)
|
|
10
|
+
* - Query-time text: "query: " prefix (improves asymmetric retrieval)
|
|
11
|
+
*
|
|
12
|
+
* First call downloads the model (~100 MB for bge-small). Subsequent calls
|
|
13
|
+
* use the local cache at ~/.cache/huggingface/hub/.
|
|
14
|
+
*/
|
|
15
|
+
/** Embedding dimension for BAAI/bge-small-en-v1.5. */
|
|
16
|
+
export declare const EMBED_DIM = 384;
|
|
17
|
+
/**
|
|
18
|
+
* Embed a passage for indexing (no query prefix).
|
|
19
|
+
* Returns a normalized Float32Array of length EMBED_DIM.
|
|
20
|
+
*/
|
|
21
|
+
export declare function embed(text: string): Promise<Float32Array>;
|
|
22
|
+
/**
|
|
23
|
+
* Embed a search query with BGE "query: " prefix.
|
|
24
|
+
* Returns a normalized Float32Array of length EMBED_DIM.
|
|
25
|
+
*/
|
|
26
|
+
export declare function embedQuery(query: string): Promise<Float32Array>;
|
|
27
|
+
/**
|
|
28
|
+
* Score (query, passage) pairs with the cross-encoder.
|
|
29
|
+
* Returns raw logits (higher = more relevant).
|
|
30
|
+
* Falls back to zeros if the reranker model is unavailable — callers
|
|
31
|
+
* should detect all-zero arrays and fall back to RRF order.
|
|
32
|
+
*/
|
|
33
|
+
export declare function rerankScores(query: string, passages: string[]): Promise<number[]>;
|
|
34
|
+
//# sourceMappingURL=embedder.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"embedder.d.ts","sourceRoot":"","sources":["../src/embedder.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAMH,sDAAsD;AACtD,eAAO,MAAM,SAAS,MAAM,CAAC;AAiB7B;;;GAGG;AACH,wBAAsB,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAI/D;AAED;;;GAGG;AACH,wBAAsB,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAIrE;AA4BD;;;;;GAKG;AACH,wBAAsB,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAiBvF"}
|
package/dist/embedder.js
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Embedder — wraps @xenova/transformers for local CPU inference.
|
|
3
|
+
*
|
|
4
|
+
* Models:
|
|
5
|
+
* Passage embedder: Xenova/bge-small-en-v1.5 (384-dim, normalized cosine)
|
|
6
|
+
* Cross-encoder: Xenova/ms-marco-MiniLM-L-6-v2 (relevance logit)
|
|
7
|
+
*
|
|
8
|
+
* BGE convention:
|
|
9
|
+
* - Index-time text: no prefix (raw passage)
|
|
10
|
+
* - Query-time text: "query: " prefix (improves asymmetric retrieval)
|
|
11
|
+
*
|
|
12
|
+
* First call downloads the model (~100 MB for bge-small). Subsequent calls
|
|
13
|
+
* use the local cache at ~/.cache/huggingface/hub/.
|
|
14
|
+
*/
|
|
15
|
+
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
|
16
|
+
// @ts-ignore — @xenova/transformers has incomplete typings
|
|
17
|
+
import { pipeline, env } from '@xenova/transformers';
|
|
18
|
+
/** Embedding dimension for BAAI/bge-small-en-v1.5. */
|
|
19
|
+
export const EMBED_DIM = 384;
|
|
20
|
+
// Disable the remote model check in test/offline environments to use cache.
|
|
21
|
+
// env.allowRemoteModels is already true by default; this line is a no-op but documents intent.
|
|
22
|
+
env.allowLocalModels = true;
|
|
23
|
+
let _embedPipeline = null;
|
|
24
|
+
async function getEmbedPipeline() {
|
|
25
|
+
if (_embedPipeline === null) {
|
|
26
|
+
_embedPipeline = (await pipeline('feature-extraction', 'Xenova/bge-small-en-v1.5'));
|
|
27
|
+
}
|
|
28
|
+
return _embedPipeline;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Embed a passage for indexing (no query prefix).
|
|
32
|
+
* Returns a normalized Float32Array of length EMBED_DIM.
|
|
33
|
+
*/
|
|
34
|
+
export async function embed(text) {
|
|
35
|
+
const pipe = await getEmbedPipeline();
|
|
36
|
+
const output = await pipe(text, { pooling: 'mean', normalize: true });
|
|
37
|
+
return new Float32Array(output.data);
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Embed a search query with BGE "query: " prefix.
|
|
41
|
+
* Returns a normalized Float32Array of length EMBED_DIM.
|
|
42
|
+
*/
|
|
43
|
+
export async function embedQuery(query) {
|
|
44
|
+
const pipe = await getEmbedPipeline();
|
|
45
|
+
const output = await pipe(`query: ${query}`, { pooling: 'mean', normalize: true });
|
|
46
|
+
return new Float32Array(output.data);
|
|
47
|
+
}
|
|
48
|
+
let _rerankPipeline = null;
|
|
49
|
+
let _rerankLoadFailed = false;
|
|
50
|
+
async function getRerankPipeline() {
|
|
51
|
+
if (_rerankLoadFailed)
|
|
52
|
+
return null;
|
|
53
|
+
if (_rerankPipeline === null) {
|
|
54
|
+
try {
|
|
55
|
+
_rerankPipeline = (await pipeline('text-classification', 'Xenova/ms-marco-MiniLM-L-6-v2'));
|
|
56
|
+
}
|
|
57
|
+
catch {
|
|
58
|
+
_rerankLoadFailed = true;
|
|
59
|
+
return null;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
return _rerankPipeline;
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Score (query, passage) pairs with the cross-encoder.
|
|
66
|
+
* Returns raw logits (higher = more relevant).
|
|
67
|
+
* Falls back to zeros if the reranker model is unavailable — callers
|
|
68
|
+
* should detect all-zero arrays and fall back to RRF order.
|
|
69
|
+
*/
|
|
70
|
+
export async function rerankScores(query, passages) {
|
|
71
|
+
const pipe = await getRerankPipeline();
|
|
72
|
+
if (pipe === null || passages.length === 0) {
|
|
73
|
+
return passages.map(() => 0);
|
|
74
|
+
}
|
|
75
|
+
const scores = [];
|
|
76
|
+
for (const passage of passages) {
|
|
77
|
+
try {
|
|
78
|
+
const result = await pipe([query, passage], { function_to_apply: 'none' });
|
|
79
|
+
const raw = (Array.isArray(result) ? result[0] : result);
|
|
80
|
+
scores.push(raw?.score ?? 0);
|
|
81
|
+
}
|
|
82
|
+
catch {
|
|
83
|
+
scores.push(0);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
return scores;
|
|
87
|
+
}
|
|
88
|
+
//# sourceMappingURL=embedder.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"embedder.js","sourceRoot":"","sources":["../src/embedder.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,6DAA6D;AAC7D,2DAA2D;AAC3D,OAAO,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,sBAAsB,CAAC;AAErD,sDAAsD;AACtD,MAAM,CAAC,MAAM,SAAS,GAAG,GAAG,CAAC;AAE7B,4EAA4E;AAC5E,+FAA+F;AAC9F,GAAqC,CAAC,gBAAgB,GAAG,IAAI,CAAC;AAI/D,IAAI,cAAc,GAAoB,IAAI,CAAC;AAE3C,KAAK,UAAU,gBAAgB;IAC7B,IAAI,cAAc,KAAK,IAAI,EAAE,CAAC;QAC5B,cAAc,GAAG,CAAC,MAAM,QAAQ,CAAC,oBAAoB,EAAE,0BAA0B,CAAC,CAAa,CAAC;IAClG,CAAC;IACD,OAAO,cAAc,CAAC;AACxB,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,KAAK,CAAC,IAAY;IACtC,MAAM,IAAI,GAAG,MAAM,gBAAgB,EAAE,CAAC;IACtC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACtE,OAAO,IAAI,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;AACvC,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,KAAa;IAC5C,MAAM,IAAI,GAAG,MAAM,gBAAgB,EAAE,CAAC;IACtC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,KAAK,EAAE,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACnF,OAAO,IAAI,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;AACvC,CAAC;AASD,IAAI,eAAe,GAA0B,IAAI,CAAC;AAClD,IAAI,iBAAiB,GAAG,KAAK,CAAC;AAE9B,KAAK,UAAU,iBAAiB;IAC9B,IAAI,iBAAiB;QAAE,OAAO,IAAI,CAAC;IACnC,IAAI,eAAe,KAAK,IAAI,EAAE,CAAC;QAC7B,IAAI,CAAC;YACH,eAAe,GAAG,CAAC,MAAM,QAAQ,CAC/B,qBAAqB,EACrB,+BAA+B,CAChC,CAAmB,CAAC;QACvB,CAAC;QAAC,MAAM,CAAC;YACP,iBAAiB,GAAG,IAAI,CAAC;YACzB,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IACD,OAAO,eAAe,CAAC;AACzB,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,KAAa,EAAE,QAAkB;IAClE,MAAM,IAAI,GAAG,MAAM,iBAAiB,EAAE,CAAC;IACvC,IAAI,IAAI,KAAK,IAAI,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3C,OAAO,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;IAC/B,CAAC;IAED,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,CAAC,KAAK,EAAE,OAAO,CAAC,EAAE,EAAE,iBAAiB,EAAE,MAAM,EAAE,CAAC,CAAC;YAC3E,MAAM,GAAG,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAkC,CAAC;YAC1F,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC;QAC/B,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACjB,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type { MemoryStore } from './store.js';
|
|
2
|
+
import { type Fact, type MessageExchange } from './types.js';
|
|
3
|
+
/**
|
|
4
|
+
* Extract facts from a conversation exchange via an LLM call.
|
|
5
|
+
*
|
|
6
|
+
* Makes one LLM call, parses the JSON array response, persists each fact via
|
|
7
|
+
* the provided store, and returns the stored Fact[].
|
|
8
|
+
*
|
|
9
|
+
* Dedup strategy: always insert as a new entry — never update an existing fact
|
|
10
|
+
* with the same subject+predicate. Decay scoring (T-006) handles ranking.
|
|
11
|
+
*
|
|
12
|
+
* @param exchange - The conversation exchange to extract facts from.
|
|
13
|
+
* @param userId - The user ID to scope facts to (passed to store.store()).
|
|
14
|
+
* NOTE: LocalStore captures userId at construction time, so
|
|
15
|
+
* this param is accepted here for documentation/future use
|
|
16
|
+
* but the store itself enforces the scope.
|
|
17
|
+
* @param store - The MemoryStore instance to persist facts into.
|
|
18
|
+
* @param llmFn - Optional LLM function to use (injectable for testing).
|
|
19
|
+
*/
|
|
20
|
+
export declare function extractFacts(exchange: MessageExchange, _userId: string, store: MemoryStore, llmFn?: (prompt: string) => Promise<string>): Promise<Fact[]>;
|
|
21
|
+
//# sourceMappingURL=extractor.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"extractor.d.ts","sourceRoot":"","sources":["../src/extractor.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAC9C,OAAO,EAAa,KAAK,IAAI,EAAE,KAAK,eAAe,EAAE,MAAM,YAAY,CAAC;AAmDxE;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAsB,YAAY,CAChC,QAAQ,EAAE,eAAe,EACzB,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,WAAW,EAClB,KAAK,GAAE,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAW,GACnD,OAAO,CAAC,IAAI,EAAE,CAAC,CAkCjB"}
|