@tryhamster/gerbil 1.0.0-rc.8 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/README.md +247 -84
- package/dist/architectures-C1I5V3Dt.mjs +6070 -0
- package/dist/architectures-C1I5V3Dt.mjs.map +1 -0
- package/dist/browser/index.d.ts +264 -588
- package/dist/browser/index.d.ts.map +1 -1
- package/dist/browser/index.js +585 -2334
- package/dist/browser/index.js.map +1 -1
- package/dist/cli.mjs +625 -1098
- package/dist/cli.mjs.map +1 -1
- package/dist/defaults-9komdrbY.mjs +24 -0
- package/dist/defaults-9komdrbY.mjs.map +1 -0
- package/dist/frameworks/express.d.mts +1 -3
- package/dist/frameworks/express.d.mts.map +1 -1
- package/dist/frameworks/express.mjs +7 -7
- package/dist/frameworks/express.mjs.map +1 -1
- package/dist/frameworks/fastify.d.mts +1 -1
- package/dist/frameworks/fastify.d.mts.map +1 -1
- package/dist/frameworks/fastify.mjs +3 -3
- package/dist/frameworks/fastify.mjs.map +1 -1
- package/dist/frameworks/hono.d.mts +1 -1
- package/dist/frameworks/hono.d.mts.map +1 -1
- package/dist/frameworks/hono.mjs +4 -4
- package/dist/frameworks/hono.mjs.map +1 -1
- package/dist/frameworks/next.d.mts +3 -2
- package/dist/frameworks/next.d.mts.map +1 -1
- package/dist/frameworks/next.mjs +4 -4
- package/dist/frameworks/next.mjs.map +1 -1
- package/dist/frameworks/react.d.mts +1 -1
- package/dist/frameworks/trpc.d.mts +1 -1
- package/dist/frameworks/trpc.d.mts.map +1 -1
- package/dist/frameworks/trpc.mjs +4 -4
- package/dist/frameworks/trpc.mjs.map +1 -1
- package/dist/gerbil-BHrJJIa4.mjs +1656 -0
- package/dist/gerbil-BHrJJIa4.mjs.map +1 -0
- package/dist/gerbil-BT9fCydo.d.mts +488 -0
- package/dist/gerbil-BT9fCydo.d.mts.map +1 -0
- package/dist/gerbil-DomNfIr1.mjs +4 -0
- package/dist/gpu/hooks.d.mts +520 -0
- package/dist/gpu/hooks.d.mts.map +1 -0
- package/dist/gpu/hooks.mjs +1188 -0
- package/dist/gpu/hooks.mjs.map +1 -0
- package/dist/gpu/index.d.mts +2 -0
- package/dist/gpu/index.mjs +6 -0
- package/dist/gpu-33qCAtHW.mjs +3615 -0
- package/dist/gpu-33qCAtHW.mjs.map +1 -0
- package/dist/index-Dgmb2kE3.d.mts +245 -0
- package/dist/index-Dgmb2kE3.d.mts.map +1 -0
- package/dist/index-jEAL2s-A.d.mts +2022 -0
- package/dist/index-jEAL2s-A.d.mts.map +1 -0
- package/dist/index.d.mts +22 -487
- package/dist/index.d.mts.map +1 -1
- package/dist/index.mjs +13 -8
- package/dist/index.mjs.map +1 -1
- package/dist/indexeddb-store-BWIMtxxH.mjs +103 -0
- package/dist/indexeddb-store-BWIMtxxH.mjs.map +1 -0
- package/dist/indexeddb-store-ClH12Xnl.mjs +4 -0
- package/dist/integrations/ai-sdk.d.mts +75 -6
- package/dist/integrations/ai-sdk.d.mts.map +1 -1
- package/dist/integrations/ai-sdk.mjs +131 -15
- package/dist/integrations/ai-sdk.mjs.map +1 -1
- package/dist/integrations/langchain.d.mts +1 -1
- package/dist/integrations/langchain.d.mts.map +1 -1
- package/dist/integrations/langchain.mjs +5 -5
- package/dist/integrations/langchain.mjs.map +1 -1
- package/dist/integrations/llamaindex.d.mts +1 -1
- package/dist/integrations/llamaindex.d.mts.map +1 -1
- package/dist/integrations/llamaindex.mjs +5 -5
- package/dist/integrations/llamaindex.mjs.map +1 -1
- package/dist/integrations/mcp-client.mjs +3 -3
- package/dist/integrations/mcp-client.mjs.map +1 -1
- package/dist/integrations/mcp.d.mts +3 -2
- package/dist/integrations/mcp.d.mts.map +1 -1
- package/dist/integrations/mcp.mjs +5 -5
- package/dist/{mcp-BvbriaBy.mjs → mcp-1DaMsaBc.mjs} +4 -4
- package/dist/mcp-1DaMsaBc.mjs.map +1 -0
- package/dist/memory/index.d.mts +3 -0
- package/dist/memory/index.mjs +6 -0
- package/dist/memory-D1P7Tmda.mjs +4 -0
- package/dist/memory-DVN0MnIG.mjs +132 -0
- package/dist/memory-DVN0MnIG.mjs.map +1 -0
- package/dist/memory-Dj0J1v88.mjs +294 -0
- package/dist/memory-Dj0J1v88.mjs.map +1 -0
- package/dist/moonshine-stt-BLyVoRpB.mjs +4 -0
- package/dist/moonshine-stt-v_P_Ci_m.mjs +11936 -0
- package/dist/moonshine-stt-v_P_Ci_m.mjs.map +1 -0
- package/dist/{one-liner-s-lD8rCC.mjs → one-liner-DnQn7HJK.mjs} +14 -16
- package/dist/one-liner-DnQn7HJK.mjs.map +1 -0
- package/dist/repl-jV5gcJFA.mjs +9 -0
- package/dist/skills/index.d.mts +270 -320
- package/dist/skills/index.d.mts.map +1 -1
- package/dist/skills/index.mjs +5 -5
- package/dist/{skills-CD3Orlex.mjs → skills-DX8D59UH.mjs} +187 -32
- package/dist/skills-DX8D59UH.mjs.map +1 -0
- package/dist/{tools-Bi1P7Xoy.mjs → tools-DQ1mPUw5.mjs} +34 -22
- package/dist/tools-DQ1mPUw5.mjs.map +1 -0
- package/dist/{types-CiTc7ez3.d.mts → types-D6FiR_oh.d.mts} +106 -12
- package/dist/types-D6FiR_oh.d.mts.map +1 -0
- package/dist/types-DQBe2lFo.d.mts +165 -0
- package/dist/types-DQBe2lFo.d.mts.map +1 -0
- package/dist/{utils-CZBZ8dgR.mjs → utils-DKO55ZmZ.mjs} +1 -1
- package/dist/{utils-CZBZ8dgR.mjs.map → utils-DKO55ZmZ.mjs.map} +1 -1
- package/dist/vector-B0panuy6.mjs +95 -0
- package/dist/vector-B0panuy6.mjs.map +1 -0
- package/docs/PROJECT-STATE.md +321 -0
- package/docs/adding-a-model-family.md +280 -0
- package/docs/ai-sdk.md +70 -61
- package/docs/architecture/overview.md +17 -7
- package/docs/browser.md +203 -8
- package/docs/embeddings.md +156 -0
- package/docs/gerbil-site-native-migration.md +217 -0
- package/docs/gpu-engine/architectures.md +398 -0
- package/docs/gpu-engine/ir.md +372 -0
- package/docs/gpu-engine/kernels.md +718 -0
- package/docs/gpu-engine/paper.html +1759 -0
- package/docs/gpu-engine/paper.md +2109 -0
- package/docs/gpu-engine/safetensors.md +312 -0
- package/docs/gpu-engine/tokenizer.md +302 -0
- package/docs/memory-rag.md +91 -0
- package/docs/metal-safari-intel.md +190 -0
- package/docs/mobile-failure-diagnosis.md +124 -0
- package/docs/mobile.md +99 -0
- package/docs/observability.md +230 -0
- package/docs/onnx-removal-plan.md +339 -0
- package/docs/research/autoresearch-portable.md +904 -0
- package/docs/research/dispatch-reduction-hivemind.md +84 -0
- package/docs/research/ios-safari-model-caching.md +117 -0
- package/docs/research/mobile-webgpu-speed-fusion.md +135 -0
- package/docs/research/native-stt-model-selection.md +49 -0
- package/docs/research/native-tts-model-selection.md +90 -0
- package/docs/research/native-vs-chromium-decision.md +152 -0
- package/docs/research/nemotron-mamba2-inference.md +910 -0
- package/docs/research/qwen35-multimodal.md +293 -0
- package/docs/research/qwen36-gemma4-targets.md +337 -0
- package/docs/research/sota-embedding-models.md +179 -0
- package/docs/research/sota-mobile-models-2026.md +263 -0
- package/docs/research/sota-modality-models.md +202 -0
- package/docs/research/tps-baselines.md +71 -0
- package/docs/research/webgpu-m4-reference.md +104 -0
- package/docs/site-update-plan.md +155 -0
- package/docs/structured-output.md +123 -0
- package/docs/stt.md +63 -446
- package/docs/tts.md +77 -499
- package/docs/vision.md +100 -338
- package/package.json +22 -7
- package/dist/chrome-backend-CORwaIyC.mjs +0 -1212
- package/dist/chrome-backend-CORwaIyC.mjs.map +0 -1
- package/dist/chrome-backend-DIKYoWj-.mjs +0 -3
- package/dist/gerbil-CJ3ifloF.mjs +0 -4
- package/dist/gerbil-Dw4Qj77e.mjs +0 -1631
- package/dist/gerbil-Dw4Qj77e.mjs.map +0 -1
- package/dist/gerbil-qOTe1nl2.d.mts +0 -431
- package/dist/gerbil-qOTe1nl2.d.mts.map +0 -1
- package/dist/kokoro-BNTb6egA.mjs +0 -20210
- package/dist/kokoro-BNTb6egA.mjs.map +0 -1
- package/dist/kokoro-DFRQ1OeM.js +0 -20212
- package/dist/kokoro-DFRQ1OeM.js.map +0 -1
- package/dist/mcp-BvbriaBy.mjs.map +0 -1
- package/dist/one-liner-s-lD8rCC.mjs.map +0 -1
- package/dist/repl-DveXw36T.mjs +0 -9
- package/dist/skills-CD3Orlex.mjs.map +0 -1
- package/dist/stt-CpLYbGFd.mjs +0 -433
- package/dist/stt-CpLYbGFd.mjs.map +0 -1
- package/dist/stt-DRPLEEHB.mjs +0 -3
- package/dist/stt-Te8Qz-Ay.js +0 -433
- package/dist/stt-Te8Qz-Ay.js.map +0 -1
- package/dist/tools-Bi1P7Xoy.mjs.map +0 -1
- package/dist/transformers.web-DokyH3rP.js +0 -3
- package/dist/transformers.web-M6mCnEYJ.js +0 -30382
- package/dist/transformers.web-M6mCnEYJ.js.map +0 -1
- package/dist/tts-C0xx3CtE.js +0 -724
- package/dist/tts-C0xx3CtE.js.map +0 -1
- package/dist/tts-DXgsKGCe.mjs +0 -3
- package/dist/tts-DeGANMNV.mjs +0 -730
- package/dist/tts-DeGANMNV.mjs.map +0 -1
- package/dist/types-CiTc7ez3.d.mts.map +0 -1
- /package/dist/{auto-update-S9s5-g0C.mjs → auto-update-BVaLXcDE.mjs} +0 -0
- /package/dist/{chunk-CkXuGtQK.mjs → chunk-B9cbKln6.mjs} +0 -0
- /package/dist/{microphone-DaMZFRuR.mjs → microphone-Bqmoz9_K.mjs} +0 -0
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
import { a as deserializeRecord, i as topK, o as matchesFilter, s as serializeRecord } from "./vector-B0panuy6.mjs";
|
|
2
|
+
import { dirname } from "node:path";
|
|
3
|
+
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
|
4
|
+
|
|
5
|
+
//#region src/memory/gerbil-embedder.ts
|
|
6
|
+
/**
|
|
7
|
+
* Build an {@link Embedder} from a Gerbil instance (or any object exposing a
|
|
8
|
+
* compatible `embedBatch`). Vectors are converted to {@link Float32Array}.
|
|
9
|
+
*
|
|
10
|
+
* @example
|
|
11
|
+
* ```ts
|
|
12
|
+
* import { Gerbil } from "@tryhamster/gerbil";
|
|
13
|
+
* import { createMemory, createGerbilEmbedder } from "@tryhamster/gerbil/memory";
|
|
14
|
+
*
|
|
15
|
+
* const g = new Gerbil();
|
|
16
|
+
* await g.loadModel("embeddinggemma-300m");
|
|
17
|
+
* const mem = createMemory({ embed: createGerbilEmbedder(g) });
|
|
18
|
+
* ```
|
|
19
|
+
*/
|
|
20
|
+
function createGerbilEmbedder(engine) {
|
|
21
|
+
return async (texts) => {
|
|
22
|
+
return (await engine.embedBatch(texts)).map((result) => Float32Array.from(result.vector));
|
|
23
|
+
};
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
//#endregion
|
|
27
|
+
//#region src/memory/stores/file-store.ts
|
|
28
|
+
/**
|
|
29
|
+
* File-backed JSON {@link MemoryStore} for simple Node persistence.
|
|
30
|
+
*
|
|
31
|
+
* Loads the whole corpus into memory and writes the full JSON file on every
|
|
32
|
+
* mutation (debounced via an awaited write queue). This is intentionally
|
|
33
|
+
* simple and aimed at small-to-medium corpora; see follow-ups for a
|
|
34
|
+
* SQLite/OPFS backend at scale. Node-only — do not import in browser code.
|
|
35
|
+
*/
|
|
36
|
+
const DEFAULT_K = 5;
|
|
37
|
+
/** JSON-on-disk vector store for Node. */
|
|
38
|
+
var FileStore = class {
|
|
39
|
+
path;
|
|
40
|
+
records = /* @__PURE__ */ new Map();
|
|
41
|
+
loaded = false;
|
|
42
|
+
writeQueue = Promise.resolve();
|
|
43
|
+
constructor(path$1) {
|
|
44
|
+
this.path = path$1;
|
|
45
|
+
}
|
|
46
|
+
async ensureLoaded() {
|
|
47
|
+
if (this.loaded) return;
|
|
48
|
+
try {
|
|
49
|
+
const raw = await readFile(this.path, "utf8");
|
|
50
|
+
const parsed = JSON.parse(raw);
|
|
51
|
+
for (const record of parsed.records) {
|
|
52
|
+
const hydrated = deserializeRecord(record);
|
|
53
|
+
this.records.set(hydrated.id, hydrated);
|
|
54
|
+
}
|
|
55
|
+
} catch (error) {
|
|
56
|
+
if (error.code !== "ENOENT") throw error;
|
|
57
|
+
}
|
|
58
|
+
this.loaded = true;
|
|
59
|
+
}
|
|
60
|
+
persist() {
|
|
61
|
+
const snapshot = {
|
|
62
|
+
version: 1,
|
|
63
|
+
records: [...this.records.values()].map(serializeRecord)
|
|
64
|
+
};
|
|
65
|
+
const payload = JSON.stringify(snapshot);
|
|
66
|
+
this.writeQueue = this.writeQueue.then(async () => {
|
|
67
|
+
await mkdir(dirname(this.path), { recursive: true });
|
|
68
|
+
await writeFile(this.path, payload, "utf8");
|
|
69
|
+
});
|
|
70
|
+
return this.writeQueue;
|
|
71
|
+
}
|
|
72
|
+
async add(record) {
|
|
73
|
+
await this.ensureLoaded();
|
|
74
|
+
this.records.set(record.id, record);
|
|
75
|
+
await this.persist();
|
|
76
|
+
}
|
|
77
|
+
async addMany(records) {
|
|
78
|
+
await this.ensureLoaded();
|
|
79
|
+
for (const record of records) this.records.set(record.id, record);
|
|
80
|
+
await this.persist();
|
|
81
|
+
}
|
|
82
|
+
async get(id) {
|
|
83
|
+
await this.ensureLoaded();
|
|
84
|
+
return this.records.get(id);
|
|
85
|
+
}
|
|
86
|
+
async search(queryVector, options = {}) {
|
|
87
|
+
await this.ensureLoaded();
|
|
88
|
+
const k = options.k ?? DEFAULT_K;
|
|
89
|
+
const candidates = [];
|
|
90
|
+
for (const record of this.records.values()) {
|
|
91
|
+
if (!record.embedding) continue;
|
|
92
|
+
if (!matchesFilter(record.metadata, options.filter)) continue;
|
|
93
|
+
candidates.push({
|
|
94
|
+
item: record,
|
|
95
|
+
vector: record.embedding
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
return topK(queryVector, candidates, k, options.minScore).map((entry) => ({
|
|
99
|
+
record: entry.item,
|
|
100
|
+
score: entry.score
|
|
101
|
+
}));
|
|
102
|
+
}
|
|
103
|
+
async delete(id) {
|
|
104
|
+
await this.ensureLoaded();
|
|
105
|
+
const existed = this.records.delete(id);
|
|
106
|
+
if (existed) await this.persist();
|
|
107
|
+
return existed;
|
|
108
|
+
}
|
|
109
|
+
async list(filter) {
|
|
110
|
+
await this.ensureLoaded();
|
|
111
|
+
const out = [];
|
|
112
|
+
for (const record of this.records.values()) if (matchesFilter(record.metadata, filter)) out.push(record);
|
|
113
|
+
return out;
|
|
114
|
+
}
|
|
115
|
+
async clear() {
|
|
116
|
+
await this.ensureLoaded();
|
|
117
|
+
this.records.clear();
|
|
118
|
+
await this.persist();
|
|
119
|
+
}
|
|
120
|
+
async size() {
|
|
121
|
+
await this.ensureLoaded();
|
|
122
|
+
return this.records.size;
|
|
123
|
+
}
|
|
124
|
+
};
|
|
125
|
+
/** Create a {@link FileStore} backed by `path` (JSON). */
|
|
126
|
+
function createFileStore(path$1) {
|
|
127
|
+
return new FileStore(path$1);
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
//#endregion
|
|
131
|
+
export { createFileStore as n, createGerbilEmbedder as r, FileStore as t };
|
|
132
|
+
//# sourceMappingURL=memory-DVN0MnIG.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"memory-DVN0MnIG.mjs","names":["path","snapshot: MemoryExport","candidates: { item: MemoryRecord; vector: Float32Array }[]","out: MemoryRecord[]"],"sources":["../src/memory/gerbil-embedder.ts","../src/memory/stores/file-store.ts"],"sourcesContent":["/**\n * Adapter that wires Gerbil's native embeddings into the memory module.\n *\n * The memory module only needs `(texts) => Promise<Float32Array[]>`. This\n * adapter accepts anything that exposes an `embedBatch` returning records with\n * a numeric `vector` — which covers a {@link Gerbil} instance, the one-liner\n * `embedBatch`, and the browser `useEmbedding().embedBatch`. Keeping the\n * contract structural avoids a hard dependency on the core engine types and\n * keeps the module engine-agnostic.\n */\n\nimport type { Embedder } from \"./types.js\";\n\n/** Minimal shape of a batch embedder returning `{ vector }` records. */\nexport type BatchEmbedderLike = {\n embedBatch(texts: string[]): Promise<{ vector: number[] }[]>;\n};\n\n/**\n * Build an {@link Embedder} from a Gerbil instance (or any object exposing a\n * compatible `embedBatch`). Vectors are converted to {@link Float32Array}.\n *\n * @example\n * ```ts\n * import { Gerbil } from \"@tryhamster/gerbil\";\n * import { createMemory, createGerbilEmbedder } from \"@tryhamster/gerbil/memory\";\n *\n * const g = new Gerbil();\n * await g.loadModel(\"embeddinggemma-300m\");\n * const mem = createMemory({ embed: createGerbilEmbedder(g) });\n * ```\n */\nexport function createGerbilEmbedder(engine: BatchEmbedderLike): Embedder {\n return async (texts: string[]) => {\n const results = await engine.embedBatch(texts);\n return results.map((result) => Float32Array.from(result.vector));\n };\n}\n","/**\n * File-backed JSON {@link MemoryStore} for simple Node persistence.\n *\n * Loads the whole corpus into memory and writes the full JSON file on every\n * mutation (debounced via an awaited write queue). This is intentionally\n * simple and aimed at small-to-medium corpora; see follow-ups for a\n * SQLite/OPFS backend at scale. Node-only — do not import in browser code.\n */\n\nimport { mkdir, readFile, writeFile } from \"node:fs/promises\";\nimport { dirname } from \"node:path\";\nimport { deserializeRecord, matchesFilter, serializeRecord } from \"../serialize.js\";\nimport type {\n MemoryExport,\n MemoryRecord,\n MemorySearchResult,\n MemoryStore,\n MetadataFilter,\n StoreSearchOptions,\n} from \"../types.js\";\nimport { type Scored, topK } from \"../vector.js\";\n\nconst DEFAULT_K = 5;\n\n/** JSON-on-disk vector store for Node. */\nexport class FileStore implements MemoryStore {\n private readonly path: string;\n private records = new Map<string, MemoryRecord>();\n private loaded = false;\n private writeQueue: Promise<void> = Promise.resolve();\n\n constructor(path: string) {\n this.path = path;\n }\n\n private async ensureLoaded(): Promise<void> {\n if (this.loaded) {\n return;\n }\n try {\n const raw = await readFile(this.path, \"utf8\");\n const parsed = JSON.parse(raw) as MemoryExport;\n for (const record of parsed.records) {\n const hydrated = deserializeRecord(record);\n this.records.set(hydrated.id, hydrated);\n }\n } catch (error) {\n // Missing file is fine — start empty. Re-throw anything else.\n if ((error as NodeJS.ErrnoException).code !== \"ENOENT\") {\n throw error;\n }\n }\n this.loaded = true;\n }\n\n private persist(): Promise<void> {\n const snapshot: MemoryExport = {\n version: 1,\n records: [...this.records.values()].map(serializeRecord),\n };\n const payload = JSON.stringify(snapshot);\n // Serialize writes so concurrent mutations don't interleave on disk.\n this.writeQueue = this.writeQueue.then(async () => {\n await mkdir(dirname(this.path), { recursive: true });\n await writeFile(this.path, payload, \"utf8\");\n });\n return this.writeQueue;\n }\n\n async add(record: MemoryRecord): Promise<void> {\n await this.ensureLoaded();\n this.records.set(record.id, record);\n await this.persist();\n }\n\n async addMany(records: MemoryRecord[]): Promise<void> {\n await this.ensureLoaded();\n for (const record of records) {\n this.records.set(record.id, record);\n }\n await this.persist();\n }\n\n async get(id: string): Promise<MemoryRecord | undefined> {\n await this.ensureLoaded();\n return this.records.get(id);\n }\n\n async search(\n queryVector: Float32Array,\n options: StoreSearchOptions = {},\n ): Promise<MemorySearchResult[]> {\n await this.ensureLoaded();\n const k = options.k ?? DEFAULT_K;\n const candidates: { item: MemoryRecord; vector: Float32Array }[] = [];\n for (const record of this.records.values()) {\n if (!record.embedding) {\n continue;\n }\n if (!matchesFilter(record.metadata, options.filter)) {\n continue;\n }\n candidates.push({ item: record, vector: record.embedding });\n }\n const ranked: Scored<MemoryRecord>[] = topK(queryVector, candidates, k, options.minScore);\n return ranked.map((entry) => ({ record: entry.item, score: entry.score }));\n }\n\n async delete(id: string): Promise<boolean> {\n await this.ensureLoaded();\n const existed = this.records.delete(id);\n if (existed) {\n await this.persist();\n }\n return existed;\n }\n\n async list(filter?: MetadataFilter): Promise<MemoryRecord[]> {\n await this.ensureLoaded();\n const out: MemoryRecord[] = [];\n for (const record of this.records.values()) {\n if (matchesFilter(record.metadata, filter)) {\n out.push(record);\n }\n }\n return out;\n }\n\n async clear(): Promise<void> {\n await this.ensureLoaded();\n this.records.clear();\n await this.persist();\n }\n\n async size(): Promise<number> {\n await this.ensureLoaded();\n return this.records.size;\n }\n}\n\n/** Create a {@link FileStore} backed by `path` (JSON). */\nexport function createFileStore(path: string): MemoryStore {\n return new FileStore(path);\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;AAgCA,SAAgB,qBAAqB,QAAqC;AACxE,QAAO,OAAO,UAAoB;AAEhC,UADgB,MAAM,OAAO,WAAW,MAAM,EAC/B,KAAK,WAAW,aAAa,KAAK,OAAO,OAAO,CAAC;;;;;;;;;;;;;;ACbpE,MAAM,YAAY;;AAGlB,IAAa,YAAb,MAA8C;CAC5C,AAAiB;CACjB,AAAQ,0BAAU,IAAI,KAA2B;CACjD,AAAQ,SAAS;CACjB,AAAQ,aAA4B,QAAQ,SAAS;CAErD,YAAY,QAAc;AACxB,OAAK,OAAOA;;CAGd,MAAc,eAA8B;AAC1C,MAAI,KAAK,OACP;AAEF,MAAI;GACF,MAAM,MAAM,MAAM,SAAS,KAAK,MAAM,OAAO;GAC7C,MAAM,SAAS,KAAK,MAAM,IAAI;AAC9B,QAAK,MAAM,UAAU,OAAO,SAAS;IACnC,MAAM,WAAW,kBAAkB,OAAO;AAC1C,SAAK,QAAQ,IAAI,SAAS,IAAI,SAAS;;WAElC,OAAO;AAEd,OAAK,MAAgC,SAAS,SAC5C,OAAM;;AAGV,OAAK,SAAS;;CAGhB,AAAQ,UAAyB;EAC/B,MAAMC,WAAyB;GAC7B,SAAS;GACT,SAAS,CAAC,GAAG,KAAK,QAAQ,QAAQ,CAAC,CAAC,IAAI,gBAAgB;GACzD;EACD,MAAM,UAAU,KAAK,UAAU,SAAS;AAExC,OAAK,aAAa,KAAK,WAAW,KAAK,YAAY;AACjD,SAAM,MAAM,QAAQ,KAAK,KAAK,EAAE,EAAE,WAAW,MAAM,CAAC;AACpD,SAAM,UAAU,KAAK,MAAM,SAAS,OAAO;IAC3C;AACF,SAAO,KAAK;;CAGd,MAAM,IAAI,QAAqC;AAC7C,QAAM,KAAK,cAAc;AACzB,OAAK,QAAQ,IAAI,OAAO,IAAI,OAAO;AACnC,QAAM,KAAK,SAAS;;CAGtB,MAAM,QAAQ,SAAwC;AACpD,QAAM,KAAK,cAAc;AACzB,OAAK,MAAM,UAAU,QACnB,MAAK,QAAQ,IAAI,OAAO,IAAI,OAAO;AAErC,QAAM,KAAK,SAAS;;CAGtB,MAAM,IAAI,IAA+C;AACvD,QAAM,KAAK,cAAc;AACzB,SAAO,KAAK,QAAQ,IAAI,GAAG;;CAG7B,MAAM,OACJ,aACA,UAA8B,EAAE,EACD;AAC/B,QAAM,KAAK,cAAc;EACzB,MAAM,IAAI,QAAQ,KAAK;EACvB,MAAMC,aAA6D,EAAE;AACrE,OAAK,MAAM,UAAU,KAAK,QAAQ,QAAQ,EAAE;AAC1C,OAAI,CAAC,OAAO,UACV;AAEF,OAAI,CAAC,cAAc,OAAO,UAAU,QAAQ,OAAO,CACjD;AAEF,cAAW,KAAK;IAAE,MAAM;IAAQ,QAAQ,OAAO;IAAW,CAAC;;AAG7D,SADuC,KAAK,aAAa,YAAY,GAAG,QAAQ,SAAS,CAC3E,KAAK,WAAW;GAAE,QAAQ,MAAM;GAAM,OAAO,MAAM;GAAO,EAAE;;CAG5E,MAAM,OAAO,IAA8B;AACzC,QAAM,KAAK,cAAc;EACzB,MAAM,UAAU,KAAK,QAAQ,OAAO,GAAG;AACvC,MAAI,QACF,OAAM,KAAK,SAAS;AAEtB,SAAO;;CAGT,MAAM,KAAK,QAAkD;AAC3D,QAAM,KAAK,cAAc;EACzB,MAAMC,MAAsB,EAAE;AAC9B,OAAK,MAAM,UAAU,KAAK,QAAQ,QAAQ,CACxC,KAAI,cAAc,OAAO,UAAU,OAAO,CACxC,KAAI,KAAK,OAAO;AAGpB,SAAO;;CAGT,MAAM,QAAuB;AAC3B,QAAM,KAAK,cAAc;AACzB,OAAK,QAAQ,OAAO;AACpB,QAAM,KAAK,SAAS;;CAGtB,MAAM,OAAwB;AAC5B,QAAM,KAAK,cAAc;AACzB,SAAO,KAAK,QAAQ;;;;AAKxB,SAAgB,gBAAgB,QAA2B;AACzD,QAAO,IAAI,UAAUH,OAAK"}
|
|
@@ -0,0 +1,294 @@
|
|
|
1
|
+
import { a as deserializeRecord, i as topK, o as matchesFilter, r as normalize, s as serializeRecord } from "./vector-B0panuy6.mjs";
|
|
2
|
+
|
|
3
|
+
//#region src/memory/chunking.ts
|
|
4
|
+
const DEFAULT_CHUNK_SIZE = 1e3;
|
|
5
|
+
const DEFAULT_OVERLAP = 200;
|
|
6
|
+
/**
|
|
7
|
+
* Split `text` into overlapping character windows.
|
|
8
|
+
*
|
|
9
|
+
* Chunks are `chunkSize` characters with `overlap` characters shared between
|
|
10
|
+
* consecutive chunks, which preserves context across boundaries. Whitespace
|
|
11
|
+
* is trimmed and empty chunks are dropped. Short text returns a single chunk.
|
|
12
|
+
*
|
|
13
|
+
* @throws if `overlap` is not less than `chunkSize` (would loop forever).
|
|
14
|
+
*/
|
|
15
|
+
function chunkText(text, options = {}) {
|
|
16
|
+
const chunkSize = options.chunkSize ?? DEFAULT_CHUNK_SIZE;
|
|
17
|
+
const overlap = options.overlap ?? DEFAULT_OVERLAP;
|
|
18
|
+
if (chunkSize <= 0) throw new Error("chunkSize must be greater than 0");
|
|
19
|
+
if (overlap < 0 || overlap >= chunkSize) throw new Error("overlap must be >= 0 and < chunkSize");
|
|
20
|
+
const trimmed = text.trim();
|
|
21
|
+
if (trimmed.length <= chunkSize) return trimmed.length > 0 ? [trimmed] : [];
|
|
22
|
+
const stride = chunkSize - overlap;
|
|
23
|
+
const chunks = [];
|
|
24
|
+
for (let start = 0; start < trimmed.length; start += stride) {
|
|
25
|
+
const chunk = trimmed.slice(start, start + chunkSize).trim();
|
|
26
|
+
if (chunk.length > 0) chunks.push(chunk);
|
|
27
|
+
if (start + chunkSize >= trimmed.length) break;
|
|
28
|
+
}
|
|
29
|
+
return chunks;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
//#endregion
|
|
33
|
+
//#region src/memory/redaction.ts
|
|
34
|
+
const REDACTION_PLACEHOLDER = "[REDACTED]";
|
|
35
|
+
/**
|
|
36
|
+
* Apply a {@link Redactor} to `text`.
|
|
37
|
+
*
|
|
38
|
+
* - A {@link RegExp} replaces all matches with `[REDACTED]` (the `g` flag is
|
|
39
|
+
* forced so replacement is global regardless of the supplied flags).
|
|
40
|
+
* - A function is invoked and its return value used.
|
|
41
|
+
* - `undefined` returns the text unchanged.
|
|
42
|
+
*/
|
|
43
|
+
function applyRedaction(text, redact) {
|
|
44
|
+
if (!redact) return text;
|
|
45
|
+
if (typeof redact === "function") return redact(text);
|
|
46
|
+
const global = redact.global ? redact : new RegExp(redact.source, `${redact.flags}g`);
|
|
47
|
+
return text.replace(global, REDACTION_PLACEHOLDER);
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
//#endregion
|
|
51
|
+
//#region src/memory/stores/memory-store.ts
|
|
52
|
+
/**
|
|
53
|
+
* In-memory {@link MemoryStore} — the default backend.
|
|
54
|
+
*
|
|
55
|
+
* Works in both Node and the browser. Holds records in a `Map` and performs
|
|
56
|
+
* a brute-force cosine top-k scan on search. Records are expected to carry
|
|
57
|
+
* pre-normalized embeddings (the {@link Memory} facade normalizes on insert).
|
|
58
|
+
*/
|
|
59
|
+
const DEFAULT_K = 5;
|
|
60
|
+
/** Brute-force, embedding-agnostic in-memory vector store. */
|
|
61
|
+
var InMemoryStore = class {
|
|
62
|
+
records = /* @__PURE__ */ new Map();
|
|
63
|
+
add(record) {
|
|
64
|
+
this.records.set(record.id, record);
|
|
65
|
+
return Promise.resolve();
|
|
66
|
+
}
|
|
67
|
+
addMany(records) {
|
|
68
|
+
for (const record of records) this.records.set(record.id, record);
|
|
69
|
+
return Promise.resolve();
|
|
70
|
+
}
|
|
71
|
+
get(id) {
|
|
72
|
+
return Promise.resolve(this.records.get(id));
|
|
73
|
+
}
|
|
74
|
+
search(queryVector, options = {}) {
|
|
75
|
+
const k = options.k ?? DEFAULT_K;
|
|
76
|
+
const candidates = [];
|
|
77
|
+
for (const record of this.records.values()) {
|
|
78
|
+
if (!record.embedding) continue;
|
|
79
|
+
if (!matchesFilter(record.metadata, options.filter)) continue;
|
|
80
|
+
candidates.push({
|
|
81
|
+
item: record,
|
|
82
|
+
vector: record.embedding
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
const ranked = topK(queryVector, candidates, k, options.minScore);
|
|
86
|
+
return Promise.resolve(ranked.map((entry) => ({
|
|
87
|
+
record: entry.item,
|
|
88
|
+
score: entry.score
|
|
89
|
+
})));
|
|
90
|
+
}
|
|
91
|
+
delete(id) {
|
|
92
|
+
return Promise.resolve(this.records.delete(id));
|
|
93
|
+
}
|
|
94
|
+
list(filter) {
|
|
95
|
+
const out = [];
|
|
96
|
+
for (const record of this.records.values()) if (matchesFilter(record.metadata, filter)) out.push(record);
|
|
97
|
+
return Promise.resolve(out);
|
|
98
|
+
}
|
|
99
|
+
clear() {
|
|
100
|
+
this.records.clear();
|
|
101
|
+
return Promise.resolve();
|
|
102
|
+
}
|
|
103
|
+
size() {
|
|
104
|
+
return Promise.resolve(this.records.size);
|
|
105
|
+
}
|
|
106
|
+
};
|
|
107
|
+
/** Create an {@link InMemoryStore}. */
|
|
108
|
+
function createInMemoryStore() {
|
|
109
|
+
return new InMemoryStore();
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
//#endregion
|
|
113
|
+
//#region src/memory/tokens.ts
|
|
114
|
+
/**
|
|
115
|
+
* Approximate token counting for context packing.
|
|
116
|
+
*
|
|
117
|
+
* We deliberately avoid a real tokenizer dependency here. The heuristic is
|
|
118
|
+
* the widely used "~4 characters per token" rule for English-ish text, with
|
|
119
|
+
* a small floor so very short strings still cost at least one token. This is
|
|
120
|
+
* an estimate used only for budgeting — slightly conservative is fine, since
|
|
121
|
+
* the goal is to stay under a model's context window, not to be exact.
|
|
122
|
+
*/
|
|
123
|
+
const CHARS_PER_TOKEN = 4;
|
|
124
|
+
/**
|
|
125
|
+
* Estimate the number of tokens in `text`.
|
|
126
|
+
*
|
|
127
|
+
* Uses the ~4-chars-per-token heuristic. Returns 0 for empty strings.
|
|
128
|
+
*/
|
|
129
|
+
function estimateTokens(text) {
|
|
130
|
+
if (text.length === 0) return 0;
|
|
131
|
+
return Math.max(1, Math.ceil(text.length / CHARS_PER_TOKEN));
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
//#endregion
|
|
135
|
+
//#region src/memory/memory.ts
|
|
136
|
+
/**
|
|
137
|
+
* The {@link Memory} facade: the small, obvious public API over a pluggable
|
|
138
|
+
* {@link MemoryStore} and an injected {@link Embedder}.
|
|
139
|
+
*
|
|
140
|
+
* Lifecycle of a write: redact -> (optional) chunk -> embed -> normalize ->
|
|
141
|
+
* store. Reads embed the query once and delegate cosine top-k to the store.
|
|
142
|
+
* {@link Memory.recall} adds token-budgeted context packing on top of search.
|
|
143
|
+
*/
|
|
144
|
+
const DEFAULT_RECALL_K = 20;
|
|
145
|
+
const DEFAULT_TOKEN_BUDGET = 1024;
|
|
146
|
+
const DEFAULT_SEPARATOR = "\n\n";
|
|
147
|
+
function generateId() {
|
|
148
|
+
const cryptoObj = globalThis.crypto;
|
|
149
|
+
if (cryptoObj?.randomUUID) return cryptoObj.randomUUID();
|
|
150
|
+
return `mem-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`;
|
|
151
|
+
}
|
|
152
|
+
/**
|
|
153
|
+
* On-device persistent memory with semantic recall.
|
|
154
|
+
*
|
|
155
|
+
* Construct via {@link createMemory}. Engine-agnostic: bring any
|
|
156
|
+
* {@link Embedder} (Gerbil's native embeddings by default) and any
|
|
157
|
+
* {@link MemoryStore} backend.
|
|
158
|
+
*/
|
|
159
|
+
var Memory = class {
|
|
160
|
+
embedder;
|
|
161
|
+
store;
|
|
162
|
+
redact;
|
|
163
|
+
defaultChunk;
|
|
164
|
+
constructor(options) {
|
|
165
|
+
if (typeof options.embed !== "function") throw new Error("createMemory requires an `embed` function");
|
|
166
|
+
this.embedder = options.embed;
|
|
167
|
+
this.store = options.store ?? createInMemoryStore();
|
|
168
|
+
this.redact = options.redact;
|
|
169
|
+
this.defaultChunk = options.chunk;
|
|
170
|
+
}
|
|
171
|
+
/** The underlying store, for advanced use (custom queries, swapping). */
|
|
172
|
+
get backend() {
|
|
173
|
+
return this.store;
|
|
174
|
+
}
|
|
175
|
+
/**
|
|
176
|
+
* Add text to memory. Long text can be split into overlapping chunks
|
|
177
|
+
* (one record per chunk). Returns the ids of the created record(s).
|
|
178
|
+
*/
|
|
179
|
+
async add(text, options = {}) {
|
|
180
|
+
const redacted = applyRedaction(text, this.redact);
|
|
181
|
+
const chunkConfig = this.resolveChunking(options.chunk);
|
|
182
|
+
const nonEmpty = (chunkConfig ? chunkText(redacted, chunkConfig) : [redacted]).filter((piece) => piece.length > 0);
|
|
183
|
+
if (nonEmpty.length === 0) return [];
|
|
184
|
+
if (options.id && nonEmpty.length > 1) throw new Error("Cannot use an explicit `id` when text is split into multiple chunks");
|
|
185
|
+
const vectors = await this.embedder(nonEmpty);
|
|
186
|
+
const createdAt = Date.now();
|
|
187
|
+
const metadata = options.metadata ?? {};
|
|
188
|
+
const records = nonEmpty.map((piece, index) => ({
|
|
189
|
+
id: index === 0 && options.id ? options.id : generateId(),
|
|
190
|
+
text: piece,
|
|
191
|
+
embedding: normalize(vectors[index]),
|
|
192
|
+
metadata,
|
|
193
|
+
createdAt
|
|
194
|
+
}));
|
|
195
|
+
await this.store.addMany(records);
|
|
196
|
+
return records.map((record) => record.id);
|
|
197
|
+
}
|
|
198
|
+
resolveChunking(chunk) {
|
|
199
|
+
if (chunk === true) return this.defaultChunk ?? {};
|
|
200
|
+
if (chunk && typeof chunk === "object") return chunk;
|
|
201
|
+
}
|
|
202
|
+
/** Fetch a record by id. */
|
|
203
|
+
get(id) {
|
|
204
|
+
return this.store.get(id);
|
|
205
|
+
}
|
|
206
|
+
/** Semantic search: returns the top-k records by cosine similarity. */
|
|
207
|
+
async search(query, options = {}) {
|
|
208
|
+
const [vector] = await this.embedder([query]);
|
|
209
|
+
return this.store.search(normalize(vector), {
|
|
210
|
+
k: options.k,
|
|
211
|
+
filter: options.filter,
|
|
212
|
+
minScore: options.minScore
|
|
213
|
+
});
|
|
214
|
+
}
|
|
215
|
+
/**
|
|
216
|
+
* Retrieve relevant memories and greedily pack them into a token-budgeted
|
|
217
|
+
* context block, highest-scoring first, stopping before the budget is
|
|
218
|
+
* exceeded. This is the per-turn context-rebuild step that turns the store
|
|
219
|
+
* into agent memory. Token counts are approximate (see {@link estimateTokens}).
|
|
220
|
+
*/
|
|
221
|
+
async recall(query, options = {}) {
|
|
222
|
+
const budget = options.tokenBudget ?? DEFAULT_TOKEN_BUDGET;
|
|
223
|
+
const separator = options.separator ?? DEFAULT_SEPARATOR;
|
|
224
|
+
const candidates = await this.search(query, {
|
|
225
|
+
k: options.k ?? DEFAULT_RECALL_K,
|
|
226
|
+
filter: options.filter,
|
|
227
|
+
minScore: options.minScore
|
|
228
|
+
});
|
|
229
|
+
const separatorTokens = estimateTokens(separator);
|
|
230
|
+
const packed = [];
|
|
231
|
+
let tokensUsed = 0;
|
|
232
|
+
for (const { record } of candidates) {
|
|
233
|
+
const recordTokens = estimateTokens(record.text);
|
|
234
|
+
const withSeparator = packed.length === 0 ? recordTokens : recordTokens + separatorTokens;
|
|
235
|
+
if (tokensUsed + withSeparator > budget) continue;
|
|
236
|
+
packed.push(record);
|
|
237
|
+
tokensUsed += withSeparator;
|
|
238
|
+
}
|
|
239
|
+
return {
|
|
240
|
+
context: packed.map((record) => record.text).join(separator),
|
|
241
|
+
records: packed,
|
|
242
|
+
tokensUsed
|
|
243
|
+
};
|
|
244
|
+
}
|
|
245
|
+
/** Delete a record by id. Returns `true` if it existed. */
|
|
246
|
+
delete(id) {
|
|
247
|
+
return this.store.delete(id);
|
|
248
|
+
}
|
|
249
|
+
/** List all records, optionally filtered by metadata. */
|
|
250
|
+
list(filter) {
|
|
251
|
+
return this.store.list(filter);
|
|
252
|
+
}
|
|
253
|
+
/** Remove all records. */
|
|
254
|
+
clear() {
|
|
255
|
+
return this.store.clear();
|
|
256
|
+
}
|
|
257
|
+
/** Total number of stored records. */
|
|
258
|
+
size() {
|
|
259
|
+
return this.store.size();
|
|
260
|
+
}
|
|
261
|
+
/** Export the full corpus as a JSON-serializable snapshot. */
|
|
262
|
+
async export() {
|
|
263
|
+
return {
|
|
264
|
+
version: 1,
|
|
265
|
+
records: (await this.store.list()).map(serializeRecord)
|
|
266
|
+
};
|
|
267
|
+
}
|
|
268
|
+
/**
|
|
269
|
+
* Import records from a snapshot produced by {@link Memory.export}.
|
|
270
|
+
* Existing records with the same id are overwritten.
|
|
271
|
+
*/
|
|
272
|
+
async import(snapshot) {
|
|
273
|
+
const records = snapshot.records.map(deserializeRecord);
|
|
274
|
+
await this.store.addMany(records);
|
|
275
|
+
}
|
|
276
|
+
};
|
|
277
|
+
/**
|
|
278
|
+
* Create a {@link Memory} instance.
|
|
279
|
+
*
|
|
280
|
+
* @example
|
|
281
|
+
* ```ts
|
|
282
|
+
* const mem = createMemory({ embed, store });
|
|
283
|
+
* await mem.add("Paris is the capital of France", { metadata: { topic: "geo" } });
|
|
284
|
+
* const hits = await mem.search("French capital", { k: 3 });
|
|
285
|
+
* const { context } = await mem.recall("French capital", { tokenBudget: 512 });
|
|
286
|
+
* ```
|
|
287
|
+
*/
|
|
288
|
+
function createMemory(options) {
|
|
289
|
+
return new Memory(options);
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
//#endregion
|
|
293
|
+
export { createInMemoryStore as a, InMemoryStore as i, createMemory as n, applyRedaction as o, estimateTokens as r, chunkText as s, Memory as t };
|
|
294
|
+
//# sourceMappingURL=memory-Dj0J1v88.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"memory-Dj0J1v88.mjs","names":["chunks: string[]","candidates: { item: MemoryRecord; vector: Float32Array }[]","ranked: Scored<MemoryRecord>[]","out: MemoryRecord[]","records: MemoryRecord[]","packed: MemoryRecord[]"],"sources":["../src/memory/chunking.ts","../src/memory/redaction.ts","../src/memory/stores/memory-store.ts","../src/memory/tokens.ts","../src/memory/memory.ts"],"sourcesContent":["/**\n * Document chunking: split long text into overlapping windows before\n * embedding so retrieval can target relevant passages.\n */\n\nimport type { ChunkOptions } from \"./types.js\";\n\nconst DEFAULT_CHUNK_SIZE = 1000;\nconst DEFAULT_OVERLAP = 200;\n\n/**\n * Split `text` into overlapping character windows.\n *\n * Chunks are `chunkSize` characters with `overlap` characters shared between\n * consecutive chunks, which preserves context across boundaries. Whitespace\n * is trimmed and empty chunks are dropped. Short text returns a single chunk.\n *\n * @throws if `overlap` is not less than `chunkSize` (would loop forever).\n */\nexport function chunkText(text: string, options: ChunkOptions = {}): string[] {\n const chunkSize = options.chunkSize ?? DEFAULT_CHUNK_SIZE;\n const overlap = options.overlap ?? DEFAULT_OVERLAP;\n\n if (chunkSize <= 0) {\n throw new Error(\"chunkSize must be greater than 0\");\n }\n if (overlap < 0 || overlap >= chunkSize) {\n throw new Error(\"overlap must be >= 0 and < chunkSize\");\n }\n\n const trimmed = text.trim();\n if (trimmed.length <= chunkSize) {\n return trimmed.length > 0 ? [trimmed] : [];\n }\n\n const stride = chunkSize - overlap;\n const chunks: string[] = [];\n for (let start = 0; start < trimmed.length; start += stride) {\n const chunk = trimmed.slice(start, start + chunkSize).trim();\n if (chunk.length > 0) {\n chunks.push(chunk);\n }\n if (start + chunkSize >= trimmed.length) {\n break;\n }\n }\n return chunks;\n}\n","/**\n * Privacy redaction applied to memory text on write.\n */\n\nimport type { Redactor } from \"./types.js\";\n\nconst REDACTION_PLACEHOLDER = \"[REDACTED]\";\n\n/**\n * Apply a {@link Redactor} to `text`.\n *\n * - A {@link RegExp} replaces all matches with `[REDACTED]` (the `g` flag is\n * forced so replacement is global regardless of the supplied flags).\n * - A function is invoked and its return value used.\n * - `undefined` returns the text unchanged.\n */\nexport function applyRedaction(text: string, redact?: Redactor): string {\n if (!redact) {\n return text;\n }\n if (typeof redact === \"function\") {\n return redact(text);\n }\n const global = redact.global ? redact : new RegExp(redact.source, `${redact.flags}g`);\n return text.replace(global, REDACTION_PLACEHOLDER);\n}\n","/**\n * In-memory {@link MemoryStore} — the default backend.\n *\n * Works in both Node and the browser. Holds records in a `Map` and performs\n * a brute-force cosine top-k scan on search. Records are expected to carry\n * pre-normalized embeddings (the {@link Memory} facade normalizes on insert).\n */\n\nimport { matchesFilter } from \"../serialize.js\";\nimport type {\n MemoryRecord,\n MemorySearchResult,\n MemoryStore,\n MetadataFilter,\n StoreSearchOptions,\n} from \"../types.js\";\nimport { type Scored, topK } from \"../vector.js\";\n\nconst DEFAULT_K = 5;\n\n/** Brute-force, embedding-agnostic in-memory vector store. */\nexport class InMemoryStore implements MemoryStore {\n private readonly records = new Map<string, MemoryRecord>();\n\n add(record: MemoryRecord): Promise<void> {\n this.records.set(record.id, record);\n return Promise.resolve();\n }\n\n addMany(records: MemoryRecord[]): Promise<void> {\n for (const record of records) {\n this.records.set(record.id, record);\n }\n return Promise.resolve();\n }\n\n get(id: string): Promise<MemoryRecord | undefined> {\n return Promise.resolve(this.records.get(id));\n }\n\n search(\n queryVector: Float32Array,\n options: StoreSearchOptions = {},\n ): Promise<MemorySearchResult[]> {\n const k = options.k ?? DEFAULT_K;\n const candidates: { item: MemoryRecord; vector: Float32Array }[] = [];\n for (const record of this.records.values()) {\n if (!record.embedding) {\n continue;\n }\n if (!matchesFilter(record.metadata, options.filter)) {\n continue;\n }\n candidates.push({ item: record, vector: record.embedding });\n }\n const ranked: Scored<MemoryRecord>[] = topK(queryVector, candidates, k, options.minScore);\n return Promise.resolve(ranked.map((entry) => ({ record: entry.item, score: entry.score })));\n }\n\n delete(id: string): Promise<boolean> {\n return Promise.resolve(this.records.delete(id));\n }\n\n list(filter?: MetadataFilter): Promise<MemoryRecord[]> {\n const out: MemoryRecord[] = [];\n for (const record of this.records.values()) {\n if (matchesFilter(record.metadata, filter)) {\n out.push(record);\n }\n }\n return Promise.resolve(out);\n }\n\n clear(): Promise<void> {\n this.records.clear();\n return Promise.resolve();\n }\n\n size(): Promise<number> {\n return Promise.resolve(this.records.size);\n }\n}\n\n/** Create an {@link InMemoryStore}. */\nexport function createInMemoryStore(): MemoryStore {\n return new InMemoryStore();\n}\n","/**\n * Approximate token counting for context packing.\n *\n * We deliberately avoid a real tokenizer dependency here. The heuristic is\n * the widely used \"~4 characters per token\" rule for English-ish text, with\n * a small floor so very short strings still cost at least one token. This is\n * an estimate used only for budgeting — slightly conservative is fine, since\n * the goal is to stay under a model's context window, not to be exact.\n */\n\nconst CHARS_PER_TOKEN = 4;\n\n/**\n * Estimate the number of tokens in `text`.\n *\n * Uses the ~4-chars-per-token heuristic. Returns 0 for empty strings.\n */\nexport function estimateTokens(text: string): number {\n if (text.length === 0) {\n return 0;\n }\n return Math.max(1, Math.ceil(text.length / CHARS_PER_TOKEN));\n}\n","/**\n * The {@link Memory} facade: the small, obvious public API over a pluggable\n * {@link MemoryStore} and an injected {@link Embedder}.\n *\n * Lifecycle of a write: redact -> (optional) chunk -> embed -> normalize ->\n * store. Reads embed the query once and delegate cosine top-k to the store.\n * {@link Memory.recall} adds token-budgeted context packing on top of search.\n */\n\nimport { chunkText } from \"./chunking.js\";\nimport { applyRedaction } from \"./redaction.js\";\nimport { deserializeRecord, serializeRecord } from \"./serialize.js\";\nimport { createInMemoryStore } from \"./stores/memory-store.js\";\nimport { estimateTokens } from \"./tokens.js\";\nimport type {\n AddOptions,\n ChunkOptions,\n Embedder,\n MemoryExport,\n MemoryOptions,\n MemoryRecord,\n MemorySearchResult,\n MemoryStore,\n MetadataFilter,\n RecallOptions,\n RecallResult,\n Redactor,\n SearchOptions,\n} from \"./types.js\";\nimport { normalize } from \"./vector.js\";\n\nconst DEFAULT_RECALL_K = 20;\nconst DEFAULT_TOKEN_BUDGET = 1024;\nconst DEFAULT_SEPARATOR = \"\\n\\n\";\n\nfunction generateId(): string {\n // crypto.randomUUID exists in modern Node (>=16) and browsers.\n const cryptoObj = globalThis.crypto;\n if (cryptoObj?.randomUUID) {\n return cryptoObj.randomUUID();\n }\n return `mem-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`;\n}\n\n/**\n * On-device persistent memory with semantic recall.\n *\n * Construct via {@link createMemory}. Engine-agnostic: bring any\n * {@link Embedder} (Gerbil's native embeddings by default) and any\n * {@link MemoryStore} backend.\n */\nexport class Memory {\n private readonly embedder: Embedder;\n private readonly store: MemoryStore;\n private readonly redact?: Redactor;\n private readonly defaultChunk?: ChunkOptions;\n\n constructor(options: MemoryOptions) {\n if (typeof options.embed !== \"function\") {\n throw new Error(\"createMemory requires an `embed` function\");\n }\n this.embedder = options.embed;\n this.store = options.store ?? createInMemoryStore();\n this.redact = options.redact;\n this.defaultChunk = options.chunk;\n }\n\n /** The underlying store, for advanced use (custom queries, swapping). */\n get backend(): MemoryStore {\n return this.store;\n }\n\n /**\n * Add text to memory. Long text can be split into overlapping chunks\n * (one record per chunk). Returns the ids of the created record(s).\n */\n async add(text: string, options: AddOptions = {}): Promise<string[]> {\n const redacted = applyRedaction(text, this.redact);\n\n const chunkConfig = this.resolveChunking(options.chunk);\n const pieces = chunkConfig ? chunkText(redacted, chunkConfig) : [redacted];\n const nonEmpty = pieces.filter((piece) => piece.length > 0);\n if (nonEmpty.length === 0) {\n return [];\n }\n\n if (options.id && nonEmpty.length > 1) {\n throw new Error(\"Cannot use an explicit `id` when text is split into multiple chunks\");\n }\n\n const vectors = await this.embedder(nonEmpty);\n const createdAt = Date.now();\n const metadata = options.metadata ?? {};\n const records: MemoryRecord[] = nonEmpty.map((piece, index) => ({\n id: index === 0 && options.id ? options.id : generateId(),\n text: piece,\n embedding: normalize(vectors[index]),\n metadata,\n createdAt,\n }));\n\n await this.store.addMany(records);\n return records.map((record) => record.id);\n }\n\n private resolveChunking(chunk: AddOptions[\"chunk\"]): ChunkOptions | undefined {\n if (chunk === true) {\n return this.defaultChunk ?? {};\n }\n if (chunk && typeof chunk === \"object\") {\n return chunk;\n }\n return;\n }\n\n /** Fetch a record by id. */\n get(id: string): Promise<MemoryRecord | undefined> {\n return this.store.get(id);\n }\n\n /** Semantic search: returns the top-k records by cosine similarity. */\n async search(query: string, options: SearchOptions = {}): Promise<MemorySearchResult[]> {\n const [vector] = await this.embedder([query]);\n return this.store.search(normalize(vector), {\n k: options.k,\n filter: options.filter,\n minScore: options.minScore,\n });\n }\n\n /**\n * Retrieve relevant memories and greedily pack them into a token-budgeted\n * context block, highest-scoring first, stopping before the budget is\n * exceeded. This is the per-turn context-rebuild step that turns the store\n * into agent memory. Token counts are approximate (see {@link estimateTokens}).\n */\n async recall(query: string, options: RecallOptions = {}): Promise<RecallResult> {\n const budget = options.tokenBudget ?? DEFAULT_TOKEN_BUDGET;\n const separator = options.separator ?? DEFAULT_SEPARATOR;\n const candidates = await this.search(query, {\n k: options.k ?? DEFAULT_RECALL_K,\n filter: options.filter,\n minScore: options.minScore,\n });\n\n const separatorTokens = estimateTokens(separator);\n const packed: MemoryRecord[] = [];\n let tokensUsed = 0;\n for (const { record } of candidates) {\n const recordTokens = estimateTokens(record.text);\n const withSeparator = packed.length === 0 ? recordTokens : recordTokens + separatorTokens;\n if (tokensUsed + withSeparator > budget) {\n continue; // Try smaller later candidates rather than stop outright.\n }\n packed.push(record);\n tokensUsed += withSeparator;\n }\n\n return {\n context: packed.map((record) => record.text).join(separator),\n records: packed,\n tokensUsed,\n };\n }\n\n /** Delete a record by id. Returns `true` if it existed. */\n delete(id: string): Promise<boolean> {\n return this.store.delete(id);\n }\n\n /** List all records, optionally filtered by metadata. */\n list(filter?: MetadataFilter): Promise<MemoryRecord[]> {\n return this.store.list(filter);\n }\n\n /** Remove all records. */\n clear(): Promise<void> {\n return this.store.clear();\n }\n\n /** Total number of stored records. */\n size(): Promise<number> {\n return this.store.size();\n }\n\n /** Export the full corpus as a JSON-serializable snapshot. */\n async export(): Promise<MemoryExport> {\n const records = await this.store.list();\n return {\n version: 1,\n records: records.map(serializeRecord),\n };\n }\n\n /**\n * Import records from a snapshot produced by {@link Memory.export}.\n * Existing records with the same id are overwritten.\n */\n async import(snapshot: MemoryExport): Promise<void> {\n const records = snapshot.records.map(deserializeRecord);\n await this.store.addMany(records);\n }\n}\n\n/**\n * Create a {@link Memory} instance.\n *\n * @example\n * ```ts\n * const mem = createMemory({ embed, store });\n * await mem.add(\"Paris is the capital of France\", { metadata: { topic: \"geo\" } });\n * const hits = await mem.search(\"French capital\", { k: 3 });\n * const { context } = await mem.recall(\"French capital\", { tokenBudget: 512 });\n * ```\n */\nexport function createMemory(options: MemoryOptions): Memory {\n return new Memory(options);\n}\n"],"mappings":";;;AAOA,MAAM,qBAAqB;AAC3B,MAAM,kBAAkB;;;;;;;;;;AAWxB,SAAgB,UAAU,MAAc,UAAwB,EAAE,EAAY;CAC5E,MAAM,YAAY,QAAQ,aAAa;CACvC,MAAM,UAAU,QAAQ,WAAW;AAEnC,KAAI,aAAa,EACf,OAAM,IAAI,MAAM,mCAAmC;AAErD,KAAI,UAAU,KAAK,WAAW,UAC5B,OAAM,IAAI,MAAM,uCAAuC;CAGzD,MAAM,UAAU,KAAK,MAAM;AAC3B,KAAI,QAAQ,UAAU,UACpB,QAAO,QAAQ,SAAS,IAAI,CAAC,QAAQ,GAAG,EAAE;CAG5C,MAAM,SAAS,YAAY;CAC3B,MAAMA,SAAmB,EAAE;AAC3B,MAAK,IAAI,QAAQ,GAAG,QAAQ,QAAQ,QAAQ,SAAS,QAAQ;EAC3D,MAAM,QAAQ,QAAQ,MAAM,OAAO,QAAQ,UAAU,CAAC,MAAM;AAC5D,MAAI,MAAM,SAAS,EACjB,QAAO,KAAK,MAAM;AAEpB,MAAI,QAAQ,aAAa,QAAQ,OAC/B;;AAGJ,QAAO;;;;;ACxCT,MAAM,wBAAwB;;;;;;;;;AAU9B,SAAgB,eAAe,MAAc,QAA2B;AACtE,KAAI,CAAC,OACH,QAAO;AAET,KAAI,OAAO,WAAW,WACpB,QAAO,OAAO,KAAK;CAErB,MAAM,SAAS,OAAO,SAAS,SAAS,IAAI,OAAO,OAAO,QAAQ,GAAG,OAAO,MAAM,GAAG;AACrF,QAAO,KAAK,QAAQ,QAAQ,sBAAsB;;;;;;;;;;;;ACNpD,MAAM,YAAY;;AAGlB,IAAa,gBAAb,MAAkD;CAChD,AAAiB,0BAAU,IAAI,KAA2B;CAE1D,IAAI,QAAqC;AACvC,OAAK,QAAQ,IAAI,OAAO,IAAI,OAAO;AACnC,SAAO,QAAQ,SAAS;;CAG1B,QAAQ,SAAwC;AAC9C,OAAK,MAAM,UAAU,QACnB,MAAK,QAAQ,IAAI,OAAO,IAAI,OAAO;AAErC,SAAO,QAAQ,SAAS;;CAG1B,IAAI,IAA+C;AACjD,SAAO,QAAQ,QAAQ,KAAK,QAAQ,IAAI,GAAG,CAAC;;CAG9C,OACE,aACA,UAA8B,EAAE,EACD;EAC/B,MAAM,IAAI,QAAQ,KAAK;EACvB,MAAMC,aAA6D,EAAE;AACrE,OAAK,MAAM,UAAU,KAAK,QAAQ,QAAQ,EAAE;AAC1C,OAAI,CAAC,OAAO,UACV;AAEF,OAAI,CAAC,cAAc,OAAO,UAAU,QAAQ,OAAO,CACjD;AAEF,cAAW,KAAK;IAAE,MAAM;IAAQ,QAAQ,OAAO;IAAW,CAAC;;EAE7D,MAAMC,SAAiC,KAAK,aAAa,YAAY,GAAG,QAAQ,SAAS;AACzF,SAAO,QAAQ,QAAQ,OAAO,KAAK,WAAW;GAAE,QAAQ,MAAM;GAAM,OAAO,MAAM;GAAO,EAAE,CAAC;;CAG7F,OAAO,IAA8B;AACnC,SAAO,QAAQ,QAAQ,KAAK,QAAQ,OAAO,GAAG,CAAC;;CAGjD,KAAK,QAAkD;EACrD,MAAMC,MAAsB,EAAE;AAC9B,OAAK,MAAM,UAAU,KAAK,QAAQ,QAAQ,CACxC,KAAI,cAAc,OAAO,UAAU,OAAO,CACxC,KAAI,KAAK,OAAO;AAGpB,SAAO,QAAQ,QAAQ,IAAI;;CAG7B,QAAuB;AACrB,OAAK,QAAQ,OAAO;AACpB,SAAO,QAAQ,SAAS;;CAG1B,OAAwB;AACtB,SAAO,QAAQ,QAAQ,KAAK,QAAQ,KAAK;;;;AAK7C,SAAgB,sBAAmC;AACjD,QAAO,IAAI,eAAe;;;;;;;;;;;;;;AC3E5B,MAAM,kBAAkB;;;;;;AAOxB,SAAgB,eAAe,MAAsB;AACnD,KAAI,KAAK,WAAW,EAClB,QAAO;AAET,QAAO,KAAK,IAAI,GAAG,KAAK,KAAK,KAAK,SAAS,gBAAgB,CAAC;;;;;;;;;;;;;ACU9D,MAAM,mBAAmB;AACzB,MAAM,uBAAuB;AAC7B,MAAM,oBAAoB;AAE1B,SAAS,aAAqB;CAE5B,MAAM,YAAY,WAAW;AAC7B,KAAI,WAAW,WACb,QAAO,UAAU,YAAY;AAE/B,QAAO,OAAO,KAAK,KAAK,CAAC,SAAS,GAAG,CAAC,GAAG,KAAK,QAAQ,CAAC,SAAS,GAAG,CAAC,MAAM,GAAG,GAAG;;;;;;;;;AAUlF,IAAa,SAAb,MAAoB;CAClB,AAAiB;CACjB,AAAiB;CACjB,AAAiB;CACjB,AAAiB;CAEjB,YAAY,SAAwB;AAClC,MAAI,OAAO,QAAQ,UAAU,WAC3B,OAAM,IAAI,MAAM,4CAA4C;AAE9D,OAAK,WAAW,QAAQ;AACxB,OAAK,QAAQ,QAAQ,SAAS,qBAAqB;AACnD,OAAK,SAAS,QAAQ;AACtB,OAAK,eAAe,QAAQ;;;CAI9B,IAAI,UAAuB;AACzB,SAAO,KAAK;;;;;;CAOd,MAAM,IAAI,MAAc,UAAsB,EAAE,EAAqB;EACnE,MAAM,WAAW,eAAe,MAAM,KAAK,OAAO;EAElD,MAAM,cAAc,KAAK,gBAAgB,QAAQ,MAAM;EAEvD,MAAM,YADS,cAAc,UAAU,UAAU,YAAY,GAAG,CAAC,SAAS,EAClD,QAAQ,UAAU,MAAM,SAAS,EAAE;AAC3D,MAAI,SAAS,WAAW,EACtB,QAAO,EAAE;AAGX,MAAI,QAAQ,MAAM,SAAS,SAAS,EAClC,OAAM,IAAI,MAAM,sEAAsE;EAGxF,MAAM,UAAU,MAAM,KAAK,SAAS,SAAS;EAC7C,MAAM,YAAY,KAAK,KAAK;EAC5B,MAAM,WAAW,QAAQ,YAAY,EAAE;EACvC,MAAMC,UAA0B,SAAS,KAAK,OAAO,WAAW;GAC9D,IAAI,UAAU,KAAK,QAAQ,KAAK,QAAQ,KAAK,YAAY;GACzD,MAAM;GACN,WAAW,UAAU,QAAQ,OAAO;GACpC;GACA;GACD,EAAE;AAEH,QAAM,KAAK,MAAM,QAAQ,QAAQ;AACjC,SAAO,QAAQ,KAAK,WAAW,OAAO,GAAG;;CAG3C,AAAQ,gBAAgB,OAAsD;AAC5E,MAAI,UAAU,KACZ,QAAO,KAAK,gBAAgB,EAAE;AAEhC,MAAI,SAAS,OAAO,UAAU,SAC5B,QAAO;;;CAMX,IAAI,IAA+C;AACjD,SAAO,KAAK,MAAM,IAAI,GAAG;;;CAI3B,MAAM,OAAO,OAAe,UAAyB,EAAE,EAAiC;EACtF,MAAM,CAAC,UAAU,MAAM,KAAK,SAAS,CAAC,MAAM,CAAC;AAC7C,SAAO,KAAK,MAAM,OAAO,UAAU,OAAO,EAAE;GAC1C,GAAG,QAAQ;GACX,QAAQ,QAAQ;GAChB,UAAU,QAAQ;GACnB,CAAC;;;;;;;;CASJ,MAAM,OAAO,OAAe,UAAyB,EAAE,EAAyB;EAC9E,MAAM,SAAS,QAAQ,eAAe;EACtC,MAAM,YAAY,QAAQ,aAAa;EACvC,MAAM,aAAa,MAAM,KAAK,OAAO,OAAO;GAC1C,GAAG,QAAQ,KAAK;GAChB,QAAQ,QAAQ;GAChB,UAAU,QAAQ;GACnB,CAAC;EAEF,MAAM,kBAAkB,eAAe,UAAU;EACjD,MAAMC,SAAyB,EAAE;EACjC,IAAI,aAAa;AACjB,OAAK,MAAM,EAAE,YAAY,YAAY;GACnC,MAAM,eAAe,eAAe,OAAO,KAAK;GAChD,MAAM,gBAAgB,OAAO,WAAW,IAAI,eAAe,eAAe;AAC1E,OAAI,aAAa,gBAAgB,OAC/B;AAEF,UAAO,KAAK,OAAO;AACnB,iBAAc;;AAGhB,SAAO;GACL,SAAS,OAAO,KAAK,WAAW,OAAO,KAAK,CAAC,KAAK,UAAU;GAC5D,SAAS;GACT;GACD;;;CAIH,OAAO,IAA8B;AACnC,SAAO,KAAK,MAAM,OAAO,GAAG;;;CAI9B,KAAK,QAAkD;AACrD,SAAO,KAAK,MAAM,KAAK,OAAO;;;CAIhC,QAAuB;AACrB,SAAO,KAAK,MAAM,OAAO;;;CAI3B,OAAwB;AACtB,SAAO,KAAK,MAAM,MAAM;;;CAI1B,MAAM,SAAgC;AAEpC,SAAO;GACL,SAAS;GACT,UAHc,MAAM,KAAK,MAAM,MAAM,EAGpB,IAAI,gBAAgB;GACtC;;;;;;CAOH,MAAM,OAAO,UAAuC;EAClD,MAAM,UAAU,SAAS,QAAQ,IAAI,kBAAkB;AACvD,QAAM,KAAK,MAAM,QAAQ,QAAQ;;;;;;;;;;;;;;AAerC,SAAgB,aAAa,SAAgC;AAC3D,QAAO,IAAI,OAAO,QAAQ"}
|