@mnemo-mcp/core 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +123 -0
- package/dist/duration.d.ts +8 -0
- package/dist/duration.d.ts.map +1 -0
- package/dist/duration.js +31 -0
- package/dist/duration.js.map +1 -0
- package/dist/embedder.d.ts +19 -0
- package/dist/embedder.d.ts.map +1 -0
- package/dist/embedder.js +33 -0
- package/dist/embedder.js.map +1 -0
- package/dist/index.d.ts +8 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +7 -0
- package/dist/index.js.map +1 -0
- package/dist/mnemo.d.ts +22 -0
- package/dist/mnemo.d.ts.map +1 -0
- package/dist/mnemo.js +266 -0
- package/dist/mnemo.js.map +1 -0
- package/dist/onnx-embedder.d.ts +10 -0
- package/dist/onnx-embedder.d.ts.map +1 -0
- package/dist/onnx-embedder.js +30 -0
- package/dist/onnx-embedder.js.map +1 -0
- package/dist/paths.d.ts +12 -0
- package/dist/paths.d.ts.map +1 -0
- package/dist/paths.js +24 -0
- package/dist/paths.js.map +1 -0
- package/dist/ranker.d.ts +9 -0
- package/dist/ranker.d.ts.map +1 -0
- package/dist/ranker.js +16 -0
- package/dist/ranker.js.map +1 -0
- package/dist/store.d.ts +32 -0
- package/dist/store.d.ts.map +1 -0
- package/dist/store.js +223 -0
- package/dist/store.js.map +1 -0
- package/dist/types.d.ts +99 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +2 -0
- package/dist/types.js.map +1 -0
- package/dist/vector-index.d.ts +31 -0
- package/dist/vector-index.d.ts.map +1 -0
- package/dist/vector-index.js +98 -0
- package/dist/vector-index.js.map +1 -0
- package/package.json +53 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Omer Maksuti
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
# Mnemo
|
|
2
|
+
|
|
3
|
+
> Persistent memory for Claude Code. Your AI never starts from scratch again.
|
|
4
|
+
|
|
5
|
+
[](https://github.com/omermaksutii/mnemo/actions)
|
|
6
|
+
[](LICENSE)
|
|
7
|
+
|
|
8
|
+
Mnemo gives Claude Code a brain that survives across sessions. It captures decisions, conventions, and preferences — and recalls them by meaning, on demand, with sub-100ms semantic search.
|
|
9
|
+
|
|
10
|
+
---
|
|
11
|
+
|
|
12
|
+
## Install
|
|
13
|
+
|
|
14
|
+
```bash
|
|
15
|
+
# install the CLI
|
|
16
|
+
npm install -g @mnemo-mcp/cli
|
|
17
|
+
|
|
18
|
+
# install Mnemo into Claude Code (skill + MCP server)
|
|
19
|
+
mnemo init
|
|
20
|
+
|
|
21
|
+
# (optional) also wire auto-capture hooks
|
|
22
|
+
mnemo init --with-hooks
|
|
23
|
+
|
|
24
|
+
# verify
|
|
25
|
+
mnemo doctor
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
After `mnemo init`, restart Claude Code. The `/mnemo` skill and the `mnemo_*` MCP tools are now available — Claude will use them automatically when relevant.
|
|
29
|
+
|
|
30
|
+
## Use it from the terminal too
|
|
31
|
+
|
|
32
|
+
```bash
|
|
33
|
+
mnemo remember "our API auth uses OAuth2 with refresh tokens every 30min"
|
|
34
|
+
mnemo remember --global "I prefer pnpm over npm"
|
|
35
|
+
mnemo recall "what's our auth pattern?"
|
|
36
|
+
mnemo list
|
|
37
|
+
mnemo stats
|
|
38
|
+
mnemo forget <id>
|
|
39
|
+
mnemo export --out memories.json
|
|
40
|
+
mnemo import memories.json
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
## What Mnemo solves
|
|
44
|
+
|
|
45
|
+
Claude Code forgets everything when the session ends. `CLAUDE.md` partially helps — it's a static blob loaded on every turn — but it grows unwieldy fast and can't answer "do we have a precedent for X?"
|
|
46
|
+
|
|
47
|
+
Mnemo gives you semantic, on-demand memory:
|
|
48
|
+
|
|
49
|
+
| | `CLAUDE.md` | Mnemo |
|
|
50
|
+
|---|---|---|
|
|
51
|
+
| Capacity | A handful of paragraphs before token cost hurts | Tens of thousands of memories |
|
|
52
|
+
| Retrieval | Always loaded, every turn | On demand, by meaning |
|
|
53
|
+
| Updates | You edit a file by hand | Captured automatically or via `/teach` |
|
|
54
|
+
| Cross-project | Per-project only | Project + global tiers |
|
|
55
|
+
| Forgetting | Manually delete lines | `mnemo forget <id>` or `/forget` |
|
|
56
|
+
|
|
57
|
+
## How it works
|
|
58
|
+
|
|
59
|
+
```
|
|
60
|
+
Claude Code session
|
|
61
|
+
├── /mnemo skill ← teaches Claude when to call the tools
|
|
62
|
+
├── @mnemo-mcp/server server ← exposes recall/remember/forget/list/stats
|
|
63
|
+
└── @mnemo-mcp/cli hooks ← session-start, pre-task, post-edit auto-wiring
|
|
64
|
+
│
|
|
65
|
+
▼
|
|
66
|
+
@mnemo-mcp/core
|
|
67
|
+
├── ONNX all-MiniLM-L6-v2 (384-dim embeddings, ~25MB, lazy)
|
|
68
|
+
├── HNSW vector index (sub-100ms recall at 50k memories)
|
|
69
|
+
└── sql.js (WASM SQLite) (~/.mnemo/memory.db)
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
Everything is local-first. No daemon. No cloud account. No telemetry.
|
|
73
|
+
|
|
74
|
+
## Auto-capture rules
|
|
75
|
+
|
|
76
|
+
By default the post-edit hook captures any change to:
|
|
77
|
+
|
|
78
|
+
- `CLAUDE.md`, `AGENTS.md`, `GEMINI.md`
|
|
79
|
+
- `*.adr.md`
|
|
80
|
+
- `docs/decisions/**`
|
|
81
|
+
- `docs/adr/**`
|
|
82
|
+
|
|
83
|
+
Anything you write to those files becomes a project memory automatically. Disable with `mnemo init` (no `--with-hooks`).
|
|
84
|
+
|
|
85
|
+
## Recall scoring
|
|
86
|
+
|
|
87
|
+
```
|
|
88
|
+
score = 0.7 × cosine_similarity + 0.2 × recency + 0.1 × access_boost
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
Recency decays with a 30-day half-life. Access boost saturates at 20 reads.
|
|
92
|
+
|
|
93
|
+
## Layout
|
|
94
|
+
|
|
95
|
+
```
|
|
96
|
+
@mnemo-mcp/core — pure TS memory engine
|
|
97
|
+
@mnemo-mcp/server — MCP server (drop into Claude Code)
|
|
98
|
+
@mnemo-mcp/cli — terminal commands + hook handlers + init
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
## Roadmap
|
|
102
|
+
|
|
103
|
+
- ✅ **v0.1** — core engine + CLI
|
|
104
|
+
- ✅ **v0.2** — MCP server (5 tools)
|
|
105
|
+
- ✅ **v0.3** — hooks (session-start, pre-task, post-edit)
|
|
106
|
+
- ✅ **v0.4** — `/mnemo` skill
|
|
107
|
+
- ✅ **v1.0** — `mnemo init`, beefier doctor, CI, polish
|
|
108
|
+
- ⏳ **v1.1** — team mode (git-synced shared memory)
|
|
109
|
+
- ⏳ **v2.0** — opt-in hosted sync, web UI
|
|
110
|
+
|
|
111
|
+
## Development
|
|
112
|
+
|
|
113
|
+
```bash
|
|
114
|
+
npm install
|
|
115
|
+
npm test # fast (44 tests, hash embedder)
|
|
116
|
+
MNEMO_TEST_ONNX=1 MNEMO_E2E=1 npm test # full incl. real ONNX (~5s)
|
|
117
|
+
npm run build
|
|
118
|
+
npm run lint
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
## License
|
|
122
|
+
|
|
123
|
+
MIT — see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Parse a duration string like "30d", "12h", "45m", "10s", "2w" into milliseconds.
|
|
3
|
+
* Returns null if the input is invalid.
|
|
4
|
+
*/
|
|
5
|
+
export declare function parseDuration(input: string): number | null;
|
|
6
|
+
export declare function expiresAtFromTtl(ttl: string | undefined, now?: number): number | null;
|
|
7
|
+
export declare function sinceFromAgo(ago: string | undefined, now?: number): number | undefined;
|
|
8
|
+
//# sourceMappingURL=duration.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"duration.d.ts","sourceRoot":"","sources":["../src/duration.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,wBAAgB,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAY1D;AAED,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,EAAE,GAAG,SAAa,GAAG,MAAM,GAAG,IAAI,CAIzF;AAED,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,EAAE,GAAG,SAAa,GAAG,MAAM,GAAG,SAAS,CAI1F"}
|
package/dist/duration.js
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Parse a duration string like "30d", "12h", "45m", "10s", "2w" into milliseconds.
|
|
3
|
+
* Returns null if the input is invalid.
|
|
4
|
+
*/
|
|
5
|
+
export function parseDuration(input) {
|
|
6
|
+
const m = /^(\d+)(s|m|h|d|w)$/i.exec(input.trim());
|
|
7
|
+
if (!m)
|
|
8
|
+
return null;
|
|
9
|
+
const n = Number(m[1]);
|
|
10
|
+
switch (m[2].toLowerCase()) {
|
|
11
|
+
case 's': return n * 1000;
|
|
12
|
+
case 'm': return n * 60_000;
|
|
13
|
+
case 'h': return n * 3_600_000;
|
|
14
|
+
case 'd': return n * 86_400_000;
|
|
15
|
+
case 'w': return n * 7 * 86_400_000;
|
|
16
|
+
}
|
|
17
|
+
return null;
|
|
18
|
+
}
|
|
19
|
+
export function expiresAtFromTtl(ttl, now = Date.now()) {
|
|
20
|
+
if (!ttl)
|
|
21
|
+
return null;
|
|
22
|
+
const ms = parseDuration(ttl);
|
|
23
|
+
return ms === null ? null : now + ms;
|
|
24
|
+
}
|
|
25
|
+
export function sinceFromAgo(ago, now = Date.now()) {
|
|
26
|
+
if (!ago)
|
|
27
|
+
return undefined;
|
|
28
|
+
const ms = parseDuration(ago);
|
|
29
|
+
return ms === null ? undefined : now - ms;
|
|
30
|
+
}
|
|
31
|
+
//# sourceMappingURL=duration.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"duration.js","sourceRoot":"","sources":["../src/duration.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,KAAa;IACzC,MAAM,CAAC,GAAG,qBAAqB,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;IACnD,IAAI,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IACpB,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACvB,QAAQ,CAAC,CAAC,CAAC,CAAE,CAAC,WAAW,EAAE,EAAE,CAAC;QAC5B,KAAK,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC;QAC1B,KAAK,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,MAAM,CAAC;QAC5B,KAAK,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,SAAS,CAAC;QAC/B,KAAK,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,UAAU,CAAC;QAChC,KAAK,GAAG,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,UAAU,CAAC;IACtC,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,GAAuB,EAAE,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE;IACxE,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IACtB,MAAM,EAAE,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;IAC9B,OAAO,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,GAAG,EAAE,CAAC;AACvC,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,GAAuB,EAAE,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE;IACpE,IAAI,CAAC,GAAG;QAAE,OAAO,SAAS,CAAC;IAC3B,MAAM,EAAE,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;IAC9B,OAAO,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,GAAG,EAAE,CAAC;AAC5C,CAAC"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
export interface Embedder {
|
|
2
|
+
readonly dimension: number;
|
|
3
|
+
embed(text: string): Promise<Float32Array>;
|
|
4
|
+
embedBatch(texts: string[]): Promise<Float32Array[]>;
|
|
5
|
+
}
|
|
6
|
+
export type HashEmbedderOpts = {
|
|
7
|
+
dimension?: number;
|
|
8
|
+
};
|
|
9
|
+
/**
|
|
10
|
+
* Deterministic, dependency-free embedder for tests.
|
|
11
|
+
* Real semantic similarity is provided by OnnxEmbedder.
|
|
12
|
+
*/
|
|
13
|
+
export declare class HashEmbedder implements Embedder {
|
|
14
|
+
readonly dimension: number;
|
|
15
|
+
constructor(opts?: HashEmbedderOpts);
|
|
16
|
+
embed(text: string): Promise<Float32Array>;
|
|
17
|
+
embedBatch(texts: string[]): Promise<Float32Array[]>;
|
|
18
|
+
}
|
|
19
|
+
//# sourceMappingURL=embedder.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"embedder.d.ts","sourceRoot":"","sources":["../src/embedder.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,QAAQ;IACvB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;IAC3C,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC;CACtD;AAED,MAAM,MAAM,gBAAgB,GAAG;IAC7B,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF;;;GAGG;AACH,qBAAa,YAAa,YAAW,QAAQ;IAC3C,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;gBAEf,IAAI,GAAE,gBAAqB;IAIjC,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IAiB1C,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;CAG3D"}
|
package/dist/embedder.js
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
/**
|
|
3
|
+
* Deterministic, dependency-free embedder for tests.
|
|
4
|
+
* Real semantic similarity is provided by OnnxEmbedder.
|
|
5
|
+
*/
|
|
6
|
+
export class HashEmbedder {
|
|
7
|
+
dimension;
|
|
8
|
+
constructor(opts = {}) {
|
|
9
|
+
this.dimension = opts.dimension ?? 384;
|
|
10
|
+
}
|
|
11
|
+
async embed(text) {
|
|
12
|
+
const out = new Float32Array(this.dimension);
|
|
13
|
+
const tokens = text.toLowerCase().split(/\W+/).filter(Boolean);
|
|
14
|
+
for (const tok of tokens) {
|
|
15
|
+
const h = createHash('sha256').update(tok).digest();
|
|
16
|
+
for (let i = 0; i < this.dimension; i++) {
|
|
17
|
+
const byte = h[i % h.length] ?? 0;
|
|
18
|
+
out[i] += (byte / 255) * 2 - 1;
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
let mag = 0;
|
|
22
|
+
for (let i = 0; i < out.length; i++)
|
|
23
|
+
mag += out[i] * out[i];
|
|
24
|
+
mag = Math.sqrt(mag) || 1;
|
|
25
|
+
for (let i = 0; i < out.length; i++)
|
|
26
|
+
out[i] = out[i] / mag;
|
|
27
|
+
return out;
|
|
28
|
+
}
|
|
29
|
+
async embedBatch(texts) {
|
|
30
|
+
return Promise.all(texts.map(t => this.embed(t)));
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
//# sourceMappingURL=embedder.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"embedder.js","sourceRoot":"","sources":["../src/embedder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAYzC;;;GAGG;AACH,MAAM,OAAO,YAAY;IACd,SAAS,CAAS;IAE3B,YAAY,OAAyB,EAAE;QACrC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,GAAG,CAAC;IACzC,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY;QACtB,MAAM,GAAG,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC7C,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC/D,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;YACzB,MAAM,CAAC,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC;YACpD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;gBACxC,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBAClC,GAAG,CAAC,CAAC,CAAE,IAAI,CAAC,IAAI,GAAG,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAClC,CAAC;QACH,CAAC;QACD,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE;YAAE,GAAG,IAAI,GAAG,CAAC,CAAC,CAAE,GAAG,GAAG,CAAC,CAAC,CAAE,CAAC;QAC9D,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC1B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE;YAAE,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAE,GAAG,GAAG,CAAC;QAC5D,OAAO,GAAG,CAAC;IACb,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,KAAe;QAC9B,OAAO,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACpD,CAAC;CACF"}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export declare const VERSION = "1.0.0";
|
|
2
|
+
export * from './types.js';
|
|
3
|
+
export { Mnemo } from './mnemo.js';
|
|
4
|
+
export { HashEmbedder } from './embedder.js';
|
|
5
|
+
export type { Embedder } from './embedder.js';
|
|
6
|
+
export { resolveDataDir, projectHashOf, paths } from './paths.js';
|
|
7
|
+
export { parseDuration, expiresAtFromTtl, sinceFromAgo } from './duration.js';
|
|
8
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,OAAO,UAAU,CAAC;AAC/B,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,YAAY,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAClE,OAAO,EAAE,aAAa,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
export const VERSION = '1.0.0';
|
|
2
|
+
export * from './types.js';
|
|
3
|
+
export { Mnemo } from './mnemo.js';
|
|
4
|
+
export { HashEmbedder } from './embedder.js';
|
|
5
|
+
export { resolveDataDir, projectHashOf, paths } from './paths.js';
|
|
6
|
+
export { parseDuration, expiresAtFromTtl, sinceFromAgo } from './duration.js';
|
|
7
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,OAAO,GAAG,OAAO,CAAC;AAC/B,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAE7C,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAClE,OAAO,EAAE,aAAa,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC"}
|
package/dist/mnemo.d.ts
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import type { CaptureInput, ListFilter, MemoryHit, MemoryRecord, MnemoOpts, MnemoStats, PruneOpts, PruneResult, RecallOpts, UpdateInput } from './types.js';
|
|
2
|
+
export declare class Mnemo {
|
|
3
|
+
private store;
|
|
4
|
+
private index;
|
|
5
|
+
private embedder;
|
|
6
|
+
private dataDir;
|
|
7
|
+
private constructor();
|
|
8
|
+
static open(opts?: MnemoOpts): Promise<Mnemo>;
|
|
9
|
+
/** True when the most recent `capture()` updated an existing memory instead of inserting a new one. */
|
|
10
|
+
lastCaptureDeduped: boolean;
|
|
11
|
+
capture(input: CaptureInput): Promise<MemoryRecord>;
|
|
12
|
+
recall(query: string, opts?: RecallOpts): Promise<MemoryHit[]>;
|
|
13
|
+
forget(id: string): Promise<void>;
|
|
14
|
+
update(id: string, fields: UpdateInput): Promise<MemoryRecord | null>;
|
|
15
|
+
list(filter?: ListFilter): Promise<MemoryRecord[]>;
|
|
16
|
+
stats(): Promise<MnemoStats>;
|
|
17
|
+
export(): Promise<MemoryRecord[]>;
|
|
18
|
+
import(records: MemoryRecord[]): Promise<void>;
|
|
19
|
+
prune(opts?: PruneOpts): Promise<PruneResult>;
|
|
20
|
+
close(): Promise<void>;
|
|
21
|
+
}
|
|
22
|
+
//# sourceMappingURL=mnemo.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"mnemo.d.ts","sourceRoot":"","sources":["../src/mnemo.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EACV,YAAY,EACZ,UAAU,EACV,SAAS,EACT,YAAY,EACZ,SAAS,EACT,UAAU,EACV,SAAS,EACT,WAAW,EACX,UAAU,EACV,WAAW,EACZ,MAAM,YAAY,CAAC;AAGpB,qBAAa,KAAK;IAEd,OAAO,CAAC,KAAK;IACb,OAAO,CAAC,KAAK;IACb,OAAO,CAAC,QAAQ;IAChB,OAAO,CAAC,OAAO;IAJjB,OAAO;WAOM,IAAI,CAAC,IAAI,GAAE,SAAc,GAAG,OAAO,CAAC,KAAK,CAAC;IAiCvD,uGAAuG;IACvG,kBAAkB,UAAS;IAErB,OAAO,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;IAkDnD,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,GAAE,UAAe,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;IA8BlE,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAMjC,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC;IAWrE,IAAI,CAAC,MAAM,GAAE,UAAe,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IAItD,KAAK,IAAI,OAAO,CAAC,UAAU,CAAC;IAiB5B,MAAM,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;IAIjC,MAAM,CAAC,OAAO,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAa9C,KAAK,CAAC,IAAI,GAAE,SAAc,GAAG,OAAO,CAAC,WAAW,CAAC;IAqEjD,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAI7B"}
|
package/dist/mnemo.js
ADDED
|
@@ -0,0 +1,266 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
2
|
+
import { Store } from './store.js';
|
|
3
|
+
import { VectorIndex } from './vector-index.js';
|
|
4
|
+
import { HashEmbedder } from './embedder.js';
|
|
5
|
+
import { score } from './ranker.js';
|
|
6
|
+
import { paths, resolveDataDir } from './paths.js';
|
|
7
|
+
import { stat } from 'node:fs/promises';
|
|
8
|
+
export class Mnemo {
|
|
9
|
+
store;
|
|
10
|
+
index;
|
|
11
|
+
embedder;
|
|
12
|
+
dataDir;
|
|
13
|
+
constructor(store, index, embedder, dataDir) {
|
|
14
|
+
this.store = store;
|
|
15
|
+
this.index = index;
|
|
16
|
+
this.embedder = embedder;
|
|
17
|
+
this.dataDir = dataDir;
|
|
18
|
+
}
|
|
19
|
+
static async open(opts = {}) {
|
|
20
|
+
const dataDir = resolveDataDir(opts.dataDir);
|
|
21
|
+
const p = paths(dataDir);
|
|
22
|
+
let embedder;
|
|
23
|
+
if (opts.embedderType === 'onnx') {
|
|
24
|
+
const { OnnxEmbedder } = await import('./onnx-embedder.js');
|
|
25
|
+
embedder = await OnnxEmbedder.load(p.modelDir);
|
|
26
|
+
}
|
|
27
|
+
else {
|
|
28
|
+
embedder = new HashEmbedder({ dimension: 384 });
|
|
29
|
+
}
|
|
30
|
+
const store = await Store.open(p.dbFile);
|
|
31
|
+
const index = await VectorIndex.open({
|
|
32
|
+
path: p.indexFile,
|
|
33
|
+
dimension: embedder.dimension,
|
|
34
|
+
maxElements: 100_000,
|
|
35
|
+
});
|
|
36
|
+
// Rebuild missing index entries from store (covers cold start where the
|
|
37
|
+
// index was deleted but the database survived).
|
|
38
|
+
const all = await store.list({ includeExpired: true });
|
|
39
|
+
if (index.size() < all.length) {
|
|
40
|
+
for (const rec of all) {
|
|
41
|
+
const v = await embedder.embed(rec.content);
|
|
42
|
+
await index.add(rec.id, v);
|
|
43
|
+
}
|
|
44
|
+
await index.save();
|
|
45
|
+
}
|
|
46
|
+
return new Mnemo(store, index, embedder, dataDir);
|
|
47
|
+
}
|
|
48
|
+
/** True when the most recent `capture()` updated an existing memory instead of inserting a new one. */
|
|
49
|
+
lastCaptureDeduped = false;
|
|
50
|
+
async capture(input) {
|
|
51
|
+
const dedupThreshold = input.dedupThreshold ?? 0.95;
|
|
52
|
+
const v = await this.embedder.embed(input.content);
|
|
53
|
+
this.lastCaptureDeduped = false;
|
|
54
|
+
if (dedupThreshold > 0 && this.index.size() > 0) {
|
|
55
|
+
const candidates = await this.index.query(v, 5);
|
|
56
|
+
for (const cand of candidates) {
|
|
57
|
+
if (cand.similarity < dedupThreshold)
|
|
58
|
+
break;
|
|
59
|
+
const existing = await this.store.get(cand.id);
|
|
60
|
+
if (!existing)
|
|
61
|
+
continue;
|
|
62
|
+
const sameScope = existing.scope === (input.scope ?? 'project') &&
|
|
63
|
+
existing.projectHash === (input.projectHash ?? null);
|
|
64
|
+
if (!sameScope)
|
|
65
|
+
continue;
|
|
66
|
+
const updated = await this.store.update(existing.id, {
|
|
67
|
+
content: input.content,
|
|
68
|
+
tags: dedupeTags([...(existing.tags ?? []), ...(input.tags ?? [])]),
|
|
69
|
+
expiresAt: input.expiresAt !== undefined ? input.expiresAt : existing.expiresAt,
|
|
70
|
+
});
|
|
71
|
+
if (updated) {
|
|
72
|
+
const v2 = await this.embedder.embed(updated.content);
|
|
73
|
+
await this.index.add(updated.id, v2);
|
|
74
|
+
await this.index.save();
|
|
75
|
+
this.lastCaptureDeduped = true;
|
|
76
|
+
return updated;
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
const now = Date.now();
|
|
81
|
+
const rec = {
|
|
82
|
+
id: randomUUID(),
|
|
83
|
+
scope: input.scope ?? 'project',
|
|
84
|
+
projectHash: input.projectHash ?? null,
|
|
85
|
+
source: input.source ?? 'manual',
|
|
86
|
+
content: input.content,
|
|
87
|
+
tags: input.tags ?? [],
|
|
88
|
+
createdAt: now,
|
|
89
|
+
updatedAt: now,
|
|
90
|
+
accessCount: 0,
|
|
91
|
+
lastAccessedAt: now,
|
|
92
|
+
expiresAt: input.expiresAt ?? null,
|
|
93
|
+
};
|
|
94
|
+
await this.store.upsert(rec);
|
|
95
|
+
await this.index.add(rec.id, v);
|
|
96
|
+
await this.index.save();
|
|
97
|
+
return rec;
|
|
98
|
+
}
|
|
99
|
+
async recall(query, opts = {}) {
|
|
100
|
+
const k = opts.k ?? 10;
|
|
101
|
+
const minScore = opts.minScore ?? 0;
|
|
102
|
+
const v = await this.embedder.embed(query);
|
|
103
|
+
const candidates = await this.index.query(v, Math.max(k * 4, 20));
|
|
104
|
+
const requiredTags = opts.tags ?? [];
|
|
105
|
+
const sources = opts.source ? (Array.isArray(opts.source) ? opts.source : [opts.source]) : null;
|
|
106
|
+
const since = opts.since ?? 0;
|
|
107
|
+
const now = Date.now();
|
|
108
|
+
const out = [];
|
|
109
|
+
for (const cand of candidates) {
|
|
110
|
+
const rec = await this.store.get(cand.id);
|
|
111
|
+
if (!rec)
|
|
112
|
+
continue;
|
|
113
|
+
if (opts.scope && opts.scope !== 'all' && rec.scope !== opts.scope)
|
|
114
|
+
continue;
|
|
115
|
+
if (opts.projectHash !== undefined && rec.projectHash !== opts.projectHash)
|
|
116
|
+
continue;
|
|
117
|
+
if (!opts.includeExpired && rec.expiresAt !== null && rec.expiresAt <= now)
|
|
118
|
+
continue;
|
|
119
|
+
if (since && rec.updatedAt < since)
|
|
120
|
+
continue;
|
|
121
|
+
if (sources && !sources.includes(rec.source))
|
|
122
|
+
continue;
|
|
123
|
+
if (requiredTags.length && !requiredTags.every(t => rec.tags.includes(t)))
|
|
124
|
+
continue;
|
|
125
|
+
const s = score(cand.similarity, rec);
|
|
126
|
+
if (s < minScore)
|
|
127
|
+
continue;
|
|
128
|
+
out.push({ record: rec, score: s, similarity: cand.similarity });
|
|
129
|
+
}
|
|
130
|
+
out.sort((a, b) => b.score - a.score);
|
|
131
|
+
const top = out.slice(0, k);
|
|
132
|
+
for (const hit of top)
|
|
133
|
+
await this.store.bumpAccess(hit.record.id);
|
|
134
|
+
return top;
|
|
135
|
+
}
|
|
136
|
+
async forget(id) {
|
|
137
|
+
await this.store.delete(id);
|
|
138
|
+
await this.index.remove(id);
|
|
139
|
+
await this.index.save();
|
|
140
|
+
}
|
|
141
|
+
async update(id, fields) {
|
|
142
|
+
const updated = await this.store.update(id, fields);
|
|
143
|
+
if (!updated)
|
|
144
|
+
return null;
|
|
145
|
+
if (fields.content !== undefined) {
|
|
146
|
+
const v = await this.embedder.embed(updated.content);
|
|
147
|
+
await this.index.add(updated.id, v);
|
|
148
|
+
await this.index.save();
|
|
149
|
+
}
|
|
150
|
+
return updated;
|
|
151
|
+
}
|
|
152
|
+
async list(filter = {}) {
|
|
153
|
+
return this.store.list(filter);
|
|
154
|
+
}
|
|
155
|
+
async stats() {
|
|
156
|
+
const counts = await this.store.count();
|
|
157
|
+
let storageBytes = 0;
|
|
158
|
+
try {
|
|
159
|
+
const s = await stat(paths(this.dataDir).dbFile);
|
|
160
|
+
storageBytes = s.size;
|
|
161
|
+
}
|
|
162
|
+
catch { }
|
|
163
|
+
return {
|
|
164
|
+
totalMemories: counts.total,
|
|
165
|
+
byScope: counts.byScope,
|
|
166
|
+
indexSize: this.index.size(),
|
|
167
|
+
embeddingDimension: this.embedder.dimension,
|
|
168
|
+
storageBytes,
|
|
169
|
+
expired: counts.expired,
|
|
170
|
+
};
|
|
171
|
+
}
|
|
172
|
+
async export() {
|
|
173
|
+
return this.store.list({ includeExpired: true });
|
|
174
|
+
}
|
|
175
|
+
async import(records) {
|
|
176
|
+
for (const rec of records) {
|
|
177
|
+
await this.store.upsert({
|
|
178
|
+
...rec,
|
|
179
|
+
source: 'imported',
|
|
180
|
+
expiresAt: rec.expiresAt ?? null,
|
|
181
|
+
});
|
|
182
|
+
const v = await this.embedder.embed(rec.content);
|
|
183
|
+
await this.index.add(rec.id, v);
|
|
184
|
+
}
|
|
185
|
+
await this.index.save();
|
|
186
|
+
}
|
|
187
|
+
async prune(opts = {}) {
|
|
188
|
+
const expiredOn = opts.expired !== false;
|
|
189
|
+
const dupThr = opts.duplicateThreshold ?? 0.97;
|
|
190
|
+
const minAccess = opts.minAccessCount ?? 0;
|
|
191
|
+
const staleDays = opts.staleAfterDays ?? 0;
|
|
192
|
+
const dryRun = !!opts.dryRun;
|
|
193
|
+
const now = Date.now();
|
|
194
|
+
const all = await this.store.list({ includeExpired: true });
|
|
195
|
+
const expired = [];
|
|
196
|
+
const stale = [];
|
|
197
|
+
const duplicates = [];
|
|
198
|
+
if (expiredOn) {
|
|
199
|
+
for (const r of all) {
|
|
200
|
+
if (r.expiresAt !== null && r.expiresAt <= now)
|
|
201
|
+
expired.push(r);
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
if (minAccess > 0 && staleDays > 0) {
|
|
205
|
+
const cutoff = now - staleDays * 86400_000;
|
|
206
|
+
for (const r of all) {
|
|
207
|
+
if (r.accessCount < minAccess && r.updatedAt < cutoff && !expired.includes(r)) {
|
|
208
|
+
stale.push(r);
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
if (dupThr > 0) {
|
|
213
|
+
const seen = new Set();
|
|
214
|
+
// Walk by most-recently-accessed first; a more recently accessed record wins.
|
|
215
|
+
const sorted = [...all].sort((a, b) => b.lastAccessedAt - a.lastAccessedAt);
|
|
216
|
+
for (const keep of sorted) {
|
|
217
|
+
if (seen.has(keep.id) || expired.includes(keep))
|
|
218
|
+
continue;
|
|
219
|
+
const v = await this.embedder.embed(keep.content);
|
|
220
|
+
const peers = await this.index.query(v, 8);
|
|
221
|
+
for (const peer of peers) {
|
|
222
|
+
if (peer.id === keep.id)
|
|
223
|
+
continue;
|
|
224
|
+
if (peer.similarity < dupThr)
|
|
225
|
+
continue;
|
|
226
|
+
const dropCandidate = await this.store.get(peer.id);
|
|
227
|
+
if (!dropCandidate)
|
|
228
|
+
continue;
|
|
229
|
+
if (dropCandidate.scope !== keep.scope || dropCandidate.projectHash !== keep.projectHash)
|
|
230
|
+
continue;
|
|
231
|
+
if (dropCandidate.lastAccessedAt > keep.lastAccessedAt)
|
|
232
|
+
continue;
|
|
233
|
+
if (seen.has(dropCandidate.id) || expired.includes(dropCandidate))
|
|
234
|
+
continue;
|
|
235
|
+
duplicates.push({ kept: keep, dropped: dropCandidate });
|
|
236
|
+
seen.add(dropCandidate.id);
|
|
237
|
+
}
|
|
238
|
+
seen.add(keep.id);
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
let totalDeleted = 0;
|
|
242
|
+
if (!dryRun) {
|
|
243
|
+
const toDelete = new Set([
|
|
244
|
+
...expired.map(r => r.id),
|
|
245
|
+
...stale.map(r => r.id),
|
|
246
|
+
...duplicates.map(d => d.dropped.id),
|
|
247
|
+
]);
|
|
248
|
+
for (const id of toDelete) {
|
|
249
|
+
await this.store.delete(id);
|
|
250
|
+
await this.index.remove(id);
|
|
251
|
+
}
|
|
252
|
+
if (toDelete.size > 0)
|
|
253
|
+
await this.index.save();
|
|
254
|
+
totalDeleted = toDelete.size;
|
|
255
|
+
}
|
|
256
|
+
return { expired, duplicates, stale, totalDeleted };
|
|
257
|
+
}
|
|
258
|
+
async close() {
|
|
259
|
+
await this.index.save();
|
|
260
|
+
this.store.close();
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
function dedupeTags(tags) {
|
|
264
|
+
return Array.from(new Set(tags.map(t => t.trim()).filter(Boolean)));
|
|
265
|
+
}
|
|
266
|
+
//# sourceMappingURL=mnemo.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"mnemo.js","sourceRoot":"","sources":["../src/mnemo.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,YAAY,EAAiB,MAAM,eAAe,CAAC;AAC5D,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AACpC,OAAO,EAAE,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAanD,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAExC,MAAM,OAAO,KAAK;IAEN;IACA;IACA;IACA;IAJV,YACU,KAAY,EACZ,KAAkB,EAClB,QAAkB,EAClB,OAAe;QAHf,UAAK,GAAL,KAAK,CAAO;QACZ,UAAK,GAAL,KAAK,CAAa;QAClB,aAAQ,GAAR,QAAQ,CAAU;QAClB,YAAO,GAAP,OAAO,CAAQ;IACtB,CAAC;IAEJ,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,OAAkB,EAAE;QACpC,MAAM,OAAO,GAAG,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC7C,MAAM,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC;QAEzB,IAAI,QAAkB,CAAC;QACvB,IAAI,IAAI,CAAC,YAAY,KAAK,MAAM,EAAE,CAAC;YACjC,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,MAAM,CAAC,oBAAoB,CAAC,CAAC;YAC5D,QAAQ,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;QACjD,CAAC;aAAM,CAAC;YACN,QAAQ,GAAG,IAAI,YAAY,CAAC,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC;QAClD,CAAC;QAED,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QACzC,MAAM,KAAK,GAAG,MAAM,WAAW,CAAC,IAAI,CAAC;YACnC,IAAI,EAAE,CAAC,CAAC,SAAS;YACjB,SAAS,EAAE,QAAQ,CAAC,SAAS;YAC7B,WAAW,EAAE,OAAO;SACrB,CAAC,CAAC;QAEH,wEAAwE;QACxE,gDAAgD;QAChD,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC;QACvD,IAAI,KAAK,CAAC,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC;YAC9B,KAAK,MAAM,GAAG,IAAI,GAAG,EAAE,CAAC;gBACtB,MAAM,CAAC,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;gBAC5C,MAAM,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;YAC7B,CAAC;YACD,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;QACrB,CAAC;QAED,OAAO,IAAI,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IACpD,CAAC;IAED,uGAAuG;IACvG,kBAAkB,GAAG,KAAK,CAAC;IAE3B,KAAK,CAAC,OAAO,CAAC,KAAmB;QAC/B,MAAM,cAAc,GAAG,KAAK,CAAC,cAAc,IAAI,IAAI,CAAC;QACpD,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACnD,IAAI,CAAC,kBAAkB,GAAG,KAAK,CAAC;QAEhC,IAAI,cAAc,GAAG,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC;YAChD,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YAChD,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;gBAC9B,IAAI,IAAI,CAAC,UAAU,GAAG,cAAc;oBAAE,MAAM;gBAC5C,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBAC/C,IAAI,CAAC,QAAQ;oBAAE,SAAS;gBACxB,MAAM,SAAS,GACb,QAAQ,CAAC,KAAK,KAAK,CAAC,KAAK,CAAC,KAAK,IAAI,SAAS,CAAC;oBAC7C,QAAQ,CAAC,WAAW,KAAK,CAAC,KAAK,CAAC,WAAW,IAAI,IAAI,CAAC,CAAC;gBACvD,IAAI,CAAC,SAAS;oBAAE,SAAS;gBACzB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,EAAE;oBACnD,OAAO,EAAE,KAAK,CAAC,OAAO;oBACtB,IAAI,EAAE,UAAU,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC;oBACnE,SAAS,EAAE,KAAK,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,SAAS;iBAChF,CAAC,CAAC;gBACH,IAAI,OAAO,EAAE,CAAC;oBACZ,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;oBACtD,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;oBACrC,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;oBACxB,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC;oBAC/B,OAAO,OAAO,CAAC;gBACjB,CAAC;YACH,CAAC;QACH,CAAC;QAED,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,GAAG,GAAiB;YACxB,EAAE,EAAE,UAAU,EAAE;YAChB,KAAK,EAAE,KAAK,CAAC,KAAK,IAAI,SAAS;YAC/B,WAAW,EAAE,KAAK,CAAC,WAAW,IAAI,IAAI;YACtC,MAAM,EAAE,KAAK,CAAC,MAAM,IAAI,QAAQ;YAChC,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,IAAI,EAAE,KAAK,CAAC,IAAI,IAAI,EAAE;YACtB,SAAS,EAAE,GAAG;YACd,SAAS,EAAE,GAAG;YACd,WAAW,EAAE,CAAC;YACd,cAAc,EAAE,GAAG;YACnB,SAAS,EAAE,KAAK,CAAC,SAAS,IAAI,IAAI;SACnC,CAAC;QACF,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC7B,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QAChC,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QACxB,OAAO,GAAG,CAAC;IACb,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,OAAmB,EAAE;QAC/C,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;QACvB,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;QACpC,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC3C,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QAClE,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QACrC,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAChG,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC;QAC9B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,GAAG,GAAgB,EAAE,CAAC;QAE5B,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;YAC9B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC1C,IAAI,CAAC,GAAG;gBAAE,SAAS;YACnB,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,KAAK,KAAK,IAAI,GAAG,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK;gBAAE,SAAS;YAC7E,IAAI,IAAI,CAAC,WAAW,KAAK,SAAS,IAAI,GAAG,CAAC,WAAW,KAAK,IAAI,CAAC,WAAW;gBAAE,SAAS;YACrF,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,GAAG,CAAC,SAAS,KAAK,IAAI,IAAI,GAAG,CAAC,SAAS,IAAI,GAAG;gBAAE,SAAS;YACrF,IAAI,KAAK,IAAI,GAAG,CAAC,SAAS,GAAG,KAAK;gBAAE,SAAS;YAC7C,IAAI,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC;gBAAE,SAAS;YACvD,IAAI,YAAY,CAAC,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;gBAAE,SAAS;YACpF,MAAM,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;YACtC,IAAI,CAAC,GAAG,QAAQ;gBAAE,SAAS;YAC3B,GAAG,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;QACnE,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;QACtC,MAAM,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAC5B,KAAK,MAAM,GAAG,IAAI,GAAG;YAAE,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAClE,OAAO,GAAG,CAAC;IACb,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,EAAU;QACrB,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAC5B,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAC5B,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;IAC1B,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,EAAU,EAAE,MAAmB;QAC1C,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;QACpD,IAAI,CAAC,OAAO;YAAE,OAAO,IAAI,CAAC;QAC1B,IAAI,MAAM,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;YACjC,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YACrD,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;YACpC,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QAC1B,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,SAAqB,EAAE;QAChC,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACjC,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACxC,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,IAAI,CAAC;YACH,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC;YACjD,YAAY,GAAG,CAAC,CAAC,IAAI,CAAC;QACxB,CAAC;QAAC,MAAM,CAAC,CAAA,CAAC;QACV,OAAO;YACL,aAAa,EAAE,MAAM,CAAC,KAAK;YAC3B,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE;YAC5B,kBAAkB,EAAE,IAAI,CAAC,QAAQ,CAAC,SAAS;YAC3C,YAAY;YACZ,OAAO,EAAE,MAAM,CAAC,OAAO;SACxB,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,MAAM;QACV,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC;IACnD,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,OAAuB;QAClC,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;YAC1B,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;gBACtB,GAAG,GAAG;gBACN,MAAM,EAAE,UAAU;gBAClB,SAAS,EAAE,GAAG,CAAC,SAAS,IAAI,IAAI;aACjC,CAAC,CAAC;YACH,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACjD,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QAClC,CAAC;QACD,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;IAC1B,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,OAAkB,EAAE;QAC9B,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,KAAK,KAAK,CAAC;QACzC,MAAM,MAAM,GAAG,IAAI,CAAC,kBAAkB,IAAI,IAAI,CAAC;QAC/C,MAAM,SAAS,GAAG,IAAI,CAAC,cAAc,IAAI,CAAC,CAAC;QAC3C,MAAM,SAAS,GAAG,IAAI,CAAC,cAAc,IAAI,CAAC,CAAC;QAC3C,MAAM,MAAM,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;QAC7B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEvB,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5D,MAAM,OAAO,GAAmB,EAAE,CAAC;QACnC,MAAM,KAAK,GAAmB,EAAE,CAAC;QACjC,MAAM,UAAU,GAAoD,EAAE,CAAC;QAEvE,IAAI,SAAS,EAAE,CAAC;YACd,KAAK,MAAM,CAAC,IAAI,GAAG,EAAE,CAAC;gBACpB,IAAI,CAAC,CAAC,SAAS,KAAK,IAAI,IAAI,CAAC,CAAC,SAAS,IAAI,GAAG;oBAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClE,CAAC;QACH,CAAC;QAED,IAAI,SAAS,GAAG,CAAC,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;YACnC,MAAM,MAAM,GAAG,GAAG,GAAG,SAAS,GAAG,SAAS,CAAC;YAC3C,KAAK,MAAM,CAAC,IAAI,GAAG,EAAE,CAAC;gBACpB,IAAI,CAAC,CAAC,WAAW,GAAG,SAAS,IAAI,CAAC,CAAC,SAAS,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;oBAC9E,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBAChB,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;YACf,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;YAC/B,8EAA8E;YAC9E,MAAM,MAAM,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,GAAG,CAAC,CAAC,cAAc,CAAC,CAAC;YAC5E,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;gBAC1B,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC;oBAAE,SAAS;gBAC1D,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAClD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;gBAC3C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;oBACzB,IAAI,IAAI,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE;wBAAE,SAAS;oBAClC,IAAI,IAAI,CAAC,UAAU,GAAG,MAAM;wBAAE,SAAS;oBACvC,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;oBACpD,IAAI,CAAC,aAAa;wBAAE,SAAS;oBAC7B,IAAI,aAAa,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK,IAAI,aAAa,CAAC,WAAW,KAAK,IAAI,CAAC,WAAW;wBAAE,SAAS;oBACnG,IAAI,aAAa,CAAC,cAAc,GAAG,IAAI,CAAC,cAAc;wBAAE,SAAS;oBACjE,IAAI,IAAI,CAAC,GAAG,CAAC,aAAa,CAAC,EAAE,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC;wBAAE,SAAS;oBAC5E,UAAU,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC,CAAC;oBACxD,IAAI,CAAC,GAAG,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;gBAC7B,CAAC;gBACD,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACpB,CAAC;QACH,CAAC;QAED,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAS;gBAC/B,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACzB,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACvB,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;aACrC,CAAC,CAAC;YACH,KAAK,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;gBAC1B,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;gBAC5B,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YAC9B,CAAC;YACD,IAAI,QAAQ,CAAC,IAAI,GAAG,CAAC;gBAAE,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;YAC/C,YAAY,GAAG,QAAQ,CAAC,IAAI,CAAC;QAC/B,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC;IACtD,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QACxB,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;CACF;AAED,SAAS,UAAU,CAAC,IAAc;IAChC,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;AACtE,CAAC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { Embedder } from './embedder.js';
|
|
2
|
+
export declare class OnnxEmbedder implements Embedder {
|
|
3
|
+
private pipe;
|
|
4
|
+
readonly dimension = 384;
|
|
5
|
+
private constructor();
|
|
6
|
+
static load(modelCacheDir: string): Promise<OnnxEmbedder>;
|
|
7
|
+
embed(text: string): Promise<Float32Array>;
|
|
8
|
+
embedBatch(texts: string[]): Promise<Float32Array[]>;
|
|
9
|
+
}
|
|
10
|
+
//# sourceMappingURL=onnx-embedder.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"onnx-embedder.d.ts","sourceRoot":"","sources":["../src/onnx-embedder.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAS9C,qBAAa,YAAa,YAAW,QAAQ;IAGvB,OAAO,CAAC,IAAI;IAFhC,QAAQ,CAAC,SAAS,OAAO;IAEzB,OAAO;WAEM,IAAI,CAAC,aAAa,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IAYzD,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IAK1C,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;CAQ3D"}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import { mkdir } from 'node:fs/promises';
|
|
2
|
+
const MODEL_ID = 'Xenova/all-MiniLM-L6-v2';
|
|
3
|
+
export class OnnxEmbedder {
|
|
4
|
+
pipe;
|
|
5
|
+
dimension = 384;
|
|
6
|
+
constructor(pipe) {
|
|
7
|
+
this.pipe = pipe;
|
|
8
|
+
}
|
|
9
|
+
static async load(modelCacheDir) {
|
|
10
|
+
await mkdir(modelCacheDir, { recursive: true });
|
|
11
|
+
const transformers = (await import('@huggingface/transformers'));
|
|
12
|
+
transformers.env.cacheDir = modelCacheDir;
|
|
13
|
+
transformers.env.allowLocalModels = false;
|
|
14
|
+
const pipe = await transformers.pipeline('feature-extraction', MODEL_ID);
|
|
15
|
+
return new OnnxEmbedder(pipe);
|
|
16
|
+
}
|
|
17
|
+
async embed(text) {
|
|
18
|
+
const out = await this.pipe(text, { pooling: 'mean', normalize: true });
|
|
19
|
+
return new Float32Array(out.data.slice(0, this.dimension));
|
|
20
|
+
}
|
|
21
|
+
async embedBatch(texts) {
|
|
22
|
+
const out = await this.pipe(texts, { pooling: 'mean', normalize: true });
|
|
23
|
+
const result = [];
|
|
24
|
+
for (let i = 0; i < texts.length; i++) {
|
|
25
|
+
result.push(new Float32Array(out.data.slice(i * this.dimension, (i + 1) * this.dimension)));
|
|
26
|
+
}
|
|
27
|
+
return result;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
//# sourceMappingURL=onnx-embedder.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"onnx-embedder.js","sourceRoot":"","sources":["../src/onnx-embedder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAGzC,MAAM,QAAQ,GAAG,yBAAyB,CAAC;AAO3C,MAAM,OAAO,YAAY;IAGK;IAFnB,SAAS,GAAG,GAAG,CAAC;IAEzB,YAA4B,IAA+B;QAA/B,SAAI,GAAJ,IAAI,CAA2B;IAAG,CAAC;IAE/D,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,aAAqB;QACrC,MAAM,KAAK,CAAC,aAAa,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAChD,MAAM,YAAY,GAAG,CAAC,MAAM,MAAM,CAAC,2BAA2B,CAAC,CAG9D,CAAC;QACF,YAAY,CAAC,GAAG,CAAC,QAAQ,GAAG,aAAa,CAAC;QAC1C,YAAY,CAAC,GAAG,CAAC,gBAAgB,GAAG,KAAK,CAAC;QAC1C,MAAM,IAAI,GAAG,MAAM,YAAY,CAAC,QAAQ,CAAC,oBAAoB,EAAE,QAAQ,CAAC,CAAC;QACzE,OAAO,IAAI,YAAY,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY;QACtB,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACxE,OAAO,IAAI,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;IAC7D,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,KAAe;QAC9B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACzE,MAAM,MAAM,GAAmB,EAAE,CAAC;QAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,MAAM,CAAC,IAAI,CAAC,IAAI,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;QAC9F,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;CACF"}
|