daftari 1.5.0 → 1.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +14 -1
- package/dist/index.js.map +1 -1
- package/dist/search/reindex.d.ts +4 -1
- package/dist/search/reindex.d.ts.map +1 -1
- package/dist/search/reindex.js +7 -6
- package/dist/search/reindex.js.map +1 -1
- package/dist/search/vector.d.ts +2 -1
- package/dist/search/vector.d.ts.map +1 -1
- package/dist/search/vector.js +37 -11
- package/dist/search/vector.js.map +1 -1
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -5,6 +5,22 @@ All notable changes to this project are documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
+
## [1.5.1] - 2026-05-18
|
|
9
|
+
|
|
10
|
+
### Fixed
|
|
11
|
+
|
|
12
|
+
- **Reindex no longer exhausts memory on mid-sized vaults** (#25). `reindexVault`
|
|
13
|
+
embedded every chunk across the whole vault in a single model call, which
|
|
14
|
+
padded the batch to its longest sequence and allocated activation tensors
|
|
15
|
+
proportional to the total chunk count — so peak memory scaled with vault
|
|
16
|
+
size. Past ~200 documents the allocation exceeded RAM and the process
|
|
17
|
+
stalled in a GC/swap death spiral with no output. Embedding now runs in
|
|
18
|
+
fixed-size sub-batches, keeping peak memory flat regardless of vault size
|
|
19
|
+
(a 600-chunk embed dropped from ~3.5 GB to ~325 MB peak RSS).
|
|
20
|
+
- **Reindex reports progress instead of running silent.** On an interactive
|
|
21
|
+
terminal, `--reindex` now prints a single-line `embedding N/M chunks`
|
|
22
|
+
counter, so a large-vault reindex can be distinguished from a hang.
|
|
23
|
+
|
|
8
24
|
## [1.5.0] - 2026-05-18
|
|
9
25
|
|
|
10
26
|
### Added
|
|
@@ -140,6 +156,7 @@ vault to AI agents, exposing 13 tools over stdio.
|
|
|
140
156
|
example documents, git history, search index); `daftari --vault` serves it.
|
|
141
157
|
- 160 tests covering all 13 tools and their supporting modules.
|
|
142
158
|
|
|
159
|
+
[1.5.1]: https://github.com/mavaali/daftari/releases/tag/v1.5.1
|
|
143
160
|
[1.4.0]: https://github.com/mavaali/daftari/releases/tag/v1.4.0
|
|
144
161
|
[1.1.1]: https://github.com/mavaali/daftari/releases/tag/v1.1.1
|
|
145
162
|
[1.1.0]: https://github.com/mavaali/daftari/releases/tag/v1.1.0
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAmBA,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CASrE;AAED,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,GAAG,IAAI,CAE3D;AAED,wBAAsB,IAAI,CAAC,IAAI,GAAE,MAAM,EAA0B,GAAG,OAAO,CAAC,IAAI,CAAC,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAmBA,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CASrE;AAED,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,GAAG,IAAI,CAE3D;AAED,wBAAsB,IAAI,CAAC,IAAI,GAAE,MAAM,EAA0B,GAAG,OAAO,CAAC,IAAI,CAAC,CAyEhF"}
|
package/dist/index.js
CHANGED
|
@@ -59,7 +59,20 @@ export async function main(argv = process.argv.slice(2)) {
|
|
|
59
59
|
process.stderr.write(`daftari: warning: role '${roleName}' not found in config — running as deny-all guest\n`);
|
|
60
60
|
}
|
|
61
61
|
// Build (or rebuild) the search index. With --reindex this is the whole job.
|
|
62
|
-
|
|
62
|
+
// On an interactive terminal, report embedding progress on a single line so
|
|
63
|
+
// a large-vault reindex is not silent; in non-TTY contexts (CI, pipes) the
|
|
64
|
+
// final summary line below is the only output.
|
|
65
|
+
let progressShown = false;
|
|
66
|
+
const reindexed = await reindexVault(vaultRoot, {
|
|
67
|
+
onProgress: process.stderr.isTTY
|
|
68
|
+
? (done, total) => {
|
|
69
|
+
progressShown = true;
|
|
70
|
+
process.stderr.write(`\rdaftari: embedding ${done}/${total} chunks`);
|
|
71
|
+
}
|
|
72
|
+
: undefined,
|
|
73
|
+
});
|
|
74
|
+
if (progressShown)
|
|
75
|
+
process.stderr.write("\n");
|
|
63
76
|
if (reindexed.ok) {
|
|
64
77
|
const r = reindexed.value;
|
|
65
78
|
process.stderr.write(`daftari: indexed ${r.documentCount} docs, ${r.chunkCount} chunks ` +
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,kCAAkC;AAClC,EAAE;AACF,+EAA+E;AAC/E,8EAA8E;AAC9E,4EAA4E;AAC5E,0DAA0D;AAC1D,EAAE;AACF,+EAA+E;AAE/E,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAC;AACjF,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAC7D,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAE/C,oEAAoE;AACpE,MAAM,UAAU,SAAS,CAAC,IAAc,EAAE,IAAY;IACpD,MAAM,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;IACzB,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACrC,IAAI,SAAS,KAAK,CAAC,CAAC,IAAI,SAAS,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QACpD,OAAO,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC;IACrC,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC;IAC1D,IAAI,MAAM;QAAE,OAAO,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;IACnD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,IAAc;IAC1C,OAAO,SAAS,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;AAClC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,IAAI,CAAC,OAAiB,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IAC/D,MAAM,QAAQ,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;IACrC,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,qDAAqD,CAAC,CAAC;QAC5E,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;QACrB,OAAO;IACT,CAAC;IAED,MAAM,SAAS,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;IACpC,IAAI,CAAC,CAAC,MAAM,eAAe,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC;QACxC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,uCAAuC,SAAS,IAAI,CAAC,CAAC;QAC3E,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;QACrB,OAAO;IACT,CAAC;IAED,2EAA2E;IAC3E,2DAA2D;IAC3D,MAAM,MAAM,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC;IACrC,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;QACf,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,MAAM,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC;QAC3D,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;QACrB,OAAO;IACT,CAAC;IAED,qEAAqE;IACrE,8DAA8D;IAC9D,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC;IAChD,MAAM,QAAQ,GAAG,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,UAAU,CAAC;IACvD,MAAM,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAC;IAC3D,IAAI,MAAM,CAAC,IAAI,KAAK,IAAI,IAAI,QAAQ,KAAK,UAAU,EAAE,CAAC;QACpD,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,2BAA2B,QAAQ,qDAAqD,CACzF,CAAC;IACJ,CAAC;IAED,6EAA6E;IAC7E,MAAM,SAAS,GAAG,MAAM,YAAY,CAAC,SAAS,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,kCAAkC;AAClC,EAAE;AACF,+EAA+E;AAC/E,8EAA8E;AAC9E,4EAA4E;AAC5E,0DAA0D;AAC1D,EAAE;AACF,+EAA+E;AAE/E,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAC;AACjF,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAC7D,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAE/C,oEAAoE;AACpE,MAAM,UAAU,SAAS,CAAC,IAAc,EAAE,IAAY;IACpD,MAAM,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;IACzB,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACrC,IAAI,SAAS,KAAK,CAAC,CAAC,IAAI,SAAS,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QACpD,OAAO,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC;IACrC,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC;IAC1D,IAAI,MAAM;QAAE,OAAO,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;IACnD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,IAAc;IAC1C,OAAO,SAAS,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;AAClC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,IAAI,CAAC,OAAiB,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IAC/D,MAAM,QAAQ,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;IACrC,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,qDAAqD,CAAC,CAAC;QAC5E,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;QACrB,OAAO;IACT,CAAC;IAED,MAAM,SAAS,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;IACpC,IAAI,CAAC,CAAC,MAAM,eAAe,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC;QACxC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,uCAAuC,SAAS,IAAI,CAAC,CAAC;QAC3E,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;QACrB,OAAO;IACT,CAAC;IAED,2EAA2E;IAC3E,2DAA2D;IAC3D,MAAM,MAAM,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC;IACrC,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;QACf,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,MAAM,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC;QAC3D,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;QACrB,OAAO;IACT,CAAC;IAED,qEAAqE;IACrE,8DAA8D;IAC9D,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC;IAChD,MAAM,QAAQ,GAAG,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,UAAU,CAAC;IACvD,MAAM,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAC;IAC3D,IAAI,MAAM,CAAC,IAAI,KAAK,IAAI,IAAI,QAAQ,KAAK,UAAU,EAAE,CAAC;QACpD,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,2BAA2B,QAAQ,qDAAqD,CACzF,CAAC;IACJ,CAAC;IAED,6EAA6E;IAC7E,4EAA4E;IAC5E,2EAA2E;IAC3E,+CAA+C;IAC/C,IAAI,aAAa,GAAG,KAAK,CAAC;IAC1B,MAAM,SAAS,GAAG,MAAM,YAAY,CAAC,SAAS,EAAE;QAC9C,UAAU,EAAE,OAAO,CAAC,MAAM,CAAC,KAAK;YAC9B,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;gBACd,aAAa,GAAG,IAAI,CAAC;gBACrB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,wBAAwB,IAAI,IAAI,KAAK,SAAS,CAAC,CAAC;YACvE,CAAC;YACH,CAAC,CAAC,SAAS;KACd,CAAC,CAAC;IACH,IAAI,aAAa;QAAE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC9C,IAAI,SAAS,CAAC,EAAE,EAAE,CAAC;QACjB,MAAM,CAAC,GAAG,SAAS,CAAC,KAAK,CAAC;QAC1B,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,oBAAoB,CAAC,CAAC,aAAa,UAAU,CAAC,CAAC,UAAU,UAAU;YACjE,YAAY,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,KAAK,CAClD,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,yEAAyE;QACzE,4CAA4C;QAC5C,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,yCAAyC,SAAS,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC;IAC7F,CAAC;IAED,IAAI,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;QAC/B,IAAI,CAAC,SAAS,CAAC,EAAE;YAAE,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;QACxC,OAAO;IACT,CAAC;IAED,MAAM,MAAM,GAAG,YAAY,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAC/C,MAAM,SAAS,GAAG,IAAI,oBAAoB,EAAE,CAAC;IAC7C,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAChC,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,6BAA6B,SAAS,aAAa;QACjD,QAAQ,MAAM,CAAC,IAAI,SAAS,MAAM,CAAC,QAAQ,IAAI,CAClD,CAAC;AACJ,CAAC;AAED,uEAAuE;AACvE,qEAAqE;AACrE,MAAM,QAAQ,GAAG,aAAa,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC;AAC3D,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,QAAQ,EAAE,CAAC;IACjC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;QACjB,MAAM,MAAM,GAAG,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QACvE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,mBAAmB,MAAM,IAAI,CAAC,CAAC;QACpD,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;IACvB,CAAC,CAAC,CAAC;AACL,CAAC"}
|
package/dist/search/reindex.d.ts
CHANGED
|
@@ -6,7 +6,10 @@ export interface ReindexResult {
|
|
|
6
6
|
skipped: string[];
|
|
7
7
|
indexedAt: string;
|
|
8
8
|
}
|
|
9
|
-
export
|
|
9
|
+
export interface ReindexOptions {
|
|
10
|
+
onProgress?: (done: number, total: number) => void;
|
|
11
|
+
}
|
|
12
|
+
export declare function reindexVault(vaultRoot: string, opts?: ReindexOptions): Promise<Result<ReindexResult, Error>>;
|
|
10
13
|
export interface IndexDocumentResult {
|
|
11
14
|
chunkCount: number;
|
|
12
15
|
vectorEnabled: boolean;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"reindex.d.ts","sourceRoot":"","sources":["../../src/search/reindex.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"reindex.d.ts","sourceRoot":"","sources":["../../src/search/reindex.ts"],"names":[],"mappings":"AAYA,OAAO,EAAW,KAAK,MAAM,EAAE,MAAM,yBAAyB,CAAC;AAgB/D,MAAM,WAAW,aAAa;IAC5B,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,OAAO,CAAC;IACvB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,cAAc;IAI7B,UAAU,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;CACpD;AA4FD,wBAAsB,YAAY,CAChC,SAAS,EAAE,MAAM,EACjB,IAAI,GAAE,cAAmB,GACxB,OAAO,CAAC,MAAM,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC,CAsCvC;AAED,MAAM,WAAW,mBAAmB;IAClC,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,OAAO,CAAC;CACxB;AAQD,wBAAsB,aAAa,CACjC,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,MAAM,CAAC,mBAAmB,EAAE,KAAK,CAAC,CAAC,CAmD7C"}
|
package/dist/search/reindex.js
CHANGED
|
@@ -4,9 +4,10 @@
|
|
|
4
4
|
// from scratch — there is no incremental update path to drift out of sync.
|
|
5
5
|
// Called on server start and by the vault_reindex tool / --reindex CLI flag.
|
|
6
6
|
//
|
|
7
|
-
// Embedding is best-effort: every chunk across the whole vault is embedded
|
|
8
|
-
//
|
|
9
|
-
//
|
|
7
|
+
// Embedding is best-effort: every chunk across the whole vault is embedded,
|
|
8
|
+
// in fixed-size sub-batches so peak memory stays flat as the vault grows. If
|
|
9
|
+
// the model is unavailable the documents (and their BM25 tokens) still index;
|
|
10
|
+
// only the vector column is left NULL and vectorEnabled is false.
|
|
10
11
|
import { parseDocument } from "../frontmatter/parser.js";
|
|
11
12
|
import { err, ok } from "../frontmatter/types.js";
|
|
12
13
|
import { clearIndex, deleteDocument, documentCount, insertChunk, insertDocument, openIndexDb, setMeta, } from "../storage/index-db.js";
|
|
@@ -90,16 +91,16 @@ function writeIndex(db, staged, embeddings) {
|
|
|
90
91
|
write();
|
|
91
92
|
return chunkCount;
|
|
92
93
|
}
|
|
93
|
-
export async function reindexVault(vaultRoot) {
|
|
94
|
+
export async function reindexVault(vaultRoot, opts = {}) {
|
|
94
95
|
const staging = await stageDocuments(vaultRoot);
|
|
95
96
|
if (!staging.ok)
|
|
96
97
|
return staging;
|
|
97
98
|
const { staged, skipped } = staging.value;
|
|
98
|
-
// One flat list of every chunk's text
|
|
99
|
+
// One flat list of every chunk's text; embed() processes it in sub-batches.
|
|
99
100
|
const allChunkTexts = [];
|
|
100
101
|
for (const s of staged)
|
|
101
102
|
allChunkTexts.push(...s.chunks);
|
|
102
|
-
const embedResult = await embed(allChunkTexts);
|
|
103
|
+
const embedResult = await embed(allChunkTexts, opts.onProgress);
|
|
103
104
|
const vectorEnabled = embedResult.ok;
|
|
104
105
|
const embeddings = embedResult.ok
|
|
105
106
|
? embedResult.value
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"reindex.js","sourceRoot":"","sources":["../../src/search/reindex.ts"],"names":[],"mappings":"AAAA,oEAAoE;AACpE,EAAE;AACF,2EAA2E;AAC3E,2EAA2E;AAC3E,6EAA6E;AAC7E,EAAE;AACF,8EAA8E;AAC9E
|
|
1
|
+
{"version":3,"file":"reindex.js","sourceRoot":"","sources":["../../src/search/reindex.ts"],"names":[],"mappings":"AAAA,oEAAoE;AACpE,EAAE;AACF,2EAA2E;AAC3E,2EAA2E;AAC3E,6EAA6E;AAC7E,EAAE;AACF,4EAA4E;AAC5E,6EAA6E;AAC7E,8EAA8E;AAC9E,kEAAkE;AAElE,OAAO,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AACzD,OAAO,EAAE,GAAG,EAAE,EAAE,EAAe,MAAM,yBAAyB,CAAC;AAC/D,OAAO,EACL,UAAU,EACV,cAAc,EACd,aAAa,EAGb,WAAW,EACX,cAAc,EACd,WAAW,EACX,OAAO,GACR,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAC5E,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AAsB9D,0EAA0E;AAC1E,8EAA8E;AAC9E,0DAA0D;AAC1D,KAAK,UAAU,QAAQ,CAAC,SAAiB,EAAE,OAAe;IACxD,MAAM,QAAQ,GAAG,gBAAgB,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IACtD,IAAI,CAAC,QAAQ,CAAC,EAAE;QAAE,OAAO,IAAI,CAAC;IAC9B,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC5C,IAAI,CAAC,IAAI,CAAC,EAAE;QAAE,OAAO,IAAI,CAAC;IAC1B,MAAM,MAAM,GAAG,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACzC,IAAI,CAAC,MAAM,CAAC,EAAE;QAAE,OAAO,IAAI,CAAC;IAE5B,MAAM,EAAE,GAAG,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC;IACpC,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC;IAClC,sEAAsE;IACtE,qBAAqB;IACrB,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,EAAE,CAAC,KAAK,IAAI,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;IAEpE,OAAO;QACL,GAAG,EAAE;YACH,IAAI,EAAE,OAAO;YACb,KAAK,EAAE,EAAE,CAAC,KAAK;YACf,UAAU,EAAE,EAAE,CAAC,UAAU,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC1D,MAAM,EAAE,EAAE,CAAC,MAAM;YACjB,MAAM,EAAE,EAAE,CAAC,MAAM;YACjB,UAAU,EAAE,EAAE,CAAC,UAAU;YACzB,OAAO,EAAE,EAAE,CAAC,OAAO;YACnB,IAAI,EAAE,EAAE,CAAC,IAAI;YACb,OAAO,EAAE,IAAI;YACb,MAAM;YACN,OAAO,EAAE,EAAE,CAAC,QAAQ;YACpB,OAAO,EAAE,EAAE,CAAC,OAAO;YACnB,YAAY,EAAE,EAAE,CAAC,aAAa;SAC/B;QACD,MAAM,EAAE,SAAS,CAAC,IAAI,CAAC;KACxB,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,uEAAuE;AACvE,8BAA8B;AAC9B,KAAK,UAAU,cAAc,CAC3B,SAAiB;IAEjB,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,SAAS,CAAC,CAAC;IACxC,IAAI,CAAC,IAAI,CAAC,EAAE;QAAE,OAAO,IAAI,CAAC;IAE1B,MAAM,MAAM,GAAqB,EAAE,CAAC;IACpC,MAAM,OAAO,GAAa,EAAE,CAAC;IAE7B,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;QACjC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QAC/C,IAAI,GAAG;YAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;;YACrB,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC7B,CAAC;IAED,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;AACjC,CAAC;AAED,SAAS,UAAU,CACjB,EAAW,EACX,MAAwB,EACxB,UAAmC;IAEnC,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,MAAM,KAAK,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE;QAChC,UAAU,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,MAAM,EAAE,CAAC;YACrC,cAAc,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;YACxB,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,UAAU,EAAE,EAAE;gBAClC,WAAW,CAAC,EAAE,EAAE;oBACd,IAAI,EAAE,GAAG,CAAC,IAAI;oBACd,UAAU;oBACV,IAAI;oBACJ,SAAS,EAAE,UAAU,CAAC,MAAM,CAAC,IAAI,IAAI;iBACtC,CAAC,CAAC;gBACH,MAAM,IAAI,CAAC,CAAC;gBACZ,UAAU,IAAI,CAAC,CAAC;YAClB,CAAC,CAAC,CAAC;QACL,CAAC;IACH,CAAC,CAAC,CAAC;IACH,KAAK,EAAE,CAAC;IACR,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,SAAiB,EACjB,OAAuB,EAAE;IAEzB,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC,SAAS,CAAC,CAAC;IAChD,IAAI,CAAC,OAAO,CAAC,EAAE;QAAE,OAAO,OAAO,CAAC;IAChC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC;IAE1C,4EAA4E;IAC5E,MAAM,aAAa,GAAa,EAAE,CAAC;IACnC,KAAK,MAAM,CAAC,IAAI,MAAM;QAAE,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC;IAExD,MAAM,WAAW,GAAG,MAAM,KAAK,CAAC,aAAa,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;IAChE,MAAM,aAAa,GAAG,WAAW,CAAC,EAAE,CAAC;IACrC,MAAM,UAAU,GAA4B,WAAW,CAAC,EAAE;QACxD,CAAC,CAAC,WAAW,CAAC,KAAK;QACnB,CAAC,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;IAElC,MAAM,QAAQ,GAAG,WAAW,CAAC,SAAS,CAAC,CAAC;IACxC,IAAI,CAAC,QAAQ,CAAC,EAAE;QAAE,OAAO,QAAQ,CAAC;IAClC,MAAM,EAAE,GAAG,QAAQ,CAAC,KAAK,CAAC;IAE1B,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC3C,IAAI,CAAC;QACH,MAAM,UAAU,GAAG,UAAU,CAAC,EAAE,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;QACtD,OAAO,CAAC,EAAE,EAAE,YAAY,EAAE,SAAS,CAAC,CAAC;QACrC,OAAO,CAAC,EAAE,EAAE,gBAAgB,EAAE,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC;QACrD,OAAO,CAAC,EAAE,EAAE,eAAe,EAAE,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC;QACpD,OAAO,EAAE,CAAC;YACR,aAAa,EAAE,MAAM,CAAC,MAAM;YAC5B,UAAU;YACV,aAAa;YACb,OAAO;YACP,SAAS;SACV,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,MAAM,GAAG,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC1D,OAAO,GAAG,CAAC,IAAI,KAAK,CAAC,yBAAyB,MAAM,EAAE,CAAC,CAAC,CAAC;IAC3D,CAAC;YAAS,CAAC;QACT,EAAE,CAAC,KAAK,EAAE,CAAC;IACb,CAAC;AACH,CAAC;AAOD,uEAAuE;AACvE,EAAE;AACF,4EAA4E;AAC5E,6EAA6E;AAC7E,4EAA4E;AAC5E,gEAAgE;AAChE,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,SAAiB,EACjB,OAAe;IAEf,MAAM,OAAO,GAAG,WAAW,CAAC,SAAS,CAAC,CAAC;IACvC,IAAI,CAAC,OAAO,CAAC,EAAE;QAAE,OAAO,OAAO,CAAC;IAChC,MAAM,UAAU,GAAG,aAAa,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACtD,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IAEtB,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,IAAI,GAAG,MAAM,YAAY,CAAC,SAAS,CAAC,CAAC;QAC3C,IAAI,CAAC,IAAI,CAAC,EAAE;YAAE,OAAO,IAAI,CAAC;QAC1B,OAAO,EAAE,CAAC;YACR,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU;YACjC,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC,aAAa;SACxC,CAAC,CAAC;IACL,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IAClD,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO,GAAG,CAAC,IAAI,KAAK,CAAC,0BAA0B,OAAO,EAAE,CAAC,CAAC,CAAC;IAC7D,CAAC;IACD,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC;IAE/B,MAAM,WAAW,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,CAAC;IACxC,MAAM,aAAa,GAAG,WAAW,CAAC,EAAE,CAAC;IACrC,MAAM,UAAU,GAA4B,WAAW,CAAC,EAAE;QACxD,CAAC,CAAC,WAAW,CAAC,KAAK;QACnB,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;IAE3B,MAAM,QAAQ,GAAG,WAAW,CAAC,SAAS,CAAC,CAAC;IACxC,IAAI,CAAC,QAAQ,CAAC,EAAE;QAAE,OAAO,QAAQ,CAAC;IAClC,MAAM,EAAE,GAAG,QAAQ,CAAC,KAAK,CAAC;IAC1B,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE;YAChC,cAAc,CAAC,EAAE,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;YAC7B,cAAc,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;YACxB,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,UAAU,EAAE,EAAE;gBAClC,WAAW,CAAC,EAAE,EAAE;oBACd,IAAI,EAAE,GAAG,CAAC,IAAI;oBACd,UAAU;oBACV,IAAI;oBACJ,SAAS,EAAE,UAAU,CAAC,UAAU,CAAC,IAAI,IAAI;iBAC1C,CAAC,CAAC;YACL,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QACH,KAAK,EAAE,CAAC;QACR,OAAO,EAAE,CAAC,EAAE,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC;IAC1D,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,MAAM,GAAG,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC1D,OAAO,GAAG,CAAC,IAAI,KAAK,CAAC,wBAAwB,MAAM,EAAE,CAAC,CAAC,CAAC;IAC1D,CAAC;YAAS,CAAC;QACT,EAAE,CAAC,KAAK,EAAE,CAAC;IACb,CAAC;AACH,CAAC"}
|
package/dist/search/vector.d.ts
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import { type Result } from "../frontmatter/types.js";
|
|
2
2
|
export declare const EMBEDDING_MODEL = "Xenova/all-MiniLM-L6-v2";
|
|
3
3
|
export declare const EMBEDDING_DIM = 384;
|
|
4
|
+
export declare const EMBED_BATCH_SIZE = 8;
|
|
4
5
|
export declare function chunkText(text: string): string[];
|
|
5
6
|
export declare function cosineSimilarity(a: Float32Array, b: Float32Array): number;
|
|
6
7
|
export declare function meanEmbedding(vectors: Float32Array[]): Float32Array | null;
|
|
7
|
-
export declare function embed(texts: string[]): Promise<Result<Float32Array[], Error>>;
|
|
8
|
+
export declare function embed(texts: string[], onProgress?: (done: number, total: number) => void): Promise<Result<Float32Array[], Error>>;
|
|
8
9
|
export declare function embedQuery(text: string): Promise<Result<Float32Array, Error>>;
|
|
9
10
|
//# sourceMappingURL=vector.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"vector.d.ts","sourceRoot":"","sources":["../../src/search/vector.ts"],"names":[],"mappings":"AAYA,OAAO,EAAW,KAAK,MAAM,EAAE,MAAM,yBAAyB,CAAC;AAE/D,eAAO,MAAM,eAAe,4BAA4B,CAAC;AACzD,eAAO,MAAM,aAAa,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"vector.d.ts","sourceRoot":"","sources":["../../src/search/vector.ts"],"names":[],"mappings":"AAYA,OAAO,EAAW,KAAK,MAAM,EAAE,MAAM,yBAAyB,CAAC;AAE/D,eAAO,MAAM,eAAe,4BAA4B,CAAC;AACzD,eAAO,MAAM,aAAa,MAAM,CAAC;AAYjC,eAAO,MAAM,gBAAgB,IAAI,CAAC;AAQlC,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CA4BhD;AAGD,wBAAgB,gBAAgB,CAAC,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,YAAY,GAAG,MAAM,CAczE;AAID,wBAAgB,aAAa,CAAC,OAAO,EAAE,YAAY,EAAE,GAAG,YAAY,GAAG,IAAI,CAW1E;AAuBD,wBAAsB,KAAK,CACzB,KAAK,EAAE,MAAM,EAAE,EACf,UAAU,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,GACjD,OAAO,CAAC,MAAM,CAAC,YAAY,EAAE,EAAE,KAAK,CAAC,CAAC,CA8BxC;AAGD,wBAAsB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC,CAMnF"}
|
package/dist/search/vector.js
CHANGED
|
@@ -12,6 +12,17 @@
|
|
|
12
12
|
import { err, ok } from "../frontmatter/types.js";
|
|
13
13
|
export const EMBEDDING_MODEL = "Xenova/all-MiniLM-L6-v2";
|
|
14
14
|
export const EMBEDDING_DIM = 384;
|
|
15
|
+
// Texts are embedded in fixed-size sub-batches rather than one call. The model
|
|
16
|
+
// pads every batch to its longest sequence and allocates activation tensors
|
|
17
|
+
// proportional to the batch size, so an unbounded batch makes peak memory
|
|
18
|
+
// scale with the whole vault — a few hundred documents is enough to exhaust
|
|
19
|
+
// RAM and stall in a GC death spiral. A small fixed batch keeps peak memory
|
|
20
|
+
// flat regardless of vault size.
|
|
21
|
+
//
|
|
22
|
+
// 8 was measured as the sweet spot: on CPU inference larger batches were both
|
|
23
|
+
// heavier (more activation memory) and slower (more compute wasted padding
|
|
24
|
+
// short chunks up to the batch's longest sequence), not faster.
|
|
25
|
+
export const EMBED_BATCH_SIZE = 8;
|
|
15
26
|
const CHUNK_MAX_CHARS = 800;
|
|
16
27
|
// Splits a document body into embeddable chunks. Paragraphs (blank-line
|
|
17
28
|
// separated) are packed greedily up to CHUNK_MAX_CHARS; a single paragraph
|
|
@@ -91,22 +102,37 @@ async function getExtractor() {
|
|
|
91
102
|
}
|
|
92
103
|
return extractorPromise;
|
|
93
104
|
}
|
|
94
|
-
// Embeds
|
|
95
|
-
//
|
|
96
|
-
// array without loading
|
|
97
|
-
|
|
105
|
+
// Embeds texts in EMBED_BATCH_SIZE sub-batches so peak memory stays flat
|
|
106
|
+
// regardless of how many texts are passed. Returns one Float32Array per input
|
|
107
|
+
// text, in input order. An empty input yields an empty array without loading
|
|
108
|
+
// the model. `onProgress` (if given) fires after each sub-batch with the count
|
|
109
|
+
// embedded so far and the total — used to drive reindex progress output.
|
|
110
|
+
export async function embed(texts, onProgress) {
|
|
98
111
|
if (texts.length === 0)
|
|
99
112
|
return ok([]);
|
|
100
113
|
try {
|
|
101
114
|
const extractor = await getExtractor();
|
|
102
|
-
const output = await extractor(texts, {
|
|
103
|
-
pooling: "mean",
|
|
104
|
-
normalize: true,
|
|
105
|
-
});
|
|
106
|
-
const dim = output.dims[output.dims.length - 1] ?? EMBEDDING_DIM;
|
|
107
115
|
const vectors = [];
|
|
108
|
-
for (let
|
|
109
|
-
|
|
116
|
+
for (let start = 0; start < texts.length; start += EMBED_BATCH_SIZE) {
|
|
117
|
+
const batch = texts.slice(start, start + EMBED_BATCH_SIZE);
|
|
118
|
+
const output = await extractor(batch, {
|
|
119
|
+
pooling: "mean",
|
|
120
|
+
normalize: true,
|
|
121
|
+
});
|
|
122
|
+
const dim = output.dims[output.dims.length - 1] ?? EMBEDDING_DIM;
|
|
123
|
+
for (let i = 0; i < batch.length; i++) {
|
|
124
|
+
vectors.push(output.data.slice(i * dim, (i + 1) * dim));
|
|
125
|
+
}
|
|
126
|
+
// Progress is a best-effort side channel: a failing reporter (e.g. a
|
|
127
|
+
// closed stderr pipe) must never abort embedding the vault.
|
|
128
|
+
if (onProgress) {
|
|
129
|
+
try {
|
|
130
|
+
onProgress(vectors.length, texts.length);
|
|
131
|
+
}
|
|
132
|
+
catch {
|
|
133
|
+
// ignore — progress reporting is not load-bearing
|
|
134
|
+
}
|
|
135
|
+
}
|
|
110
136
|
}
|
|
111
137
|
return ok(vectors);
|
|
112
138
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"vector.js","sourceRoot":"","sources":["../../src/search/vector.ts"],"names":[],"mappings":"AAAA,kDAAkD;AAClD,EAAE;AACF,yEAAyE;AACzE,yEAAyE;AACzE,kFAAkF;AAClF,+EAA+E;AAC/E,2EAA2E;AAC3E,EAAE;AACF,2EAA2E;AAC3E,+EAA+E;AAC/E,0EAA0E;AAE1E,OAAO,EAAE,GAAG,EAAE,EAAE,EAAe,MAAM,yBAAyB,CAAC;AAE/D,MAAM,CAAC,MAAM,eAAe,GAAG,yBAAyB,CAAC;AACzD,MAAM,CAAC,MAAM,aAAa,GAAG,GAAG,CAAC;AAEjC,MAAM,eAAe,GAAG,GAAG,CAAC;AAE5B,wEAAwE;AACxE,2EAA2E;AAC3E,6EAA6E;AAC7E,4DAA4D;AAC5D,MAAM,UAAU,SAAS,CAAC,IAAY;IACpC,MAAM,UAAU,GAAG,IAAI;SACpB,KAAK,CAAC,SAAS,CAAC;SAChB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SACpB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAE/B,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,OAAO,GAAG,EAAE,CAAC;IACjB,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,IAAI,IAAI,CAAC,MAAM,GAAG,eAAe,EAAE,CAAC;YAClC,IAAI,OAAO,EAAE,CAAC;gBACZ,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACrB,OAAO,GAAG,EAAE,CAAC;YACf,CAAC;YACD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,eAAe,EAAE,CAAC;gBACtD,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,eAAe,CAAC,CAAC,CAAC;YAClD,CAAC;YACD,SAAS;QACX,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,GAAG,eAAe,IAAI,OAAO,EAAE,CAAC;YAClE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACrB,OAAO,GAAG,IAAI,CAAC;QACjB,CAAC;aAAM,CAAC;YACN,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,OAAO,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QACrD,CAAC;IACH,CAAC;IACD,IAAI,OAAO;QAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAClC,OAAO,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;AACpD,CAAC;AAED,6EAA6E;AAC7E,MAAM,UAAU,gBAAgB,CAAC,CAAe,EAAE,CAAe;IAC/D,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACtD,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAW,CAAC;QACzB,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAW,CAAC;QACzB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;QACb,KAAK,IAAI,CAAC,GAAG,CAAC,CAAC;QACf,KAAK,IAAI,CAAC,GAAG,CAAC,CAAC;IACjB,CAAC;IACD,IAAI,KAAK,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACzC,OAAO,GAAG,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;AACrD,CAAC;AAED,6EAA6E;AAC7E,uEAAuE;AACvE,MAAM,UAAU,aAAa,CAAC,OAAuB;IACnD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACtC,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC;IACpC,IAAI,GAAG,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAC3B,MAAM,GAAG,GAAG,IAAI,YAAY,CAAC,GAAG,CAAC,CAAC;IAClC,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,IAAI,CAAC,CAAC,MAAM,KAAK,GAAG;YAAE,SAAS;QAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE;YAAE,GAAG,CAAC,CAAC,CAAC,GAAI,GAAG,CAAC,CAAC,CAAY,GAAI,CAAC,CAAC,CAAC,CAAY,CAAC;IAC/E,CAAC;IACD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE;QAAE,GAAG,CAAC,CAAC,CAAC,GAAI,GAAG,CAAC,CAAC,CAAY,GAAG,OAAO,CAAC,MAAM,CAAC;IAC3E,OAAO,GAAG,CAAC;AACb,CAAC;AAOD,IAAI,gBAAgB,GAA8B,IAAI,CAAC;AAEvD,KAAK,UAAU,YAAY;IACzB,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACtB,gBAAgB,GAAG,MAAM,CAAC,2BAA2B,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAC3E,QAAQ,CAAC,oBAAoB,EAAE,eAAe,CAAC,CAC1B,CAAC;IAC1B,CAAC;IACD,OAAO,gBAAgB,CAAC;AAC1B,CAAC;AAED,+EAA+E;AAC/E,
|
|
1
|
+
{"version":3,"file":"vector.js","sourceRoot":"","sources":["../../src/search/vector.ts"],"names":[],"mappings":"AAAA,kDAAkD;AAClD,EAAE;AACF,yEAAyE;AACzE,yEAAyE;AACzE,kFAAkF;AAClF,+EAA+E;AAC/E,2EAA2E;AAC3E,EAAE;AACF,2EAA2E;AAC3E,+EAA+E;AAC/E,0EAA0E;AAE1E,OAAO,EAAE,GAAG,EAAE,EAAE,EAAe,MAAM,yBAAyB,CAAC;AAE/D,MAAM,CAAC,MAAM,eAAe,GAAG,yBAAyB,CAAC;AACzD,MAAM,CAAC,MAAM,aAAa,GAAG,GAAG,CAAC;AAEjC,+EAA+E;AAC/E,4EAA4E;AAC5E,0EAA0E;AAC1E,4EAA4E;AAC5E,4EAA4E;AAC5E,iCAAiC;AACjC,EAAE;AACF,8EAA8E;AAC9E,2EAA2E;AAC3E,gEAAgE;AAChE,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC;AAElC,MAAM,eAAe,GAAG,GAAG,CAAC;AAE5B,wEAAwE;AACxE,2EAA2E;AAC3E,6EAA6E;AAC7E,4DAA4D;AAC5D,MAAM,UAAU,SAAS,CAAC,IAAY;IACpC,MAAM,UAAU,GAAG,IAAI;SACpB,KAAK,CAAC,SAAS,CAAC;SAChB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SACpB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAE/B,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,OAAO,GAAG,EAAE,CAAC;IACjB,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,IAAI,IAAI,CAAC,MAAM,GAAG,eAAe,EAAE,CAAC;YAClC,IAAI,OAAO,EAAE,CAAC;gBACZ,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACrB,OAAO,GAAG,EAAE,CAAC;YACf,CAAC;YACD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,eAAe,EAAE,CAAC;gBACtD,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,eAAe,CAAC,CAAC,CAAC;YAClD,CAAC;YACD,SAAS;QACX,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,GAAG,eAAe,IAAI,OAAO,EAAE,CAAC;YAClE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACrB,OAAO,GAAG,IAAI,CAAC;QACjB,CAAC;aAAM,CAAC;YACN,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,OAAO,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QACrD,CAAC;IACH,CAAC;IACD,IAAI,OAAO;QAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAClC,OAAO,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;AACpD,CAAC;AAED,6EAA6E;AAC7E,MAAM,UAAU,gBAAgB,CAAC,CAAe,EAAE,CAAe;IAC/D,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACtD,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAW,CAAC;QACzB,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAW,CAAC;QACzB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;QACb,KAAK,IAAI,CAAC,GAAG,CAAC,CAAC;QACf,KAAK,IAAI,CAAC,GAAG,CAAC,CAAC;IACjB,CAAC;IACD,IAAI,KAAK,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACzC,OAAO,GAAG,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;AACrD,CAAC;AAED,6EAA6E;AAC7E,uEAAuE;AACvE,MAAM,UAAU,aAAa,CAAC,OAAuB;IACnD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACtC,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC;IACpC,IAAI,GAAG,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAC3B,MAAM,GAAG,GAAG,IAAI,YAAY,CAAC,GAAG,CAAC,CAAC;IAClC,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,IAAI,CAAC,CAAC,MAAM,KAAK,GAAG;YAAE,SAAS;QAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE;YAAE,GAAG,CAAC,CAAC,CAAC,GAAI,GAAG,CAAC,CAAC,CAAY,GAAI,CAAC,CAAC,CAAC,CAAY,CAAC;IAC/E,CAAC;IACD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE;QAAE,GAAG,CAAC,CAAC,CAAC,GAAI,GAAG,CAAC,CAAC,CAAY,GAAG,OAAO,CAAC,MAAM,CAAC;IAC3E,OAAO,GAAG,CAAC;AACb,CAAC;AAOD,IAAI,gBAAgB,GAA8B,IAAI,CAAC;AAEvD,KAAK,UAAU,YAAY;IACzB,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACtB,gBAAgB,GAAG,MAAM,CAAC,2BAA2B,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAC3E,QAAQ,CAAC,oBAAoB,EAAE,eAAe,CAAC,CAC1B,CAAC;IAC1B,CAAC;IACD,OAAO,gBAAgB,CAAC;AAC1B,CAAC;AAED,yEAAyE;AACzE,8EAA8E;AAC9E,6EAA6E;AAC7E,+EAA+E;AAC/E,yEAAyE;AACzE,MAAM,CAAC,KAAK,UAAU,KAAK,CACzB,KAAe,EACf,UAAkD;IAElD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC;IACtC,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,MAAM,YAAY,EAAE,CAAC;QACvC,MAAM,OAAO,GAAmB,EAAE,CAAC;QACnC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,KAAK,IAAI,gBAAgB,EAAE,CAAC;YACpE,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,gBAAgB,CAAC,CAAC;YAC3D,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,KAAK,EAAE;gBACpC,OAAO,EAAE,MAAM;gBACf,SAAS,EAAE,IAAI;aAChB,CAAC,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,aAAa,CAAC;YACjE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACtC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;YAC1D,CAAC;YACD,qEAAqE;YACrE,4DAA4D;YAC5D,IAAI,UAAU,EAAE,CAAC;gBACf,IAAI,CAAC;oBACH,UAAU,CAAC,OAAO,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;gBAC3C,CAAC;gBAAC,MAAM,CAAC;oBACP,kDAAkD;gBACpD,CAAC;YACH,CAAC;QACH,CAAC;QACD,OAAO,EAAE,CAAC,OAAO,CAAC,CAAC;IACrB,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,MAAM,GAAG,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC1D,OAAO,GAAG,CAAC,IAAI,KAAK,CAAC,qBAAqB,MAAM,EAAE,CAAC,CAAC,CAAC;IACvD,CAAC;AACH,CAAC;AAED,2DAA2D;AAC3D,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,IAAY;IAC3C,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IACnC,IAAI,CAAC,MAAM,CAAC,EAAE;QAAE,OAAO,MAAM,CAAC;IAC9B,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAC9B,IAAI,CAAC,KAAK;QAAE,OAAO,GAAG,CAAC,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC,CAAC;IAClE,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC;AACnB,CAAC"}
|