@intentic/iq-engine 1.176.2 → 1.209.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +61 -3
- package/dist/embed/vector-cache.d.ts +9 -0
- package/dist/embed/vector-cache.d.ts.map +1 -0
- package/dist/embed/vector-cache.js +103 -0
- package/dist/embed/vector-cache.js.map +1 -0
- package/dist/engines/semantic.d.ts +2 -1
- package/dist/engines/semantic.d.ts.map +1 -1
- package/dist/engines/semantic.js +51 -24
- package/dist/engines/semantic.js.map +1 -1
- package/dist/index.d.ts +17 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +49 -16
- package/dist/index.js.map +1 -1
- package/dist/indexer/index-worker.d.ts +3 -0
- package/dist/indexer/index-worker.d.ts.map +1 -1
- package/dist/indexer/index-worker.js +26 -10
- package/dist/indexer/index-worker.js.map +1 -1
- package/dist/indexer/indexer.d.ts.map +1 -1
- package/dist/indexer/indexer.js +2 -1
- package/dist/indexer/indexer.js.map +1 -1
- package/dist/query/query-worker.d.ts +34 -0
- package/dist/query/query-worker.d.ts.map +1 -0
- package/dist/query/query-worker.js +38 -0
- package/dist/query/query-worker.js.map +1 -0
- package/dist/query/scorer.d.ts +22 -0
- package/dist/query/scorer.d.ts.map +1 -0
- package/dist/query/scorer.js +18 -0
- package/dist/query/scorer.js.map +1 -0
- package/dist/query/worker-scorer.d.ts +15 -0
- package/dist/query/worker-scorer.d.ts.map +1 -0
- package/dist/query/worker-scorer.js +85 -0
- package/dist/query/worker-scorer.js.map +1 -0
- package/dist/store/db.d.ts +1 -0
- package/dist/store/db.d.ts.map +1 -1
- package/dist/store/db.js +51 -5
- package/dist/store/db.js.map +1 -1
- package/dist/store/index-store.d.ts.map +1 -1
- package/dist/store/index-store.js +8 -6
- package/dist/store/index-store.js.map +1 -1
- package/dist/store/vectors.d.ts +11 -0
- package/dist/store/vectors.d.ts.map +1 -0
- package/dist/store/vectors.js +42 -0
- package/dist/store/vectors.js.map +1 -0
- package/dist/testing.d.ts.map +1 -1
- package/dist/testing.js +6 -5
- package/dist/testing.js.map +1 -1
- package/dist/types.d.ts +2 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/verbs/dispatch.d.ts +2 -5
- package/dist/verbs/dispatch.d.ts.map +1 -1
- package/dist/verbs/dispatch.js +30 -18
- package/dist/verbs/dispatch.js.map +1 -1
- package/dist/workspace/floor.d.ts +1 -1
- package/dist/workspace/floor.d.ts.map +1 -1
- package/dist/workspace/floor.js +18 -4
- package/dist/workspace/floor.js.map +1 -1
- package/dist/workspace/glob.d.ts.map +1 -1
- package/dist/workspace/glob.js +3 -1
- package/dist/workspace/glob.js.map +1 -1
- package/dist/workspace/scan.d.ts.map +1 -1
- package/dist/workspace/scan.js +2 -1
- package/dist/workspace/scan.js.map +1 -1
- package/package.json +9 -8
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { parentPort, workerData } from "node:worker_threads";
|
|
2
|
-
import { loadEmbedder } from "../embed/embedder.js";
|
|
2
|
+
import { loadEmbedder, MODEL_ID } from "../embed/embedder.js";
|
|
3
|
+
import { openVectorCache, vectorCachePath } from "../embed/vector-cache.js";
|
|
3
4
|
import { embedPending } from "../engines/semantic.js";
|
|
4
|
-
import { openIndex } from "../store/db.js";
|
|
5
|
+
import { compactIndex, openIndex } from "../store/db.js";
|
|
5
6
|
import { readIndexStatus } from "../store/index-store.js";
|
|
6
7
|
import { sweep } from "../workspace/scan.js";
|
|
7
8
|
import { revalidate, syncModel } from "./indexer.js";
|
|
@@ -16,27 +17,31 @@ const post = (event) => {
|
|
|
16
17
|
};
|
|
17
18
|
const db = openIndex(indexDir, "write");
|
|
18
19
|
syncModel(db, modelDir);
|
|
20
|
+
const vectorCache = openVectorCache(vectorCachePath(indexDir), MODEL_ID);
|
|
19
21
|
let embedderPromise;
|
|
20
22
|
const getEmbedder = () => (embedderPromise ??= loadEmbedder(modelDir));
|
|
21
23
|
let requested = 0;
|
|
22
24
|
let applied = 0;
|
|
23
25
|
let draining = false;
|
|
24
26
|
let warmed = false;
|
|
27
|
+
let lastGeneration = 0;
|
|
28
|
+
const EMBED_SLICE_CHUNKS = 512;
|
|
29
|
+
const EMBED_SLICE_MS = 3_000;
|
|
25
30
|
const pass = async (target) => {
|
|
26
31
|
const sweepStart = Date.now();
|
|
27
32
|
const entries = await sweep(root, false);
|
|
28
33
|
post({ type: "swept", entries, sweepStart });
|
|
29
34
|
const { generation } = await revalidate(db, entries, parseEntry);
|
|
35
|
+
lastGeneration = generation;
|
|
30
36
|
applied = target;
|
|
31
37
|
post({ type: "indexed", generation, seq: applied });
|
|
38
|
+
};
|
|
39
|
+
const embedSlice = async () => {
|
|
32
40
|
const embedder = await getEmbedder();
|
|
33
|
-
if (embedder
|
|
34
|
-
|
|
35
|
-
}
|
|
36
|
-
if (!warmed) {
|
|
37
|
-
warmed = true;
|
|
38
|
-
post({ type: "warmed", status: readIndexStatus(db, generation) });
|
|
41
|
+
if (embedder === undefined) {
|
|
42
|
+
return 0;
|
|
39
43
|
}
|
|
44
|
+
return embedPending(db, embedder, vectorCache, EMBED_SLICE_CHUNKS, EMBED_SLICE_MS);
|
|
40
45
|
};
|
|
41
46
|
const drain = async () => {
|
|
42
47
|
if (draining) {
|
|
@@ -46,10 +51,21 @@ const drain = async () => {
|
|
|
46
51
|
try {
|
|
47
52
|
for (;;) {
|
|
48
53
|
const target = requested;
|
|
49
|
-
if (applied
|
|
54
|
+
if (applied < target) {
|
|
55
|
+
await pass(target);
|
|
56
|
+
continue;
|
|
57
|
+
}
|
|
58
|
+
const remaining = await embedSlice();
|
|
59
|
+
post({ type: "embedding", remaining });
|
|
60
|
+
if (remaining === 0) {
|
|
61
|
+
compactIndex(db);
|
|
62
|
+
vectorCache?.compact();
|
|
63
|
+
if (!warmed) {
|
|
64
|
+
warmed = true;
|
|
65
|
+
post({ type: "warmed", status: readIndexStatus(db, lastGeneration) });
|
|
66
|
+
}
|
|
50
67
|
return;
|
|
51
68
|
}
|
|
52
|
-
await pass(target);
|
|
53
69
|
}
|
|
54
70
|
}
|
|
55
71
|
catch (error) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index-worker.js","sourceRoot":"","sources":["../../src/indexer/index-worker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAE7D,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;
|
|
1
|
+
{"version":3,"file":"index-worker.js","sourceRoot":"","sources":["../../src/indexer/index-worker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAE7D,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAC9D,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAC5E,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACtD,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AACzD,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAE1D,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAC7C,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAgD9C,MAAM,IAAI,GAAG,UAAU,CAAC;AACxB,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;IAChB,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;AACvE,CAAC;AAED,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,UAA6B,CAAC;AAEnE,MAAM,IAAI,GAAG,CAAC,KAAuB,EAAQ,EAAE;IAC3C,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;AAC5B,CAAC,CAAC;AAIF,MAAM,EAAE,GAAG,SAAS,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;AACxC,SAAS,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;AAIxB,MAAM,WAAW,GAAG,eAAe,CAAC,eAAe,CAAC,QAAQ,CAAC,EAAE,QAAQ,CAAC,CAAC;AAEzE,IAAI,eAA0D,CAAC;AAC/D,MAAM,WAAW,GAAG,GAAkC,EAAE,CAAC,CAAC,eAAe,KAAK,YAAY,CAAC,QAAQ,CAAC,CAAC,CAAC;AAEtG,IAAI,SAAS,GAAG,CAAC,CAAC;AAClB,IAAI,OAAO,GAAG,CAAC,CAAC;AAChB,IAAI,QAAQ,GAAG,KAAK,CAAC;AACrB,IAAI,MAAM,GAAG,KAAK,CAAC;AAGnB,IAAI,cAAc,GAAG,CAAC,CAAC;AAqBvB,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAC/B,MAAM,cAAc,GAAG,KAAK,CAAC;AAI7B,MAAM,IAAI,GAAG,KAAK,EAAE,MAAc,EAAiB,EAAE;IACjD,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC9B,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACzC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC,CAAC;IAC7C,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,UAAU,CAAC,EAAE,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;IACjE,cAAc,GAAG,UAAU,CAAC;IAC5B,OAAO,GAAG,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,UAAU,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC;AACxD,CAAC,CAAC;AAIF,MAAM,UAAU,GAAG,KAAK,IAAqB,EAAE;IAC3C,MAAM,QAAQ,GAAG,MAAM,WAAW,EAAE,CAAC;IACrC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QACzB,OAAO,CAAC,CAAC;IACb,CAAC;IACD,OAAO,YAAY,CAAC,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,kBAAkB,EAAE,cAAc,CAAC,CAAC;AACvF,CAAC,CAAC;AAMF,MAAM,KAAK,GAAG,KAAK,IAAmB,EAAE;IACpC,IAAI,QAAQ,EAAE,CAAC;QACX,OAAO;IACX,CAAC;IACD,QAAQ,GAAG,IAAI,CAAC;IAChB,IAAI,CAAC;QACD,SAAS,CAAC;YAIN,MAAM,MAAM,GAAG,SAAS,CAAC;YACzB,IAAI,OAAO,GAAG,MAAM,EAAE,CAAC;gBACnB,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC;gBACnB,SAAS;YACb,CAAC;YACD,MAAM,SAAS,GAAG,MAAM,UAAU,EAAE,CAAC;YAGrC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,CAAC,CAAC;YACvC,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;gBAGlB,YAAY,CAAC,EAAE,CAAC,CAAC;gBACjB,WAAW,EAAE,OAAO,EAAE,CAAC;gBAQvB,IAAI,CAAC,MAAM,EAAE,CAAC;oBACV,MAAM,GAAG,IAAI,CAAC;oBACd,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,eAAe,CAAC,EAAE,EAAE,cAAc,CAAC,EAAE,CAAC,CAAC;gBAC1E,CAAC;gBACD,OAAO;YACX,CAAC;QACL,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACb,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;IAC/F,CAAC;YAAS,CAAC;QACP,QAAQ,GAAG,KAAK,CAAC;IACrB,CAAC;AACL,CAAC,CAAC;AAEF,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,OAA2B,EAAE,EAAE;IAC/C,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC;IAC7C,KAAK,KAAK,EAAE,CAAC;AACjB,CAAC,CAAC,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"indexer.d.ts","sourceRoot":"","sources":["../../src/indexer/indexer.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"indexer.d.ts","sourceRoot":"","sources":["../../src/indexer/indexer.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AAa9C,OAAO,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAYlE,MAAM,MAAM,SAAS,GAAG,CACpB,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,MAAM,GAAG,SAAS,EACxB,OAAO,EAAE,MAAM,KACd;IAAE,OAAO,EAAE,SAAS,EAAE,CAAC;IAAC,MAAM,EAAE,QAAQ,EAAE,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,EAAE,CAAA;CAAE,CAAC;AAEzF,MAAM,WAAW,gBAAgB;IAC7B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC5B;AAMD,eAAO,MAAM,SAAS,OAAQ,OAAO,YAAY,MAAM,GAAG,SAAS,KAAG,IAQrE,CAAC;AAsBF,eAAO,MAAM,QAAQ,OAAQ,OAAO,WAAW,SAAS,SAAS,EAAE,KAAG,MAgBrE,CAAC;AAIF,eAAO,MAAM,UAAU,OAAc,OAAO,WAAW,SAAS,SAAS,EAAE,UAAU,SAAS,KAAG,OAAO,CAAC,gBAAgB,CAqFxH,CAAC"}
|
package/dist/indexer/indexer.js
CHANGED
|
@@ -2,6 +2,7 @@ import { createHash } from "node:crypto";
|
|
|
2
2
|
import { readFile } from "node:fs/promises";
|
|
3
3
|
import { MODEL_ID } from "../embed/embedder.js";
|
|
4
4
|
import { bumpGeneration, deleteFile, generationOf, getMeta, listFiles, replaceFile, setMeta, touchFile, } from "../store/index-store.js";
|
|
5
|
+
import { clearVectors } from "../store/vectors.js";
|
|
5
6
|
import { langOf } from "../workspace/scan.js";
|
|
6
7
|
const MAX_FILE_BYTES = 1024 * 1024;
|
|
7
8
|
const PARSER_VERSION = "3";
|
|
@@ -11,7 +12,7 @@ export const syncModel = (db, modelDir) => {
|
|
|
11
12
|
return;
|
|
12
13
|
}
|
|
13
14
|
if (getMeta(db, "model_id") !== MODEL_ID) {
|
|
14
|
-
db
|
|
15
|
+
clearVectors(db);
|
|
15
16
|
setMeta(db, "model_id", MODEL_ID);
|
|
16
17
|
}
|
|
17
18
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"indexer.js","sourceRoot":"","sources":["../../src/indexer/indexer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAEhD,OAAO,EACH,cAAc,EACd,UAAU,EACV,YAAY,EACZ,OAAO,EACP,SAAS,EACT,WAAW,EACX,OAAO,EAEP,SAAS,GACZ,MAAM,yBAAyB,CAAC;
|
|
1
|
+
{"version":3,"file":"indexer.js","sourceRoot":"","sources":["../../src/indexer/indexer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAEhD,OAAO,EACH,cAAc,EACd,UAAU,EACV,YAAY,EACZ,OAAO,EACP,SAAS,EACT,WAAW,EACX,OAAO,EAEP,SAAS,GACZ,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAEnD,OAAO,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAE9C,MAAM,cAAc,GAAG,IAAI,GAAG,IAAI,CAAC;AAKnC,MAAM,cAAc,GAAG,GAAG,CAAC;AAgB3B,MAAM,QAAQ,GAAG,CAAC,GAAW,EAAW,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;AAI3D,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,EAAW,EAAE,QAA4B,EAAQ,EAAE;IACzE,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QACzB,OAAO;IACX,CAAC;IACD,IAAI,OAAO,CAAC,EAAE,EAAE,UAAU,CAAC,KAAK,QAAQ,EAAE,CAAC;QACvC,YAAY,CAAC,EAAE,CAAC,CAAC;QACjB,OAAO,CAAC,EAAE,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC;IACtC,CAAC;AACL,CAAC,CAAC;AAIF,MAAM,OAAO,GAAG,CAAC,KAAgB,EAAE,QAAgC,EAAW,EAAE,CAC5E,QAAQ,KAAK,SAAS,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,OAAO,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC,IAAI,CAAC;AAY7G,MAAM,UAAU,GAAG,CAAC,KAAgB,EAAE,QAAgC,EAAW,EAAE,CAAC,QAAQ,KAAK,SAAS,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC,IAAI,CAAC;AAK3I,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,EAAW,EAAE,OAA6B,EAAU,EAAE;IAC3E,MAAM,MAAM,GAAG,SAAS,CAAC,EAAE,CAAC,CAAC;IAC7B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC1B,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACrB,IAAI,UAAU,CAAC,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;YAC5C,GAAG,IAAI,CAAC,CAAC;QACb,CAAC;IACL,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC;QAC/B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAClB,GAAG,IAAI,CAAC,CAAC;QACb,CAAC;IACL,CAAC;IACD,OAAO,GAAG,CAAC;AACf,CAAC,CAAC;AAIF,MAAM,CAAC,MAAM,UAAU,GAAG,KAAK,EAAE,EAAW,EAAE,OAA6B,EAAE,KAAiB,EAA6B,EAAE;IACzH,MAAM,MAAM,GAAG,SAAS,CAAC,EAAE,CAAC,CAAC;IAC7B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,UAAU,GAAG,KAAK,KAAK,SAAS,IAAI,OAAO,CAAC,EAAE,EAAE,gBAAgB,CAAC,KAAK,cAAc,CAAC;IAC3F,IAAI,OAAO,GAAG,CAAC,CAAC;IAGhB,MAAM,SAAS,GAAG,CAAC,KAAgB,EAAQ,EAAE;QACzC,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,CAChB,WAAW,CACP,EAAE,EACF,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC,EAAE,EAC3H,EAAE,EACF,EAAE,EACF,EAAE,CACL,CACJ,CAAC;QACF,OAAO,EAAE,CAAC;IACd,CAAC,CAAC;IAEF,MAAM,SAAS,GAAG,CAAC,KAAgB,EAAE,QAA4C,EAAE,GAAuB,EAAQ,EAAE;QAChH,MAAM,IAAI,GAAG,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAGhE,IAAI,GAAG,KAAK,SAAS,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,IAAI,KAAK,SAAS,CAAC,EAAE,CAAC;YAC7D,SAAS,CAAC,KAAK,CAAC,CAAC;YACjB,OAAO;QACX,CAAC;QACD,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC5D,IAAI,CAAC,UAAU,IAAI,QAAQ,KAAK,SAAS,IAAI,QAAQ,CAAC,IAAI,KAAK,IAAI,EAAE,CAAC;YAClE,SAAS,CAAC,EAAE,EAAE,QAAQ,CAAC,EAAE,EAAE,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YACtD,OAAO;QACX,CAAC;QACD,MAAM,OAAO,GAAG,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACpG,MAAM,MAAM,GAAG,KAAK,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,UAAU,EAAE,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;QAC7G,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,CAChB,WAAW,CACP,EAAE,EACF,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,EAC3H,MAAM,CAAC,OAAO,EACd,MAAM,CAAC,MAAM,EACb,MAAM,CAAC,OAAO,CACjB,CACJ,CAAC;QACF,OAAO,EAAE,CAAC;IACd,CAAC,CAAC;IAEF,MAAM,MAAM,GAAwH,EAAE,CAAC;IACvI,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC1B,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACrB,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACxC,IAAI,CAAC,UAAU,IAAI,OAAO,CAAC,KAAK,EAAE,QAAQ,CAAC,EAAE,CAAC;YAC1C,SAAS;QACb,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;IACtD,CAAC;IAID,MAAM,UAAU,GAAG,EAAE,CAAC;IACtB,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;QAC3C,KAAK,IAAI,KAAK,GAAG,KAAK,EAAE,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC;YACnF,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAE,CAAC;YAChC,IAAI,QAAQ,CAAC,IAAI,KAAK,SAAS,IAAI,QAAQ,CAAC,KAAK,CAAC,IAAI,IAAI,cAAc,EAAE,CAAC;gBACvE,QAAQ,CAAC,IAAI,GAAG,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;YACxE,CAAC;QACL,CAAC;QACD,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,cAAc,EAAE,CAAC;YACnC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACtB,SAAS;QACb,CAAC;QACD,SAAS,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAE,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC;QACtD,IAAI,CAAC,IAAI,GAAG,SAAS,CAAC;IAC1B,CAAC;IACD,KAAK,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,MAAM,EAAE,CAAC;QAChC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAClB,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,EAAE,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;YAC9C,OAAO,EAAE,CAAC;QACd,CAAC;IACL,CAAC;IACD,IAAI,UAAU,EAAE,CAAC;QACb,OAAO,CAAC,EAAE,EAAE,gBAAgB,EAAE,cAAc,CAAC,CAAC;IAClD,CAAC;IACD,MAAM,UAAU,GAAG,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;IACvE,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,CAAC;AACzD,CAAC,CAAC"}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import type { EngineHit } from "../types.js";
|
|
2
|
+
export interface QueryWorkerData {
|
|
3
|
+
readonly indexDir: string;
|
|
4
|
+
readonly modelDir: string | undefined;
|
|
5
|
+
}
|
|
6
|
+
export type QueryWorkerRequest = {
|
|
7
|
+
readonly type: "semantic";
|
|
8
|
+
readonly id: number;
|
|
9
|
+
readonly query: string;
|
|
10
|
+
readonly allowed: string[];
|
|
11
|
+
} | {
|
|
12
|
+
readonly type: "rerank";
|
|
13
|
+
readonly id: number;
|
|
14
|
+
readonly query: string;
|
|
15
|
+
readonly passages: string[];
|
|
16
|
+
};
|
|
17
|
+
export type QueryWorkerResponse = {
|
|
18
|
+
readonly type: "semantic";
|
|
19
|
+
readonly id: number;
|
|
20
|
+
readonly hits: EngineHit[];
|
|
21
|
+
readonly pending: number;
|
|
22
|
+
} | {
|
|
23
|
+
readonly type: "rerank";
|
|
24
|
+
readonly id: number;
|
|
25
|
+
readonly scores: number[];
|
|
26
|
+
} | {
|
|
27
|
+
readonly type: "absent";
|
|
28
|
+
readonly id: number;
|
|
29
|
+
} | {
|
|
30
|
+
readonly type: "failed";
|
|
31
|
+
readonly id: number;
|
|
32
|
+
readonly error: string;
|
|
33
|
+
};
|
|
34
|
+
//# sourceMappingURL=query-worker.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"query-worker.d.ts","sourceRoot":"","sources":["../../src/query/query-worker.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAwB7C,MAAM,WAAW,eAAe;IAC5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;CACzC;AAGD,MAAM,MAAM,kBAAkB,GACxB;IAAE,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAAC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,CAAA;CAAE,GACtG;IAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IAAC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAA;CAAE,CAAC;AAI5G,MAAM,MAAM,mBAAmB,GACzB;IAAE,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAAC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,SAAS,EAAE,CAAC;IAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;CAAE,GACxG;IAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IAAC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,CAAA;CAAE,GAC3E;IAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IAAC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;CAAE,GAChD;IAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IAAC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC"}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { parentPort, workerData } from "node:worker_threads";
|
|
2
|
+
import { loadEmbedder } from "../embed/embedder.js";
|
|
3
|
+
import { loadReranker } from "../embed/reranker.js";
|
|
4
|
+
import { semanticSearch } from "../engines/semantic.js";
|
|
5
|
+
import { openIndex } from "../store/db.js";
|
|
6
|
+
const port = parentPort;
|
|
7
|
+
if (port === null) {
|
|
8
|
+
throw new Error("iq query worker: not started as a worker thread");
|
|
9
|
+
}
|
|
10
|
+
const { indexDir, modelDir } = workerData;
|
|
11
|
+
const db = openIndex(indexDir, "read");
|
|
12
|
+
const embedderReady = loadEmbedder(modelDir);
|
|
13
|
+
const rerankerReady = loadReranker(modelDir);
|
|
14
|
+
const answer = async (request) => {
|
|
15
|
+
const { id } = request;
|
|
16
|
+
if (request.type === "semantic") {
|
|
17
|
+
const embedder = await embedderReady;
|
|
18
|
+
if (embedder === undefined) {
|
|
19
|
+
return { type: "absent", id };
|
|
20
|
+
}
|
|
21
|
+
const pending = Number(db.get("SELECT COUNT(*) AS n FROM chunks WHERE embedded = 0")?.["n"] ?? 0);
|
|
22
|
+
const hits = semanticSearch(db, await embedder.embedQuery(request.query), new Set(request.allowed));
|
|
23
|
+
return { type: "semantic", id, hits, pending };
|
|
24
|
+
}
|
|
25
|
+
const reranker = await rerankerReady;
|
|
26
|
+
if (reranker === undefined) {
|
|
27
|
+
return { type: "absent", id };
|
|
28
|
+
}
|
|
29
|
+
return { type: "rerank", id, scores: await reranker.rerank(request.query, request.passages) };
|
|
30
|
+
};
|
|
31
|
+
port.on("message", (request) => {
|
|
32
|
+
void answer(request).then((response) => {
|
|
33
|
+
port.postMessage(response);
|
|
34
|
+
}, (error) => {
|
|
35
|
+
port.postMessage({ type: "failed", id: request.id, error: error instanceof Error ? error.message : String(error) });
|
|
36
|
+
});
|
|
37
|
+
});
|
|
38
|
+
//# sourceMappingURL=query-worker.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"query-worker.js","sourceRoot":"","sources":["../../src/query/query-worker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAC7D,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACpD,OAAO,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AACxD,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AA2C3C,MAAM,IAAI,GAAG,UAAU,CAAC;AACxB,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;IAChB,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;AACvE,CAAC;AAED,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,UAA6B,CAAC;AAE7D,MAAM,EAAE,GAAG,SAAS,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;AACvC,MAAM,aAAa,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;AAC7C,MAAM,aAAa,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;AAE7C,MAAM,MAAM,GAAG,KAAK,EAAE,OAA2B,EAAgC,EAAE;IAC/E,MAAM,EAAE,EAAE,EAAE,GAAG,OAAO,CAAC;IACvB,IAAI,OAAO,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;QAC9B,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC;QACrC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YACzB,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;QAClC,CAAC;QAGD,MAAM,OAAO,GAAG,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,qDAAqD,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;QAClG,MAAM,IAAI,GAAG,cAAc,CAAC,EAAE,EAAE,MAAM,QAAQ,CAAC,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;QACpG,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;IACnD,CAAC;IACD,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC;IACrC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QACzB,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;IAClC,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;AAClG,CAAC,CAAC;AAEF,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,OAA2B,EAAE,EAAE;IAC/C,KAAK,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CACrB,CAAC,QAAQ,EAAE,EAAE;QACT,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;IAC/B,CAAC,EACD,CAAC,KAAc,EAAE,EAAE;QAGf,IAAI,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,OAAO,CAAC,EAAE,EAAE,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACxH,CAAC,CACJ,CAAC;AACN,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import type { Embedder } from "../embed/embedder.js";
|
|
2
|
+
import type { Reranker } from "../embed/reranker.js";
|
|
3
|
+
import type { VectorCache } from "../embed/vector-cache.js";
|
|
4
|
+
import type { IndexDb } from "../store/db.js";
|
|
5
|
+
import type { EngineHit } from "../types.js";
|
|
6
|
+
export interface SemanticOutcome {
|
|
7
|
+
readonly hits: EngineHit[];
|
|
8
|
+
readonly pending: number;
|
|
9
|
+
}
|
|
10
|
+
export interface QueryScorer {
|
|
11
|
+
semantic(query: string, allowed: ReadonlySet<string>): Promise<SemanticOutcome | undefined>;
|
|
12
|
+
rerank(query: string, passages: readonly string[]): Promise<number[] | undefined>;
|
|
13
|
+
}
|
|
14
|
+
export interface InThreadScorerOptions {
|
|
15
|
+
readonly db: IndexDb;
|
|
16
|
+
readonly embedder: () => Promise<Embedder | undefined>;
|
|
17
|
+
readonly reranker: () => Promise<Reranker | undefined>;
|
|
18
|
+
readonly topUpEmbeddings: boolean;
|
|
19
|
+
readonly cache: () => VectorCache | undefined;
|
|
20
|
+
}
|
|
21
|
+
export declare const inThreadScorer: (options: InThreadScorerOptions) => QueryScorer;
|
|
22
|
+
//# sourceMappingURL=scorer.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"scorer.d.ts","sourceRoot":"","sources":["../../src/query/scorer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAE5D,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAE7C,MAAM,WAAW,eAAe;IAC5B,QAAQ,CAAC,IAAI,EAAE,SAAS,EAAE,CAAC;IAE3B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC5B;AAeD,MAAM,WAAW,WAAW;IAGxB,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,eAAe,GAAG,SAAS,CAAC,CAAC;IAG5F,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,GAAG,SAAS,CAAC,CAAC;CACrF;AAED,MAAM,WAAW,qBAAqB;IAGlC,QAAQ,CAAC,EAAE,EAAE,OAAO,CAAC;IACrB,QAAQ,CAAC,QAAQ,EAAE,MAAM,OAAO,CAAC,QAAQ,GAAG,SAAS,CAAC,CAAC;IACvD,QAAQ,CAAC,QAAQ,EAAE,MAAM,OAAO,CAAC,QAAQ,GAAG,SAAS,CAAC,CAAC;IAIvD,QAAQ,CAAC,eAAe,EAAE,OAAO,CAAC;IAGlC,QAAQ,CAAC,KAAK,EAAE,MAAM,WAAW,GAAG,SAAS,CAAC;CACjD;AAGD,eAAO,MAAM,cAAc,YAAa,qBAAqB,KAAG,WAe9D,CAAC"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { embedPending, semanticSearch } from "../engines/semantic.js";
|
|
2
|
+
export const inThreadScorer = (options) => ({
|
|
3
|
+
async semantic(query, allowed) {
|
|
4
|
+
const embedder = await options.embedder();
|
|
5
|
+
if (embedder === undefined) {
|
|
6
|
+
return undefined;
|
|
7
|
+
}
|
|
8
|
+
const pending = options.topUpEmbeddings
|
|
9
|
+
? await embedPending(options.db, embedder, options.cache())
|
|
10
|
+
: Number(options.db.get("SELECT COUNT(*) AS n FROM chunks WHERE embedded = 0")?.["n"] ?? 0);
|
|
11
|
+
return { hits: semanticSearch(options.db, await embedder.embedQuery(query), allowed), pending };
|
|
12
|
+
},
|
|
13
|
+
async rerank(query, passages) {
|
|
14
|
+
const reranker = await options.reranker();
|
|
15
|
+
return reranker === undefined ? undefined : reranker.rerank(query, passages);
|
|
16
|
+
},
|
|
17
|
+
});
|
|
18
|
+
//# sourceMappingURL=scorer.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"scorer.js","sourceRoot":"","sources":["../../src/query/scorer.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAgDtE,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,OAA8B,EAAe,EAAE,CAAC,CAAC;IAC5E,KAAK,CAAC,QAAQ,CAAC,KAAK,EAAE,OAAO;QACzB,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,QAAQ,EAAE,CAAC;QAC1C,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YACzB,OAAO,SAAS,CAAC;QACrB,CAAC;QACD,MAAM,OAAO,GAAG,OAAO,CAAC,eAAe;YACnC,CAAC,CAAC,MAAM,YAAY,CAAC,OAAO,CAAC,EAAE,EAAE,QAAQ,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC;YAC3D,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,GAAG,CAAC,qDAAqD,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;QAChG,OAAO,EAAE,IAAI,EAAE,cAAc,CAAC,OAAO,CAAC,EAAE,EAAE,MAAM,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC,EAAE,OAAO,EAAE,CAAC;IACpG,CAAC;IACD,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,QAAQ;QACxB,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,QAAQ,EAAE,CAAC;QAC1C,OAAO,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;IACjF,CAAC;CACJ,CAAC,CAAC"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { QueryScorer } from "./scorer.js";
|
|
2
|
+
export interface WorkerScorerOptions {
|
|
3
|
+
readonly indexDir: string;
|
|
4
|
+
readonly modelDir: string | undefined;
|
|
5
|
+
readonly onError?: (error: Error) => void;
|
|
6
|
+
}
|
|
7
|
+
export interface WorkerScorer extends QueryScorer {
|
|
8
|
+
metrics(): {
|
|
9
|
+
readonly live: boolean;
|
|
10
|
+
readonly pendingRequests: number;
|
|
11
|
+
};
|
|
12
|
+
close(): Promise<void>;
|
|
13
|
+
}
|
|
14
|
+
export declare const workerScorer: (options: WorkerScorerOptions) => WorkerScorer;
|
|
15
|
+
//# sourceMappingURL=worker-scorer.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"worker-scorer.d.ts","sourceRoot":"","sources":["../../src/query/worker-scorer.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAO/C,MAAM,WAAW,mBAAmB;IAChC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAItC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,CAAC;CAC7C;AAED,MAAM,WAAW,YAAa,SAAQ,WAAW;IAC7C,OAAO,IAAI;QAAE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;QAAC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAA;KAAE,CAAC;IACxE,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B;AAQD,eAAO,MAAM,YAAY,YAAa,mBAAmB,KAAG,YAiG3D,CAAC"}
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
import { join } from "node:path";
|
|
2
|
+
import { pathToFileURL } from "node:url";
|
|
3
|
+
import { Worker } from "node:worker_threads";
|
|
4
|
+
import { packageRoot } from "@intentic/constants/node";
|
|
5
|
+
const WORKER_URL = pathToFileURL(join(packageRoot(import.meta.url), "dist/query/query-worker.js"));
|
|
6
|
+
export const workerScorer = (options) => {
|
|
7
|
+
const pending = new Map();
|
|
8
|
+
let worker;
|
|
9
|
+
let nextId = 0;
|
|
10
|
+
let closed = false;
|
|
11
|
+
const settleAll = () => {
|
|
12
|
+
const waiting = [...pending.values()];
|
|
13
|
+
pending.clear();
|
|
14
|
+
for (const settle of waiting) {
|
|
15
|
+
settle(undefined);
|
|
16
|
+
}
|
|
17
|
+
};
|
|
18
|
+
const abandon = (dead, error) => {
|
|
19
|
+
if (worker === dead) {
|
|
20
|
+
worker = undefined;
|
|
21
|
+
}
|
|
22
|
+
settleAll();
|
|
23
|
+
options.onError?.(error);
|
|
24
|
+
};
|
|
25
|
+
const live = () => {
|
|
26
|
+
if (worker !== undefined) {
|
|
27
|
+
return worker;
|
|
28
|
+
}
|
|
29
|
+
const spawned = new Worker(WORKER_URL, {
|
|
30
|
+
workerData: { indexDir: options.indexDir, modelDir: options.modelDir },
|
|
31
|
+
});
|
|
32
|
+
spawned.on("message", (response) => {
|
|
33
|
+
const settle = pending.get(response.id);
|
|
34
|
+
pending.delete(response.id);
|
|
35
|
+
settle?.(response);
|
|
36
|
+
});
|
|
37
|
+
spawned.on("error", (error) => {
|
|
38
|
+
abandon(spawned, error);
|
|
39
|
+
});
|
|
40
|
+
spawned.on("exit", () => {
|
|
41
|
+
if (worker === spawned) {
|
|
42
|
+
abandon(spawned, new Error("iq query worker exited"));
|
|
43
|
+
}
|
|
44
|
+
});
|
|
45
|
+
worker = spawned;
|
|
46
|
+
return spawned;
|
|
47
|
+
};
|
|
48
|
+
live();
|
|
49
|
+
const send = async (request) => {
|
|
50
|
+
if (closed) {
|
|
51
|
+
return undefined;
|
|
52
|
+
}
|
|
53
|
+
const id = nextId++;
|
|
54
|
+
const message = request(id);
|
|
55
|
+
const answered = new Promise((resolve) => {
|
|
56
|
+
pending.set(id, resolve);
|
|
57
|
+
});
|
|
58
|
+
live().postMessage(message);
|
|
59
|
+
const response = await answered;
|
|
60
|
+
if (response?.type === "failed") {
|
|
61
|
+
options.onError?.(new Error(`iq query worker: ${message.type} failed: ${response.error}`));
|
|
62
|
+
return undefined;
|
|
63
|
+
}
|
|
64
|
+
return response;
|
|
65
|
+
};
|
|
66
|
+
return {
|
|
67
|
+
metrics: () => ({ live: worker !== undefined, pendingRequests: pending.size }),
|
|
68
|
+
async semantic(query, allowed) {
|
|
69
|
+
const response = await send((id) => ({ type: "semantic", id, query, allowed: [...allowed] }));
|
|
70
|
+
return response?.type === "semantic" ? { hits: response.hits, pending: response.pending } : undefined;
|
|
71
|
+
},
|
|
72
|
+
async rerank(query, passages) {
|
|
73
|
+
const response = await send((id) => ({ type: "rerank", id, query, passages: [...passages] }));
|
|
74
|
+
return response?.type === "rerank" ? response.scores : undefined;
|
|
75
|
+
},
|
|
76
|
+
async close() {
|
|
77
|
+
closed = true;
|
|
78
|
+
const running = worker;
|
|
79
|
+
worker = undefined;
|
|
80
|
+
settleAll();
|
|
81
|
+
await running?.terminate();
|
|
82
|
+
},
|
|
83
|
+
};
|
|
84
|
+
};
|
|
85
|
+
//# sourceMappingURL=worker-scorer.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"worker-scorer.js","sourceRoot":"","sources":["../../src/query/worker-scorer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAC7C,OAAO,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAOvD,MAAM,UAAU,GAAG,aAAa,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,IAAI,CAAC,GAAG,CAAC,EAAE,4BAA4B,CAAC,CAAC,CAAC;AAsBnG,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,OAA4B,EAAgB,EAAE;IACvE,MAAM,OAAO,GAAG,IAAI,GAAG,EAA+D,CAAC;IACvF,IAAI,MAA0B,CAAC;IAC/B,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,MAAM,GAAG,KAAK,CAAC;IAGnB,MAAM,SAAS,GAAG,GAAS,EAAE;QACzB,MAAM,OAAO,GAAG,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QACtC,OAAO,CAAC,KAAK,EAAE,CAAC;QAChB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC3B,MAAM,CAAC,SAAS,CAAC,CAAC;QACtB,CAAC;IACL,CAAC,CAAC;IAIF,MAAM,OAAO,GAAG,CAAC,IAAY,EAAE,KAAY,EAAQ,EAAE;QACjD,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;YAClB,MAAM,GAAG,SAAS,CAAC;QACvB,CAAC;QACD,SAAS,EAAE,CAAC;QACZ,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC;IAC7B,CAAC,CAAC;IAEF,MAAM,IAAI,GAAG,GAAW,EAAE;QACtB,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACvB,OAAO,MAAM,CAAC;QAClB,CAAC;QAGD,MAAM,OAAO,GAAG,IAAI,MAAM,CAAC,UAAU,EAAE;YACnC,UAAU,EAAE,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAA4B;SACnG,CAAC,CAAC;QACH,OAAO,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,QAA6B,EAAE,EAAE;YACpD,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;YACxC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;YAC5B,MAAM,EAAE,CAAC,QAAQ,CAAC,CAAC;QACvB,CAAC,CAAC,CAAC;QACH,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAY,EAAE,EAAE;YACjC,OAAO,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QAC5B,CAAC,CAAC,CAAC;QACH,OAAO,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE;YAEpB,IAAI,MAAM,KAAK,OAAO,EAAE,CAAC;gBACrB,OAAO,CAAC,OAAO,EAAE,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC,CAAC;YAC1D,CAAC;QACL,CAAC,CAAC,CAAC;QACH,MAAM,GAAG,OAAO,CAAC;QACjB,OAAO,OAAO,CAAC;IACnB,CAAC,CAAC;IAKF,IAAI,EAAE,CAAC;IAEP,MAAM,IAAI,GAAG,KAAK,EAAE,OAA2C,EAA4C,EAAE;QACzG,IAAI,MAAM,EAAE,CAAC;YACT,OAAO,SAAS,CAAC;QACrB,CAAC;QACD,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC;QACpB,MAAM,OAAO,GAAG,OAAO,CAAC,EAAE,CAAC,CAAC;QAC5B,MAAM,QAAQ,GAAG,IAAI,OAAO,CAAkC,CAAC,OAAO,EAAE,EAAE;YACtE,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;QAC7B,CAAC,CAAC,CAAC;QAEH,IAAI,EAAE,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;QAC5B,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC;QAChC,IAAI,QAAQ,EAAE,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC9B,OAAO,CAAC,OAAO,EAAE,CAAC,IAAI,KAAK,CAAC,oBAAoB,OAAO,CAAC,IAAI,YAAY,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;YAC3F,OAAO,SAAS,CAAC;QACrB,CAAC;QACD,OAAO,QAAQ,CAAC;IACpB,CAAC,CAAC;IAEF,OAAO;QACH,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,KAAK,SAAS,EAAE,eAAe,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC;QAC9E,KAAK,CAAC,QAAQ,CAAC,KAAK,EAAE,OAAO;YACzB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,GAAG,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC;YAC9F,OAAO,QAAQ,EAAE,IAAI,KAAK,UAAU,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;QAC1G,CAAC;QACD,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,QAAQ;YACxB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,GAAG,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;YAC9F,OAAO,QAAQ,EAAE,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;QACrE,CAAC;QACD,KAAK,CAAC,KAAK;YACP,MAAM,GAAG,IAAI,CAAC;YACd,MAAM,OAAO,GAAG,MAAM,CAAC;YACvB,MAAM,GAAG,SAAS,CAAC;YAInB,SAAS,EAAE,CAAC;YACZ,MAAM,OAAO,EAAE,SAAS,EAAE,CAAC;QAC/B,CAAC;KACJ,CAAC;AACN,CAAC,CAAC"}
|
package/dist/store/db.d.ts
CHANGED
|
@@ -6,6 +6,7 @@ export interface IndexDb {
|
|
|
6
6
|
transaction(fn: () => void): void;
|
|
7
7
|
close(): void;
|
|
8
8
|
}
|
|
9
|
+
export declare const compactIndex: (db: IndexDb) => boolean;
|
|
9
10
|
export type IndexMode = "write" | "read";
|
|
10
11
|
export declare const isIndexBusy: (error: unknown) => boolean;
|
|
11
12
|
export declare const openIndex: (dir: string, mode: IndexMode) => IndexDb;
|
package/dist/store/db.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"db.d.ts","sourceRoot":"","sources":["../../src/store/db.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"db.d.ts","sourceRoot":"","sources":["../../src/store/db.ts"],"names":[],"mappings":"AAsGA,MAAM,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,GAAG,UAAU,GAAG,IAAI,CAAC,CAAC;AAI/E,MAAM,WAAW,OAAO;IACpB,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC,MAAM,GAAG,MAAM,GAAG,MAAM,GAAG,UAAU,GAAG,IAAI,CAAC,EAAE,GAAG,GAAG,EAAE,CAAC;IACrF,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC,MAAM,GAAG,MAAM,GAAG,MAAM,GAAG,UAAU,GAAG,IAAI,CAAC,EAAE,GAAG,GAAG,GAAG,SAAS,CAAC;IAC/F,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC,MAAM,GAAG,MAAM,GAAG,MAAM,GAAG,UAAU,GAAG,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC;IACpF,WAAW,CAAC,EAAE,EAAE,MAAM,IAAI,GAAG,IAAI,CAAC;IAClC,KAAK,IAAI,IAAI,CAAC;CACjB;AAKD,eAAO,MAAM,YAAY,OAAQ,OAAO,KAAG,OAQ1C,CAAC;AAKF,MAAM,MAAM,SAAS,GAAG,OAAO,GAAG,MAAM,CAAC;AA8EzC,eAAO,MAAM,WAAW,UAAW,OAAO,KAAG,OACuD,CAAC;AAOrG,eAAO,MAAM,SAAS,QAAS,MAAM,QAAQ,SAAS,KAAG,OAUxD,CAAC"}
|
package/dist/store/db.js
CHANGED
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
import { mkdirSync, rmSync } from "node:fs";
|
|
2
2
|
import { join } from "node:path";
|
|
3
3
|
import { DatabaseSync } from "node:sqlite";
|
|
4
|
-
|
|
4
|
+
import { getLoadablePath } from "sqlite-vec";
|
|
5
|
+
const SCHEMA_VERSION = "7";
|
|
6
|
+
const EMBEDDING_DIM = 384;
|
|
7
|
+
const COMPACT_FREELIST_RATIO = 0.25;
|
|
5
8
|
const DDL = `
|
|
6
9
|
CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
|
|
7
10
|
CREATE TABLE IF NOT EXISTS files (
|
|
@@ -43,10 +46,17 @@ CREATE TABLE IF NOT EXISTS chunks (
|
|
|
43
46
|
end_line INTEGER NOT NULL,
|
|
44
47
|
hash TEXT NOT NULL,
|
|
45
48
|
text TEXT NOT NULL,
|
|
46
|
-
|
|
49
|
+
-- Whether chunk_vectors holds a row for this chunk. Denormalized because the two questions asked of it are
|
|
50
|
+
-- asked constantly and neither is cheap against the vector table: "which chunks still need embedding" runs
|
|
51
|
+
-- every top-up batch, and "how many are left" runs on every natural-language query, for the "embeddings
|
|
52
|
+
-- 87%" note. Counting the vector table instead measured 7.5ms; the partial index below answers both in
|
|
53
|
+
-- microseconds and shrinks to nothing as coverage completes. Written in the same statement pair as the
|
|
54
|
+
-- vector row, so the two cannot disagree.
|
|
55
|
+
embedded INTEGER NOT NULL DEFAULT 0
|
|
47
56
|
);
|
|
48
57
|
CREATE INDEX IF NOT EXISTS chunks_file ON chunks(file_id);
|
|
49
58
|
CREATE INDEX IF NOT EXISTS chunks_hash ON chunks(hash);
|
|
59
|
+
CREATE INDEX IF NOT EXISTS chunks_unembedded ON chunks(id) WHERE embedded = 0;
|
|
50
60
|
-- BM25 over chunk text (the sparse tier of hybrid retrieval). External-content: rows live in chunks; the
|
|
51
61
|
-- triggers keep the FTS index in sync — verified to fire on FK-cascade deletes too. tokenchars keeps
|
|
52
62
|
-- snake_case/$identifiers whole.
|
|
@@ -57,7 +67,32 @@ END;
|
|
|
57
67
|
CREATE TRIGGER IF NOT EXISTS chunks_fts_ad AFTER DELETE ON chunks BEGIN
|
|
58
68
|
INSERT INTO chunks_fts(chunks_fts, rowid, text) VALUES ('delete', old.id, old.text);
|
|
59
69
|
END;
|
|
70
|
+
-- The dense tier, and the counterpart to chunks_fts above: sqlite-vec ranks the whole corpus inside SQLite and
|
|
71
|
+
-- returns only the k rows asked for, where this used to hand every vector to JavaScript and score them there.
|
|
72
|
+
-- file_id is a metadata column rather than an auxiliary (+) one specifically so a scoped query can filter on it
|
|
73
|
+
-- DURING the ranking pass — filtering afterwards would return the top k of the workspace and then discard the
|
|
74
|
+
-- ones out of scope, which is a different (and wrong) answer.
|
|
75
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS chunk_vectors USING vec0(
|
|
76
|
+
chunk_id INTEGER PRIMARY KEY,
|
|
77
|
+
embedding int8[${EMBEDDING_DIM}] distance_metric=cosine,
|
|
78
|
+
file_id INTEGER
|
|
79
|
+
);
|
|
80
|
+
-- vec0 is not a real table, so it is outside foreign keys: nothing cascades into it. This trigger is what
|
|
81
|
+
-- deletes a vector when its chunk goes, including on the FK cascade from files that chunks_fts_ad also relies on.
|
|
82
|
+
CREATE TRIGGER IF NOT EXISTS chunks_vec_ad AFTER DELETE ON chunks BEGIN
|
|
83
|
+
DELETE FROM chunk_vectors WHERE chunk_id = old.id;
|
|
84
|
+
END;
|
|
60
85
|
`;
|
|
86
|
+
const pragmaNumber = (db, name) => Number(db.get(`PRAGMA ${name}`)?.[name] ?? 0);
|
|
87
|
+
export const compactIndex = (db) => {
|
|
88
|
+
const pageCount = pragmaNumber(db, "page_count");
|
|
89
|
+
const freePages = pragmaNumber(db, "freelist_count");
|
|
90
|
+
if (pageCount === 0 || freePages / pageCount < COMPACT_FREELIST_RATIO) {
|
|
91
|
+
return false;
|
|
92
|
+
}
|
|
93
|
+
db.run("PRAGMA incremental_vacuum");
|
|
94
|
+
return true;
|
|
95
|
+
};
|
|
61
96
|
const wrap = (db) => ({
|
|
62
97
|
all: (sql, ...params) => db.prepare(sql).all(...params),
|
|
63
98
|
get: (sql, ...params) => db.prepare(sql).get(...params),
|
|
@@ -77,16 +112,27 @@ const wrap = (db) => ({
|
|
|
77
112
|
},
|
|
78
113
|
close: () => db.close(),
|
|
79
114
|
});
|
|
115
|
+
const loadVectorExtension = (db) => {
|
|
116
|
+
db.enableLoadExtension(true);
|
|
117
|
+
db.loadExtension(getLoadablePath());
|
|
118
|
+
db.enableLoadExtension(false);
|
|
119
|
+
};
|
|
80
120
|
const open = (dir, mode) => {
|
|
81
121
|
if (mode === "read") {
|
|
82
|
-
const readOnly = new DatabaseSync(join(dir, "index.db"), { readOnly: true });
|
|
122
|
+
const readOnly = new DatabaseSync(join(dir, "index.db"), { readOnly: true, allowExtension: true });
|
|
83
123
|
readOnly.exec("PRAGMA busy_timeout = 5000;");
|
|
124
|
+
loadVectorExtension(readOnly);
|
|
84
125
|
return wrap(readOnly);
|
|
85
126
|
}
|
|
86
|
-
mkdirSync(
|
|
87
|
-
const db = new DatabaseSync(join(dir, "index.db"));
|
|
127
|
+
mkdirSync(dir, { recursive: true });
|
|
128
|
+
const db = new DatabaseSync(join(dir, "index.db"), { allowExtension: true });
|
|
88
129
|
db.exec("PRAGMA busy_timeout = 5000;");
|
|
130
|
+
const pageCount = Number(db.prepare("PRAGMA page_count").get()?.["page_count"] ?? 0);
|
|
131
|
+
if (pageCount === 0) {
|
|
132
|
+
db.exec("PRAGMA auto_vacuum = INCREMENTAL;");
|
|
133
|
+
}
|
|
89
134
|
db.exec("PRAGMA journal_mode = WAL; PRAGMA synchronous = NORMAL; PRAGMA foreign_keys = ON;");
|
|
135
|
+
loadVectorExtension(db);
|
|
90
136
|
db.exec(DDL);
|
|
91
137
|
const wrapped = wrap(db);
|
|
92
138
|
const version = wrapped.get("SELECT value FROM meta WHERE key = 'schema_version'")?.["value"];
|
package/dist/store/db.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"db.js","sourceRoot":"","sources":["../../src/store/db.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AAC5C,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"db.js","sourceRoot":"","sources":["../../src/store/db.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AAC5C,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAI7C,MAAM,cAAc,GAAG,GAAG,CAAC;AAQ3B,MAAM,aAAa,GAAG,GAAG,CAAC;AAM1B,MAAM,sBAAsB,GAAG,IAAI,CAAC;AAEpC,MAAM,GAAG,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAqES,aAAa;;;;;;;;CAQjC,CAAC;AAcF,MAAM,YAAY,GAAG,CAAC,EAAW,EAAE,IAAqC,EAAU,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,UAAU,IAAI,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;AAGnI,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,EAAW,EAAW,EAAE;IACjD,MAAM,SAAS,GAAG,YAAY,CAAC,EAAE,EAAE,YAAY,CAAC,CAAC;IACjD,MAAM,SAAS,GAAG,YAAY,CAAC,EAAE,EAAE,gBAAgB,CAAC,CAAC;IACrD,IAAI,SAAS,KAAK,CAAC,IAAI,SAAS,GAAG,SAAS,GAAG,sBAAsB,EAAE,CAAC;QACpE,OAAO,KAAK,CAAC;IACjB,CAAC;IACD,EAAE,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAC;IACpC,OAAO,IAAI,CAAC;AAChB,CAAC,CAAC;AAOF,MAAM,IAAI,GAAG,CAAC,EAAgB,EAAW,EAAE,CAAC,CAAC;IACzC,GAAG,EAAE,CAAC,GAAG,EAAE,GAAG,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,MAAM,CAAU;IAChE,GAAG,EAAE,CAAC,GAAG,EAAE,GAAG,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,MAAM,CAAoB;IAC1E,GAAG,EAAE,CAAC,GAAG,EAAE,GAAG,MAAM,EAAE,EAAE;QACpB,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC;IACnC,CAAC;IACD,WAAW,EAAE,CAAC,EAAE,EAAE,EAAE;QAChB,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACjB,IAAI,CAAC;YACD,EAAE,EAAE,CAAC;YACL,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACtB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YACpB,MAAM,KAAK,CAAC;QAChB,CAAC;IACL,CAAC;IACD,KAAK,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE;CAC1B,CAAC,CAAC;AAMH,MAAM,mBAAmB,GAAG,CAAC,EAAgB,EAAQ,EAAE;IACnD,EAAE,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC;IAC7B,EAAE,CAAC,aAAa,CAAC,eAAe,EAAE,CAAC,CAAC;IACpC,EAAE,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAC;AAClC,CAAC,CAAC;AAEF,MAAM,IAAI,GAAG,CAAC,GAAW,EAAE,IAAe,EAAW,EAAE;IACnD,IAAI,IAAI,KAAK,MAAM,EAAE,CAAC;QAClB,MAAM,QAAQ,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,CAAC,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC;QAGnG,QAAQ,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;QAC7C,mBAAmB,CAAC,QAAQ,CAAC,CAAC;QAG9B,OAAO,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC1B,CAAC;IAKD,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACpC,MAAM,EAAE,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,CAAC,EAAE,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC;IAI7E,EAAE,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;IAIvC,MAAM,SAAS,GAAG,MAAM,CAAE,EAAE,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAC,GAAG,EAAsB,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC;IAC1G,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;QAClB,EAAE,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAC;IACjD,CAAC;IACD,EAAE,CAAC,IAAI,CAAC,mFAAmF,CAAC,CAAC;IAE7F,mBAAmB,CAAC,EAAE,CAAC,CAAC;IACxB,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACb,MAAM,OAAO,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC;IACzB,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,qDAAqD,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC;IAC9F,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;QACxB,OAAO,CAAC,GAAG,CAAC,4DAA4D,EAAE,cAAc,CAAC,CAAC;QAC1F,OAAO,OAAO,CAAC;IACnB,CAAC;IACD,IAAI,OAAO,KAAK,cAAc,EAAE,CAAC;QAC7B,EAAE,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,mBAAmB,MAAM,CAAC,OAAO,CAAC,OAAO,cAAc,EAAE,CAAC,CAAC;IAC/E,CAAC;IACD,OAAO,OAAO,CAAC;AACnB,CAAC,CAAC;AAIF,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,KAAc,EAAW,EAAE,CACnD,KAAK,YAAY,KAAK,IAAI,kDAAkD,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;AAOrG,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,GAAW,EAAE,IAAe,EAAW,EAAE;IAC/D,IAAI,CAAC;QACD,OAAO,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;IAC3B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACb,IAAI,IAAI,KAAK,MAAM,IAAI,WAAW,CAAC,KAAK,CAAC,EAAE,CAAC;YACxC,MAAM,KAAK,CAAC;QAChB,CAAC;QACD,MAAM,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9C,OAAO,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;IAC3B,CAAC;AACL,CAAC,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index-store.d.ts","sourceRoot":"","sources":["../../src/store/index-store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACpE,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"index-store.d.ts","sourceRoot":"","sources":["../../src/store/index-store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACpE,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAGvC,MAAM,WAAW,UAAU;IACvB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,CAAC;IAClC,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,CAAC;IAClC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACzB;AAED,eAAO,MAAM,SAAS,OAAQ,OAAO,KAAG,GAAG,CAAC,MAAM,EAAE,UAAU,CAc7D,CAAC;AAEF,eAAO,MAAM,SAAS,OAAQ,OAAO,MAAM,MAAM,WAAW,MAAM,QAAQ,MAAM,KAAG,IAElF,CAAC;AAEF,eAAO,MAAM,UAAU,OAAQ,OAAO,MAAM,MAAM,KAAG,IAEpD,CAAC;AAIF,eAAO,MAAM,WAAW,OAChB,OAAO,QACL;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,GAAG,SAAS,CAAC;IAAC,IAAI,EAAE,MAAM,GAAG,SAAS,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,WAClI,SAAS,SAAS,EAAE,UACrB,SAAS,QAAQ,EAAE,WAClB,SAAS,MAAM,EAAE,KAC3B,IA8CF,CAAC;AAEF,eAAO,MAAM,OAAO,OAAQ,OAAO,OAAO,MAAM,KAAG,MAAM,GAAG,SAC4B,CAAC;AAEzF,eAAO,MAAM,OAAO,OAAQ,OAAO,OAAO,MAAM,SAAS,MAAM,KAAG,IAEjE,CAAC;AAEF,eAAO,MAAM,cAAc,OAAQ,OAAO,KAAG,MAI5C,CAAC;AAEF,eAAO,MAAM,YAAY,OAAQ,OAAO,KAAG,MAAkD,CAAC;AAI9F,eAAO,MAAM,eAAe,OAAQ,OAAO,cAAc,MAAM,KAAG,WAUjE,CAAC"}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { copyVector, vectorsOfFile } from "./vectors.js";
|
|
1
2
|
export const listFiles = (db) => {
|
|
2
3
|
const map = new Map();
|
|
3
4
|
for (const row of db.all("SELECT id, path, repo, lang, mtime_ms, size, hash FROM files")) {
|
|
@@ -20,10 +21,7 @@ export const deleteFile = (db, id) => {
|
|
|
20
21
|
db.run("DELETE FROM files WHERE id = ?", id);
|
|
21
22
|
};
|
|
22
23
|
export const replaceFile = (db, file, symbols, chunks, imports) => {
|
|
23
|
-
const previousEmbeddings =
|
|
24
|
-
for (const row of db.all("SELECT c.hash, c.embedding FROM chunks c JOIN files f ON f.id = c.file_id WHERE f.path = ? AND c.embedding IS NOT NULL", file.path)) {
|
|
25
|
-
previousEmbeddings.set(row["hash"], row["embedding"]);
|
|
26
|
-
}
|
|
24
|
+
const previousEmbeddings = vectorsOfFile(db, file.path);
|
|
27
25
|
db.run("DELETE FROM files WHERE path = ?", file.path);
|
|
28
26
|
db.run("INSERT INTO files (path, repo, lang, mtime_ms, size, hash, complexity) VALUES (?, ?, ?, ?, ?, ?, ?)", file.path, file.repo ?? null, file.lang ?? null, Math.round(file.mtimeMs), file.size, file.hash, file.complexity);
|
|
29
27
|
const id = Number(db.get("SELECT id FROM files WHERE path = ?", file.path)["id"]);
|
|
@@ -34,7 +32,11 @@ export const replaceFile = (db, file, symbols, chunks, imports) => {
|
|
|
34
32
|
db.run("INSERT INTO imports (file_id, specifier) VALUES (?, ?)", id, specifier);
|
|
35
33
|
}
|
|
36
34
|
for (const chunk of chunks) {
|
|
37
|
-
db.
|
|
35
|
+
const inserted = db.get("INSERT INTO chunks (file_id, start_line, end_line, hash, text) VALUES (?, ?, ?, ?, ?) RETURNING id", id, chunk.startLine, chunk.endLine, chunk.hash, chunk.text);
|
|
36
|
+
const carried = previousEmbeddings.get(chunk.hash);
|
|
37
|
+
if (carried !== undefined) {
|
|
38
|
+
copyVector(db, Number(inserted["id"]), id, carried);
|
|
39
|
+
}
|
|
38
40
|
}
|
|
39
41
|
};
|
|
40
42
|
export const getMeta = (db, key) => db.get("SELECT value FROM meta WHERE key = ?", key)?.["value"];
|
|
@@ -53,7 +55,7 @@ export const readIndexStatus = (db, generation) => {
|
|
|
53
55
|
files: count("SELECT COUNT(*) AS n FROM files"),
|
|
54
56
|
symbols: count("SELECT COUNT(*) AS n FROM symbols"),
|
|
55
57
|
chunks: count("SELECT COUNT(*) AS n FROM chunks"),
|
|
56
|
-
embedded: count("SELECT COUNT(*) AS n FROM chunks WHERE
|
|
58
|
+
embedded: count("SELECT COUNT(*) AS n FROM chunks WHERE embedded = 1"),
|
|
57
59
|
generation,
|
|
58
60
|
freshness: { state: "fresh", ageMs: 0 },
|
|
59
61
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index-store.js","sourceRoot":"","sources":["../../src/store/index-store.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index-store.js","sourceRoot":"","sources":["../../src/store/index-store.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAYzD,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,EAAW,EAA2B,EAAE;IAC9D,MAAM,GAAG,GAAG,IAAI,GAAG,EAAsB,CAAC;IAC1C,KAAK,MAAM,GAAG,IAAI,EAAE,CAAC,GAAG,CAAC,8DAA8D,CAAC,EAAE,CAAC;QACvF,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAW,EAAE;YAC3B,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACrB,IAAI,EAAE,GAAG,CAAC,MAAM,CAAW;YAC3B,IAAI,EAAG,GAAG,CAAC,MAAM,CAAmB,IAAI,SAAS;YACjD,IAAI,EAAG,GAAG,CAAC,MAAM,CAAmB,IAAI,SAAS;YACjD,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YAChC,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACzB,IAAI,EAAE,GAAG,CAAC,MAAM,CAAW;SAC9B,CAAC,CAAC;IACP,CAAC;IACD,OAAO,GAAG,CAAC;AACf,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,EAAW,EAAE,EAAU,EAAE,OAAe,EAAE,IAAY,EAAQ,EAAE;IACtF,EAAE,CAAC,GAAG,CAAC,sDAAsD,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;AAClG,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,EAAW,EAAE,EAAU,EAAQ,EAAE;IACxD,EAAE,CAAC,GAAG,CAAC,gCAAgC,EAAE,EAAE,CAAC,CAAC;AACjD,CAAC,CAAC;AAIF,MAAM,CAAC,MAAM,WAAW,GAAG,CACvB,EAAW,EACX,IAA2I,EAC3I,OAA6B,EAC7B,MAA2B,EAC3B,OAA0B,EACtB,EAAE;IACN,MAAM,kBAAkB,GAAG,aAAa,CAAC,EAAE,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IACxD,EAAE,CAAC,GAAG,CAAC,kCAAkC,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IACtD,EAAE,CAAC,GAAG,CACF,qGAAqG,EACrG,IAAI,CAAC,IAAI,EACT,IAAI,CAAC,IAAI,IAAI,IAAI,EACjB,IAAI,CAAC,IAAI,IAAI,IAAI,EACjB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,EACxB,IAAI,CAAC,IAAI,EACT,IAAI,CAAC,IAAI,EACT,IAAI,CAAC,UAAU,CAClB,CAAC;IACF,MAAM,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,qCAAqC,EAAE,IAAI,CAAC,IAAI,CAAE,CAAC,IAAI,CAAC,CAAC,CAAC;IACnF,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC3B,EAAE,CAAC,GAAG,CACF,2HAA2H,EAC3H,EAAE,EACF,MAAM,CAAC,IAAI,EACX,MAAM,CAAC,IAAI,EACX,MAAM,CAAC,IAAI,EACX,MAAM,CAAC,OAAO,EACd,MAAM,CAAC,SAAS,EAChB,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EACvB,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAC3B,CAAC;IACN,CAAC;IACD,KAAK,MAAM,SAAS,IAAI,OAAO,EAAE,CAAC;QAC9B,EAAE,CAAC,GAAG,CAAC,wDAAwD,EAAE,EAAE,EAAE,SAAS,CAAC,CAAC;IACpF,CAAC;IACD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAGzB,MAAM,QAAQ,GAAG,EAAE,CAAC,GAAG,CACnB,oGAAoG,EACpG,EAAE,EACF,KAAK,CAAC,SAAS,EACf,KAAK,CAAC,OAAO,EACb,KAAK,CAAC,IAAI,EACV,KAAK,CAAC,IAAI,CACb,CAAC;QACF,MAAM,OAAO,GAAG,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACnD,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;YACxB,UAAU,CAAC,EAAE,EAAE,MAAM,CAAC,QAAS,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,OAAO,CAAC,CAAC;QACzD,CAAC;IACL,CAAC;AACL,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,EAAW,EAAE,GAAW,EAAsB,EAAE,CACpE,EAAE,CAAC,GAAG,CAAC,sCAAsC,EAAE,GAAG,CAAC,EAAE,CAAC,OAAO,CAAuB,CAAC;AAEzF,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,EAAW,EAAE,GAAW,EAAE,KAAa,EAAQ,EAAE;IACrE,EAAE,CAAC,GAAG,CAAC,mGAAmG,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC;AAC5H,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,EAAW,EAAU,EAAE;IAClD,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC,EAAE,EAAE,YAAY,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IAC1D,OAAO,CAAC,EAAE,EAAE,YAAY,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IACxC,OAAO,IAAI,CAAC;AAChB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,EAAW,EAAU,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,EAAE,YAAY,CAAC,IAAI,GAAG,CAAC,CAAC;AAI9F,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,EAAW,EAAE,UAAkB,EAAe,EAAE;IAC5E,MAAM,KAAK,GAAG,CAAC,GAAW,EAAU,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;IACvE,OAAO;QACH,KAAK,EAAE,KAAK,CAAC,iCAAiC,CAAC;QAC/C,OAAO,EAAE,KAAK,CAAC,mCAAmC,CAAC;QACnD,MAAM,EAAE,KAAK,CAAC,kCAAkC,CAAC;QACjD,QAAQ,EAAE,KAAK,CAAC,qDAAqD,CAAC;QACtE,UAAU;QACV,SAAS,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE;KAC1C,CAAC;AACN,CAAC,CAAC"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { IndexDb } from "./db.js";
|
|
2
|
+
export declare const putVector: (db: IndexDb, chunkId: number, fileId: number, vec: Float32Array) => void;
|
|
3
|
+
export declare const copyVector: (db: IndexDb, chunkId: number, fileId: number, quantized: Uint8Array) => void;
|
|
4
|
+
export declare const vectorsOfFile: (db: IndexDb, path: string) => Map<string, Uint8Array>;
|
|
5
|
+
export declare const clearVectors: (db: IndexDb) => void;
|
|
6
|
+
export interface VectorHit {
|
|
7
|
+
readonly chunkId: number;
|
|
8
|
+
readonly score: number;
|
|
9
|
+
}
|
|
10
|
+
export declare const nearestChunks: (db: IndexDb, queryVec: Float32Array, k: number, allowedFileIds: readonly number[] | undefined) => VectorHit[];
|
|
11
|
+
//# sourceMappingURL=vectors.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"vectors.d.ts","sourceRoot":"","sources":["../../src/store/vectors.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAuCvC,eAAO,MAAM,SAAS,OAAQ,OAAO,WAAW,MAAM,UAAU,MAAM,OAAO,YAAY,KAAG,IAQ3F,CAAC;AAGF,eAAO,MAAM,UAAU,OAAQ,OAAO,WAAW,MAAM,UAAU,MAAM,aAAa,UAAU,KAAG,IAKhG,CAAC;AAGF,eAAO,MAAM,aAAa,OAAQ,OAAO,QAAQ,MAAM,KAAG,GAAG,CAAC,MAAM,EAAE,UAAU,CAS/E,CAAC;AAGF,eAAO,MAAM,YAAY,OAAQ,OAAO,KAAG,IAG1C,CAAC;AAEF,MAAM,WAAW,SAAS;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IAEzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CAC1B;AAMD,eAAO,MAAM,aAAa,OAAQ,OAAO,YAAY,YAAY,KAAK,MAAM,kBAAkB,SAAS,MAAM,EAAE,GAAG,SAAS,KAAG,SAAS,EAetI,CAAC"}
|