daftari 1.7.0 → 1.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +53 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +97 -32
- package/dist/index.js.map +1 -1
- package/dist/search/hybrid.js +3 -3
- package/dist/search/hybrid.js.map +1 -1
- package/dist/search/index-state.d.ts +17 -0
- package/dist/search/index-state.d.ts.map +1 -0
- package/dist/search/index-state.js +68 -0
- package/dist/search/index-state.js.map +1 -0
- package/dist/search/reindex.d.ts +4 -0
- package/dist/search/reindex.d.ts.map +1 -1
- package/dist/search/reindex.js +238 -34
- package/dist/search/reindex.js.map +1 -1
- package/dist/storage/index-db.d.ts +14 -3
- package/dist/storage/index-db.d.ts.map +1 -1
- package/dist/storage/index-db.js +98 -20
- package/dist/storage/index-db.js.map +1 -1
- package/dist/tools/search.d.ts.map +1 -1
- package/dist/tools/search.js +37 -8
- package/dist/tools/search.js.map +1 -1
- package/dist/tools/write.d.ts.map +1 -1
- package/dist/tools/write.js +28 -0
- package/dist/tools/write.js.map +1 -1
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -7,6 +7,59 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
|
|
8
8
|
## [Unreleased]
|
|
9
9
|
|
|
10
|
+
## [1.8.0] - 2026-05-20
|
|
11
|
+
|
|
12
|
+
### Changed
|
|
13
|
+
|
|
14
|
+
- **Content-addressed embedding cache** (#38, PR #39). Embeddings are no
|
|
15
|
+
longer keyed by `(path, chunk_index)` — they now live in a separate
|
|
16
|
+
`embeddings` table keyed by `(content_hash, model)`, where
|
|
17
|
+
`content_hash` is the SHA-256 of the chunk's text. `chunks` rows carry
|
|
18
|
+
a `content_hash` column and join to `embeddings` for the current model.
|
|
19
|
+
A reindex now hashes every chunk, asks the cache which hashes already
|
|
20
|
+
have a row, and only embeds the misses — so the cost of a reindex
|
|
21
|
+
scales with the number of *changed chunks*, not the size of the vault.
|
|
22
|
+
An edit to one paragraph re-embeds one chunk; a rename re-embeds zero;
|
|
23
|
+
a paragraph moved verbatim to another file re-embeds zero. The
|
|
24
|
+
composite primary key on `(content_hash, model)` is intentional: a
|
|
25
|
+
future model migration can keep both the old and new model's
|
|
26
|
+
embeddings present under the same hash. After writing chunks, the
|
|
27
|
+
reindex runs an internal `vault_gc` step that drops embeddings rows
|
|
28
|
+
whose `content_hash` is no longer referenced by any chunk, so the
|
|
29
|
+
cache does not accumulate orphans. `index.db` rebuilds cleanly on the
|
|
30
|
+
schema bump (the index is a derived cache); the first reindex after
|
|
31
|
+
upgrade is a one-time full embed that populates the cache, and every
|
|
32
|
+
reindex after that is incremental. This is PR 1 of the #38 unbundle;
|
|
33
|
+
fs.watch reactive indexing, lazy model load, FTS5, sqlite-vec, and
|
|
34
|
+
pluggable embedding backends are tracked as separate follow-ups.
|
|
35
|
+
|
|
36
|
+
## [1.7.1] - 2026-05-19
|
|
37
|
+
|
|
38
|
+
### Fixed
|
|
39
|
+
|
|
40
|
+
- **MCP server hang at startup** (#35, PR #36). The server no longer re-embeds
|
|
41
|
+
the entire vault on every launch and no longer waits for indexing to finish
|
|
42
|
+
before opening the stdio transport. Three compounding bugs are fixed:
|
|
43
|
+
(1) `main()` always called `reindexVault` even when `.daftari/index.db`
|
|
44
|
+
already reflected the files on disk — every restart re-embedded the whole
|
|
45
|
+
vault (~25 minutes on a 3,500-file vault); now a path→mtime manifest is
|
|
46
|
+
persisted in the SQLite meta table and compared on startup, so an
|
|
47
|
+
unchanged vault skips the embedding pass entirely. (2) The
|
|
48
|
+
`StdioServerTransport` opened only after indexing completed, so MCP
|
|
49
|
+
clients could not answer `initialize` for the whole duration; the
|
|
50
|
+
transport now opens first and indexing — when required — runs as a
|
|
51
|
+
background task. (3) Progress was emitted only on TTY stderr, leaving
|
|
52
|
+
every real (non-TTY) MCP client with zero output during a cold start;
|
|
53
|
+
progress now streams on stderr in both TTY (\\r-updated) and pipe (full
|
|
54
|
+
line every ~5%) modes. A new in-process `IndexState`
|
|
55
|
+
(`ready`/`indexing`/`error` + progress) gates `vault_search`,
|
|
56
|
+
`vault_search_related`, `vault_reindex`, `vault_write`, `vault_append`,
|
|
57
|
+
`vault_promote`, and `vault_deprecate` while indexing — those tools
|
|
58
|
+
return a progress-bearing busy error so clients can retry. Read tools
|
|
59
|
+
(`vault_read`, `vault_index`, `vault_status`) are unaffected because
|
|
60
|
+
they go to the filesystem, not the index. `--reindex` remains the one
|
|
61
|
+
synchronous mode (rebuild, exit).
|
|
62
|
+
|
|
10
63
|
## [1.7.0] - 2026-05-19
|
|
11
64
|
|
|
12
65
|
### Added
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAgCA,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CASrE;AAED,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,GAAG,IAAI,CAE3D;AAED,wBAAsB,IAAI,CAAC,IAAI,GAAE,MAAM,EAA0B,GAAG,OAAO,CAAC,IAAI,CAAC,CAuFhF"}
|
package/dist/index.js
CHANGED
|
@@ -1,16 +1,24 @@
|
|
|
1
1
|
// Daftari MCP server entry point.
|
|
2
2
|
//
|
|
3
3
|
// Parses `--vault <path>`, verifies the vault directory exists, loads the RBAC
|
|
4
|
-
// config,
|
|
5
|
-
//
|
|
6
|
-
//
|
|
4
|
+
// config, opens the MCP stdio transport, then — if the index isn't already
|
|
5
|
+
// current — runs a reindex in the background. Diagnostics go to stderr so they
|
|
6
|
+
// never corrupt the stdio JSON-RPC stream on stdout.
|
|
7
7
|
//
|
|
8
|
-
//
|
|
8
|
+
// The transport opens before indexing on purpose: a cold reindex on a large
|
|
9
|
+
// vault is minutes long, and a client must be able to answer `initialize` and
|
|
10
|
+
// list tools immediately. Tools that depend on the index consult
|
|
11
|
+
// `getIndexStatus()` and reply "still indexing — N/M chunks" until the
|
|
12
|
+
// background pass finishes.
|
|
13
|
+
//
|
|
14
|
+
// `--reindex` is the one synchronous mode: rebuild the index, exit, do not
|
|
15
|
+
// start the server.
|
|
9
16
|
import { resolve } from "node:path";
|
|
10
17
|
import { pathToFileURL } from "node:url";
|
|
11
18
|
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
12
19
|
import { GUEST_ROLE, resolveAccess } from "./access/rbac.js";
|
|
13
|
-
import {
|
|
20
|
+
import { markIndexError, markIndexing, markIndexReady, setIndexProgress, } from "./search/index-state.js";
|
|
21
|
+
import { isIndexFresh, reindexVault } from "./search/reindex.js";
|
|
14
22
|
import { createServer } from "./server.js";
|
|
15
23
|
import { directoryExists } from "./storage/local.js";
|
|
16
24
|
import { loadConfig } from "./utils/config.js";
|
|
@@ -58,41 +66,98 @@ export async function main(argv = process.argv.slice(2)) {
|
|
|
58
66
|
if (access.role === null && roleName !== GUEST_ROLE) {
|
|
59
67
|
process.stderr.write(`daftari: warning: role '${roleName}' not found in config — running as deny-all guest\n`);
|
|
60
68
|
}
|
|
61
|
-
//
|
|
62
|
-
//
|
|
63
|
-
//
|
|
64
|
-
//
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
else {
|
|
82
|
-
// A failed index is not fatal: lexical search still works and the search
|
|
83
|
-
// tools retry indexing lazily on first use.
|
|
84
|
-
process.stderr.write(`daftari: warning: index build failed: ${reindexed.error.message}\n`);
|
|
85
|
-
}
|
|
86
|
-
if (argv.includes("--reindex")) {
|
|
87
|
-
if (!reindexed.ok)
|
|
69
|
+
// The persisted index is a derived cache: if every file on disk matches the
|
|
70
|
+
// manifest written by the last reindex, the on-disk index already reflects
|
|
71
|
+
// the vault and we can skip the embedding pass entirely (~25 min on a
|
|
72
|
+
// multi-thousand-file vault). --reindex forces a rebuild even when fresh.
|
|
73
|
+
const forceReindex = argv.includes("--reindex");
|
|
74
|
+
// --reindex is the one synchronous mode: rebuild and exit. No transport,
|
|
75
|
+
// no background work. The IndexState is updated for completeness but no
|
|
76
|
+
// tool runs against it in this mode.
|
|
77
|
+
if (forceReindex) {
|
|
78
|
+
markIndexing();
|
|
79
|
+
const reindexed = await reindexVault(vaultRoot, makeProgressReporter());
|
|
80
|
+
if (reindexed.ok) {
|
|
81
|
+
const r = reindexed.value;
|
|
82
|
+
markIndexReady();
|
|
83
|
+
process.stderr.write(`daftari: indexed ${r.documentCount} docs, ${r.chunkCount} chunks ` +
|
|
84
|
+
`(vectors ${r.vectorEnabled ? "on" : "off"})\n`);
|
|
85
|
+
}
|
|
86
|
+
else {
|
|
87
|
+
markIndexError(reindexed.error.message);
|
|
88
|
+
process.stderr.write(`daftari: warning: index build failed: ${reindexed.error.message}\n`);
|
|
88
89
|
process.exitCode = 1;
|
|
90
|
+
}
|
|
89
91
|
return;
|
|
90
92
|
}
|
|
93
|
+
// Open MCP transport first so the client can answer `initialize` and
|
|
94
|
+
// `tools/list` immediately. Indexing — if needed — runs as a background
|
|
95
|
+
// task; tools that depend on the index will respond "still indexing" until
|
|
96
|
+
// it completes.
|
|
91
97
|
const server = createServer(vaultRoot, access);
|
|
92
98
|
const transport = new StdioServerTransport();
|
|
93
99
|
await server.connect(transport);
|
|
94
100
|
process.stderr.write(`daftari: serving vault at ${vaultRoot} (stdio) — ` +
|
|
95
101
|
`user=${access.user} role=${access.roleName}\n`);
|
|
102
|
+
const fresh = await isIndexFresh(vaultRoot);
|
|
103
|
+
if (fresh) {
|
|
104
|
+
process.stderr.write(`daftari: index is up to date — skipping reindex\n`);
|
|
105
|
+
markIndexReady();
|
|
106
|
+
return;
|
|
107
|
+
}
|
|
108
|
+
// Background reindex. The promise is intentionally not awaited — main()
|
|
109
|
+
// returns once the transport is up, and the indexing pass runs to
|
|
110
|
+
// completion alongside the live server.
|
|
111
|
+
markIndexing();
|
|
112
|
+
process.stderr.write(`daftari: starting background reindex…\n`);
|
|
113
|
+
void runBackgroundReindex(vaultRoot);
|
|
114
|
+
}
|
|
115
|
+
async function runBackgroundReindex(vaultRoot) {
|
|
116
|
+
try {
|
|
117
|
+
const reindexed = await reindexVault(vaultRoot, makeProgressReporter());
|
|
118
|
+
if (reindexed.ok) {
|
|
119
|
+
const r = reindexed.value;
|
|
120
|
+
markIndexReady();
|
|
121
|
+
process.stderr.write(`daftari: indexed ${r.documentCount} docs, ${r.chunkCount} chunks ` +
|
|
122
|
+
`(vectors ${r.vectorEnabled ? "on" : "off"})\n`);
|
|
123
|
+
}
|
|
124
|
+
else {
|
|
125
|
+
markIndexError(reindexed.error.message);
|
|
126
|
+
process.stderr.write(`daftari: warning: background index build failed: ${reindexed.error.message}\n`);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
catch (e) {
|
|
130
|
+
const reason = e instanceof Error ? (e.stack ?? e.message) : String(e);
|
|
131
|
+
markIndexError(reason);
|
|
132
|
+
process.stderr.write(`daftari: warning: background indexer crashed: ${reason}\n`);
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
// Builds a ReindexOptions whose onProgress streams to both stderr (for
|
|
136
|
+
// operator visibility) and the in-process IndexState (so tools can return
|
|
137
|
+
// progress to MCP clients). TTY stderr gets a \r-updated single line; piped
|
|
138
|
+
// stderr gets a full line every ~5% so MCP-client logs stay readable instead
|
|
139
|
+
// of going silent for tens of minutes.
|
|
140
|
+
function makeProgressReporter() {
|
|
141
|
+
const PIPE_STEP = 0.05;
|
|
142
|
+
let nextPipeMark = 0;
|
|
143
|
+
return {
|
|
144
|
+
onProgress: (done, total) => {
|
|
145
|
+
setIndexProgress(done, total);
|
|
146
|
+
if (total === 0)
|
|
147
|
+
return;
|
|
148
|
+
if (process.stderr.isTTY) {
|
|
149
|
+
process.stderr.write(`\rdaftari: embedding ${done}/${total} chunks`);
|
|
150
|
+
if (done === total)
|
|
151
|
+
process.stderr.write("\n");
|
|
152
|
+
return;
|
|
153
|
+
}
|
|
154
|
+
const ratio = done / total;
|
|
155
|
+
if (ratio >= nextPipeMark || done === total) {
|
|
156
|
+
process.stderr.write(`daftari: embedding ${done}/${total} chunks\n`);
|
|
157
|
+
nextPipeMark = Math.floor(ratio / PIPE_STEP + 1) * PIPE_STEP;
|
|
158
|
+
}
|
|
159
|
+
},
|
|
160
|
+
};
|
|
96
161
|
}
|
|
97
162
|
// Auto-run only when this module is the process entry point (e.g. `tsx
|
|
98
163
|
// src/index.ts`). When imported (by cli.ts or tests) it stays inert.
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,kCAAkC;AAClC,EAAE;AACF,+EAA+E;AAC/E,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,kCAAkC;AAClC,EAAE;AACF,+EAA+E;AAC/E,2EAA2E;AAC3E,+EAA+E;AAC/E,qDAAqD;AACrD,EAAE;AACF,4EAA4E;AAC5E,8EAA8E;AAC9E,iEAAiE;AACjE,uEAAuE;AACvE,4BAA4B;AAC5B,EAAE;AACF,2EAA2E;AAC3E,oBAAoB;AAEpB,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAC;AACjF,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAC7D,OAAO,EACL,cAAc,EACd,YAAY,EACZ,cAAc,EACd,gBAAgB,GACjB,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,YAAY,EAAuB,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACtF,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAE/C,oEAAoE;AACpE,MAAM,UAAU,SAAS,CAAC,IAAc,EAAE,IAAY;IACpD,MAAM,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;IACzB,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACrC,IAAI,SAAS,KAAK,CAAC,CAAC,IAAI,SAAS,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QACpD,OAAO,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC;IACrC,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC;IAC1D,IAAI,MAAM;QAAE,OAAO,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;IACnD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,IAAc;IAC1C,OAAO,SAAS,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;AAClC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,IAAI,CAAC,OAAiB,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IAC/D,MAAM,QAAQ,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;IACrC,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,qDAAqD,CAAC,CAAC;QAC5E,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;QACrB,OAAO;IACT,CAAC;IAED,MAAM,SAAS,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;IACpC,IAAI,CAAC,CAAC,MAAM,eAAe,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC;QACxC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,uCAAuC,SAAS,IAAI,CAAC,CAAC;QAC3E,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;QACrB,OAAO;IACT,CAAC;IAED,2EAA2E;IAC3E,2DAA2D;IAC3D,MAAM,MAAM,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC;IACrC,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;QACf,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,MAAM,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC;QAC3D,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;QACrB,OAAO;IACT,CAAC;IAED,qEAAqE;IACrE,8DAA8D;IAC9D,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC;IAChD,MAAM,QAAQ,GAAG,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,UAAU,CAAC;IACvD,MAAM,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAC;IAC3D,IAAI,MAAM,CAAC,IAAI,KAAK,IAAI,IAAI,QAAQ,KAAK,UAAU,EAAE,CAAC;QACpD,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,2BAA2B,QAAQ,qDAAqD,CACzF,CAAC;IACJ,CAAC;IAED,4EAA4E;IAC5E,2EAA2E;IAC3E,sEAAsE;IACtE,0EAA0E;IAC1E,MAAM,YAAY,GAAG,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IAEhD,yEAAyE;IACzE,wEAAwE;IACxE,qCAAqC;IACrC,IAAI,YAAY,EAAE,CAAC;QACjB,YAAY,EAAE,CAAC;QACf,MAAM,SAAS,GAAG,MAAM,YAAY,CAAC,SAAS,EAAE,oBAAoB,EAAE,CAAC,CAAC;QACxE,IAAI,SAAS,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,CAAC,GAAG,SAAS,CAAC,KAAK,CAAC;YAC1B,cAAc,EAAE,CAAC;YACjB,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,oBAAoB,CAAC,CAAC,aAAa,UAAU,CAAC,CAAC,UAAU,UAAU;gBACjE,YAAY,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,KAAK,CAClD,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,cAAc,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YACxC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,yCAAyC,SAAS,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC;YAC3F,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;QACvB,CAAC;QACD,OAAO;IACT,CAAC;IAED,qEAAqE;IACrE,wEAAwE;IACxE,2EAA2E;IAC3E,gBAAgB;IAChB,MAAM,MAAM,GAAG,YAAY,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAC/C,MAAM,SAAS,GAAG,IAAI,oBAAoB,EAAE,CAAC;IAC7C,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAChC,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,6BAA6B,SAAS,aAAa;QACjD,QAAQ,MAAM,CAAC,IAAI,SAAS,MAAM,CAAC,QAAQ,IAAI,CAClD,CAAC;IAEF,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,SAAS,CAAC,CAAC;IAC5C,IAAI,KAAK,EAAE,CAAC;QACV,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,mDAAmD,CAAC,CAAC;QAC1E,cAAc,EAAE,CAAC;QACjB,OAAO;IACT,CAAC;IAED,wEAAwE;IACxE,kEAAkE;IAClE,wCAAwC;IACxC,YAAY,EAAE,CAAC;IACf,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,yCAAyC,CAAC,CAAC;IAChE,KAAK,oBAAoB,CAAC,SAAS,CAAC,CAAC;AACvC,CAAC;AAED,KAAK,UAAU,oBAAoB,CAAC,SAAiB;IACnD,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,MAAM,YAAY,CAAC,SAAS,EAAE,oBAAoB,EAAE,CAAC,CAAC;QACxE,IAAI,SAAS,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,CAAC,GAAG,SAAS,CAAC,KAAK,CAAC;YAC1B,cAAc,EAAE,CAAC;YACjB,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,oBAAoB,CAAC,CAAC,aAAa,UAAU,CAAC,CAAC,UAAU,UAAU;gBACjE,YAAY,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,KAAK,CAClD,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,cAAc,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YACxC,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,oDAAoD,SAAS,CAAC,KAAK,CAAC,OAAO,IAAI,CAChF,CAAC;QACJ,CAAC;IACH,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,MAAM,GAAG,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QACvE,cAAc,CAAC,MAAM,CAAC,CAAC;QACvB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,iDAAiD,MAAM,IAAI,CAAC,CAAC;IACpF,CAAC;AACH,CAAC;AAED,uEAAuE;AACvE,0EAA0E;AAC1E,4EAA4E;AAC5E,6EAA6E;AAC7E,uCAAuC;AACvC,SAAS,oBAAoB;IAC3B,MAAM,SAAS,GAAG,IAAI,CAAC;IACvB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,OAAO;QACL,UAAU,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;YAC1B,gBAAgB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YAC9B,IAAI,KAAK,KAAK,CAAC;gBAAE,OAAO;YACxB,IAAI,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;gBACzB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,wBAAwB,IAAI,IAAI,KAAK,SAAS,CAAC,CAAC;gBACrE,IAAI,IAAI,KAAK,KAAK;oBAAE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAC/C,OAAO;YACT,CAAC;YACD,MAAM,KAAK,GAAG,IAAI,GAAG,KAAK,CAAC;YAC3B,IAAI,KAAK,IAAI,YAAY,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;gBAC5C,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,sBAAsB,IAAI,IAAI,KAAK,WAAW,CAAC,CAAC;gBACrE,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,SAAS,GAAG,CAAC,CAAC,GAAG,SAAS,CAAC;YAC/D,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC;AAED,uEAAuE;AACvE,qEAAqE;AACrE,MAAM,QAAQ,GAAG,aAAa,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC;AAC3D,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,QAAQ,EAAE,CAAC;IACjC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;QACjB,MAAM,MAAM,GAAG,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QACvE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,mBAAmB,MAAM,IAAI,CAAC,CAAC;QACpD,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;IACvB,CAAC,CAAC,CAAC;AACL,CAAC"}
|
package/dist/search/hybrid.js
CHANGED
|
@@ -11,7 +11,7 @@ import { computeDecay } from "../curation/decay.js";
|
|
|
11
11
|
import { ok } from "../frontmatter/types.js";
|
|
12
12
|
import { getAllChunks, getAllDocuments, getChunksForPath, getDocument, } from "../storage/index-db.js";
|
|
13
13
|
import { buildBm25, searchBm25, tokenize } from "./bm25.js";
|
|
14
|
-
import { cosineSimilarity, embedQuery, meanEmbedding } from "./vector.js";
|
|
14
|
+
import { cosineSimilarity, EMBEDDING_MODEL, embedQuery, meanEmbedding } from "./vector.js";
|
|
15
15
|
export const DEFAULT_WEIGHTS = { bm25: 0.5, vector: 0.5 };
|
|
16
16
|
const SNIPPET_RADIUS = 140;
|
|
17
17
|
// Pulls a readable excerpt from a document body, centred on the earliest
|
|
@@ -67,7 +67,7 @@ function rankDocuments(db, queryTokens, queryEmbedding, opts) {
|
|
|
67
67
|
const vectorRaw = new Map();
|
|
68
68
|
let vectorUsed = false;
|
|
69
69
|
if (queryEmbedding) {
|
|
70
|
-
for (const chunk of getAllChunks(db)) {
|
|
70
|
+
for (const chunk of getAllChunks(db, EMBEDDING_MODEL)) {
|
|
71
71
|
if (!chunk.embedding)
|
|
72
72
|
continue;
|
|
73
73
|
vectorUsed = true;
|
|
@@ -151,7 +151,7 @@ export function relatedSearch(db, path, options = {}) {
|
|
|
151
151
|
error: new Error(`document not indexed: ${path} (try vault_reindex)`),
|
|
152
152
|
};
|
|
153
153
|
}
|
|
154
|
-
const chunkVectors = getChunksForPath(db, path)
|
|
154
|
+
const chunkVectors = getChunksForPath(db, path, EMBEDDING_MODEL)
|
|
155
155
|
.map((c) => c.embedding)
|
|
156
156
|
.filter((e) => e !== null);
|
|
157
157
|
const queryEmbedding = meanEmbedding(chunkVectors);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"hybrid.js","sourceRoot":"","sources":["../../src/search/hybrid.ts"],"names":[],"mappings":"AAAA,4EAA4E;AAC5E,EAAE;AACF,+EAA+E;AAC/E,4EAA4E;AAC5E,wCAAwC;AACxC,EAAE;AACF,wEAAwE;AACxE,wEAAwE;AACxE,kEAAkE;AAElE,OAAO,EAAE,YAAY,EAAmB,MAAM,sBAAsB,CAAC;AACrE,OAAO,EAAE,EAAE,EAAe,MAAM,yBAAyB,CAAC;AAC1D,OAAO,EACL,YAAY,EACZ,eAAe,EACf,gBAAgB,EAChB,WAAW,GAEZ,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAC5D,OAAO,EAAE,gBAAgB,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"hybrid.js","sourceRoot":"","sources":["../../src/search/hybrid.ts"],"names":[],"mappings":"AAAA,4EAA4E;AAC5E,EAAE;AACF,+EAA+E;AAC/E,4EAA4E;AAC5E,wCAAwC;AACxC,EAAE;AACF,wEAAwE;AACxE,wEAAwE;AACxE,kEAAkE;AAElE,OAAO,EAAE,YAAY,EAAmB,MAAM,sBAAsB,CAAC;AACrE,OAAO,EAAE,EAAE,EAAe,MAAM,yBAAyB,CAAC;AAC1D,OAAO,EACL,YAAY,EACZ,eAAe,EACf,gBAAgB,EAChB,WAAW,GAEZ,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAC5D,OAAO,EAAE,gBAAgB,EAAE,eAAe,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAO3F,MAAM,CAAC,MAAM,eAAe,GAAkB,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;AAsBzE,MAAM,cAAc,GAAG,GAAG,CAAC;AAE3B,yEAAyE;AACzE,6EAA6E;AAC7E,2CAA2C;AAC3C,SAAS,WAAW,CAAC,OAAe,EAAE,WAAqB;IACzD,MAAM,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IACtD,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACtC,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,EAAE,CAAC;IAEtC,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC;IACf,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;QAC/B,MAAM,EAAE,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAC/B,IAAI,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,CAAC,IAAI,EAAE,GAAG,KAAK,CAAC;YAAE,KAAK,GAAG,EAAE,CAAC;IAC5D,CAAC;IAED,IAAI,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;QACjB,OAAO,SAAS,CAAC,MAAM,GAAG,cAAc,GAAG,CAAC;YAC1C,CAAC,CAAC,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,cAAc,GAAG,CAAC,CAAC,GAAG;YAC9C,CAAC,CAAC,SAAS,CAAC;IAChB,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,GAAG,cAAc,CAAC,CAAC;IAClD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,MAAM,EAAE,KAAK,GAAG,cAAc,CAAC,CAAC;IAC/D,IAAI,OAAO,GAAG,SAAS,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IAC1C,IAAI,KAAK,GAAG,CAAC;QAAE,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;IACvC,IAAI,GAAG,GAAG,SAAS,CAAC,MAAM;QAAE,OAAO,GAAG,GAAG,OAAO,GAAG,CAAC;IACpD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,6EAA6E;AAC7E,wCAAwC;AACxC,SAAS,SAAS,CAAC,MAA2B;IAC5C,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE;QAAE,IAAI,CAAC,GAAG,GAAG;YAAE,GAAG,GAAG,CAAC,CAAC;IACtD,IAAI,GAAG,KAAK,CAAC;QAAE,OAAO,IAAI,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAChE,OAAO,IAAI,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;AAC5D,CAAC;AAQD,8EAA8E;AAC9E,8EAA8E;AAC9E,oEAAoE;AACpE,SAAS,aAAa,CACpB,EAAW,EACX,WAAqB,EACrB,cAAmC,EACnC,IAAiB;IAEjB,MAAM,SAAS,GAAG,eAAe,CAAC,EAAE,CAAC,CAAC;IACtC,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAE1D,MAAM,SAAS,GAAG,SAAS,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC;IACxF,MAAM,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC1C,KAAK,MAAM,GAAG,IAAI,UAAU,CAAC,SAAS,EAAE,WAAW,CAAC,EAAE,CAAC;QACrD,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC;IACnC,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC5C,IAAI,UAAU,GAAG,KAAK,CAAC;IACvB,IAAI,cAAc,EAAE,CAAC;QACnB,KAAK,MAAM,KAAK,IAAI,YAAY,CAAC,EAAE,EAAE,eAAe,CAAC,EAAE,CAAC;YACtD,IAAI,CAAC,KAAK,CAAC,SAAS;gBAAE,SAAS;YAC/B,UAAU,GAAG,IAAI,CAAC;YAClB,MAAM,GAAG,GAAG,gBAAgB,CAAC,cAAc,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;YAC9D,MAAM,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC;YACpD,IAAI,GAAG,GAAG,IAAI;gBAAE,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QACjD,CAAC;IACH,CAAC;IAED,MAAM,QAAQ,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;IACpC,MAAM,UAAU,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC;IAExC,yEAAyE;IACzE,MAAM,OAAO,GAAkB,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;IAElF,MAAM,UAAU,GAAG,IAAI,GAAG,CAAS,CAAC,GAAG,QAAQ,CAAC,IAAI,EAAE,EAAE,GAAG,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAE/E,MAAM,IAAI,GAAgB,EAAE,CAAC;IAC7B,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,IAAI,IAAI,KAAK,IAAI,CAAC,WAAW;YAAE,SAAS;QACxC,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC7B,IAAI,CAAC,GAAG;YAAE,SAAS;QACnB,MAAM,SAAS,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1C,MAAM,WAAW,GAAG,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC9C,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,GAAG,SAAS,GAAG,OAAO,CAAC,MAAM,GAAG,WAAW,CAAC;QACtE,IAAI,KAAK,IAAI,CAAC;YAAE,SAAS;QACzB,IAAI,CAAC,IAAI,CAAC;YACR,IAAI;YACJ,KAAK,EAAE,GAAG,CAAC,KAAK;YAChB,UAAU,EAAE,GAAG,CAAC,UAAU;YAC1B,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,KAAK;YACL,SAAS;YACT,WAAW;YACX,OAAO,EAAE,WAAW,CAAC,GAAG,CAAC,OAAO,EAAE,WAAW,CAAC;YAC9C,KAAK,EAAE,YAAY,CAAC;gBAClB,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,UAAU,EAAE,GAAG,CAAC,UAAU;gBAC1B,OAAO,EAAE,GAAG,CAAC,OAAO;gBACpB,OAAO,EAAE,GAAG,CAAC,OAAO;gBACpB,QAAQ,EAAE,GAAG,CAAC,OAAO;gBACrB,aAAa,EAAE,GAAG,CAAC,YAAY;aAChC,CAAC;SACH,CAAC,CAAC;IACL,CAAC;IAED,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IACvC,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,UAAU,EAAE,CAAC;AACzD,CAAC;AAOD,mDAAmD;AACnD,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,EAAW,EACX,KAAa,EACb,UAA+B,EAAE;IAEjC,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,eAAe,CAAC;IACnD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC;IAClC,MAAM,WAAW,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;IAEpC,MAAM,WAAW,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,CAAC;IAC5C,MAAM,cAAc,GAAG,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;IAEjE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,aAAa,CAAC,EAAE,EAAE,WAAW,EAAE,cAAc,EAAE;QAC1E,OAAO;QACP,KAAK;QACL,WAAW,EAAE,SAAS;KACvB,CAAC,CAAC;IAEH,OAAO,EAAE,CAAC;QACR,KAAK;QACL,KAAK,EAAE,IAAI,CAAC,MAAM;QAClB,UAAU;QACV,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;QACtD,IAAI;KACL,CAAC,CAAC;AACL,CAAC;AAUD,8EAA8E;AAC9E,6EAA6E;AAC7E,6EAA6E;AAC7E,6EAA6E;AAC7E,uBAAuB;AACvB,MAAM,UAAU,aAAa,CAC3B,EAAW,EACX,IAAY,EACZ,UAA+B,EAAE;IAEjC,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,eAAe,CAAC;IACnD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC;IAElC,MAAM,GAAG,GAAG,WAAW,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;IAClC,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,OAAO;YACL,EAAE,EAAE,KAAK;YACT,KAAK,EAAE,IAAI,KAAK,CAAC,yBAAyB,IAAI,sBAAsB,CAAC;SACtE,CAAC;IACJ,CAAC;IAED,MAAM,YAAY,GAAG,gBAAgB,CAAC,EAAE,EAAE,IAAI,EAAE,eAAe,CAAC;SAC7D,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;SACvB,MAAM,CAAC,CAAC,CAAC,EAAqB,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;IAChD,MAAM,cAAc,GAAG,aAAa,CAAC,YAAY,CAAC,CAAC;IAEnD,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,aAAa,CAAC,EAAE,EAAE,GAAG,CAAC,MAAM,EAAE,cAAc,EAAE;QACzE,OAAO;QACP,KAAK;QACL,WAAW,EAAE,IAAI;KAClB,CAAC,CAAC;IAEH,OAAO,EAAE,CAAC;QACR,IAAI;QACJ,KAAK,EAAE,IAAI,CAAC,MAAM;QAClB,UAAU;QACV,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;QACtD,IAAI;KACL,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
export type IndexStatus = "ready" | "indexing" | "error";
|
|
2
|
+
export interface IndexSnapshot {
|
|
3
|
+
status: IndexStatus;
|
|
4
|
+
done: number;
|
|
5
|
+
total: number;
|
|
6
|
+
error: string | null;
|
|
7
|
+
startedAt: string | null;
|
|
8
|
+
finishedAt: string | null;
|
|
9
|
+
}
|
|
10
|
+
export declare function getIndexStatus(): IndexSnapshot;
|
|
11
|
+
export declare function markIndexing(): void;
|
|
12
|
+
export declare function setIndexProgress(done: number, total: number): void;
|
|
13
|
+
export declare function markIndexReady(): void;
|
|
14
|
+
export declare function markIndexError(message: string): void;
|
|
15
|
+
export declare function resetIndexState(): void;
|
|
16
|
+
export declare function indexingBusyMessage(snapshot: IndexSnapshot): string;
|
|
17
|
+
//# sourceMappingURL=index-state.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index-state.d.ts","sourceRoot":"","sources":["../../src/search/index-state.ts"],"names":[],"mappings":"AASA,MAAM,MAAM,WAAW,GAAG,OAAO,GAAG,UAAU,GAAG,OAAO,CAAC;AAEzD,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,WAAW,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;CAC3B;AAeD,wBAAgB,cAAc,IAAI,aAAa,CAE9C;AAED,wBAAgB,YAAY,IAAI,IAAI,CASnC;AAED,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAGlE;AAED,wBAAgB,cAAc,IAAI,IAAI,CAOrC;AAED,wBAAgB,cAAc,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAOpD;AAID,wBAAgB,eAAe,IAAI,IAAI,CAEtC;AAKD,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,aAAa,GAAG,MAAM,CAKnE"}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
// Process-wide indexing state.
|
|
2
|
+
//
|
|
3
|
+
// The MCP server opens the stdio transport before running a cold-start reindex
|
|
4
|
+
// so `initialize` / `tools/list` answer immediately. While that background pass
|
|
5
|
+
// runs the server is alive but search and write tools cannot trust the index;
|
|
6
|
+
// they consult this module to decide whether to serve or to reply "still
|
|
7
|
+
// indexing — N/M chunks". A single in-process snapshot is enough because the
|
|
8
|
+
// server is one Node process per vault.
|
|
9
|
+
function freshState() {
|
|
10
|
+
return {
|
|
11
|
+
status: "ready",
|
|
12
|
+
done: 0,
|
|
13
|
+
total: 0,
|
|
14
|
+
error: null,
|
|
15
|
+
startedAt: null,
|
|
16
|
+
finishedAt: null,
|
|
17
|
+
};
|
|
18
|
+
}
|
|
19
|
+
let state = freshState();
|
|
20
|
+
export function getIndexStatus() {
|
|
21
|
+
return { ...state };
|
|
22
|
+
}
|
|
23
|
+
export function markIndexing() {
|
|
24
|
+
state = {
|
|
25
|
+
status: "indexing",
|
|
26
|
+
done: 0,
|
|
27
|
+
total: 0,
|
|
28
|
+
error: null,
|
|
29
|
+
startedAt: new Date().toISOString(),
|
|
30
|
+
finishedAt: null,
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
export function setIndexProgress(done, total) {
|
|
34
|
+
if (state.status !== "indexing")
|
|
35
|
+
return;
|
|
36
|
+
state = { ...state, done, total };
|
|
37
|
+
}
|
|
38
|
+
export function markIndexReady() {
|
|
39
|
+
state = {
|
|
40
|
+
...state,
|
|
41
|
+
status: "ready",
|
|
42
|
+
error: null,
|
|
43
|
+
finishedAt: new Date().toISOString(),
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
export function markIndexError(message) {
|
|
47
|
+
state = {
|
|
48
|
+
...state,
|
|
49
|
+
status: "error",
|
|
50
|
+
error: message,
|
|
51
|
+
finishedAt: new Date().toISOString(),
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
// Tests load tools without running main(); resetting the singleton between
|
|
55
|
+
// suites keeps cross-test pollution out of the state machine.
|
|
56
|
+
export function resetIndexState() {
|
|
57
|
+
state = freshState();
|
|
58
|
+
}
|
|
59
|
+
// Formatted message tools return to clients while indexing is in progress.
|
|
60
|
+
// One place so the phrasing is consistent across vault_search, vault_write,
|
|
61
|
+
// vault_reindex, etc.
|
|
62
|
+
export function indexingBusyMessage(snapshot) {
|
|
63
|
+
if (snapshot.total > 0) {
|
|
64
|
+
return `vault is still indexing (${snapshot.done}/${snapshot.total} chunks) — try again shortly`;
|
|
65
|
+
}
|
|
66
|
+
return `vault is still indexing — try again shortly`;
|
|
67
|
+
}
|
|
68
|
+
//# sourceMappingURL=index-state.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index-state.js","sourceRoot":"","sources":["../../src/search/index-state.ts"],"names":[],"mappings":"AAAA,+BAA+B;AAC/B,EAAE;AACF,+EAA+E;AAC/E,gFAAgF;AAChF,8EAA8E;AAC9E,yEAAyE;AACzE,6EAA6E;AAC7E,wCAAwC;AAaxC,SAAS,UAAU;IACjB,OAAO;QACL,MAAM,EAAE,OAAO;QACf,IAAI,EAAE,CAAC;QACP,KAAK,EAAE,CAAC;QACR,KAAK,EAAE,IAAI;QACX,SAAS,EAAE,IAAI;QACf,UAAU,EAAE,IAAI;KACjB,CAAC;AACJ,CAAC;AAED,IAAI,KAAK,GAAkB,UAAU,EAAE,CAAC;AAExC,MAAM,UAAU,cAAc;IAC5B,OAAO,EAAE,GAAG,KAAK,EAAE,CAAC;AACtB,CAAC;AAED,MAAM,UAAU,YAAY;IAC1B,KAAK,GAAG;QACN,MAAM,EAAE,UAAU;QAClB,IAAI,EAAE,CAAC;QACP,KAAK,EAAE,CAAC;QACR,KAAK,EAAE,IAAI;QACX,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,UAAU,EAAE,IAAI;KACjB,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,IAAY,EAAE,KAAa;IAC1D,IAAI,KAAK,CAAC,MAAM,KAAK,UAAU;QAAE,OAAO;IACxC,KAAK,GAAG,EAAE,GAAG,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;AACpC,CAAC;AAED,MAAM,UAAU,cAAc;IAC5B,KAAK,GAAG;QACN,GAAG,KAAK;QACR,MAAM,EAAE,OAAO;QACf,KAAK,EAAE,IAAI;QACX,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KACrC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,OAAe;IAC5C,KAAK,GAAG;QACN,GAAG,KAAK;QACR,MAAM,EAAE,OAAO;QACf,KAAK,EAAE,OAAO;QACd,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KACrC,CAAC;AACJ,CAAC;AAED,2EAA2E;AAC3E,8DAA8D;AAC9D,MAAM,UAAU,eAAe;IAC7B,KAAK,GAAG,UAAU,EAAE,CAAC;AACvB,CAAC;AAED,2EAA2E;AAC3E,4EAA4E;AAC5E,sBAAsB;AACtB,MAAM,UAAU,mBAAmB,CAAC,QAAuB;IACzD,IAAI,QAAQ,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;QACvB,OAAO,4BAA4B,QAAQ,CAAC,IAAI,IAAI,QAAQ,CAAC,KAAK,8BAA8B,CAAC;IACnG,CAAC;IACD,OAAO,6CAA6C,CAAC;AACvD,CAAC"}
|
package/dist/search/reindex.d.ts
CHANGED
|
@@ -1,10 +1,14 @@
|
|
|
1
1
|
import { type Result } from "../frontmatter/types.js";
|
|
2
|
+
export declare function isIndexFresh(vaultRoot: string): Promise<boolean>;
|
|
2
3
|
export interface ReindexResult {
|
|
3
4
|
documentCount: number;
|
|
4
5
|
chunkCount: number;
|
|
5
6
|
vectorEnabled: boolean;
|
|
6
7
|
skipped: string[];
|
|
7
8
|
indexedAt: string;
|
|
9
|
+
embeddedCount: number;
|
|
10
|
+
cacheHits: number;
|
|
11
|
+
orphansRemoved: number;
|
|
8
12
|
}
|
|
9
13
|
export interface ReindexOptions {
|
|
10
14
|
onProgress?: (done: number, total: number) => void;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"reindex.d.ts","sourceRoot":"","sources":["../../src/search/reindex.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"reindex.d.ts","sourceRoot":"","sources":["../../src/search/reindex.ts"],"names":[],"mappings":"AAqBA,OAAO,EAAW,KAAK,MAAM,EAAE,MAAM,yBAAyB,CAAC;AA8E/D,wBAAsB,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CActE;AAED,MAAM,WAAW,aAAa;IAC5B,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,OAAO,CAAC;IACvB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAIlB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,cAAc;IAM7B,UAAU,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;CACpD;AA+FD,wBAAsB,YAAY,CAChC,SAAS,EAAE,MAAM,EACjB,IAAI,GAAE,cAAmB,GACxB,OAAO,CAAC,MAAM,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC,CAmGvC;AAED,MAAM,WAAW,mBAAmB;IAClC,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,OAAO,CAAC;CACxB;AASD,wBAAsB,aAAa,CACjC,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,MAAM,CAAC,mBAAmB,EAAE,KAAK,CAAC,CAAC,CA8F7C"}
|