@owrede/vault-memory 2.1.0 → 2.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +20 -0
- package/README.md +35 -11
- package/dist/cli.js +617 -147
- package/dist/cli.js.map +1 -1
- package/package.json +2 -1
package/dist/cli.js
CHANGED
|
@@ -89,7 +89,7 @@ function extractResourceLength(handle) {
|
|
|
89
89
|
if (firstSlash === -1) return 0;
|
|
90
90
|
return afterScheme.length - (firstSlash + 1);
|
|
91
91
|
}
|
|
92
|
-
var ServerConfigSchema, VaultConfigSchema, BriefOllamaConfigSchema, BriefConfigSchema, ContractsMcpClientConfigSchema, ContractsConfigSchema, DEFAULT_CONTRACTS_CONFIG, PluginConfigSchema, DEFAULT_PLUGIN_CONFIG, MemorySinkConfigSchema, MemoryConfigSchema, AppConfigSchema, DEFAULT_CONFIG;
|
|
92
|
+
var ServerConfigSchema, ContextFitConfigSchema, VaultConfigSchema, BriefOllamaConfigSchema, BriefConfigSchema, ContractsMcpClientConfigSchema, ContractsConfigSchema, DEFAULT_CONTRACTS_CONFIG, PluginConfigSchema, DEFAULT_PLUGIN_CONFIG, MemorySinkConfigSchema, MemoryConfigSchema, AppConfigSchema, DEFAULT_CONFIG;
|
|
93
93
|
var init_loader = __esm({
|
|
94
94
|
"src/config/loader.ts"() {
|
|
95
95
|
"use strict";
|
|
@@ -102,9 +102,17 @@ var init_loader = __esm({
|
|
|
102
102
|
reranker_backend: z.enum(["onnx", "ollama"]).optional(),
|
|
103
103
|
reranker_model_dir: z.string().optional()
|
|
104
104
|
});
|
|
105
|
+
ContextFitConfigSchema = z.object({
|
|
106
|
+
command: z.string().min(1).optional(),
|
|
107
|
+
tokenizer: z.string().min(1).optional(),
|
|
108
|
+
method: z.enum(["exact", "bm25", "sid", "graph", "hierarchy", "hybrid"]).optional()
|
|
109
|
+
});
|
|
105
110
|
VaultConfigSchema = z.object({
|
|
106
111
|
name: z.string().min(1),
|
|
107
112
|
path: z.string().min(1),
|
|
113
|
+
// ADR-008: retrieval engine. Omitted ⇒ "ollama" (back-compat default).
|
|
114
|
+
backend: z.enum(["ollama", "contextfit"]).optional(),
|
|
115
|
+
contextfit: ContextFitConfigSchema.optional(),
|
|
108
116
|
embedding_model: z.string().optional(),
|
|
109
117
|
secondary_embedding_model: z.string().optional(),
|
|
110
118
|
write_enabled: z.boolean().optional(),
|
|
@@ -228,7 +236,8 @@ async function addVault(opts) {
|
|
|
228
236
|
name: proposedName,
|
|
229
237
|
path: resolvedPath,
|
|
230
238
|
writeEnabled: opts.writeEnabled ?? false,
|
|
231
|
-
excludeGlobs: opts.excludeGlobs ?? DEFAULT_EXCLUDE_GLOBS
|
|
239
|
+
excludeGlobs: opts.excludeGlobs ?? DEFAULT_EXCLUDE_GLOBS,
|
|
240
|
+
...opts.backend ? { backend: opts.backend } : {}
|
|
232
241
|
});
|
|
233
242
|
await ensureFileExists(cfgFile);
|
|
234
243
|
await appendToFile(cfgFile, block);
|
|
@@ -252,13 +261,21 @@ function renderVaultBlock(input) {
|
|
|
252
261
|
`# Added by vault-memory add-vault on ${(/* @__PURE__ */ new Date()).toISOString()}`,
|
|
253
262
|
"[[vaults]]",
|
|
254
263
|
`name = ${JSON.stringify(input.name)}`,
|
|
255
|
-
`path = ${JSON.stringify(input.path)}
|
|
264
|
+
`path = ${JSON.stringify(input.path)}`
|
|
265
|
+
];
|
|
266
|
+
if (input.backend === "contextfit") {
|
|
267
|
+
lines.push(
|
|
268
|
+
`# ADR-008: CPU-only, token-native engine (no Ollama/embeddings/GPU).`,
|
|
269
|
+
`backend = "contextfit"`
|
|
270
|
+
);
|
|
271
|
+
}
|
|
272
|
+
lines.push(
|
|
256
273
|
`write_enabled = ${input.writeEnabled}`,
|
|
257
274
|
`exclude_globs = [`,
|
|
258
275
|
...input.excludeGlobs.map((g) => ` ${JSON.stringify(g)},`),
|
|
259
276
|
`]`,
|
|
260
277
|
""
|
|
261
|
-
|
|
278
|
+
);
|
|
262
279
|
return lines.join("\n");
|
|
263
280
|
}
|
|
264
281
|
async function ensureFileExists(path7) {
|
|
@@ -1303,7 +1320,7 @@ function backfillSectionsFromChunks(db) {
|
|
|
1303
1320
|
@parent_id, @ord, @chunk_id_first, @chunk_id_last, @created_at)
|
|
1304
1321
|
`);
|
|
1305
1322
|
const lookupExistingSection = db.prepare(
|
|
1306
|
-
"SELECT id FROM sections WHERE note_id = ? AND anchor = ?"
|
|
1323
|
+
"SELECT id FROM sections WHERE note_id = ? AND heading_path = ? AND anchor = ?"
|
|
1307
1324
|
);
|
|
1308
1325
|
let backfilled = 0;
|
|
1309
1326
|
const now = Date.now();
|
|
@@ -1339,7 +1356,11 @@ function backfillSectionsFromChunks(db) {
|
|
|
1339
1356
|
if (info.changes > 0) {
|
|
1340
1357
|
insertedIds.push(Number(info.lastInsertRowid));
|
|
1341
1358
|
} else {
|
|
1342
|
-
const existing2 = lookupExistingSection.get(
|
|
1359
|
+
const existing2 = lookupExistingSection.get(
|
|
1360
|
+
note.id,
|
|
1361
|
+
JSON.stringify(s.heading_path),
|
|
1362
|
+
s.anchor
|
|
1363
|
+
);
|
|
1343
1364
|
insertedIds.push(existing2 ? Number(existing2.id) : null);
|
|
1344
1365
|
}
|
|
1345
1366
|
}
|
|
@@ -1671,6 +1692,11 @@ function runMigration014(db, _ctx) {
|
|
|
1671
1692
|
ON contract_audit(verb);
|
|
1672
1693
|
`);
|
|
1673
1694
|
}
|
|
1695
|
+
function runMigration015(db, _ctx) {
|
|
1696
|
+
db.exec(
|
|
1697
|
+
"DROP INDEX IF EXISTS sections_note_anchor; CREATE UNIQUE INDEX IF NOT EXISTS sections_note_headingpath_anchor ON sections(note_id, heading_path, anchor);"
|
|
1698
|
+
);
|
|
1699
|
+
}
|
|
1674
1700
|
var INITIAL_SCHEMA, MIGRATION_002_ALIASES, MIGRATION_003_FIX_DELETE_FKS, MIGRATION_004_VARIABLE_DIMS, MIGRATION_006_BODY_HASH, MIGRATION_007_DOC_URI_ADD, MIGRATIONS;
|
|
1675
1701
|
var init_schema = __esm({
|
|
1676
1702
|
"src/db/schema.ts"() {
|
|
@@ -1946,6 +1972,11 @@ CREATE INDEX IF NOT EXISTS idx_notes_doc_uri ON notes(doc_uri);
|
|
|
1946
1972
|
version: 14,
|
|
1947
1973
|
description: "contract_audit table \u2014 Phase 6 / CON-* / Q-AUD",
|
|
1948
1974
|
run: runMigration014
|
|
1975
|
+
},
|
|
1976
|
+
{
|
|
1977
|
+
version: 15,
|
|
1978
|
+
description: "section identity = (note_id, heading_path, anchor) \u2014 context-aware, no longer collapse byte-identical siblings in different contexts (ADR-032 revised)",
|
|
1979
|
+
run: runMigration015
|
|
1949
1980
|
}
|
|
1950
1981
|
];
|
|
1951
1982
|
}
|
|
@@ -3050,6 +3081,9 @@ var init_sections = __esm({
|
|
|
3050
3081
|
this._getByAnchor = db.prepare(
|
|
3051
3082
|
"SELECT * FROM sections WHERE note_id = ? AND anchor = ?"
|
|
3052
3083
|
);
|
|
3084
|
+
this._getByIdentity = db.prepare(
|
|
3085
|
+
"SELECT * FROM sections WHERE note_id = ? AND heading_path = ? AND anchor = ?"
|
|
3086
|
+
);
|
|
3053
3087
|
this._findContainingChunk = db.prepare(
|
|
3054
3088
|
// `chunk_id` is monotonically increasing per note; chunk_id_first
|
|
3055
3089
|
// and chunk_id_last carve disjoint ranges (or both NULL for a
|
|
@@ -3073,6 +3107,7 @@ var init_sections = __esm({
|
|
|
3073
3107
|
_deleteByNote;
|
|
3074
3108
|
_getByNote;
|
|
3075
3109
|
_getByAnchor;
|
|
3110
|
+
_getByIdentity;
|
|
3076
3111
|
_findContainingChunk;
|
|
3077
3112
|
_countByNote;
|
|
3078
3113
|
/**
|
|
@@ -3105,14 +3140,15 @@ var init_sections = __esm({
|
|
|
3105
3140
|
return ids;
|
|
3106
3141
|
}
|
|
3107
3142
|
/**
|
|
3108
|
-
* Insert one section, collision-safe. Returns the id of the row that
|
|
3109
|
-
*
|
|
3110
|
-
* same-
|
|
3111
|
-
* row's id (so callers can resolve parent_id
|
|
3112
|
-
*
|
|
3113
|
-
*
|
|
3114
|
-
*
|
|
3115
|
-
*
|
|
3143
|
+
* Insert one section, collision-safe. Returns the id of the row that now
|
|
3144
|
+
* owns the identity (note_id, heading_path, anchor): the freshly inserted
|
|
3145
|
+
* row, or — when a same-context byte-identical sibling already won the
|
|
3146
|
+
* unique slot — that surviving row's id (so callers can resolve parent_id
|
|
3147
|
+
* linkage). Per ADR-032 (revised), a collision now requires BOTH same anchor
|
|
3148
|
+
* AND same heading_path, so differently-placed identical sections persist as
|
|
3149
|
+
* distinct rows. Mirrors src/sections/backfill.ts. The live indexer uses
|
|
3150
|
+
* this instead of `insertMany` so duplicate sibling headings can't abort the
|
|
3151
|
+
* whole index run (see ISSUE-indexer-duplicate-anchor.md).
|
|
3116
3152
|
*/
|
|
3117
3153
|
insertOneResolving(r) {
|
|
3118
3154
|
const info = this._insert.run({
|
|
@@ -3128,7 +3164,7 @@ var init_sections = __esm({
|
|
|
3128
3164
|
created_at: Date.now()
|
|
3129
3165
|
});
|
|
3130
3166
|
if (info.changes > 0) return Number(info.lastInsertRowid);
|
|
3131
|
-
const existing = this.
|
|
3167
|
+
const existing = this._getByIdentity.get(r.note_id, r.heading_path, r.anchor);
|
|
3132
3168
|
return existing ? Number(existing.id) : null;
|
|
3133
3169
|
}
|
|
3134
3170
|
deleteByNote(noteId) {
|
|
@@ -4922,6 +4958,290 @@ var init_hybrid = __esm({
|
|
|
4922
4958
|
}
|
|
4923
4959
|
});
|
|
4924
4960
|
|
|
4961
|
+
// src/adapters/retrieval/contextfit/cli.ts
|
|
4962
|
+
import spawn from "cross-spawn";
|
|
4963
|
+
function runContextFit(cfg, subcommandArgs, timeoutMs) {
|
|
4964
|
+
const globalArgs = ["--kb", cfg.kbPath];
|
|
4965
|
+
if (cfg.tokenizer) globalArgs.push("--tokenizer", cfg.tokenizer);
|
|
4966
|
+
const args2 = [...globalArgs, ...subcommandArgs];
|
|
4967
|
+
return new Promise((resolve7, reject) => {
|
|
4968
|
+
let child;
|
|
4969
|
+
try {
|
|
4970
|
+
child = spawn(cfg.command, args2, { stdio: ["pipe", "pipe", "pipe"] });
|
|
4971
|
+
} catch (err) {
|
|
4972
|
+
const e = err;
|
|
4973
|
+
reject(
|
|
4974
|
+
new ContextFitError(
|
|
4975
|
+
`contextfit spawn failed: ${e.message}`,
|
|
4976
|
+
e.code === "ENOENT" ? "ENOENT" : "NONZERO_EXIT"
|
|
4977
|
+
)
|
|
4978
|
+
);
|
|
4979
|
+
return;
|
|
4980
|
+
}
|
|
4981
|
+
child.stdin?.end();
|
|
4982
|
+
let stdout = "";
|
|
4983
|
+
let stderr = "";
|
|
4984
|
+
let settled = false;
|
|
4985
|
+
const timer = setTimeout(() => {
|
|
4986
|
+
if (settled) return;
|
|
4987
|
+
settled = true;
|
|
4988
|
+
child.kill("SIGKILL");
|
|
4989
|
+
reject(new ContextFitError(`contextfit timed out after ${timeoutMs}ms`, "TIMEOUT"));
|
|
4990
|
+
}, timeoutMs);
|
|
4991
|
+
child.stdout?.on("data", (d) => {
|
|
4992
|
+
stdout += d.toString();
|
|
4993
|
+
});
|
|
4994
|
+
child.stderr?.on("data", (d) => {
|
|
4995
|
+
stderr += d.toString();
|
|
4996
|
+
});
|
|
4997
|
+
child.on("error", (err) => {
|
|
4998
|
+
if (settled) return;
|
|
4999
|
+
settled = true;
|
|
5000
|
+
clearTimeout(timer);
|
|
5001
|
+
if (err.code === "ENOENT") {
|
|
5002
|
+
reject(
|
|
5003
|
+
new ContextFitError(
|
|
5004
|
+
`contextfit not found (tried '${cfg.command}'). Install it with \`pipx install contextfit\` (or pip), or set the command path.`,
|
|
5005
|
+
"ENOENT"
|
|
5006
|
+
)
|
|
5007
|
+
);
|
|
5008
|
+
} else {
|
|
5009
|
+
reject(new ContextFitError(`contextfit spawn failed: ${err.message}`));
|
|
5010
|
+
}
|
|
5011
|
+
});
|
|
5012
|
+
child.on("close", (codeNum) => {
|
|
5013
|
+
if (settled) return;
|
|
5014
|
+
settled = true;
|
|
5015
|
+
clearTimeout(timer);
|
|
5016
|
+
if (codeNum === 0) {
|
|
5017
|
+
resolve7({ stdout, stderr });
|
|
5018
|
+
} else {
|
|
5019
|
+
reject(
|
|
5020
|
+
new ContextFitError(
|
|
5021
|
+
`contextfit exited ${codeNum}: ${stderr.trim() || stdout.trim() || "(no output)"}`,
|
|
5022
|
+
"NONZERO_EXIT"
|
|
5023
|
+
)
|
|
5024
|
+
);
|
|
5025
|
+
}
|
|
5026
|
+
});
|
|
5027
|
+
});
|
|
5028
|
+
}
|
|
5029
|
+
async function runContextFitWithRetry(cfg, subcommandArgs, timeoutMs) {
|
|
5030
|
+
try {
|
|
5031
|
+
return await runContextFit(cfg, subcommandArgs, timeoutMs);
|
|
5032
|
+
} catch (err) {
|
|
5033
|
+
const isEbadf = err instanceof ContextFitError && /EBADF/.test(err.message);
|
|
5034
|
+
if (!isEbadf) throw err;
|
|
5035
|
+
await new Promise((r) => setTimeout(r, 50));
|
|
5036
|
+
return runContextFit(cfg, subcommandArgs, timeoutMs);
|
|
5037
|
+
}
|
|
5038
|
+
}
|
|
5039
|
+
async function contextFitIngest(cfg, source, opts = {}) {
|
|
5040
|
+
const args2 = ["ingest", source, "--rebuild-index-after-ingest"];
|
|
5041
|
+
if (opts.chunkSize !== void 0) args2.push("--chunk-size", String(opts.chunkSize));
|
|
5042
|
+
if (opts.overlap !== void 0) args2.push("--overlap", String(opts.overlap));
|
|
5043
|
+
const { stdout } = await runContextFitWithRetry(cfg, args2, cfg.timeoutMs ?? 6e5);
|
|
5044
|
+
return stdout;
|
|
5045
|
+
}
|
|
5046
|
+
async function contextFitQuery(cfg, query, opts = {}) {
|
|
5047
|
+
const args2 = ["query", query, "--json"];
|
|
5048
|
+
if (opts.topK !== void 0) args2.push("--top-k", String(opts.topK));
|
|
5049
|
+
if (opts.method !== void 0) args2.push("--method", opts.method);
|
|
5050
|
+
const { stdout } = await runContextFitWithRetry(cfg, args2, cfg.timeoutMs ?? 3e4);
|
|
5051
|
+
return parseQueryOutput(stdout);
|
|
5052
|
+
}
|
|
5053
|
+
function parseQueryOutput(stdout) {
|
|
5054
|
+
const start = stdout.indexOf("{");
|
|
5055
|
+
if (start === -1) {
|
|
5056
|
+
throw new ContextFitError(
|
|
5057
|
+
`contextfit query produced no JSON: ${stdout.slice(0, 200)}`,
|
|
5058
|
+
"BAD_JSON"
|
|
5059
|
+
);
|
|
5060
|
+
}
|
|
5061
|
+
let parsed;
|
|
5062
|
+
try {
|
|
5063
|
+
parsed = JSON.parse(stdout.slice(start));
|
|
5064
|
+
} catch (err) {
|
|
5065
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
5066
|
+
throw new ContextFitError(`contextfit query JSON parse failed: ${msg}`, "BAD_JSON");
|
|
5067
|
+
}
|
|
5068
|
+
const obj = parsed;
|
|
5069
|
+
if (!Array.isArray(obj.chunks)) {
|
|
5070
|
+
throw new ContextFitError(
|
|
5071
|
+
`contextfit query JSON missing 'chunks' array (got keys: ${Object.keys(obj ?? {}).join(", ")})`,
|
|
5072
|
+
"BAD_JSON"
|
|
5073
|
+
);
|
|
5074
|
+
}
|
|
5075
|
+
return {
|
|
5076
|
+
query: typeof obj.query === "string" ? obj.query : "",
|
|
5077
|
+
method: typeof obj.method === "string" ? obj.method : "hybrid",
|
|
5078
|
+
retrieved_chunks: typeof obj.retrieved_chunks === "number" ? obj.retrieved_chunks : obj.chunks.length,
|
|
5079
|
+
chunks: obj.chunks
|
|
5080
|
+
};
|
|
5081
|
+
}
|
|
5082
|
+
async function contextFitProbe(cfg) {
|
|
5083
|
+
try {
|
|
5084
|
+
await new Promise((resolve7, reject) => {
|
|
5085
|
+
const child = spawn(cfg.command, ["--help"], { stdio: ["pipe", "pipe", "pipe"] });
|
|
5086
|
+
child.stdin?.end();
|
|
5087
|
+
child.on("error", reject);
|
|
5088
|
+
child.on(
|
|
5089
|
+
"close",
|
|
5090
|
+
(c) => c === 0 ? resolve7() : reject(new Error(`exit ${c}`))
|
|
5091
|
+
);
|
|
5092
|
+
});
|
|
5093
|
+
return true;
|
|
5094
|
+
} catch {
|
|
5095
|
+
return false;
|
|
5096
|
+
}
|
|
5097
|
+
}
|
|
5098
|
+
var ContextFitError;
|
|
5099
|
+
var init_cli = __esm({
|
|
5100
|
+
"src/adapters/retrieval/contextfit/cli.ts"() {
|
|
5101
|
+
"use strict";
|
|
5102
|
+
init_esm_shims();
|
|
5103
|
+
ContextFitError = class extends Error {
|
|
5104
|
+
constructor(message, code = "NONZERO_EXIT") {
|
|
5105
|
+
super(message);
|
|
5106
|
+
this.code = code;
|
|
5107
|
+
}
|
|
5108
|
+
code;
|
|
5109
|
+
name = "ContextFitError";
|
|
5110
|
+
};
|
|
5111
|
+
}
|
|
5112
|
+
});
|
|
5113
|
+
|
|
5114
|
+
// src/adapters/retrieval/contextfit/index.ts
|
|
5115
|
+
var contextfit_exports = {};
|
|
5116
|
+
__export(contextfit_exports, {
|
|
5117
|
+
cliConfigForVault: () => cliConfigForVault,
|
|
5118
|
+
contextFitKbDir: () => contextFitKbDir,
|
|
5119
|
+
indexVaultWithContextFit: () => indexVaultWithContextFit,
|
|
5120
|
+
searchVaultWithContextFit: () => searchVaultWithContextFit,
|
|
5121
|
+
sourceToNotePath: () => sourceToNotePath
|
|
5122
|
+
});
|
|
5123
|
+
import { homedir as homedir4 } from "os";
|
|
5124
|
+
import { rm } from "fs/promises";
|
|
5125
|
+
import { join as join4, relative, isAbsolute } from "path";
|
|
5126
|
+
function contextFitKbDir(vaultName) {
|
|
5127
|
+
return join4(homedir4(), ".vault-memory", "contextfit", vaultName);
|
|
5128
|
+
}
|
|
5129
|
+
function cliConfigForVault(vault) {
|
|
5130
|
+
const cfg = {
|
|
5131
|
+
command: vault.contextfit?.command ?? DEFAULT_COMMAND,
|
|
5132
|
+
kbPath: contextFitKbDir(vault.name)
|
|
5133
|
+
};
|
|
5134
|
+
if (vault.contextfit?.tokenizer) cfg.tokenizer = vault.contextfit.tokenizer;
|
|
5135
|
+
return cfg;
|
|
5136
|
+
}
|
|
5137
|
+
async function indexVaultWithContextFit(vault, opts = {}) {
|
|
5138
|
+
const log = opts.onProgress ?? (() => {
|
|
5139
|
+
});
|
|
5140
|
+
const cfg = cliConfigForVault(vault);
|
|
5141
|
+
const start = Date.now();
|
|
5142
|
+
log(`ContextFit: ingesting ${vault.path} \u2192 ${cfg.kbPath}`);
|
|
5143
|
+
const available = await contextFitProbe({ command: cfg.command });
|
|
5144
|
+
if (!available) {
|
|
5145
|
+
return {
|
|
5146
|
+
status: "failed",
|
|
5147
|
+
stats: "",
|
|
5148
|
+
durationMs: Date.now() - start,
|
|
5149
|
+
error: `ContextFit CLI not runnable (tried '${cfg.command}'). Install with \`pipx install contextfit\` or set [[vaults]].contextfit.command.`
|
|
5150
|
+
};
|
|
5151
|
+
}
|
|
5152
|
+
try {
|
|
5153
|
+
await rm(cfg.kbPath, { recursive: true, force: true });
|
|
5154
|
+
const stats = await contextFitIngest(cfg, vault.path);
|
|
5155
|
+
log(stats.trim().split("\n").slice(-3).join(" \xB7 "));
|
|
5156
|
+
return { status: "completed", stats, durationMs: Date.now() - start };
|
|
5157
|
+
} catch (err) {
|
|
5158
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
5159
|
+
return { status: "failed", stats: "", durationMs: Date.now() - start, error: message };
|
|
5160
|
+
}
|
|
5161
|
+
}
|
|
5162
|
+
function sourceToNotePath(source, vaultPath) {
|
|
5163
|
+
if (!source) return null;
|
|
5164
|
+
const rel = isAbsolute(source) ? relative(vaultPath, source) : source;
|
|
5165
|
+
if (rel.startsWith("..")) return null;
|
|
5166
|
+
return rel.split(/[\\/]/).join("/");
|
|
5167
|
+
}
|
|
5168
|
+
function chunkToHit(chunk, vault) {
|
|
5169
|
+
const notePath = sourceToNotePath(chunk.metadata?.source, vault.path);
|
|
5170
|
+
if (notePath === null) return null;
|
|
5171
|
+
const base = notePath.split("/").pop() ?? notePath;
|
|
5172
|
+
const noteTitle = base.replace(/\.md$/i, "");
|
|
5173
|
+
const hit = {
|
|
5174
|
+
vault: vault.name,
|
|
5175
|
+
notePath,
|
|
5176
|
+
noteTitle,
|
|
5177
|
+
chunkText: chunk.preview ?? "",
|
|
5178
|
+
chunkIdx: chunk.chunk_id,
|
|
5179
|
+
headingPath: null,
|
|
5180
|
+
score: chunk.score,
|
|
5181
|
+
scoreBreakdown: { contextfit: chunk.score }
|
|
5182
|
+
};
|
|
5183
|
+
return hit;
|
|
5184
|
+
}
|
|
5185
|
+
async function searchVaultWithContextFit(vault, query, opts = {}) {
|
|
5186
|
+
const cfg = cliConfigForVault(vault);
|
|
5187
|
+
const method = vault.contextfit?.method ?? "hybrid";
|
|
5188
|
+
const result = await contextFitQuery(cfg, query, {
|
|
5189
|
+
topK: opts.topK ?? 10,
|
|
5190
|
+
method
|
|
5191
|
+
});
|
|
5192
|
+
const hits = [];
|
|
5193
|
+
for (const chunk of result.chunks) {
|
|
5194
|
+
const hit = chunkToHit(chunk, vault);
|
|
5195
|
+
if (hit) hits.push(hit);
|
|
5196
|
+
}
|
|
5197
|
+
return hits;
|
|
5198
|
+
}
|
|
5199
|
+
var DEFAULT_COMMAND;
|
|
5200
|
+
var init_contextfit = __esm({
|
|
5201
|
+
"src/adapters/retrieval/contextfit/index.ts"() {
|
|
5202
|
+
"use strict";
|
|
5203
|
+
init_esm_shims();
|
|
5204
|
+
init_cli();
|
|
5205
|
+
DEFAULT_COMMAND = "contextfit";
|
|
5206
|
+
}
|
|
5207
|
+
});
|
|
5208
|
+
|
|
5209
|
+
// src/search/dispatch.ts
|
|
5210
|
+
function isContextFit(vault) {
|
|
5211
|
+
return vault.config.backend === "contextfit";
|
|
5212
|
+
}
|
|
5213
|
+
async function searchVaults(opts) {
|
|
5214
|
+
const topK = opts.topK ?? 10;
|
|
5215
|
+
const cfVaults = opts.vaults.filter(isContextFit);
|
|
5216
|
+
const ollamaVaults = opts.vaults.filter((v) => !isContextFit(v));
|
|
5217
|
+
if (cfVaults.length === 0) {
|
|
5218
|
+
return hybridSearch(opts);
|
|
5219
|
+
}
|
|
5220
|
+
const { searchVaultWithContextFit: searchVaultWithContextFit2 } = await Promise.resolve().then(() => (init_contextfit(), contextfit_exports));
|
|
5221
|
+
const cfPromise = Promise.all(
|
|
5222
|
+
cfVaults.map(
|
|
5223
|
+
(v) => searchVaultWithContextFit2(v.config, opts.query, { topK }).catch((err) => {
|
|
5224
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
5225
|
+
console.error(`[search:${v.config.name}] ContextFit query failed: ${msg}`);
|
|
5226
|
+
return [];
|
|
5227
|
+
})
|
|
5228
|
+
)
|
|
5229
|
+
);
|
|
5230
|
+
const ollamaPromise = ollamaVaults.length > 0 ? hybridSearch({ ...opts, vaults: ollamaVaults }) : Promise.resolve([]);
|
|
5231
|
+
const [cfResultsNested, ollamaResults] = await Promise.all([cfPromise, ollamaPromise]);
|
|
5232
|
+
const cfResults = cfResultsNested.flat();
|
|
5233
|
+
const merged = [...ollamaResults, ...cfResults];
|
|
5234
|
+
merged.sort((a, b) => b.score - a.score);
|
|
5235
|
+
return merged.slice(0, topK);
|
|
5236
|
+
}
|
|
5237
|
+
var init_dispatch = __esm({
|
|
5238
|
+
"src/search/dispatch.ts"() {
|
|
5239
|
+
"use strict";
|
|
5240
|
+
init_esm_shims();
|
|
5241
|
+
init_hybrid();
|
|
5242
|
+
}
|
|
5243
|
+
});
|
|
5244
|
+
|
|
4925
5245
|
// src/search/glob.ts
|
|
4926
5246
|
function compile(pattern) {
|
|
4927
5247
|
const cached = cache.get(pattern);
|
|
@@ -4969,6 +5289,7 @@ var init_search = __esm({
|
|
|
4969
5289
|
"use strict";
|
|
4970
5290
|
init_esm_shims();
|
|
4971
5291
|
init_hybrid();
|
|
5292
|
+
init_dispatch();
|
|
4972
5293
|
init_glob();
|
|
4973
5294
|
}
|
|
4974
5295
|
});
|
|
@@ -5014,7 +5335,7 @@ var init_reranker = __esm({
|
|
|
5014
5335
|
// src/rerank/onnx-reranker.ts
|
|
5015
5336
|
import { readFile as readFile3 } from "fs/promises";
|
|
5016
5337
|
import { existsSync } from "fs";
|
|
5017
|
-
import { join as
|
|
5338
|
+
import { join as join5 } from "path";
|
|
5018
5339
|
function sigmoid(x) {
|
|
5019
5340
|
return 1 / (1 + Math.exp(-x));
|
|
5020
5341
|
}
|
|
@@ -5092,8 +5413,8 @@ var init_onnx_reranker = __esm({
|
|
|
5092
5413
|
if (this.loaded) return this.loaded;
|
|
5093
5414
|
if (this.loading) return this.loading;
|
|
5094
5415
|
this.loading = (async () => {
|
|
5095
|
-
const modelPath =
|
|
5096
|
-
const tokenizerPath =
|
|
5416
|
+
const modelPath = join5(this.modelDir, "model_quantized.onnx");
|
|
5417
|
+
const tokenizerPath = join5(this.modelDir, "tokenizer.json");
|
|
5097
5418
|
if (!existsSync(modelPath)) {
|
|
5098
5419
|
throw new Error(
|
|
5099
5420
|
`OnnxReranker: model file not found at ${modelPath}. Run: curl -L https://huggingface.co/onnx-community/bge-reranker-v2-m3-ONNX/resolve/main/onnx/model_quantized.onnx -o ${modelPath}`
|
|
@@ -6419,53 +6740,62 @@ async function indexVault(vault, options) {
|
|
|
6419
6740
|
const mode = options.mode ?? "incremental";
|
|
6420
6741
|
const log = options.onProgress ?? (() => {
|
|
6421
6742
|
});
|
|
6422
|
-
|
|
6423
|
-
const
|
|
6424
|
-
|
|
6425
|
-
|
|
6426
|
-
}
|
|
6427
|
-
const modelExists = await options.ollama.modelExists(options.embeddingModel);
|
|
6428
|
-
if (!modelExists) {
|
|
6429
|
-
throw new Error(
|
|
6430
|
-
`Embedding model "${options.embeddingModel}" not found in Ollama. Available: ${health.models?.join(", ") ?? "(none)"}. Run: ollama pull ${options.embeddingModel}`
|
|
6431
|
-
);
|
|
6432
|
-
}
|
|
6433
|
-
const probe = await options.ollama.embed({
|
|
6434
|
-
model: options.embeddingModel,
|
|
6435
|
-
texts: ["probe"]
|
|
6436
|
-
});
|
|
6437
|
-
const dim = probe.dim;
|
|
6438
|
-
const modelRow = vault.db.models.upsert({
|
|
6439
|
-
name: options.embeddingModel,
|
|
6440
|
-
provider: "ollama",
|
|
6441
|
-
dim
|
|
6442
|
-
});
|
|
6743
|
+
const embedMode = options.embeddings ?? "ollama";
|
|
6744
|
+
const ollama = options.ollama;
|
|
6745
|
+
let dim = 0;
|
|
6746
|
+
let modelRow = null;
|
|
6443
6747
|
let secondaryModelRow = null;
|
|
6444
|
-
if (
|
|
6445
|
-
|
|
6446
|
-
|
|
6447
|
-
|
|
6448
|
-
|
|
6748
|
+
if (embedMode === "ollama") {
|
|
6749
|
+
if (!ollama) {
|
|
6750
|
+
throw new Error("indexVault: embeddings='ollama' requires an OllamaClient (options.ollama).");
|
|
6751
|
+
}
|
|
6752
|
+
log(`Probing Ollama model: ${options.embeddingModel}`);
|
|
6753
|
+
const health = await ollama.healthCheck();
|
|
6754
|
+
if (!health.ok) {
|
|
6755
|
+
throw new Error(`Ollama unreachable: ${health.error ?? "unknown error"}`);
|
|
6756
|
+
}
|
|
6757
|
+
const modelExists = await ollama.modelExists(options.embeddingModel);
|
|
6758
|
+
if (!modelExists) {
|
|
6449
6759
|
throw new Error(
|
|
6450
|
-
`
|
|
6760
|
+
`Embedding model "${options.embeddingModel}" not found in Ollama. Available: ${health.models?.join(", ") ?? "(none)"}. Run: ollama pull ${options.embeddingModel}`
|
|
6451
6761
|
);
|
|
6452
6762
|
}
|
|
6453
|
-
const
|
|
6454
|
-
model:
|
|
6763
|
+
const probe = await ollama.embed({
|
|
6764
|
+
model: options.embeddingModel,
|
|
6455
6765
|
texts: ["probe"]
|
|
6456
6766
|
});
|
|
6457
|
-
|
|
6458
|
-
|
|
6767
|
+
dim = probe.dim;
|
|
6768
|
+
modelRow = vault.db.models.upsert({
|
|
6769
|
+
name: options.embeddingModel,
|
|
6459
6770
|
provider: "ollama",
|
|
6460
|
-
dim
|
|
6461
|
-
active: false
|
|
6771
|
+
dim
|
|
6462
6772
|
});
|
|
6463
|
-
|
|
6773
|
+
if (options.secondaryEmbeddingModel) {
|
|
6774
|
+
const secName = options.secondaryEmbeddingModel;
|
|
6775
|
+
log(`Probing secondary (shadow) model: ${secName}`);
|
|
6776
|
+
const secExists = await ollama.modelExists(secName);
|
|
6777
|
+
if (!secExists) {
|
|
6778
|
+
throw new Error(
|
|
6779
|
+
`Secondary embedding model "${secName}" not found in Ollama. Run: ollama pull ${secName}`
|
|
6780
|
+
);
|
|
6781
|
+
}
|
|
6782
|
+
const secProbe = await ollama.embed({
|
|
6783
|
+
model: secName,
|
|
6784
|
+
texts: ["probe"]
|
|
6785
|
+
});
|
|
6786
|
+
const row = vault.db.models.upsert({
|
|
6787
|
+
name: secName,
|
|
6788
|
+
provider: "ollama",
|
|
6789
|
+
dim: secProbe.dim,
|
|
6790
|
+
active: false
|
|
6791
|
+
});
|
|
6792
|
+
secondaryModelRow = { id: row.id, dim: row.dim };
|
|
6793
|
+
}
|
|
6464
6794
|
}
|
|
6465
6795
|
vault.db.audit.startRun({
|
|
6466
6796
|
runId,
|
|
6467
6797
|
vaultName: vault.config.name,
|
|
6468
|
-
modelId: modelRow
|
|
6798
|
+
modelId: modelRow?.id ?? null,
|
|
6469
6799
|
trigger: mode === "full" ? "manual-full" : "manual-incremental"
|
|
6470
6800
|
});
|
|
6471
6801
|
let notesIndexed = 0;
|
|
@@ -6556,36 +6886,38 @@ async function indexVault(vault, options) {
|
|
|
6556
6886
|
`[indexer:${vault.config.name}] section build failed for ${parsed.relativePath}: ${message} \u2014 skipping sections for this note`
|
|
6557
6887
|
);
|
|
6558
6888
|
}
|
|
6559
|
-
|
|
6560
|
-
|
|
6561
|
-
|
|
6562
|
-
});
|
|
6563
|
-
if (embedResult.dim !== dim) {
|
|
6564
|
-
throw new Error(`Embedding dimension mismatch: expected ${dim}, got ${embedResult.dim}`);
|
|
6565
|
-
}
|
|
6566
|
-
const embeddingInputs = chunkIds.map((chunkId, i) => ({
|
|
6567
|
-
chunkId,
|
|
6568
|
-
modelId: modelRow.id,
|
|
6569
|
-
vector: embedResult.vectors[i]
|
|
6570
|
-
}));
|
|
6571
|
-
vault.db.embeddings.insertBatch(embeddingInputs);
|
|
6572
|
-
if (secondaryModelRow) {
|
|
6573
|
-
const secEmbed = await options.ollama.embed({
|
|
6574
|
-
model: options.secondaryEmbeddingModel,
|
|
6889
|
+
if (embedMode === "ollama") {
|
|
6890
|
+
const embedResult = await ollama.embed({
|
|
6891
|
+
model: options.embeddingModel,
|
|
6575
6892
|
texts: chunks.map((c) => c.text)
|
|
6576
6893
|
});
|
|
6577
|
-
if (
|
|
6578
|
-
throw new Error(
|
|
6579
|
-
|
|
6894
|
+
if (embedResult.dim !== dim) {
|
|
6895
|
+
throw new Error(`Embedding dimension mismatch: expected ${dim}, got ${embedResult.dim}`);
|
|
6896
|
+
}
|
|
6897
|
+
const embeddingInputs = chunkIds.map((chunkId, i) => ({
|
|
6898
|
+
chunkId,
|
|
6899
|
+
modelId: modelRow.id,
|
|
6900
|
+
vector: embedResult.vectors[i]
|
|
6901
|
+
}));
|
|
6902
|
+
vault.db.embeddings.insertBatch(embeddingInputs);
|
|
6903
|
+
if (secondaryModelRow) {
|
|
6904
|
+
const secEmbed = await ollama.embed({
|
|
6905
|
+
model: options.secondaryEmbeddingModel,
|
|
6906
|
+
texts: chunks.map((c) => c.text)
|
|
6907
|
+
});
|
|
6908
|
+
if (secEmbed.dim !== secondaryModelRow.dim) {
|
|
6909
|
+
throw new Error(
|
|
6910
|
+
`Secondary embedding dimension mismatch: expected ${secondaryModelRow.dim}, got ${secEmbed.dim}`
|
|
6911
|
+
);
|
|
6912
|
+
}
|
|
6913
|
+
vault.db.embeddings.insertBatch(
|
|
6914
|
+
chunkIds.map((chunkId, i) => ({
|
|
6915
|
+
chunkId,
|
|
6916
|
+
modelId: secondaryModelRow.id,
|
|
6917
|
+
vector: secEmbed.vectors[i]
|
|
6918
|
+
}))
|
|
6580
6919
|
);
|
|
6581
6920
|
}
|
|
6582
|
-
vault.db.embeddings.insertBatch(
|
|
6583
|
-
chunkIds.map((chunkId, i) => ({
|
|
6584
|
-
chunkId,
|
|
6585
|
-
modelId: secondaryModelRow.id,
|
|
6586
|
-
vector: secEmbed.vectors[i]
|
|
6587
|
-
}))
|
|
6588
|
-
);
|
|
6589
6921
|
}
|
|
6590
6922
|
insertWikilinks(vault, noteId, parsed.wikilinks, firstPassResolver);
|
|
6591
6923
|
writeAllEdges(vault, noteId, parsed, firstPassResolver);
|
|
@@ -6850,16 +7182,24 @@ async function indexNote(options) {
|
|
|
6850
7182
|
isNew: false
|
|
6851
7183
|
};
|
|
6852
7184
|
}
|
|
6853
|
-
const
|
|
6854
|
-
|
|
6855
|
-
|
|
6856
|
-
|
|
6857
|
-
|
|
6858
|
-
|
|
6859
|
-
|
|
6860
|
-
|
|
6861
|
-
|
|
6862
|
-
|
|
7185
|
+
const embedMode = options.embeddings ?? "ollama";
|
|
7186
|
+
let activeModel = null;
|
|
7187
|
+
if (embedMode === "ollama") {
|
|
7188
|
+
if (!ollama) {
|
|
7189
|
+
throw new Error("single-indexer: embeddings='ollama' requires an OllamaClient.");
|
|
7190
|
+
}
|
|
7191
|
+
const am = vault.db.models.getActive();
|
|
7192
|
+
if (!am) {
|
|
7193
|
+
throw new Error(
|
|
7194
|
+
`single-indexer: no active embedding model in DB. Run a full index first to register "${embeddingModel}".`
|
|
7195
|
+
);
|
|
7196
|
+
}
|
|
7197
|
+
if (am.name !== embeddingModel) {
|
|
7198
|
+
throw new Error(
|
|
7199
|
+
`single-indexer: active model "${am.name}" does not match requested "${embeddingModel}". Run a full re-index to switch models.`
|
|
7200
|
+
);
|
|
7201
|
+
}
|
|
7202
|
+
activeModel = am;
|
|
6863
7203
|
}
|
|
6864
7204
|
const upsert = vault.db.notes.upsertByPath({
|
|
6865
7205
|
path: parsed.relativePath,
|
|
@@ -6872,6 +7212,7 @@ async function indexNote(options) {
|
|
|
6872
7212
|
wordCount: parsed.wordCount
|
|
6873
7213
|
});
|
|
6874
7214
|
vault.db.aliases.setForNote(upsert.id, extractAliases(parsed.frontmatter));
|
|
7215
|
+
vault.db.sections.deleteByNote(upsert.id);
|
|
6875
7216
|
vault.db.chunks.deleteByNote(upsert.id);
|
|
6876
7217
|
vault.db.wikilinks.deleteByNote(upsert.id);
|
|
6877
7218
|
vault.db.edges.deleteByNote(upsert.id);
|
|
@@ -6901,41 +7242,52 @@ async function indexNote(options) {
|
|
|
6901
7242
|
chunkIdFragment: computeChunkIdFragment(c.text)
|
|
6902
7243
|
}))
|
|
6903
7244
|
);
|
|
6904
|
-
|
|
6905
|
-
|
|
6906
|
-
|
|
6907
|
-
|
|
6908
|
-
|
|
6909
|
-
|
|
6910
|
-
|
|
7245
|
+
try {
|
|
7246
|
+
buildSectionsForNote(vault, upsert.id, parsed.content, chunkIds);
|
|
7247
|
+
} catch (err) {
|
|
7248
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
7249
|
+
process.stderr.write(
|
|
7250
|
+
`[single-indexer:${vault.config.name}] section build failed for ${parsed.relativePath}: ${message}
|
|
7251
|
+
`
|
|
6911
7252
|
);
|
|
6912
7253
|
}
|
|
6913
|
-
|
|
6914
|
-
|
|
6915
|
-
|
|
6916
|
-
|
|
6917
|
-
|
|
6918
|
-
|
|
6919
|
-
|
|
6920
|
-
|
|
6921
|
-
|
|
6922
|
-
|
|
6923
|
-
|
|
6924
|
-
|
|
6925
|
-
|
|
6926
|
-
|
|
6927
|
-
|
|
6928
|
-
|
|
6929
|
-
|
|
7254
|
+
if (embedMode === "ollama") {
|
|
7255
|
+
const embedResult = await ollama.embed({
|
|
7256
|
+
model: embeddingModel,
|
|
7257
|
+
texts: chunks.map((c) => c.text)
|
|
7258
|
+
});
|
|
7259
|
+
if (embedResult.dim !== activeModel.dim) {
|
|
7260
|
+
throw new Error(
|
|
7261
|
+
`single-indexer: embedding dim ${embedResult.dim} does not match registered dim ${activeModel.dim} for model "${embeddingModel}".`
|
|
7262
|
+
);
|
|
7263
|
+
}
|
|
7264
|
+
vault.db.embeddings.insertBatch(
|
|
7265
|
+
chunkIds.map((chunkId, i) => ({
|
|
7266
|
+
chunkId,
|
|
7267
|
+
modelId: activeModel.id,
|
|
7268
|
+
vector: embedResult.vectors[i]
|
|
7269
|
+
}))
|
|
7270
|
+
);
|
|
7271
|
+
if (secondaryName) {
|
|
7272
|
+
const secondaryModel = vault.db.models.getByName(secondaryName);
|
|
7273
|
+
if (secondaryModel && secondaryModel.id !== activeModel.id) {
|
|
7274
|
+
const secEmbed = await ollama.embed({
|
|
7275
|
+
model: secondaryName,
|
|
7276
|
+
texts: chunks.map((c) => c.text)
|
|
7277
|
+
});
|
|
7278
|
+
if (secEmbed.dim !== secondaryModel.dim) {
|
|
7279
|
+
throw new Error(
|
|
7280
|
+
`single-indexer: shadow embedding dim ${secEmbed.dim} does not match registered dim ${secondaryModel.dim} for "${secondaryName}".`
|
|
7281
|
+
);
|
|
7282
|
+
}
|
|
7283
|
+
vault.db.embeddings.insertBatch(
|
|
7284
|
+
chunkIds.map((chunkId, i) => ({
|
|
7285
|
+
chunkId,
|
|
7286
|
+
modelId: secondaryModel.id,
|
|
7287
|
+
vector: secEmbed.vectors[i]
|
|
7288
|
+
}))
|
|
6930
7289
|
);
|
|
6931
7290
|
}
|
|
6932
|
-
vault.db.embeddings.insertBatch(
|
|
6933
|
-
chunkIds.map((chunkId, i) => ({
|
|
6934
|
-
chunkId,
|
|
6935
|
-
modelId: secondaryModel.id,
|
|
6936
|
-
vector: secEmbed.vectors[i]
|
|
6937
|
-
}))
|
|
6938
|
-
);
|
|
6939
7291
|
}
|
|
6940
7292
|
}
|
|
6941
7293
|
insertWikilinks2(vault, upsert.id, parsed.wikilinks);
|
|
@@ -7027,6 +7379,7 @@ async function catchupVault(options) {
|
|
|
7027
7379
|
});
|
|
7028
7380
|
let reindexed = 0;
|
|
7029
7381
|
const knownPaths = /* @__PURE__ */ new Set();
|
|
7382
|
+
const isContextFit2 = vault.config.backend === "contextfit";
|
|
7030
7383
|
for (const file of files) {
|
|
7031
7384
|
const parsed = await parseNote(file, vault.config.path).catch(() => null);
|
|
7032
7385
|
if (!parsed) continue;
|
|
@@ -7039,7 +7392,7 @@ async function catchupVault(options) {
|
|
|
7039
7392
|
vault,
|
|
7040
7393
|
absolutePath: file,
|
|
7041
7394
|
embeddingModel: options.embeddingModel,
|
|
7042
|
-
ollama: options.ollama
|
|
7395
|
+
...isContextFit2 ? { embeddings: "none" } : { ollama: options.ollama }
|
|
7043
7396
|
});
|
|
7044
7397
|
if (result.status === "indexed") {
|
|
7045
7398
|
reindexed++;
|
|
@@ -7056,6 +7409,13 @@ async function catchupVault(options) {
|
|
|
7056
7409
|
}
|
|
7057
7410
|
}
|
|
7058
7411
|
}
|
|
7412
|
+
if (isContextFit2 && (reindexed > 0 || removed > 0)) {
|
|
7413
|
+
const { indexVaultWithContextFit: indexVaultWithContextFit2 } = await Promise.resolve().then(() => (init_contextfit(), contextfit_exports));
|
|
7414
|
+
const r = await indexVaultWithContextFit2(vault.config, { onProgress: log });
|
|
7415
|
+
log(
|
|
7416
|
+
r.status === "completed" ? `catch-up: ContextFit KB rebuilt (${r.durationMs}ms)` : `catch-up: ContextFit KB rebuild failed: ${r.error}`
|
|
7417
|
+
);
|
|
7418
|
+
}
|
|
7059
7419
|
return {
|
|
7060
7420
|
scanned: files.length,
|
|
7061
7421
|
reindexed,
|
|
@@ -7063,9 +7423,9 @@ async function catchupVault(options) {
|
|
|
7063
7423
|
durationMs: Date.now() - started
|
|
7064
7424
|
};
|
|
7065
7425
|
}
|
|
7066
|
-
function joinAbs(root,
|
|
7067
|
-
if (root.endsWith("/")) return `${root}${
|
|
7068
|
-
return `${root}/${
|
|
7426
|
+
function joinAbs(root, relative6) {
|
|
7427
|
+
if (root.endsWith("/")) return `${root}${relative6}`;
|
|
7428
|
+
return `${root}/${relative6}`;
|
|
7069
7429
|
}
|
|
7070
7430
|
var init_catchup = __esm({
|
|
7071
7431
|
"src/indexer/catchup.ts"() {
|
|
@@ -7317,10 +7677,10 @@ var init_indexer2 = __esm({
|
|
|
7317
7677
|
|
|
7318
7678
|
// src/adapters/delivery/obsidian-fs/fs.ts
|
|
7319
7679
|
import { promises as fs5 } from "fs";
|
|
7320
|
-
import { dirname, isAbsolute, resolve as resolve6, sep as sep5 } from "path";
|
|
7680
|
+
import { dirname, isAbsolute as isAbsolute2, resolve as resolve6, sep as sep5 } from "path";
|
|
7321
7681
|
import { randomBytes } from "crypto";
|
|
7322
7682
|
async function atomicWriteFile(absPath, content) {
|
|
7323
|
-
if (!
|
|
7683
|
+
if (!isAbsolute2(absPath)) {
|
|
7324
7684
|
throw new Error(`atomicWriteFile requires an absolute path: ${absPath}`);
|
|
7325
7685
|
}
|
|
7326
7686
|
const parent = dirname(absPath);
|
|
@@ -7342,7 +7702,7 @@ async function safeJoinInsideVault(vaultRoot, relativePath) {
|
|
|
7342
7702
|
if (typeof relativePath !== "string" || relativePath.length === 0) {
|
|
7343
7703
|
throw new OutsideVaultError(relativePath, vaultRoot);
|
|
7344
7704
|
}
|
|
7345
|
-
if (
|
|
7705
|
+
if (isAbsolute2(relativePath)) {
|
|
7346
7706
|
throw new OutsideVaultError(relativePath, vaultRoot);
|
|
7347
7707
|
}
|
|
7348
7708
|
const root = resolve6(vaultRoot);
|
|
@@ -9902,14 +10262,14 @@ var init_get = __esm({
|
|
|
9902
10262
|
|
|
9903
10263
|
// src/brief/lock.ts
|
|
9904
10264
|
import { open, readFile as readFile5, unlink, mkdir as mkdir2 } from "fs/promises";
|
|
9905
|
-
import { homedir as
|
|
9906
|
-
import { join as
|
|
10265
|
+
import { homedir as homedir5 } from "os";
|
|
10266
|
+
import { join as join7 } from "path";
|
|
9907
10267
|
function lockDir(rootOverride) {
|
|
9908
|
-
if (rootOverride !== void 0) return
|
|
9909
|
-
return
|
|
10268
|
+
if (rootOverride !== void 0) return join7(rootOverride, "locks");
|
|
10269
|
+
return join7(homedir5(), ".vault-memory", "locks");
|
|
9910
10270
|
}
|
|
9911
10271
|
function lockPath(vaultName, rootOverride) {
|
|
9912
|
-
return
|
|
10272
|
+
return join7(lockDir(rootOverride), `${vaultName}.lock`);
|
|
9913
10273
|
}
|
|
9914
10274
|
function isProcessAlive(pid) {
|
|
9915
10275
|
try {
|
|
@@ -10415,7 +10775,7 @@ async function searchSections(deps, args2) {
|
|
|
10415
10775
|
const resolution = deps.sectionForHit(hit.vault, hit.notePath, hit.chunkIdx);
|
|
10416
10776
|
if (!resolution) continue;
|
|
10417
10777
|
if (resolution.headingPath.length === 0) continue;
|
|
10418
|
-
const key = `${resolution.noteId}#${resolution.anchor}`;
|
|
10778
|
+
const key = `${resolution.noteId}#${resolution.headingPath.join("\0")}#${resolution.anchor}`;
|
|
10419
10779
|
const existing = sectionMap.get(key);
|
|
10420
10780
|
if (!existing) {
|
|
10421
10781
|
sectionMap.set(key, {
|
|
@@ -10746,6 +11106,9 @@ var init_watcher = __esm({
|
|
|
10746
11106
|
queue;
|
|
10747
11107
|
opts;
|
|
10748
11108
|
started = false;
|
|
11109
|
+
/** ADR-008: debounce timer for ContextFit KB re-ingest (coalesces bursts). */
|
|
11110
|
+
cfReingestTimer = null;
|
|
11111
|
+
cfReingestInFlight = false;
|
|
10749
11112
|
constructor(options) {
|
|
10750
11113
|
this.opts = {
|
|
10751
11114
|
vault: options.vault,
|
|
@@ -10793,12 +11156,51 @@ var init_watcher = __esm({
|
|
|
10793
11156
|
if (!this.started) return;
|
|
10794
11157
|
this.started = false;
|
|
10795
11158
|
this.queue.shutdown();
|
|
11159
|
+
if (this.cfReingestTimer) {
|
|
11160
|
+
clearTimeout(this.cfReingestTimer);
|
|
11161
|
+
this.cfReingestTimer = null;
|
|
11162
|
+
}
|
|
10796
11163
|
if (this.fsWatcher) {
|
|
10797
11164
|
await this.fsWatcher.close();
|
|
10798
11165
|
this.fsWatcher = null;
|
|
10799
11166
|
}
|
|
10800
11167
|
}
|
|
10801
11168
|
// ─── internal ──────────────────────────────────────────────────────────
|
|
11169
|
+
/**
|
|
11170
|
+
* ADR-008: schedule a debounced full ContextFit KB re-ingest. Per-note
|
|
11171
|
+
* changes update the SQLite layer immediately (via indexNote); the ContextFit
|
|
11172
|
+
* search KB is rebuilt in one coalesced pass ~1.5s after the last change so a
|
|
11173
|
+
* burst of edits triggers a single re-ingest. CPU-only and fast.
|
|
11174
|
+
*/
|
|
11175
|
+
scheduleContextFitReingest() {
|
|
11176
|
+
if (this.cfReingestTimer) clearTimeout(this.cfReingestTimer);
|
|
11177
|
+
this.cfReingestTimer = setTimeout(() => {
|
|
11178
|
+
this.cfReingestTimer = null;
|
|
11179
|
+
void this.runContextFitReingest();
|
|
11180
|
+
}, 1500);
|
|
11181
|
+
}
|
|
11182
|
+
async runContextFitReingest() {
|
|
11183
|
+
if (this.cfReingestInFlight) {
|
|
11184
|
+
this.scheduleContextFitReingest();
|
|
11185
|
+
return;
|
|
11186
|
+
}
|
|
11187
|
+
this.cfReingestInFlight = true;
|
|
11188
|
+
try {
|
|
11189
|
+
const { indexVaultWithContextFit: indexVaultWithContextFit2 } = await Promise.resolve().then(() => (init_contextfit(), contextfit_exports));
|
|
11190
|
+
const r = await indexVaultWithContextFit2(this.opts.vault.config, {});
|
|
11191
|
+
if (r.status === "completed") {
|
|
11192
|
+
this.opts.log(`ContextFit KB refreshed (${r.durationMs}ms)`);
|
|
11193
|
+
} else {
|
|
11194
|
+
this.opts.log(`ContextFit KB refresh failed: ${r.error}`);
|
|
11195
|
+
}
|
|
11196
|
+
} catch (err) {
|
|
11197
|
+
this.opts.log(
|
|
11198
|
+
`ContextFit KB refresh error: ${err instanceof Error ? err.message : String(err)}`
|
|
11199
|
+
);
|
|
11200
|
+
} finally {
|
|
11201
|
+
this.cfReingestInFlight = false;
|
|
11202
|
+
}
|
|
11203
|
+
}
|
|
10802
11204
|
onFsEvent(absolutePath, kind) {
|
|
10803
11205
|
if (!absolutePath.endsWith(".md")) return;
|
|
10804
11206
|
const relativePath = this.toRelative(absolutePath);
|
|
@@ -10817,10 +11219,12 @@ var init_watcher = __esm({
|
|
|
10817
11219
|
}
|
|
10818
11220
|
async handleFlush(event) {
|
|
10819
11221
|
const relativePath = this.toRelative(event.path);
|
|
11222
|
+
const isContextFit2 = this.opts.vault.config.backend === "contextfit";
|
|
10820
11223
|
if (event.kind === "delete") {
|
|
10821
11224
|
const result2 = removeNote(this.opts.vault, event.path);
|
|
10822
11225
|
if (result2.removed) {
|
|
10823
11226
|
this.opts.log(`removed ${relativePath}`);
|
|
11227
|
+
if (isContextFit2) this.scheduleContextFitReingest();
|
|
10824
11228
|
} else {
|
|
10825
11229
|
this.opts.log(`delete event for unknown ${relativePath} (skip)`);
|
|
10826
11230
|
}
|
|
@@ -10831,13 +11235,16 @@ var init_watcher = __esm({
|
|
|
10831
11235
|
absolutePath: event.path,
|
|
10832
11236
|
embeddingModel: this.opts.embeddingModel,
|
|
10833
11237
|
secondaryEmbeddingModel: this.opts.secondaryEmbeddingModel,
|
|
10834
|
-
|
|
11238
|
+
// ADR-008: ContextFit vaults build the SQLite layer without embeddings;
|
|
11239
|
+
// their search KB is refreshed by the debounced re-ingest below.
|
|
11240
|
+
...isContextFit2 ? { embeddings: "none" } : { ollama: this.opts.ollama }
|
|
10835
11241
|
});
|
|
10836
11242
|
switch (result.status) {
|
|
10837
11243
|
case "indexed":
|
|
10838
11244
|
this.opts.log(
|
|
10839
11245
|
`indexed ${relativePath} (${result.isNew ? "new" : "updated"}, ${result.chunksCreated} chunks)`
|
|
10840
11246
|
);
|
|
11247
|
+
if (isContextFit2) this.scheduleContextFitReingest();
|
|
10841
11248
|
break;
|
|
10842
11249
|
case "unchanged":
|
|
10843
11250
|
break;
|
|
@@ -14521,6 +14928,13 @@ async function handleWriteNote(registry, vault, parsed) {
|
|
|
14521
14928
|
if (res.observedValue !== void 0) out.observedValue = res.observedValue;
|
|
14522
14929
|
return out;
|
|
14523
14930
|
}
|
|
14931
|
+
if (vault.config.backend === "contextfit") {
|
|
14932
|
+
try {
|
|
14933
|
+
const { indexVaultWithContextFit: indexVaultWithContextFit2 } = await Promise.resolve().then(() => (init_contextfit(), contextfit_exports));
|
|
14934
|
+
await indexVaultWithContextFit2(vault.config, {});
|
|
14935
|
+
} catch {
|
|
14936
|
+
}
|
|
14937
|
+
}
|
|
14524
14938
|
const noteRow = vault.db.notes.getByPath(parsed.path);
|
|
14525
14939
|
return {
|
|
14526
14940
|
ok: true,
|
|
@@ -14734,7 +15148,12 @@ function handleSearchText(manager, activeVault, query, vaultFilter, topK, exclud
|
|
|
14734
15148
|
const fanK = hasExclude ? topK * 3 : topK;
|
|
14735
15149
|
const sanitized = FtsQueries.sanitize(query);
|
|
14736
15150
|
const allHits = [];
|
|
15151
|
+
const skippedContextFit = [];
|
|
14737
15152
|
for (const vault of targets) {
|
|
15153
|
+
if (vault.config.backend === "contextfit") {
|
|
15154
|
+
skippedContextFit.push(vault.config.name);
|
|
15155
|
+
continue;
|
|
15156
|
+
}
|
|
14738
15157
|
const ftsHits = vault.db.fts.search(sanitized, fanK, true);
|
|
14739
15158
|
for (const hit of ftsHits) {
|
|
14740
15159
|
const chunk = vault.db.chunks.getById(hit.chunkId);
|
|
@@ -14759,9 +15178,14 @@ function handleSearchText(manager, activeVault, query, vaultFilter, topK, exclud
|
|
|
14759
15178
|
hits: allHits.slice(0, topK),
|
|
14760
15179
|
count: allHits.length
|
|
14761
15180
|
};
|
|
14762
|
-
|
|
14763
|
-
|
|
15181
|
+
const notes = [];
|
|
15182
|
+
if (skipped.length > 0) notes.push(`Skipped vault(s) currently indexing: ${skipped.join(", ")}.`);
|
|
15183
|
+
if (skippedContextFit.length > 0) {
|
|
15184
|
+
notes.push(
|
|
15185
|
+
`search_text is not supported for ContextFit vault(s): ${skippedContextFit.join(", ")} \u2014 use search_hybrid or search_semantic instead.`
|
|
15186
|
+
);
|
|
14764
15187
|
}
|
|
15188
|
+
if (notes.length > 0) out.note = notes.join(" ");
|
|
14765
15189
|
return out;
|
|
14766
15190
|
}
|
|
14767
15191
|
async function handleSearchHybrid(manager, ollama, defaultModel, activeVault, query, vaultFilter, topK, rrfK, excludePaths, reranker, recencyWeight = 0, authorityWeight = 0, halfLifeDays = 30, includeSuperseded = false, displayUrlFor2, expandOpts, expandDeps) {
|
|
@@ -14774,7 +15198,7 @@ async function handleSearchHybrid(manager, ollama, defaultModel, activeVault, qu
|
|
|
14774
15198
|
}
|
|
14775
15199
|
const hasExclude = excludePaths !== void 0 && excludePaths.length > 0;
|
|
14776
15200
|
const innerTopK = hasExclude ? topK * 3 : topK;
|
|
14777
|
-
const hits = await
|
|
15201
|
+
const hits = await searchVaults({
|
|
14778
15202
|
query,
|
|
14779
15203
|
embeddingModel: defaultModel,
|
|
14780
15204
|
ollama,
|
|
@@ -14812,7 +15236,7 @@ async function handleSearchCompat(manager, registry, ollama, defaultModel, activ
|
|
|
14812
15236
|
note: skipped.length > 0 ? `All eligible vaults are indexing; skipped: ${skipped.join(", ")}.` : "No vaults configured."
|
|
14813
15237
|
};
|
|
14814
15238
|
}
|
|
14815
|
-
const hits = await
|
|
15239
|
+
const hits = await searchVaults({
|
|
14816
15240
|
query,
|
|
14817
15241
|
embeddingModel: defaultModel,
|
|
14818
15242
|
ollama,
|
|
@@ -15729,7 +16153,7 @@ __export(server_exports, {
|
|
|
15729
16153
|
});
|
|
15730
16154
|
import { McpServer, ResourceTemplate } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
15731
16155
|
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
15732
|
-
import { homedir as
|
|
16156
|
+
import { homedir as homedir6 } from "os";
|
|
15733
16157
|
import { join as joinPath } from "path";
|
|
15734
16158
|
async function discoverMemorySinks(configured, vaults) {
|
|
15735
16159
|
if (configured.length > 0) {
|
|
@@ -15796,19 +16220,20 @@ async function serve(options = {}) {
|
|
|
15796
16220
|
const activeVault = process.env.VAULT_MEMORY_ACTIVE_VAULT?.trim() || void 0;
|
|
15797
16221
|
const rerankerBackend = config.server.reranker_backend ?? (config.server.reranker_model ? "onnx" : void 0);
|
|
15798
16222
|
const reranker = config.server.reranker_model ? rerankerBackend === "ollama" ? new OllamaReranker({ ollama, model: config.server.reranker_model }) : new OnnxReranker({
|
|
15799
|
-
modelDir: config.server.reranker_model_dir ?? joinPath(
|
|
16223
|
+
modelDir: config.server.reranker_model_dir ?? joinPath(homedir6(), ".vault-memory", "models", "bge-reranker-v2-m3")
|
|
15800
16224
|
}) : void 0;
|
|
15801
16225
|
const watchers = /* @__PURE__ */ new Map();
|
|
15802
16226
|
const briefDaemons = /* @__PURE__ */ new Map();
|
|
15803
16227
|
const startCatchupAndWatchers = async () => {
|
|
15804
16228
|
for (const vault of manager.list()) {
|
|
15805
|
-
|
|
16229
|
+
const isContextFit2 = vault.config.backend === "contextfit";
|
|
16230
|
+
if (!isContextFit2 && !vault.config.embedding_model && !vault.db.models.getActive()) continue;
|
|
15806
16231
|
const modelName = vault.config.embedding_model ?? defaultModel;
|
|
15807
16232
|
try {
|
|
15808
16233
|
const result = await catchupVault({
|
|
15809
16234
|
vault,
|
|
15810
16235
|
embeddingModel: modelName,
|
|
15811
|
-
ollama,
|
|
16236
|
+
...isContextFit2 ? {} : { ollama },
|
|
15812
16237
|
log: (m) => process.stderr.write(`[catchup:${vault.config.name}] ${m}
|
|
15813
16238
|
`)
|
|
15814
16239
|
});
|
|
@@ -16891,6 +17316,36 @@ async function runIndex(rest) {
|
|
|
16891
17316
|
});
|
|
16892
17317
|
const targets = vaultName ? [manager.require(vaultName)] : manager.list();
|
|
16893
17318
|
for (const vault of targets) {
|
|
17319
|
+
if (vault.config.backend === "contextfit") {
|
|
17320
|
+
const { indexVaultWithContextFit: indexVaultWithContextFit2 } = await Promise.resolve().then(() => (init_contextfit(), contextfit_exports));
|
|
17321
|
+
console.error(
|
|
17322
|
+
`
|
|
17323
|
+
\u2192 Indexing "${vault.config.name}" with ContextFit (CPU-only, no embeddings)`
|
|
17324
|
+
);
|
|
17325
|
+
const sqlite = await indexVault2(vault, {
|
|
17326
|
+
mode,
|
|
17327
|
+
embeddingModel: "contextfit",
|
|
17328
|
+
embeddings: "none",
|
|
17329
|
+
onProgress: (msg) => console.error(` ${msg}`)
|
|
17330
|
+
});
|
|
17331
|
+
if (sqlite.status !== "completed") {
|
|
17332
|
+
console.error(`\u2717 ${vault.config.name}: SQLite layer failed \u2014 ${sqlite.error}`);
|
|
17333
|
+
process.exitCode = 1;
|
|
17334
|
+
continue;
|
|
17335
|
+
}
|
|
17336
|
+
const cfResult = await indexVaultWithContextFit2(vault.config, {
|
|
17337
|
+
onProgress: (msg) => console.error(` ${msg}`)
|
|
17338
|
+
});
|
|
17339
|
+
if (cfResult.status === "completed") {
|
|
17340
|
+
console.error(
|
|
17341
|
+
`\u2713 ${vault.config.name}: ${sqlite.notesIndexed} notes (SQLite) + ContextFit KB \xB7 ${sqlite.durationMs + cfResult.durationMs}ms`
|
|
17342
|
+
);
|
|
17343
|
+
} else {
|
|
17344
|
+
console.error(`\u2717 ${vault.config.name}: ContextFit KB failed \u2014 ${cfResult.error}`);
|
|
17345
|
+
process.exitCode = 1;
|
|
17346
|
+
}
|
|
17347
|
+
continue;
|
|
17348
|
+
}
|
|
16894
17349
|
const model = vault.config.embedding_model ?? config.server.default_embedding_model ?? "qwen3-embedding";
|
|
16895
17350
|
console.error(`
|
|
16896
17351
|
\u2192 Indexing "${vault.config.name}" (${mode}) with ${model}`);
|
|
@@ -16918,6 +17373,8 @@ async function runAddVault(rest) {
|
|
|
16918
17373
|
let name;
|
|
16919
17374
|
let writeEnabled = false;
|
|
16920
17375
|
let skipIndex = false;
|
|
17376
|
+
let backend;
|
|
17377
|
+
const USAGE = "Usage: vault-memory add-vault <path> [--name <name>] [--write] [--backend ollama|contextfit] [--no-index]";
|
|
16921
17378
|
for (let i = 0; i < rest.length; i++) {
|
|
16922
17379
|
const arg = rest[i];
|
|
16923
17380
|
if (arg === "--name") {
|
|
@@ -16925,24 +17382,37 @@ async function runAddVault(rest) {
|
|
|
16925
17382
|
i++;
|
|
16926
17383
|
} else if (arg === "--write" || arg === "--write-enabled") {
|
|
16927
17384
|
writeEnabled = true;
|
|
17385
|
+
} else if (arg === "--backend") {
|
|
17386
|
+
const v = rest[i + 1];
|
|
17387
|
+
i++;
|
|
17388
|
+
if (v !== "ollama" && v !== "contextfit") {
|
|
17389
|
+
console.error(`--backend must be "ollama" or "contextfit" (got: ${v ?? "<missing>"})`);
|
|
17390
|
+
process.exit(2);
|
|
17391
|
+
}
|
|
17392
|
+
backend = v;
|
|
16928
17393
|
} else if (arg === "--no-index") {
|
|
16929
17394
|
skipIndex = true;
|
|
16930
17395
|
} else if (arg === "--help" || arg === "-h") {
|
|
16931
|
-
console.error(
|
|
17396
|
+
console.error(`${USAGE}
|
|
16932
17397
|
|
|
16933
17398
|
Registers a vault in ~/.vault-memory/config.toml, writes a .mcp.json
|
|
16934
|
-
into the vault root, and runs an initial index. Idempotent
|
|
17399
|
+
into the vault root, and runs an initial index. Idempotent.
|
|
17400
|
+
|
|
17401
|
+
--backend contextfit Use the CPU-only, token-native ContextFit engine
|
|
17402
|
+
(no Ollama / embeddings / GPU). Requires the
|
|
17403
|
+
\`contextfit\` CLI (pipx install contextfit). Ideal for
|
|
17404
|
+
resource-limited / non-GPU hosts (e.g. a Synology NAS).`);
|
|
16935
17405
|
return;
|
|
16936
17406
|
} else if (arg && !arg.startsWith("--") && path7 === null) {
|
|
16937
17407
|
path7 = arg;
|
|
16938
17408
|
}
|
|
16939
17409
|
}
|
|
16940
17410
|
if (path7 === null) {
|
|
16941
|
-
console.error(
|
|
17411
|
+
console.error(USAGE);
|
|
16942
17412
|
process.exit(2);
|
|
16943
17413
|
}
|
|
16944
|
-
console.error(`\u2192 Registering vault: ${path7}`);
|
|
16945
|
-
const result = await addVault2({ path: path7, name, writeEnabled });
|
|
17414
|
+
console.error(`\u2192 Registering vault: ${path7}${backend ? ` (backend: ${backend})` : ""}`);
|
|
17415
|
+
const result = await addVault2({ path: path7, name, writeEnabled, ...backend ? { backend } : {} });
|
|
16946
17416
|
for (const step of result.steps) {
|
|
16947
17417
|
switch (step.kind) {
|
|
16948
17418
|
case "config-added":
|