@owrede/vault-memory 2.2.1 → 2.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +54 -0
- package/README.md +4 -2
- package/dist/cli.js +364 -62
- package/dist/cli.js.map +1 -1
- package/package.json +2 -2
package/dist/cli.js
CHANGED
|
@@ -203,13 +203,13 @@ async function addVault(opts) {
|
|
|
203
203
|
const cfgFile = opts.configFile ?? configPath();
|
|
204
204
|
const binary = opts.binary ?? "vault-memory";
|
|
205
205
|
const steps = [];
|
|
206
|
-
const
|
|
206
|
+
const stat2 = await fs.stat(resolvedPath).catch((err) => {
|
|
207
207
|
if (err.code === "ENOENT") {
|
|
208
208
|
throw new Error(`Vault path does not exist: ${resolvedPath}`);
|
|
209
209
|
}
|
|
210
210
|
throw err;
|
|
211
211
|
});
|
|
212
|
-
if (!
|
|
212
|
+
if (!stat2.isDirectory()) {
|
|
213
213
|
throw new Error(`Vault path is not a directory: ${resolvedPath}`);
|
|
214
214
|
}
|
|
215
215
|
const proposedName = opts.name ?? slugifyVaultName(basename(resolvedPath));
|
|
@@ -1697,6 +1697,12 @@ function runMigration015(db, _ctx) {
|
|
|
1697
1697
|
"DROP INDEX IF EXISTS sections_note_anchor; CREATE UNIQUE INDEX IF NOT EXISTS sections_note_headingpath_anchor ON sections(note_id, heading_path, anchor);"
|
|
1698
1698
|
);
|
|
1699
1699
|
}
|
|
1700
|
+
function runMigration016(db, _ctx) {
|
|
1701
|
+
const cols = db.prepare("PRAGMA table_info(notes)").all();
|
|
1702
|
+
if (!cols.some((c) => c.name === "rendered_source_hash")) {
|
|
1703
|
+
db.exec("ALTER TABLE notes ADD COLUMN rendered_source_hash TEXT");
|
|
1704
|
+
}
|
|
1705
|
+
}
|
|
1700
1706
|
var INITIAL_SCHEMA, MIGRATION_002_ALIASES, MIGRATION_003_FIX_DELETE_FKS, MIGRATION_004_VARIABLE_DIMS, MIGRATION_006_BODY_HASH, MIGRATION_007_DOC_URI_ADD, MIGRATIONS;
|
|
1701
1707
|
var init_schema = __esm({
|
|
1702
1708
|
"src/db/schema.ts"() {
|
|
@@ -1977,6 +1983,11 @@ CREATE INDEX IF NOT EXISTS idx_notes_doc_uri ON notes(doc_uri);
|
|
|
1977
1983
|
version: 15,
|
|
1978
1984
|
description: "section identity = (note_id, heading_path, anchor) \u2014 context-aware, no longer collapse byte-identical siblings in different contexts (ADR-032 revised)",
|
|
1979
1985
|
run: runMigration015
|
|
1986
|
+
},
|
|
1987
|
+
{
|
|
1988
|
+
version: 16,
|
|
1989
|
+
description: "notes.rendered_source_hash \u2014 overlay marker for plugin-rendered Datacore content (ADR-033)",
|
|
1990
|
+
run: runMigration016
|
|
1980
1991
|
}
|
|
1981
1992
|
];
|
|
1982
1993
|
}
|
|
@@ -5111,6 +5122,103 @@ var init_cli = __esm({
|
|
|
5111
5122
|
}
|
|
5112
5123
|
});
|
|
5113
5124
|
|
|
5125
|
+
// src/adapters/retrieval/contextfit/ingest-lock.ts
|
|
5126
|
+
var ingest_lock_exports = {};
|
|
5127
|
+
__export(ingest_lock_exports, {
|
|
5128
|
+
clearIngestDirty: () => clearIngestDirty,
|
|
5129
|
+
isIngestDirty: () => isIngestDirty,
|
|
5130
|
+
markIngestDirty: () => markIngestDirty,
|
|
5131
|
+
releaseIngestLock: () => releaseIngestLock,
|
|
5132
|
+
tryAcquireIngestLock: () => tryAcquireIngestLock
|
|
5133
|
+
});
|
|
5134
|
+
import { open, readFile as readFile3, unlink, mkdir as mkdir2, writeFile as writeFile2, stat } from "fs/promises";
|
|
5135
|
+
import { homedir as homedir4 } from "os";
|
|
5136
|
+
import { join as join4 } from "path";
|
|
5137
|
+
function lockDir(rootOverride) {
|
|
5138
|
+
if (rootOverride !== void 0) return join4(rootOverride, "locks");
|
|
5139
|
+
return join4(homedir4(), ".vault-memory", "locks");
|
|
5140
|
+
}
|
|
5141
|
+
function lockPath(vaultName, rootOverride) {
|
|
5142
|
+
return join4(lockDir(rootOverride), `${vaultName}.ingest.lock`);
|
|
5143
|
+
}
|
|
5144
|
+
function dirtyPath(vaultName, rootOverride) {
|
|
5145
|
+
return join4(lockDir(rootOverride), `${vaultName}.ingest.dirty`);
|
|
5146
|
+
}
|
|
5147
|
+
function isProcessAlive(pid) {
|
|
5148
|
+
try {
|
|
5149
|
+
process.kill(pid, 0);
|
|
5150
|
+
return true;
|
|
5151
|
+
} catch (err) {
|
|
5152
|
+
if (err.code === "ESRCH") return false;
|
|
5153
|
+
return true;
|
|
5154
|
+
}
|
|
5155
|
+
}
|
|
5156
|
+
async function readOwnerPid(path7) {
|
|
5157
|
+
try {
|
|
5158
|
+
const buf = await readFile3(path7, "utf8");
|
|
5159
|
+
const pid = parseInt(buf.trim(), 10);
|
|
5160
|
+
return Number.isFinite(pid) && pid > 0 ? pid : null;
|
|
5161
|
+
} catch {
|
|
5162
|
+
return null;
|
|
5163
|
+
}
|
|
5164
|
+
}
|
|
5165
|
+
async function tryAcquireIngestLock(vaultName, options = {}) {
|
|
5166
|
+
const dir = lockDir(options.rootOverride);
|
|
5167
|
+
await mkdir2(dir, { recursive: true });
|
|
5168
|
+
const path7 = lockPath(vaultName, options.rootOverride);
|
|
5169
|
+
const MAX_ATTEMPTS = 3;
|
|
5170
|
+
const attempt = async (n) => {
|
|
5171
|
+
if (n > MAX_ATTEMPTS) return { acquired: false, ownerPid: -1, path: path7 };
|
|
5172
|
+
try {
|
|
5173
|
+
const handle = await open(path7, "wx");
|
|
5174
|
+
try {
|
|
5175
|
+
await handle.writeFile(`${process.pid}
|
|
5176
|
+
`);
|
|
5177
|
+
} finally {
|
|
5178
|
+
await handle.close();
|
|
5179
|
+
}
|
|
5180
|
+
return { acquired: true, path: path7 };
|
|
5181
|
+
} catch (err) {
|
|
5182
|
+
if (err.code !== "EEXIST") throw err;
|
|
5183
|
+
const ownerPid = await readOwnerPid(path7);
|
|
5184
|
+
if (ownerPid === null || !isProcessAlive(ownerPid)) {
|
|
5185
|
+
await unlink(path7).catch(() => void 0);
|
|
5186
|
+
return attempt(n + 1);
|
|
5187
|
+
}
|
|
5188
|
+
return { acquired: false, ownerPid, path: path7 };
|
|
5189
|
+
}
|
|
5190
|
+
};
|
|
5191
|
+
return attempt(1);
|
|
5192
|
+
}
|
|
5193
|
+
async function releaseIngestLock(vaultName, options = {}) {
|
|
5194
|
+
await unlink(lockPath(vaultName, options.rootOverride)).catch(() => void 0);
|
|
5195
|
+
}
|
|
5196
|
+
async function markIngestDirty(vaultName, options = {}) {
|
|
5197
|
+
const dir = lockDir(options.rootOverride);
|
|
5198
|
+
await mkdir2(dir, { recursive: true });
|
|
5199
|
+
await writeFile2(dirtyPath(vaultName, options.rootOverride), `${process.pid}
|
|
5200
|
+
`).catch(
|
|
5201
|
+
() => void 0
|
|
5202
|
+
);
|
|
5203
|
+
}
|
|
5204
|
+
async function isIngestDirty(vaultName, options = {}) {
|
|
5205
|
+
try {
|
|
5206
|
+
await stat(dirtyPath(vaultName, options.rootOverride));
|
|
5207
|
+
return true;
|
|
5208
|
+
} catch {
|
|
5209
|
+
return false;
|
|
5210
|
+
}
|
|
5211
|
+
}
|
|
5212
|
+
async function clearIngestDirty(vaultName, options = {}) {
|
|
5213
|
+
await unlink(dirtyPath(vaultName, options.rootOverride)).catch(() => void 0);
|
|
5214
|
+
}
|
|
5215
|
+
var init_ingest_lock = __esm({
|
|
5216
|
+
"src/adapters/retrieval/contextfit/ingest-lock.ts"() {
|
|
5217
|
+
"use strict";
|
|
5218
|
+
init_esm_shims();
|
|
5219
|
+
}
|
|
5220
|
+
});
|
|
5221
|
+
|
|
5114
5222
|
// src/adapters/retrieval/contextfit/index.ts
|
|
5115
5223
|
var contextfit_exports = {};
|
|
5116
5224
|
__export(contextfit_exports, {
|
|
@@ -5120,11 +5228,11 @@ __export(contextfit_exports, {
|
|
|
5120
5228
|
searchVaultWithContextFit: () => searchVaultWithContextFit,
|
|
5121
5229
|
sourceToNotePath: () => sourceToNotePath
|
|
5122
5230
|
});
|
|
5123
|
-
import { homedir as
|
|
5231
|
+
import { homedir as homedir5 } from "os";
|
|
5124
5232
|
import { rm } from "fs/promises";
|
|
5125
|
-
import { join as
|
|
5233
|
+
import { join as join5, relative, isAbsolute } from "path";
|
|
5126
5234
|
function contextFitKbDir(vaultName) {
|
|
5127
|
-
return
|
|
5235
|
+
return join5(homedir5(), ".vault-memory", "contextfit", vaultName);
|
|
5128
5236
|
}
|
|
5129
5237
|
function cliConfigForVault(vault) {
|
|
5130
5238
|
const cfg = {
|
|
@@ -5139,8 +5247,12 @@ async function indexVaultWithContextFit(vault, opts = {}) {
|
|
|
5139
5247
|
});
|
|
5140
5248
|
const cfg = cliConfigForVault(vault);
|
|
5141
5249
|
const start = Date.now();
|
|
5250
|
+
const lockOpts = opts.lockRootOverride !== void 0 ? { rootOverride: opts.lockRootOverride } : {};
|
|
5251
|
+
const probe = opts._deps?.probe ?? ((c) => contextFitProbe({ command: c.command }));
|
|
5252
|
+
const ingest = opts._deps?.ingest ?? contextFitIngest;
|
|
5253
|
+
const clearKb = opts._deps?.clearKb ?? ((p) => rm(p, { recursive: true, force: true }));
|
|
5142
5254
|
log(`ContextFit: ingesting ${vault.path} \u2192 ${cfg.kbPath}`);
|
|
5143
|
-
const available = await
|
|
5255
|
+
const available = await probe(cfg);
|
|
5144
5256
|
if (!available) {
|
|
5145
5257
|
return {
|
|
5146
5258
|
status: "failed",
|
|
@@ -5149,14 +5261,29 @@ async function indexVaultWithContextFit(vault, opts = {}) {
|
|
|
5149
5261
|
error: `ContextFit CLI not runnable (tried '${cfg.command}'). Install with \`pipx install contextfit\` or set [[vaults]].contextfit.command.`
|
|
5150
5262
|
};
|
|
5151
5263
|
}
|
|
5264
|
+
const lock = await tryAcquireIngestLock(vault.name, lockOpts);
|
|
5265
|
+
if (!lock.acquired) {
|
|
5266
|
+
await markIngestDirty(vault.name, lockOpts);
|
|
5267
|
+
log(`ContextFit: re-ingest already in progress (pid ${lock.ownerPid}); flagged for retry`);
|
|
5268
|
+
return { status: "skipped", stats: "", durationMs: Date.now() - start };
|
|
5269
|
+
}
|
|
5152
5270
|
try {
|
|
5153
|
-
|
|
5154
|
-
|
|
5271
|
+
const MAX_PASSES = 8;
|
|
5272
|
+
let stats = "";
|
|
5273
|
+
let passes = 0;
|
|
5274
|
+
do {
|
|
5275
|
+
await clearIngestDirty(vault.name, lockOpts);
|
|
5276
|
+
await clearKb(cfg.kbPath);
|
|
5277
|
+
stats = await ingest(cfg, vault.path);
|
|
5278
|
+
passes += 1;
|
|
5279
|
+
} while (passes < MAX_PASSES && await isIngestDirty(vault.name, lockOpts));
|
|
5155
5280
|
log(stats.trim().split("\n").slice(-3).join(" \xB7 "));
|
|
5156
5281
|
return { status: "completed", stats, durationMs: Date.now() - start };
|
|
5157
5282
|
} catch (err) {
|
|
5158
5283
|
const message = err instanceof Error ? err.message : String(err);
|
|
5159
5284
|
return { status: "failed", stats: "", durationMs: Date.now() - start, error: message };
|
|
5285
|
+
} finally {
|
|
5286
|
+
await releaseIngestLock(vault.name, lockOpts);
|
|
5160
5287
|
}
|
|
5161
5288
|
}
|
|
5162
5289
|
function sourceToNotePath(source, vaultPath) {
|
|
@@ -5202,6 +5329,7 @@ var init_contextfit = __esm({
|
|
|
5202
5329
|
"use strict";
|
|
5203
5330
|
init_esm_shims();
|
|
5204
5331
|
init_cli();
|
|
5332
|
+
init_ingest_lock();
|
|
5205
5333
|
DEFAULT_COMMAND = "contextfit";
|
|
5206
5334
|
}
|
|
5207
5335
|
});
|
|
@@ -5333,9 +5461,9 @@ var init_reranker = __esm({
|
|
|
5333
5461
|
});
|
|
5334
5462
|
|
|
5335
5463
|
// src/rerank/onnx-reranker.ts
|
|
5336
|
-
import { readFile as
|
|
5464
|
+
import { readFile as readFile4 } from "fs/promises";
|
|
5337
5465
|
import { existsSync } from "fs";
|
|
5338
|
-
import { join as
|
|
5466
|
+
import { join as join6 } from "path";
|
|
5339
5467
|
function sigmoid(x) {
|
|
5340
5468
|
return 1 / (1 + Math.exp(-x));
|
|
5341
5469
|
}
|
|
@@ -5413,8 +5541,8 @@ var init_onnx_reranker = __esm({
|
|
|
5413
5541
|
if (this.loaded) return this.loaded;
|
|
5414
5542
|
if (this.loading) return this.loading;
|
|
5415
5543
|
this.loading = (async () => {
|
|
5416
|
-
const modelPath =
|
|
5417
|
-
const tokenizerPath =
|
|
5544
|
+
const modelPath = join6(this.modelDir, "model_quantized.onnx");
|
|
5545
|
+
const tokenizerPath = join6(this.modelDir, "tokenizer.json");
|
|
5418
5546
|
if (!existsSync(modelPath)) {
|
|
5419
5547
|
throw new Error(
|
|
5420
5548
|
`OnnxReranker: model file not found at ${modelPath}. Run: curl -L https://huggingface.co/onnx-community/bge-reranker-v2-m3-ONNX/resolve/main/onnx/model_quantized.onnx -o ${modelPath}`
|
|
@@ -5428,7 +5556,7 @@ var init_onnx_reranker = __esm({
|
|
|
5428
5556
|
const [ort, tokMod, tokJson] = await Promise.all([
|
|
5429
5557
|
import("onnxruntime-node"),
|
|
5430
5558
|
import("@huggingface/tokenizers"),
|
|
5431
|
-
|
|
5559
|
+
readFile4(tokenizerPath, "utf-8")
|
|
5432
5560
|
]);
|
|
5433
5561
|
const tokenizerJson = JSON.parse(tokJson);
|
|
5434
5562
|
const config = deriveTokenizerConfig(tokenizerJson);
|
|
@@ -5892,6 +6020,73 @@ var init_wikilinks2 = __esm({
|
|
|
5892
6020
|
}
|
|
5893
6021
|
});
|
|
5894
6022
|
|
|
6023
|
+
// src/reader/datacore.ts
|
|
6024
|
+
function stripDynamicViewBlocks(body) {
|
|
6025
|
+
if (!body.includes("```") && !body.includes("~~~")) {
|
|
6026
|
+
return { content: body, replaced: 0 };
|
|
6027
|
+
}
|
|
6028
|
+
const lines = body.split("\n");
|
|
6029
|
+
const out = [];
|
|
6030
|
+
let replaced = 0;
|
|
6031
|
+
let i = 0;
|
|
6032
|
+
while (i < lines.length) {
|
|
6033
|
+
const line = lines[i];
|
|
6034
|
+
const open3 = FENCE_OPEN_RE.exec(line);
|
|
6035
|
+
if (open3) {
|
|
6036
|
+
const indent = open3[1] ?? "";
|
|
6037
|
+
const marker = open3[2] ?? "";
|
|
6038
|
+
const lang = (open3[3] ?? "").toLowerCase();
|
|
6039
|
+
const markerChar = marker[0];
|
|
6040
|
+
const isDynamic = DYNAMIC_VIEW_LANGS.has(lang);
|
|
6041
|
+
let j = i + 1;
|
|
6042
|
+
let closed = false;
|
|
6043
|
+
while (j < lines.length) {
|
|
6044
|
+
const close = FENCE_OPEN_RE.exec(lines[j]);
|
|
6045
|
+
if (close && (close[2] ?? "")[0] === markerChar && (close[2] ?? "").length >= marker.length && (close[3] ?? "") === "") {
|
|
6046
|
+
closed = true;
|
|
6047
|
+
break;
|
|
6048
|
+
}
|
|
6049
|
+
j++;
|
|
6050
|
+
}
|
|
6051
|
+
if (isDynamic) {
|
|
6052
|
+
out.push(`${indent}${DATACORE_PLACEHOLDER}`);
|
|
6053
|
+
replaced++;
|
|
6054
|
+
i = closed ? j + 1 : lines.length;
|
|
6055
|
+
} else {
|
|
6056
|
+
out.push(line);
|
|
6057
|
+
if (closed) {
|
|
6058
|
+
for (let k = i + 1; k <= j; k++) out.push(lines[k]);
|
|
6059
|
+
i = j + 1;
|
|
6060
|
+
} else {
|
|
6061
|
+
for (let k = i + 1; k < lines.length; k++) out.push(lines[k]);
|
|
6062
|
+
i = lines.length;
|
|
6063
|
+
}
|
|
6064
|
+
}
|
|
6065
|
+
} else {
|
|
6066
|
+
out.push(line);
|
|
6067
|
+
i++;
|
|
6068
|
+
}
|
|
6069
|
+
}
|
|
6070
|
+
if (replaced === 0) return { content: body, replaced: 0 };
|
|
6071
|
+
return { content: out.join("\n"), replaced };
|
|
6072
|
+
}
|
|
6073
|
+
var DYNAMIC_VIEW_LANGS, DATACORE_PLACEHOLDER, FENCE_OPEN_RE;
|
|
6074
|
+
var init_datacore = __esm({
|
|
6075
|
+
"src/reader/datacore.ts"() {
|
|
6076
|
+
"use strict";
|
|
6077
|
+
init_esm_shims();
|
|
6078
|
+
DYNAMIC_VIEW_LANGS = /* @__PURE__ */ new Set([
|
|
6079
|
+
"datacore",
|
|
6080
|
+
"datacorejsx",
|
|
6081
|
+
"datacorejs",
|
|
6082
|
+
"dataview",
|
|
6083
|
+
"dataviewjs"
|
|
6084
|
+
]);
|
|
6085
|
+
DATACORE_PLACEHOLDER = "[Datacore view]";
|
|
6086
|
+
FENCE_OPEN_RE = /^(\s*)(`{3,}|~{3,})\s*([A-Za-z0-9_-]*)\s*$/;
|
|
6087
|
+
}
|
|
6088
|
+
});
|
|
6089
|
+
|
|
5895
6090
|
// src/adapters/source/obsidian-fs/hash.ts
|
|
5896
6091
|
import { createHash as createHash3 } from "crypto";
|
|
5897
6092
|
function sha256(input) {
|
|
@@ -5930,7 +6125,7 @@ import * as path3 from "path";
|
|
|
5930
6125
|
import matter from "gray-matter";
|
|
5931
6126
|
async function parseNote(absolutePath, vaultRoot) {
|
|
5932
6127
|
const raw = await fs3.readFile(absolutePath, "utf-8");
|
|
5933
|
-
const
|
|
6128
|
+
const stat2 = await fs3.stat(absolutePath);
|
|
5934
6129
|
const parsed = matter(raw);
|
|
5935
6130
|
const content = parsed.content;
|
|
5936
6131
|
const fmData = parsed.data;
|
|
@@ -5938,15 +6133,17 @@ async function parseNote(absolutePath, vaultRoot) {
|
|
|
5938
6133
|
const title = extractTitle(content) ?? path3.basename(absolutePath, ".md");
|
|
5939
6134
|
const hash = computeNoteHash(content, frontmatter);
|
|
5940
6135
|
const bodyHash = computeBodyHash(content);
|
|
5941
|
-
const mtime = Math.floor(
|
|
6136
|
+
const mtime = Math.floor(stat2.mtimeMs);
|
|
5942
6137
|
const bodyLinks = extractWikilinks(content);
|
|
5943
6138
|
const frontmatterLinks = extractFrontmatterWikilinks(frontmatter);
|
|
5944
6139
|
const wikilinks = frontmatterLinks.length === 0 ? bodyLinks : mergeFrontmatterIntoBody(bodyLinks, frontmatterLinks);
|
|
5945
6140
|
const wordCount = countWords2(content);
|
|
5946
6141
|
const relativePath = toPosix2(path3.relative(path3.resolve(vaultRoot), path3.resolve(absolutePath)));
|
|
6142
|
+
const indexedContent = stripDynamicViewBlocks(content).content;
|
|
5947
6143
|
return {
|
|
5948
6144
|
relativePath,
|
|
5949
6145
|
content,
|
|
6146
|
+
indexedContent,
|
|
5950
6147
|
frontmatter,
|
|
5951
6148
|
title,
|
|
5952
6149
|
hash,
|
|
@@ -5990,6 +6187,7 @@ var init_parser = __esm({
|
|
|
5990
6187
|
"use strict";
|
|
5991
6188
|
init_esm_shims();
|
|
5992
6189
|
init_wikilinks2();
|
|
6190
|
+
init_datacore();
|
|
5993
6191
|
init_hash();
|
|
5994
6192
|
}
|
|
5995
6193
|
});
|
|
@@ -6045,8 +6243,8 @@ var init_obsidian_fs = __esm({
|
|
|
6045
6243
|
for (const abs of files) {
|
|
6046
6244
|
if (limit !== void 0 && yielded >= limit) break;
|
|
6047
6245
|
const rel = this.toPosix(path4.relative(path4.resolve(this.vault.path), abs));
|
|
6048
|
-
const
|
|
6049
|
-
const mtime = Math.floor(
|
|
6246
|
+
const stat2 = await fs4.stat(abs);
|
|
6247
|
+
const mtime = Math.floor(stat2.mtimeMs);
|
|
6050
6248
|
if (since !== void 0 && mtime < since) continue;
|
|
6051
6249
|
const body = await fs4.readFile(abs, "utf-8");
|
|
6052
6250
|
const hash = computeBodyHash(body);
|
|
@@ -6069,7 +6267,7 @@ var init_obsidian_fs = __esm({
|
|
|
6069
6267
|
const abs = this.absPath(rel);
|
|
6070
6268
|
if (CONTRACT_PATH_RE.test(rel)) {
|
|
6071
6269
|
const body = await fs4.readFile(abs, "utf-8");
|
|
6072
|
-
const
|
|
6270
|
+
const stat2 = await fs4.stat(abs);
|
|
6073
6271
|
const hash = computeBodyHash(body);
|
|
6074
6272
|
return {
|
|
6075
6273
|
id,
|
|
@@ -6078,7 +6276,7 @@ var init_obsidian_fs = __esm({
|
|
|
6078
6276
|
blocks: [{ kind: "paragraph", text: body }],
|
|
6079
6277
|
properties: {},
|
|
6080
6278
|
links: [],
|
|
6081
|
-
mtime: Math.floor(
|
|
6279
|
+
mtime: Math.floor(stat2.mtimeMs),
|
|
6082
6280
|
hash,
|
|
6083
6281
|
display_url: this.formatDisplayUrl(id)
|
|
6084
6282
|
};
|
|
@@ -6810,6 +7008,7 @@ async function indexVault(vault, options) {
|
|
|
6810
7008
|
vault.db.transaction(() => {
|
|
6811
7009
|
const allNotes = vault.db.notes.listAll();
|
|
6812
7010
|
for (const n of allNotes) {
|
|
7011
|
+
vault.db.sections.deleteByNote(n.id);
|
|
6813
7012
|
vault.db.chunks.deleteByNote(n.id);
|
|
6814
7013
|
vault.db.wikilinks.deleteByNote(n.id);
|
|
6815
7014
|
vault.db.edges.deleteByNote(n.id);
|
|
@@ -6833,6 +7032,9 @@ async function indexVault(vault, options) {
|
|
|
6833
7032
|
log(` skipped (parse error): ${rel} \u2014 ${msg}`);
|
|
6834
7033
|
continue;
|
|
6835
7034
|
}
|
|
7035
|
+
const previous = vault.db.notes.getByPath(parsed.relativePath);
|
|
7036
|
+
const hashUnchanged = previous != null && previous.hash === parsed.hash;
|
|
7037
|
+
const bodyUnchanged = previous != null && previous.body_hash != null && previous.body_hash === parsed.bodyHash;
|
|
6836
7038
|
const upsert = vault.db.notes.upsertByPath({
|
|
6837
7039
|
path: parsed.relativePath,
|
|
6838
7040
|
content: parsed.content,
|
|
@@ -6845,24 +7047,28 @@ async function indexVault(vault, options) {
|
|
|
6845
7047
|
});
|
|
6846
7048
|
vault.db.notes.setStatus(upsert.id, extractStatus(parsed.frontmatter));
|
|
6847
7049
|
vault.db.aliases.setForNote(upsert.id, extractAliases(parsed.frontmatter));
|
|
6848
|
-
const noteExisted = !upsert.isNew;
|
|
6849
|
-
const existing = noteExisted ? vault.db.notes.getById(upsert.id) : null;
|
|
6850
7050
|
const chunkCount = vault.db.chunks.getByNote(upsert.id).length;
|
|
6851
|
-
const
|
|
7051
|
+
const bodyChanged = !hashUnchanged && !bodyUnchanged;
|
|
7052
|
+
const needsReindex = mode === "full" || upsert.isNew || chunkCount === 0 || bodyChanged;
|
|
7053
|
+
const frontmatterOnly = !upsert.isNew && !needsReindex && !hashUnchanged;
|
|
6852
7054
|
if (upsert.isNew) notesIndexed++;
|
|
6853
|
-
else if (needsReindex) notesUpdated++;
|
|
7055
|
+
else if (needsReindex || frontmatterOnly) notesUpdated++;
|
|
6854
7056
|
if (needsReindex) {
|
|
6855
7057
|
parsedNotes.push({ parsed, noteId: upsert.id, needsReindex: true });
|
|
7058
|
+
} else if (frontmatterOnly) {
|
|
7059
|
+
vault.db.wikilinks.deleteByNote(upsert.id);
|
|
7060
|
+
vault.db.edges.deleteByNote(upsert.id);
|
|
7061
|
+
insertWikilinks(vault, upsert.id, parsed.wikilinks, firstPassResolver);
|
|
7062
|
+
writeAllEdges(vault, upsert.id, parsed, firstPassResolver);
|
|
6856
7063
|
}
|
|
6857
|
-
void existing;
|
|
6858
7064
|
}
|
|
6859
7065
|
log(`${parsedNotes.length} notes need (re-)indexing`);
|
|
6860
7066
|
for (const { parsed, noteId } of parsedNotes) {
|
|
7067
|
+
vault.db.sections.deleteByNote(noteId);
|
|
6861
7068
|
vault.db.chunks.deleteByNote(noteId);
|
|
6862
7069
|
vault.db.wikilinks.deleteByNote(noteId);
|
|
6863
7070
|
vault.db.edges.deleteByNote(noteId);
|
|
6864
|
-
|
|
6865
|
-
const chunks = chunkNote(parsed.content);
|
|
7071
|
+
const chunks = chunkNote(parsed.indexedContent);
|
|
6866
7072
|
if (chunks.length === 0) {
|
|
6867
7073
|
insertWikilinks(vault, noteId, parsed.wikilinks, firstPassResolver);
|
|
6868
7074
|
writeAllEdges(vault, noteId, parsed, firstPassResolver);
|
|
@@ -6879,7 +7085,7 @@ async function indexVault(vault, options) {
|
|
|
6879
7085
|
}));
|
|
6880
7086
|
const chunkIds = vault.db.chunks.insertBatch(noteId, chunkInputs);
|
|
6881
7087
|
try {
|
|
6882
|
-
buildSectionsForNote(vault, noteId, parsed.
|
|
7088
|
+
buildSectionsForNote(vault, noteId, parsed.indexedContent, chunkIds);
|
|
6883
7089
|
} catch (err) {
|
|
6884
7090
|
const message = err instanceof Error ? err.message : String(err);
|
|
6885
7091
|
console.error(
|
|
@@ -7216,7 +7422,7 @@ async function indexNote(options) {
|
|
|
7216
7422
|
vault.db.chunks.deleteByNote(upsert.id);
|
|
7217
7423
|
vault.db.wikilinks.deleteByNote(upsert.id);
|
|
7218
7424
|
vault.db.edges.deleteByNote(upsert.id);
|
|
7219
|
-
const chunks = chunkNote(parsed.
|
|
7425
|
+
const chunks = chunkNote(parsed.indexedContent);
|
|
7220
7426
|
if (chunks.length === 0) {
|
|
7221
7427
|
insertWikilinks2(vault, upsert.id, parsed.wikilinks);
|
|
7222
7428
|
writeAllEdges2(vault, upsert.id, parsed);
|
|
@@ -7243,7 +7449,7 @@ async function indexNote(options) {
|
|
|
7243
7449
|
}))
|
|
7244
7450
|
);
|
|
7245
7451
|
try {
|
|
7246
|
-
buildSectionsForNote(vault, upsert.id, parsed.
|
|
7452
|
+
buildSectionsForNote(vault, upsert.id, parsed.indexedContent, chunkIds);
|
|
7247
7453
|
} catch (err) {
|
|
7248
7454
|
const message = err instanceof Error ? err.message : String(err);
|
|
7249
7455
|
process.stderr.write(
|
|
@@ -7409,12 +7615,15 @@ async function catchupVault(options) {
|
|
|
7409
7615
|
}
|
|
7410
7616
|
}
|
|
7411
7617
|
}
|
|
7412
|
-
if (isContextFit2
|
|
7413
|
-
const
|
|
7414
|
-
const
|
|
7415
|
-
|
|
7416
|
-
r
|
|
7417
|
-
|
|
7618
|
+
if (isContextFit2) {
|
|
7619
|
+
const cf = await Promise.resolve().then(() => (init_contextfit(), contextfit_exports));
|
|
7620
|
+
const dirty = await (await Promise.resolve().then(() => (init_ingest_lock(), ingest_lock_exports))).isIngestDirty(vault.config.name);
|
|
7621
|
+
if (reindexed > 0 || removed > 0 || dirty) {
|
|
7622
|
+
const r = await cf.indexVaultWithContextFit(vault.config, { onProgress: log });
|
|
7623
|
+
log(
|
|
7624
|
+
r.status === "completed" ? `catch-up: ContextFit KB rebuilt (${r.durationMs}ms)` : r.status === "skipped" ? `catch-up: ContextFit KB re-ingest already in progress; skipping` : `catch-up: ContextFit KB rebuild failed: ${r.error}`
|
|
7625
|
+
);
|
|
7626
|
+
}
|
|
7418
7627
|
}
|
|
7419
7628
|
return {
|
|
7420
7629
|
scanned: files.length,
|
|
@@ -7848,14 +8057,15 @@ async function writeNote(input) {
|
|
|
7848
8057
|
};
|
|
7849
8058
|
}
|
|
7850
8059
|
}
|
|
7851
|
-
const
|
|
8060
|
+
const yamlDumpOptions = { lineWidth: -1 };
|
|
8061
|
+
const fileText = frontmatter !== null && Object.keys(frontmatter).length > 0 ? matter2.stringify(content, frontmatter, yamlDumpOptions) : content;
|
|
7852
8062
|
input.onBeforeFsWrite?.();
|
|
7853
8063
|
await atomicWriteFile(absPath, fileText);
|
|
7854
8064
|
const written = await readExistingFile(absPath);
|
|
7855
8065
|
if (written === null) {
|
|
7856
8066
|
throw new Error(`Internal error: file disappeared after write: ${relativePath}`);
|
|
7857
8067
|
}
|
|
7858
|
-
const
|
|
8068
|
+
const stat2 = await fs6.stat(absPath);
|
|
7859
8069
|
const previousNote = vault.db.notes.getByPath(relativePath);
|
|
7860
8070
|
const previousHash = previousNote?.hash ?? null;
|
|
7861
8071
|
const title = extractTitle2(written.content, relativePath);
|
|
@@ -7869,7 +8079,7 @@ async function writeNote(input) {
|
|
|
7869
8079
|
title,
|
|
7870
8080
|
hash: written.hash,
|
|
7871
8081
|
bodyHash: computeBodyHash(written.content),
|
|
7872
|
-
mtime: Math.floor(
|
|
8082
|
+
mtime: Math.floor(stat2.mtimeMs),
|
|
7873
8083
|
wordCount: countWords3(written.content)
|
|
7874
8084
|
});
|
|
7875
8085
|
vault.db.aliases.setForNote(up.id, extractAliases(written.frontmatter));
|
|
@@ -8241,7 +8451,7 @@ var init_path = __esm({
|
|
|
8241
8451
|
});
|
|
8242
8452
|
|
|
8243
8453
|
// src/adapters/delivery/obsidian-fs/contract-yaml-read.ts
|
|
8244
|
-
import { readFile as
|
|
8454
|
+
import { readFile as readFile5 } from "fs/promises";
|
|
8245
8455
|
var init_contract_yaml_read = __esm({
|
|
8246
8456
|
"src/adapters/delivery/obsidian-fs/contract-yaml-read.ts"() {
|
|
8247
8457
|
"use strict";
|
|
@@ -10261,17 +10471,17 @@ var init_get = __esm({
|
|
|
10261
10471
|
});
|
|
10262
10472
|
|
|
10263
10473
|
// src/brief/lock.ts
|
|
10264
|
-
import { open, readFile as
|
|
10265
|
-
import { homedir as
|
|
10266
|
-
import { join as
|
|
10267
|
-
function
|
|
10268
|
-
if (rootOverride !== void 0) return
|
|
10269
|
-
return
|
|
10474
|
+
import { open as open2, readFile as readFile6, unlink as unlink2, mkdir as mkdir3 } from "fs/promises";
|
|
10475
|
+
import { homedir as homedir6 } from "os";
|
|
10476
|
+
import { join as join8 } from "path";
|
|
10477
|
+
function lockDir2(rootOverride) {
|
|
10478
|
+
if (rootOverride !== void 0) return join8(rootOverride, "locks");
|
|
10479
|
+
return join8(homedir6(), ".vault-memory", "locks");
|
|
10270
10480
|
}
|
|
10271
|
-
function
|
|
10272
|
-
return
|
|
10481
|
+
function lockPath2(vaultName, rootOverride) {
|
|
10482
|
+
return join8(lockDir2(rootOverride), `${vaultName}.lock`);
|
|
10273
10483
|
}
|
|
10274
|
-
function
|
|
10484
|
+
function isProcessAlive2(pid) {
|
|
10275
10485
|
try {
|
|
10276
10486
|
process.kill(pid, 0);
|
|
10277
10487
|
return true;
|
|
@@ -10280,9 +10490,9 @@ function isProcessAlive(pid) {
|
|
|
10280
10490
|
return true;
|
|
10281
10491
|
}
|
|
10282
10492
|
}
|
|
10283
|
-
async function
|
|
10493
|
+
async function readOwnerPid2(path7) {
|
|
10284
10494
|
try {
|
|
10285
|
-
const buf = await
|
|
10495
|
+
const buf = await readFile6(path7, "utf8");
|
|
10286
10496
|
const pid = parseInt(buf.trim(), 10);
|
|
10287
10497
|
return Number.isFinite(pid) && pid > 0 ? pid : null;
|
|
10288
10498
|
} catch {
|
|
@@ -10290,16 +10500,16 @@ async function readOwnerPid(path7) {
|
|
|
10290
10500
|
}
|
|
10291
10501
|
}
|
|
10292
10502
|
async function tryAcquireLock(vaultName, options = {}) {
|
|
10293
|
-
const dir =
|
|
10294
|
-
await
|
|
10295
|
-
const path7 =
|
|
10503
|
+
const dir = lockDir2(options.rootOverride);
|
|
10504
|
+
await mkdir3(dir, { recursive: true });
|
|
10505
|
+
const path7 = lockPath2(vaultName, options.rootOverride);
|
|
10296
10506
|
const MAX_ATTEMPTS = 3;
|
|
10297
10507
|
const attempt = async (n, stolenFromPid) => {
|
|
10298
10508
|
if (n > MAX_ATTEMPTS) {
|
|
10299
10509
|
return { acquired: false, ownerPid: stolenFromPid ?? -1, path: path7 };
|
|
10300
10510
|
}
|
|
10301
10511
|
try {
|
|
10302
|
-
const handle = await
|
|
10512
|
+
const handle = await open2(path7, "wx");
|
|
10303
10513
|
try {
|
|
10304
10514
|
await handle.writeFile(`${process.pid}
|
|
10305
10515
|
`);
|
|
@@ -10311,9 +10521,9 @@ async function tryAcquireLock(vaultName, options = {}) {
|
|
|
10311
10521
|
return result;
|
|
10312
10522
|
} catch (err) {
|
|
10313
10523
|
if (err.code !== "EEXIST") throw err;
|
|
10314
|
-
const ownerPid = await
|
|
10315
|
-
if (ownerPid === null || !
|
|
10316
|
-
await
|
|
10524
|
+
const ownerPid = await readOwnerPid2(path7);
|
|
10525
|
+
if (ownerPid === null || !isProcessAlive2(ownerPid)) {
|
|
10526
|
+
await unlink2(path7).catch(() => void 0);
|
|
10317
10527
|
return attempt(n + 1, ownerPid ?? -1);
|
|
10318
10528
|
}
|
|
10319
10529
|
return { acquired: false, ownerPid, path: path7 };
|
|
@@ -10322,7 +10532,7 @@ async function tryAcquireLock(vaultName, options = {}) {
|
|
|
10322
10532
|
return attempt(1);
|
|
10323
10533
|
}
|
|
10324
10534
|
async function releaseLock(vaultName, options = {}) {
|
|
10325
|
-
await
|
|
10535
|
+
await unlink2(lockPath2(vaultName, options.rootOverride)).catch(() => void 0);
|
|
10326
10536
|
}
|
|
10327
10537
|
var init_lock = __esm({
|
|
10328
10538
|
"src/brief/lock.ts"() {
|
|
@@ -11190,6 +11400,8 @@ var init_watcher = __esm({
|
|
|
11190
11400
|
const r = await indexVaultWithContextFit2(this.opts.vault.config, {});
|
|
11191
11401
|
if (r.status === "completed") {
|
|
11192
11402
|
this.opts.log(`ContextFit KB refreshed (${r.durationMs}ms)`);
|
|
11403
|
+
} else if (r.status === "skipped") {
|
|
11404
|
+
this.opts.log(`ContextFit KB refresh skipped (another ingest in progress; flagged)`);
|
|
11193
11405
|
} else {
|
|
11194
11406
|
this.opts.log(`ContextFit KB refresh failed: ${r.error}`);
|
|
11195
11407
|
}
|
|
@@ -16138,6 +16350,92 @@ var init_contracts2 = __esm({
|
|
|
16138
16350
|
}
|
|
16139
16351
|
});
|
|
16140
16352
|
|
|
16353
|
+
// package.json
|
|
16354
|
+
var package_default;
|
|
16355
|
+
var init_package = __esm({
|
|
16356
|
+
"package.json"() {
|
|
16357
|
+
package_default = {
|
|
16358
|
+
name: "@owrede/vault-memory",
|
|
16359
|
+
version: "2.3.1",
|
|
16360
|
+
description: "Local-first semantic memory MCP server for Obsidian vaults",
|
|
16361
|
+
type: "module",
|
|
16362
|
+
license: "MIT",
|
|
16363
|
+
workspaces: [
|
|
16364
|
+
"plugin"
|
|
16365
|
+
],
|
|
16366
|
+
repository: {
|
|
16367
|
+
type: "git",
|
|
16368
|
+
url: "git+https://github.com/owrede/vault-memory.git"
|
|
16369
|
+
},
|
|
16370
|
+
bin: {
|
|
16371
|
+
"vault-memory": "dist/cli.js"
|
|
16372
|
+
},
|
|
16373
|
+
files: [
|
|
16374
|
+
"dist",
|
|
16375
|
+
"README.md",
|
|
16376
|
+
"LICENSE",
|
|
16377
|
+
"CHANGELOG.md"
|
|
16378
|
+
],
|
|
16379
|
+
engines: {
|
|
16380
|
+
node: ">=22 <26"
|
|
16381
|
+
},
|
|
16382
|
+
scripts: {
|
|
16383
|
+
build: "tsup",
|
|
16384
|
+
dev: "tsx watch src/cli.ts",
|
|
16385
|
+
start: "node dist/cli.js",
|
|
16386
|
+
test: "vitest run",
|
|
16387
|
+
"test:watch": "vitest",
|
|
16388
|
+
lint: "tsc --noEmit",
|
|
16389
|
+
"lint:adapters": "sh scripts/lint-adapters.sh",
|
|
16390
|
+
"lint:check": 'sh scripts/check-fixture-privacy.sh && sh scripts/lint-no-telemetry.sh && sh scripts/lint-adapters.sh && tsc --noEmit && prettier --check "src/**/*.ts"',
|
|
16391
|
+
format: 'prettier --write "src/**/*.ts"',
|
|
16392
|
+
"eval:baseline": "vitest run evals/v1-baseline/baseline.test.ts",
|
|
16393
|
+
"eval:snapshot": "node evals/v1-baseline/dump-tools.mjs > evals/v1-baseline/tools-list.snapshot.json && node evals/v1-baseline/dump-resources.mjs > evals/v1-baseline/resources-list.snapshot.json",
|
|
16394
|
+
"eval:smoketest": "npm run build && node scripts/smoketest-non-claude.mjs",
|
|
16395
|
+
release: "node scripts/release.mjs",
|
|
16396
|
+
"sync-marketplace": "node scripts/sync-marketplace.mjs"
|
|
16397
|
+
},
|
|
16398
|
+
dependencies: {
|
|
16399
|
+
"@huggingface/tokenizers": "^0.1.3",
|
|
16400
|
+
"@modelcontextprotocol/sdk": "^1.29.0",
|
|
16401
|
+
"better-sqlite3": "^11.7.0",
|
|
16402
|
+
chokidar: "^4.0.1",
|
|
16403
|
+
"cross-spawn": "^7.0.6",
|
|
16404
|
+
graphology: "^0.26.0",
|
|
16405
|
+
"graphology-communities-louvain": "^2.0.2",
|
|
16406
|
+
"gray-matter": "^4.0.3",
|
|
16407
|
+
"onnxruntime-node": "^1.26.0",
|
|
16408
|
+
seedrandom: "^3.0.5",
|
|
16409
|
+
"smol-toml": "^1.3.1",
|
|
16410
|
+
"sqlite-vec": "^0.1.6",
|
|
16411
|
+
yaml: "^2.9.0",
|
|
16412
|
+
zod: "^4.4.3"
|
|
16413
|
+
},
|
|
16414
|
+
devDependencies: {
|
|
16415
|
+
"@types/better-sqlite3": "^7.6.12",
|
|
16416
|
+
"@types/node": "^22.10.0",
|
|
16417
|
+
"@types/seedrandom": "^3.0.8",
|
|
16418
|
+
prettier: "^3.4.0",
|
|
16419
|
+
tsup: "^8.3.5",
|
|
16420
|
+
tsx: "^4.19.2",
|
|
16421
|
+
typescript: "^5.7.0",
|
|
16422
|
+
vitest: "^2.1.8"
|
|
16423
|
+
}
|
|
16424
|
+
};
|
|
16425
|
+
}
|
|
16426
|
+
});
|
|
16427
|
+
|
|
16428
|
+
// src/version.ts
|
|
16429
|
+
var VERSION;
|
|
16430
|
+
var init_version = __esm({
|
|
16431
|
+
"src/version.ts"() {
|
|
16432
|
+
"use strict";
|
|
16433
|
+
init_esm_shims();
|
|
16434
|
+
init_package();
|
|
16435
|
+
VERSION = package_default.version;
|
|
16436
|
+
}
|
|
16437
|
+
});
|
|
16438
|
+
|
|
16141
16439
|
// src/server.ts
|
|
16142
16440
|
var server_exports = {};
|
|
16143
16441
|
__export(server_exports, {
|
|
@@ -16153,7 +16451,7 @@ __export(server_exports, {
|
|
|
16153
16451
|
});
|
|
16154
16452
|
import { McpServer, ResourceTemplate } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
16155
16453
|
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
16156
|
-
import { homedir as
|
|
16454
|
+
import { homedir as homedir7 } from "os";
|
|
16157
16455
|
import { join as joinPath } from "path";
|
|
16158
16456
|
async function discoverMemorySinks(configured, vaults) {
|
|
16159
16457
|
if (configured.length > 0) {
|
|
@@ -16220,7 +16518,7 @@ async function serve(options = {}) {
|
|
|
16220
16518
|
const activeVault = process.env.VAULT_MEMORY_ACTIVE_VAULT?.trim() || void 0;
|
|
16221
16519
|
const rerankerBackend = config.server.reranker_backend ?? (config.server.reranker_model ? "onnx" : void 0);
|
|
16222
16520
|
const reranker = config.server.reranker_model ? rerankerBackend === "ollama" ? new OllamaReranker({ ollama, model: config.server.reranker_model }) : new OnnxReranker({
|
|
16223
|
-
modelDir: config.server.reranker_model_dir ?? joinPath(
|
|
16521
|
+
modelDir: config.server.reranker_model_dir ?? joinPath(homedir7(), ".vault-memory", "models", "bge-reranker-v2-m3")
|
|
16224
16522
|
}) : void 0;
|
|
16225
16523
|
const watchers = /* @__PURE__ */ new Map();
|
|
16226
16524
|
const briefDaemons = /* @__PURE__ */ new Map();
|
|
@@ -17219,7 +17517,7 @@ async function serve(options = {}) {
|
|
|
17219
17517
|
`);
|
|
17220
17518
|
});
|
|
17221
17519
|
}
|
|
17222
|
-
var
|
|
17520
|
+
var MEMORY_AUTO_DISCOVERY_FOLDER;
|
|
17223
17521
|
var init_server = __esm({
|
|
17224
17522
|
"src/server.ts"() {
|
|
17225
17523
|
"use strict";
|
|
@@ -17258,7 +17556,7 @@ var init_server = __esm({
|
|
|
17258
17556
|
init_brief2();
|
|
17259
17557
|
init_assembly2();
|
|
17260
17558
|
init_contracts2();
|
|
17261
|
-
|
|
17559
|
+
init_version();
|
|
17262
17560
|
MEMORY_AUTO_DISCOVERY_FOLDER = "_memory";
|
|
17263
17561
|
}
|
|
17264
17562
|
});
|
|
@@ -17340,6 +17638,10 @@ async function runIndex(rest) {
|
|
|
17340
17638
|
console.error(
|
|
17341
17639
|
`\u2713 ${vault.config.name}: ${sqlite.notesIndexed} notes (SQLite) + ContextFit KB \xB7 ${sqlite.durationMs + cfResult.durationMs}ms`
|
|
17342
17640
|
);
|
|
17641
|
+
} else if (cfResult.status === "skipped") {
|
|
17642
|
+
console.error(
|
|
17643
|
+
`\u21B7 ${vault.config.name}: ${sqlite.notesIndexed} notes (SQLite); ContextFit KB re-ingest already in progress in another process \u2014 flagged for retry, skipping`
|
|
17644
|
+
);
|
|
17343
17645
|
} else {
|
|
17344
17646
|
console.error(`\u2717 ${vault.config.name}: ContextFit KB failed \u2014 ${cfResult.error}`);
|
|
17345
17647
|
process.exitCode = 1;
|