@staix/vault-mem 0.11.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/README.md +17 -2
- package/bin/vmem.js +5 -1
- package/package.json +2 -1
- package/skills/_shared/vault-adapter.md +131 -0
- package/skills/skill-mine/SKILL.md +52 -0
- package/skills/skill-mine/vocab/actions.yaml +110 -0
- package/skills/skill-mine/vocab/objects.yaml +123 -0
- package/skills/vault-connect/SKILL.md +75 -0
- package/skills/vault-extract/README.md +20 -0
- package/skills/vault-extract/SKILL.md +139 -0
- package/skills/vault-graph/SKILL.md +120 -0
- package/skills/vault-learn/SKILL.md +111 -0
- package/skills/vault-lint/README.md +32 -0
- package/skills/vault-lint/SKILL.md +137 -0
- package/skills/vault-lint/references/lint-output.md +97 -0
- package/skills/vault-lint/references/vault-scope-rules.md +26 -0
- package/skills/vault-lint/references/work-scope-checks.md +71 -0
- package/skills/vault-refactor/SKILL.md +112 -0
- package/skills/vault-remember/SKILL.md +38 -0
- package/skills/vault-rethink/README.md +15 -0
- package/skills/vault-rethink/SKILL.md +58 -0
- package/skills/vault-stats/SKILL.md +49 -0
- package/skills/vault-sync/README.md +15 -0
- package/skills/vault-sync/SKILL.md +195 -0
- package/skills/vault-sync/references/r4-method-evidence.md +12 -0
- package/skills/vault-update/SKILL.md +68 -0
- package/skills/vmem-status/SKILL.md +30 -0
- package/src/amem/host-api.ts +5 -1
- package/src/analysis-input.ts +15 -0
- package/src/analysis-types.ts +3 -1
- package/src/assertion-policy.ts +4 -2
- package/src/assertions.ts +48 -8
- package/src/cli-output.ts +10 -3
- package/src/cli.ts +29 -3
- package/src/commands-table.ts +3 -1
- package/src/config.ts +21 -7
- package/src/enrichment-query.ts +65 -0
- package/src/enrichment-search.ts +69 -0
- package/src/enrichment-store.ts +197 -0
- package/src/enrichment.ts +245 -0
- package/src/graph-evidence.ts +2 -2
- package/src/index.ts +9 -0
- package/src/mcp-vault.ts +46 -8
- package/src/proposal-provider.ts +3 -2
- package/src/proposals.ts +10 -4
- package/src/read-output.ts +1 -1
- package/src/registry.ts +57 -2
- package/src/setup.ts +12 -3
- package/src/skills-install.ts +216 -0
- package/src/ui/panel-types.ts +55 -0
- package/src/ui/panel.js +76 -0
- package/src/ui/plugin.ts +95 -5
- package/src/vault-graph.ts +15 -7
- package/src/vault-index.ts +17 -0
- package/src/vault-lint.ts +12 -4
- package/src/write-operations.ts +38 -1
- package/src/write-types.ts +1 -0
- package/src/write-validation.ts +2 -1
- package/src/writer.ts +1 -1
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
/** Bounded hybrid ranking over current indexed notes and local stored vectors. */
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { loadRegistry } from "./registry.ts";
|
|
4
|
+
import { logBinding, matchesBinding } from "./vault-log.ts";
|
|
5
|
+
import { canonicalPath, isDenied } from "./platform.ts";
|
|
6
|
+
import { readEnrichmentStore } from "./enrichment-store.ts";
|
|
7
|
+
import { rankHybridCandidates, vectorSimilarity } from "./enrichment-search.ts";
|
|
8
|
+
import { resultLimit, type VaultIndex, type SearchOptions, type SearchResult } from "./vault-index.ts";
|
|
9
|
+
|
|
10
|
+
export interface QueryVector { model: string; values: number[] }
|
|
11
|
+
function queryContext(home: string, vaultId: string) {
|
|
12
|
+
const config = loadRegistry(home), vault = config.vaults.find(v => v.id === vaultId);
|
|
13
|
+
if (!vault) throw new Error("enrichment: vault is not registered; select a registered vault");
|
|
14
|
+
let binding: ReturnType<typeof logBinding>;
|
|
15
|
+
try { binding = logBinding(vault, config); } catch { throw new Error("enrichment: vault root is unavailable; restore the registered vault and retry"); }
|
|
16
|
+
const check = () => {
|
|
17
|
+
let current: ReturnType<typeof loadRegistry>;
|
|
18
|
+
try { current = loadRegistry(home); } catch { throw new Error("enrichment: registry is unavailable after the read; inspect the configuration and retry"); }
|
|
19
|
+
if (!matchesBinding(current, vault.id, binding)) throw new Error("enrichment: vault binding or policy changed during the read; retry with the current registry");
|
|
20
|
+
};
|
|
21
|
+
const store = vault.enrichment?.enabled ? readEnrichmentStore(home, vault, config.vaults, config.denyPaths ?? []) : undefined;
|
|
22
|
+
return { config, vault, store, check };
|
|
23
|
+
}
|
|
24
|
+
function allowed(ctx: ReturnType<typeof queryContext>, path: string): boolean {
|
|
25
|
+
const file = join(ctx.vault.path, "notes", path);
|
|
26
|
+
return !isDenied(file, ctx.config.denyPaths ?? []) && !isDenied(canonicalPath(file), ctx.config.denyPaths ?? []);
|
|
27
|
+
}
|
|
28
|
+
function candidates(ctx: ReturnType<typeof queryContext>, index: VaultIndex, query: QueryVector) {
|
|
29
|
+
const { store } = ctx;
|
|
30
|
+
if (!store) return [];
|
|
31
|
+
const vectors = store.records.filter(r => r.kind === "embedding");
|
|
32
|
+
if (!vectors.length) return [];
|
|
33
|
+
if (!query || query.model !== store.model || !Array.isArray(query.values) || query.values.length !== store.dimension) throw new Error("enrichment: query vector model or dimension differs from stored vectors; use the recorded model and dimension");
|
|
34
|
+
try { vectorSimilarity(query.values, query.values); } catch { throw new Error("enrichment: query vector is invalid or zero; provide finite nonzero values"); }
|
|
35
|
+
const metadata = new Map(index.enrichmentPointers().filter(p => allowed(ctx, p.path)).map(p => [p.path, p]));
|
|
36
|
+
return vectors.flatMap(r => {
|
|
37
|
+
const id = r.path.slice(6), note = metadata.get(id);
|
|
38
|
+
if (!note || note.hash !== r.contentHash || !Array.isArray(r.value)) return [];
|
|
39
|
+
return [{ id, score: vectorSimilarity(query.values, r.value as number[]), note }];
|
|
40
|
+
}).sort((a, b) => b.score - a.score || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
|
|
41
|
+
}
|
|
42
|
+
export function hybridSearch(home: string, vault: string, index: VaultIndex, query: string, options: SearchOptions, vector: QueryVector): SearchResult[] {
|
|
43
|
+
const limit = resultLimit(options.limit), ctx = queryContext(home, vault);
|
|
44
|
+
if (index.vault.id !== ctx.vault.id || index.vault.path !== ctx.vault.path || !index.sourceIdentityCurrent()) throw new Error("enrichment: index is unscanned or belongs to another vault; reopen and scan the selected vault before retrying");
|
|
45
|
+
const lexical = index.search(query, { ...options, limit: 500 }).filter(note => allowed(ctx, note.path));
|
|
46
|
+
const hits = candidates(ctx, index, vector).filter(h => (!options.type || h.note.type === options.type) && (!options.domain || h.note.domain === options.domain));
|
|
47
|
+
if (!hits.length) { ctx.check(); if (!index.sourceIdentityCurrent()) throw new Error("enrichment: index source changed during the read; reopen the vault and retry"); return lexical.slice(0, limit); }
|
|
48
|
+
const vectorHits = hits.slice(0, 9500), ranked = rankHybridCandidates(lexical.map((v, i) => ({ id: v.path, rank: i + 1 })), vectorHits.map((v, i) => ({ id: v.id, rank: i + 1 })));
|
|
49
|
+
const notes = new Map(vectorHits.map(h => [h.id, h.note]));
|
|
50
|
+
for (const note of lexical) notes.set(note.path, { ...note, hash: "" });
|
|
51
|
+
const result = ranked.slice(0, limit).map(r => { const { hash: _, ...note } = notes.get(r.id)!; return { ...note, score: r.score }; });
|
|
52
|
+
ctx.check(); if (!index.sourceIdentityCurrent()) throw new Error("enrichment: index source changed during the read; reopen the vault and retry"); return result;
|
|
53
|
+
}
|
|
54
|
+
export function similarNotes(home: string, vault: string, index: VaultIndex, path: string, limit = 20) {
|
|
55
|
+
if (!Number.isSafeInteger(limit) || limit < 1 || limit > 100) throw new Error("enrichment: similar limit must be 1..100; choose a smaller page");
|
|
56
|
+
const ctx = queryContext(home, vault), { vault: spec, store } = ctx;
|
|
57
|
+
if (index.vault.id !== spec.id || index.vault.path !== spec.path || !index.sourceIdentityCurrent()) throw new Error("enrichment: index is unscanned or belongs to another vault; reopen and scan the selected vault before retrying");
|
|
58
|
+
if (!store) throw new Error("enrichment: enrichment is disabled; enable it in settings");
|
|
59
|
+
const id = index.resolve(path), pointer = index.enrichmentPointers().find(p => p.path === id);
|
|
60
|
+
if (!allowed(ctx, id)) throw new Error("enrichment: source note is denied; select an allowed note");
|
|
61
|
+
const source = store.records.find(r => r.kind === "embedding" && r.path === `notes/${id}` && r.contentHash === pointer?.hash);
|
|
62
|
+
if (!source || !Array.isArray(source.value) || !store.model) throw new Error("enrichment: no current vector exists for this note; run enrichment for the current content");
|
|
63
|
+
const result = candidates(ctx, index, { model: store.model, values: source.value as number[] }).filter(hit => hit.id !== id).slice(0, limit).map(({ note, score }) => ({ path: note.path, title: note.title, score }));
|
|
64
|
+
ctx.check(); if (!index.sourceIdentityCurrent()) throw new Error("enrichment: index source changed during the read; reopen the vault and retry"); return result;
|
|
65
|
+
}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
/** Pure, bounded helpers for combining lexical and vector search results. */
|
|
2
|
+
|
|
3
|
+
export const MAX_VECTOR_DIMENSIONS = 4096;
|
|
4
|
+
export const MAX_RANK_CANDIDATES = 10_000;
|
|
5
|
+
const RRF_K = 60;
|
|
6
|
+
|
|
7
|
+
/** Cosine similarity for two finite, non-zero vectors of equal bounded size. */
|
|
8
|
+
export function vectorSimilarity(left: readonly number[], right: readonly number[]): number {
|
|
9
|
+
if (!Array.isArray(left) || !Array.isArray(right) || left.length === 0 || right.length === 0) {
|
|
10
|
+
throw new TypeError("vectors must be nonempty arrays");
|
|
11
|
+
}
|
|
12
|
+
if (left.length > MAX_VECTOR_DIMENSIONS || right.length > MAX_VECTOR_DIMENSIONS) {
|
|
13
|
+
throw new RangeError(`vector dimensions must not exceed ${MAX_VECTOR_DIMENSIONS}`);
|
|
14
|
+
}
|
|
15
|
+
if (left.length !== right.length) throw new RangeError("vector dimensions do not match");
|
|
16
|
+
let leftScale = 0, rightScale = 0;
|
|
17
|
+
for (let i = 0; i < left.length; i++) {
|
|
18
|
+
const a = left[i], b = right[i];
|
|
19
|
+
if (!Number.isFinite(a) || !Number.isFinite(b)) throw new TypeError("vector values must be finite numbers");
|
|
20
|
+
leftScale = Math.max(leftScale, Math.abs(a));
|
|
21
|
+
rightScale = Math.max(rightScale, Math.abs(b));
|
|
22
|
+
}
|
|
23
|
+
if (leftScale === 0 || rightScale === 0) throw new RangeError("zero vectors are not supported");
|
|
24
|
+
let dot = 0, leftNorm = 0, rightNorm = 0;
|
|
25
|
+
for (let i = 0; i < left.length; i++) {
|
|
26
|
+
const a = left[i] / leftScale, b = right[i] / rightScale;
|
|
27
|
+
dot += a * b;
|
|
28
|
+
leftNorm += a * a;
|
|
29
|
+
rightNorm += b * b;
|
|
30
|
+
}
|
|
31
|
+
return dot / Math.sqrt(leftNorm * rightNorm);
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export interface RankedCandidate { id: string; rank: number }
|
|
35
|
+
export interface HybridRankedCandidate { id: string; score: number; lexicalRank?: number; vectorRank?: number }
|
|
36
|
+
|
|
37
|
+
/** Reciprocal-rank fusion with lexical then id tie-breaking for stable output. */
|
|
38
|
+
export function rankHybridCandidates(
|
|
39
|
+
lexical: readonly RankedCandidate[],
|
|
40
|
+
vector: readonly RankedCandidate[],
|
|
41
|
+
): HybridRankedCandidate[] {
|
|
42
|
+
if (!Array.isArray(lexical) || !Array.isArray(vector)) throw new TypeError("candidate lists must be arrays");
|
|
43
|
+
if (lexical.length > MAX_RANK_CANDIDATES || vector.length > MAX_RANK_CANDIDATES || lexical.length + vector.length > MAX_RANK_CANDIDATES) {
|
|
44
|
+
throw new RangeError(`candidate count must not exceed ${MAX_RANK_CANDIDATES}`);
|
|
45
|
+
}
|
|
46
|
+
const merged = new Map<string, { lexicalRank?: number; vectorRank?: number }>();
|
|
47
|
+
const add = (items: readonly RankedCandidate[], field: "lexicalRank" | "vectorRank") => {
|
|
48
|
+
for (const item of items) {
|
|
49
|
+
if (!item || typeof item.id !== "string" || item.id.length === 0 || !Number.isSafeInteger(item.rank) || item.rank < 1 || item.rank > MAX_RANK_CANDIDATES) {
|
|
50
|
+
throw new TypeError(`each candidate needs a nonempty id and rank in 1..${MAX_RANK_CANDIDATES}`);
|
|
51
|
+
}
|
|
52
|
+
let entry = merged.get(item.id);
|
|
53
|
+
if (!entry) {
|
|
54
|
+
entry = {};
|
|
55
|
+
merged.set(item.id, entry);
|
|
56
|
+
}
|
|
57
|
+
if (entry[field] !== undefined) throw new TypeError(`duplicate candidate id in ${field === "lexicalRank" ? "lexical" : "vector"} results`);
|
|
58
|
+
entry[field] = item.rank;
|
|
59
|
+
}
|
|
60
|
+
};
|
|
61
|
+
add(lexical, "lexicalRank");
|
|
62
|
+
add(vector, "vectorRank");
|
|
63
|
+
return Array.from(merged, ([id, ranks]) => ({
|
|
64
|
+
id,
|
|
65
|
+
score: (ranks.lexicalRank === undefined ? 0 : 1 / (RRF_K + ranks.lexicalRank)) +
|
|
66
|
+
(ranks.vectorRank === undefined ? 0 : 1 / (RRF_K + ranks.vectorRank)),
|
|
67
|
+
...ranks,
|
|
68
|
+
})).sort((a, b) => b.score - a.score || (a.lexicalRank ?? Infinity) - (b.lexicalRank ?? Infinity) || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
|
|
69
|
+
}
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
/** Bounded, vault-bound local enrichment storage. No model is called here. */
|
|
2
|
+
import { closeSync, constants, fstatSync, fsyncSync, lstatSync, mkdirSync, openSync, readSync, realpathSync, writeFileSync } from "node:fs";
|
|
3
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
4
|
+
import { hostname } from "node:os";
|
|
5
|
+
import { isAbsolute, join, relative, resolve } from "node:path";
|
|
6
|
+
import { isDenied } from "./platform.ts";
|
|
7
|
+
import { loadRegistry, LOCK_FILE, syncDirectory } from "./registry.ts";
|
|
8
|
+
import { MUTATION_LOCK } from "./writer.ts";
|
|
9
|
+
import { logBinding, matchesBinding } from "./vault-log.ts";
|
|
10
|
+
import { capturePublicationPath, cleanupOwnedFile, publishFile } from "./write-files.ts";
|
|
11
|
+
import type { VaultSpec } from "./config.ts";
|
|
12
|
+
|
|
13
|
+
export const MAX_ENRICHMENT_STORE_BYTES = 128 * 1024 * 1024;
|
|
14
|
+
export const MAX_ENRICHMENT_RECORDS = 50000;
|
|
15
|
+
export const MAX_EMBEDDING_DIMENSION = 4096;
|
|
16
|
+
export type EnrichmentKind = "embedding" | "summary" | "community-label" | "inferred-link" | "fact";
|
|
17
|
+
export interface EnrichmentProducer { skill: string; version: string; model: string }
|
|
18
|
+
export interface EnrichmentRecord { path: string; contentHash: string; kind: EnrichmentKind; producer: EnrichmentProducer; createdAt: string; value: unknown }
|
|
19
|
+
export interface EnrichmentStore { schemaVersion: 1; binding: string; model?: string; dimension?: number; records: EnrichmentRecord[] }
|
|
20
|
+
function fail(message: string): never { throw new Error(`enrichment: ${message}`); }
|
|
21
|
+
const hash = (value: string) => createHash("sha256").update(value).digest("hex");
|
|
22
|
+
const within = (root: string, target: string) => { const r = relative(root, target); return r === "" || (!isAbsolute(r) && r !== ".." && !r.startsWith(`..${process.platform === "win32" ? "\\" : "/"}`)); };
|
|
23
|
+
const digest = /^[a-f0-9]{64}$/;
|
|
24
|
+
const kinds = new Set(["embedding", "summary", "community-label", "inferred-link", "fact"]);
|
|
25
|
+
const object = (v: unknown): v is Record<string, unknown> => !!v && typeof v === "object" && !Array.isArray(v) && Object.getPrototypeOf(v) === Object.prototype;
|
|
26
|
+
const text = (v: unknown, maximum: number) => typeof v === "string" && !!v.trim() && Buffer.byteLength(v) <= maximum && !/[\p{Cc}\p{Zl}\p{Zp}]/u.test(v);
|
|
27
|
+
function validVector(value: unknown, dimension: number): value is number[] {
|
|
28
|
+
return Array.isArray(value) && value.length === dimension && value.every(n => typeof n === "number" && Number.isFinite(n)) && value.some(n => n !== 0);
|
|
29
|
+
}
|
|
30
|
+
/** Float32 encoding keeps a 1536-dimensional vector to exactly 8192 base64 bytes. */
|
|
31
|
+
function encodeVector(value: unknown, dimension: number): { encoding: "float32le"; data: string } {
|
|
32
|
+
if (!validVector(value, dimension)) fail("embedding vector is invalid; supply finite nonzero values within the dimension limit");
|
|
33
|
+
const bytes = Buffer.alloc(dimension * 4); let nonzero = false;
|
|
34
|
+
for (let i = 0; i < dimension; i++) {
|
|
35
|
+
const rounded = Math.fround(value[i]!);
|
|
36
|
+
if (!Number.isFinite(rounded)) fail("embedding value overflows Float32; use finite Float32-range values");
|
|
37
|
+
if (rounded !== 0) nonzero = true;
|
|
38
|
+
bytes.writeFloatLE(rounded, i * 4);
|
|
39
|
+
}
|
|
40
|
+
if (!nonzero) fail("embedding becomes zero in Float32; provide a nonzero representable vector");
|
|
41
|
+
return { encoding: "float32le", data: bytes.toString("base64") };
|
|
42
|
+
}
|
|
43
|
+
function decodeVector(value: unknown, dimension: number): number[] {
|
|
44
|
+
if (Array.isArray(value)) { encodeVector(value, dimension); return value; } // unpublished v1 data
|
|
45
|
+
if (!object(value) || Object.keys(value).sort().join(",") !== "data,encoding" || value.encoding !== "float32le" || typeof value.data !== "string" || value.data.length !== Math.ceil(dimension * 4 / 3) * 4) fail("sidecar vector encoding is invalid; inspect the local enrichment store");
|
|
46
|
+
const bytes = Buffer.from(value.data, "base64");
|
|
47
|
+
if (bytes.length !== dimension * 4 || bytes.toString("base64") !== value.data) fail("sidecar vector encoding is invalid; inspect the local enrichment store");
|
|
48
|
+
const vector = Array.from({ length: dimension }, (_, i) => bytes.readFloatLE(i * 4));
|
|
49
|
+
if (!validVector(vector, dimension)) fail("sidecar vector is invalid; inspect the local enrichment store");
|
|
50
|
+
return vector;
|
|
51
|
+
}
|
|
52
|
+
function parseStore(value: unknown, binding: string): EnrichmentStore {
|
|
53
|
+
if (!object(value) || Object.keys(value).some(k => !["schemaVersion", "binding", "model", "dimension", "records"].includes(k)) || value.schemaVersion !== 1 || value.binding !== binding || !Array.isArray(value.records) || value.records.length > MAX_ENRICHMENT_RECORDS) fail("sidecar metadata is invalid or bound to another vault; inspect the local enrichment store");
|
|
54
|
+
if ((value.model === undefined) !== (value.dimension === undefined) || (value.model !== undefined && (!text(value.model, 256) || !Number.isSafeInteger(value.dimension) || (value.dimension as number) < 1 || (value.dimension as number) > MAX_EMBEDDING_DIMENSION))) fail("sidecar model metadata is invalid; inspect the local enrichment store");
|
|
55
|
+
const seen = new Set<string>();
|
|
56
|
+
for (const item of value.records) {
|
|
57
|
+
if (!object(item) || Object.keys(item).some(k => !["path", "contentHash", "kind", "producer", "createdAt", "value"].includes(k)) || typeof item.path !== "string" || !/^notes\/(?!\.)[^/\\\p{Cc}]+\.md$/u.test(item.path) || Buffer.byteLength(item.path) > 1024 || typeof item.contentHash !== "string" || !digest.test(item.contentHash) || typeof item.kind !== "string" || !kinds.has(item.kind) || !object(item.producer) || Object.keys(item.producer).sort().join(",") !== "model,skill,version" || !text(item.producer.skill, 128) || !text(item.producer.version, 128) || !text(item.producer.model, 256) || typeof item.createdAt !== "string" || Number.isNaN(Date.parse(item.createdAt)) || new Date(item.createdAt).toISOString() !== item.createdAt || !("value" in item)) fail("sidecar record is invalid; inspect the local enrichment store");
|
|
58
|
+
const key = JSON.stringify([item.path, item.kind, item.contentHash]);
|
|
59
|
+
if (seen.has(key)) fail("sidecar contains duplicate records; inspect the local enrichment store");
|
|
60
|
+
seen.add(key);
|
|
61
|
+
if (item.kind === "embedding") {
|
|
62
|
+
if (item.producer.model !== value.model || typeof value.dimension !== "number") fail("sidecar vector has a different model or dimension; inspect the local enrichment store");
|
|
63
|
+
item.value = decodeVector(item.value, value.dimension);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
return value as unknown as EnrichmentStore;
|
|
67
|
+
}
|
|
68
|
+
function location(home: string, vault: VaultSpec, roots: VaultSpec[], denyPaths: string[], create: boolean) {
|
|
69
|
+
const configured = resolve(home);
|
|
70
|
+
if (isDenied(configured, denyPaths)) fail("home is denied; choose an allowed VMEM_HOME");
|
|
71
|
+
let root: string;
|
|
72
|
+
try { root = realpathSync(configured); } catch { return fail("home is unavailable; restore the configured VMEM_HOME and retry"); }
|
|
73
|
+
let protectedRoots: string[];
|
|
74
|
+
try { protectedRoots = roots.map(v => realpathSync(v.path)); } catch { return fail("a registered vault root is unavailable; restore it or unregister it before using enrichment"); }
|
|
75
|
+
if (isDenied(root, denyPaths) || protectedRoots.some(path => within(path, root))) fail("home is denied or inside a vault; choose an allowed VMEM_HOME outside all vaults");
|
|
76
|
+
let stat: ReturnType<typeof lstatSync>;
|
|
77
|
+
try { stat = lstatSync(vault.path); } catch { return fail("vault root is unavailable; inspect the registry and restore the vault"); }
|
|
78
|
+
if (!stat.isDirectory() || stat.isSymbolicLink() || realpathSync(vault.path) !== vault.path || isDenied(vault.path, denyPaths)) fail("vault root is linked, changed or denied; inspect the registry and policy");
|
|
79
|
+
const binding = hash(JSON.stringify([vault.id, vault.path, stat.dev, stat.ino]));
|
|
80
|
+
const dir = join(root, "enrichment"), file = join(dir, `${binding}.json`);
|
|
81
|
+
if (isDenied(dir, denyPaths) || isDenied(file, denyPaths)) fail("sidecar is denied; inspect configured deny paths");
|
|
82
|
+
if (create) {
|
|
83
|
+
try { mkdirSync(dir, { mode: 0o700 }); syncDirectory(root); }
|
|
84
|
+
catch (error) { if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; }
|
|
85
|
+
}
|
|
86
|
+
try {
|
|
87
|
+
const d = lstatSync(dir);
|
|
88
|
+
if (!d.isDirectory() || d.isSymbolicLink() || (d.mode & 0o077) !== 0 || (typeof process.getuid === "function" && d.uid !== process.getuid())) fail("sidecar directory is linked or not private; inspect VMEM_HOME/enrichment");
|
|
89
|
+
} catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT" || create) throw error; }
|
|
90
|
+
return { root, dir, file, binding };
|
|
91
|
+
}
|
|
92
|
+
function readFile(loc: ReturnType<typeof location>): EnrichmentStore {
|
|
93
|
+
const empty: EnrichmentStore = { schemaVersion: 1, binding: loc.binding, records: [] };
|
|
94
|
+
let fd: number;
|
|
95
|
+
try { fd = openSync(loc.file, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); }
|
|
96
|
+
catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return empty; throw new Error("enrichment: sidecar cannot be read; inspect VMEM_HOME/enrichment", { cause: error }); }
|
|
97
|
+
try {
|
|
98
|
+
const captured = capturePublicationPath(loc.root, loc.file);
|
|
99
|
+
const before = fstatSync(fd);
|
|
100
|
+
if (!before.isFile() || before.nlink !== 1 || (before.mode & 0o077) !== 0 || (typeof process.getuid === "function" && before.uid !== process.getuid()) || before.size > MAX_ENRICHMENT_STORE_BYTES) fail("sidecar is linked, not private or exceeds its byte limit; inspect the local enrichment store");
|
|
101
|
+
const bytes = Buffer.alloc(before.size + 1); let count = 0;
|
|
102
|
+
while (count < bytes.length) { const n = readSync(fd, bytes, count, bytes.length - count, null); if (!n) break; count += n; }
|
|
103
|
+
const after = fstatSync(fd), named = lstatSync(loc.file);
|
|
104
|
+
if (count !== before.size || after.size !== before.size || before.ino !== after.ino || before.dev !== after.dev || after.ino !== named.ino || after.dev !== named.dev || before.mtimeMs !== after.mtimeMs || before.ctimeMs !== after.ctimeMs || JSON.stringify(captured.parents) !== JSON.stringify(capturePublicationPath(loc.root, loc.file).parents)) fail("sidecar changed during read; stop concurrent edits and retry");
|
|
105
|
+
const raw = bytes.subarray(0, count), content = raw.toString("utf8");
|
|
106
|
+
if (!Buffer.from(content).equals(raw)) fail("sidecar is not UTF-8; inspect the local enrichment store");
|
|
107
|
+
let parsed: unknown; try { parsed = JSON.parse(content); } catch { return fail("sidecar is not valid JSON; inspect the local enrichment store"); }
|
|
108
|
+
return parseStore(parsed, loc.binding);
|
|
109
|
+
} finally { closeSync(fd); }
|
|
110
|
+
}
|
|
111
|
+
export function readEnrichmentStore(home: string, vault: VaultSpec, roots: VaultSpec[], denyPaths: string[]): EnrichmentStore {
|
|
112
|
+
return readFile(location(home, vault, roots, denyPaths, false));
|
|
113
|
+
}
|
|
114
|
+
/** Validate complete metadata and vector shape before any side effects. */
|
|
115
|
+
export function validateEnrichmentRecord(record: EnrichmentRecord): void {
|
|
116
|
+
if (!record || typeof record !== "object") fail("intake record is missing; supply a complete result");
|
|
117
|
+
const provisional: EnrichmentStore = { schemaVersion: 1, binding: "0".repeat(64), records: [record] };
|
|
118
|
+
if (record.kind === "embedding") {
|
|
119
|
+
if (!object(record.producer) || !Array.isArray(record.value) || !validVector(record.value, record.value.length) || record.value.length < 1 || record.value.length > MAX_EMBEDDING_DIMENSION) fail("embedding is invalid; supply a producer and a finite nonzero vector");
|
|
120
|
+
encodeVector(record.value, record.value.length);
|
|
121
|
+
provisional.model = record.producer.model; provisional.dimension = record.value.length;
|
|
122
|
+
}
|
|
123
|
+
parseStore(provisional, provisional.binding);
|
|
124
|
+
}
|
|
125
|
+
/** Serialize under the registry lock; callers must recheck the note hash before returning their receipt. */
|
|
126
|
+
export function writeEnrichmentRecord(home: string, vault: VaultSpec, record: EnrichmentRecord, validateCurrent: () => void, validateStore: (store: EnrichmentStore) => void = () => {}, heldMutation?: number): { applied: true; issues: string[] } {
|
|
127
|
+
validateEnrichmentRecord(record);
|
|
128
|
+
const config = loadRegistry(home), currentVault = config.vaults.find(v => v.id === vault.id);
|
|
129
|
+
if (!currentVault || JSON.stringify(currentVault) !== JSON.stringify(vault)) fail("vault registration changed; reload settings and retry");
|
|
130
|
+
if (!vault.enrichment?.enabled) fail("enrichment is disabled for this vault; enable it in settings first");
|
|
131
|
+
location(home, vault, config.vaults, config.denyPaths ?? [], false);
|
|
132
|
+
const binding = logBinding(vault, config), lock = join(resolve(home), LOCK_FILE);
|
|
133
|
+
if (isDenied(lock, config.denyPaths ?? [])) fail("registry lock is denied; inspect configured deny paths");
|
|
134
|
+
const mutation = join(vault.path, MUTATION_LOCK);
|
|
135
|
+
if (isDenied(mutation, config.denyPaths ?? [])) fail("vault mutation lock is denied; inspect configured deny paths");
|
|
136
|
+
let mutationFd: number;
|
|
137
|
+
if (heldMutation !== undefined) mutationFd = heldMutation;
|
|
138
|
+
else {
|
|
139
|
+
try { mutationFd = openSync(mutation, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600); }
|
|
140
|
+
catch (error) { throw new Error("enrichment: vault mutation lock is unavailable; stop concurrent writers and inspect the lock", { cause: error }); }
|
|
141
|
+
}
|
|
142
|
+
const verifyMutation = () => {
|
|
143
|
+
const held = fstatSync(mutationFd), named = lstatSync(mutation);
|
|
144
|
+
if (!held.isFile() || named.isSymbolicLink() || held.dev !== named.dev || held.ino !== named.ino) fail("vault mutation lock ownership changed; inspect the lock before retrying");
|
|
145
|
+
};
|
|
146
|
+
let fd: number | undefined, published = false, failure: unknown;
|
|
147
|
+
const issues: string[] = [];
|
|
148
|
+
try {
|
|
149
|
+
if (heldMutation === undefined) { writeFileSync(mutationFd, JSON.stringify({ pid: process.pid, host: hostname(), created: new Date().toISOString() }) + "\n"); fsyncSync(mutationFd); }
|
|
150
|
+
verifyMutation();
|
|
151
|
+
try { fd = openSync(lock, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600); }
|
|
152
|
+
catch (error) { throw new Error("enrichment: registry lock is unavailable; stop concurrent writers and inspect the lock", { cause: error }); }
|
|
153
|
+
writeFileSync(fd, JSON.stringify({ pid: process.pid, host: hostname(), created: new Date().toISOString() }) + "\n"); fsyncSync(fd);
|
|
154
|
+
const current = loadRegistry(home);
|
|
155
|
+
if (!matchesBinding(current, vault.id, binding)) fail("vault binding or policy changed; reload settings and retry");
|
|
156
|
+
validateCurrent();
|
|
157
|
+
const loc = location(home, vault, current.vaults, current.denyPaths ?? [], true), store = readFile(loc);
|
|
158
|
+
validateStore(store);
|
|
159
|
+
if (record.kind === "embedding") {
|
|
160
|
+
if (!Array.isArray(record.value) || record.value.length < 1 || record.value.length > MAX_EMBEDDING_DIMENSION || !validVector(record.value, record.value.length)) fail("embedding vector is invalid; supply finite nonzero values within the dimension limit");
|
|
161
|
+
if (store.model !== undefined && (store.model !== record.producer.model || store.dimension !== record.value.length)) fail("embedding model or dimension differs from this vault's sidecar; use the recorded model and dimension");
|
|
162
|
+
store.model = record.producer.model; store.dimension = record.value.length;
|
|
163
|
+
}
|
|
164
|
+
const replaceLatest = ["embedding", "summary", "community-label"].includes(record.kind);
|
|
165
|
+
store.records = store.records.filter(r => r.path !== record.path || r.kind !== record.kind || (!replaceLatest && r.contentHash !== record.contentHash));
|
|
166
|
+
store.records.push(record);
|
|
167
|
+
parseStore(store, loc.binding);
|
|
168
|
+
const persisted = { ...store, records: store.records.map(r => r.kind === "embedding" ? { ...r, value: encodeVector(r.value, store.dimension!) } : r) };
|
|
169
|
+
const bytes = Buffer.from(JSON.stringify(persisted));
|
|
170
|
+
if (bytes.length > MAX_ENRICHMENT_STORE_BYTES) fail("sidecar exceeds the aggregate byte limit; reduce stored records before retrying");
|
|
171
|
+
validateCurrent(); verifyMutation();
|
|
172
|
+
const captured = capturePublicationPath(loc.root, loc.file), temp = join(loc.dir, `.enrichment-${randomUUID()}.tmp`);
|
|
173
|
+
if (isDenied(temp, current.denyPaths ?? [])) fail("temporary sidecar is denied; inspect configured deny paths");
|
|
174
|
+
const out = openSync(temp, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600);
|
|
175
|
+
try { writeFileSync(out, bytes); fsyncSync(out); publishFile(captured, { kind: "rename", temp }, () => { published = true; }); syncDirectory(loc.dir); }
|
|
176
|
+
finally { try { cleanupOwnedFile(temp, out); } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; } finally { closeSync(out); } }
|
|
177
|
+
try { validateCurrent(); } catch { issues.push("source changed after the sidecar was published; inspect the note and regenerate the result"); }
|
|
178
|
+
if (!matchesBinding(loadRegistry(home), vault.id, binding)) issues.push("vault binding or policy changed after publication; inspect the registry and sidecar before retrying");
|
|
179
|
+
} catch (error) { failure = error; }
|
|
180
|
+
finally {
|
|
181
|
+
if (fd !== undefined) {
|
|
182
|
+
try { cleanupOwnedFile(lock, fd); } catch { issues.push("registry lock cleanup failed; inspect the lock before retrying"); }
|
|
183
|
+
try { closeSync(fd); } catch { issues.push("registry lock close failed; inspect the lock before retrying"); }
|
|
184
|
+
}
|
|
185
|
+
if (heldMutation === undefined) {
|
|
186
|
+
try { cleanupOwnedFile(mutation, mutationFd); } catch { issues.push("vault mutation lock cleanup failed; inspect the lock before retrying"); }
|
|
187
|
+
try { closeSync(mutationFd); } catch { issues.push("vault mutation lock close failed; inspect the lock before retrying"); }
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
if (!published && failure) {
|
|
191
|
+
if (issues.length) throw new Error(`enrichment: sidecar was not published; ${issues.join("; ")}`, { cause: failure });
|
|
192
|
+
throw failure;
|
|
193
|
+
}
|
|
194
|
+
if (!published) fail("sidecar was not published; inspect the local store before retrying");
|
|
195
|
+
if (failure) issues.push("sidecar was published but finalization failed; inspect the local store before retrying");
|
|
196
|
+
return { applied: true, issues };
|
|
197
|
+
}
|
|
@@ -0,0 +1,245 @@
|
|
|
1
|
+
/** Explicit, offline enrichment work and intake. */
|
|
2
|
+
import { closeSync, constants, fstatSync, fsyncSync, lstatSync, openSync, readSync, realpathSync, writeFileSync } from "node:fs";
|
|
3
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
4
|
+
import { join } from "node:path";
|
|
5
|
+
import { hostname } from "node:os";
|
|
6
|
+
import { cleanupOwnedFile } from "./write-files.ts";
|
|
7
|
+
import { applyWrite, MUTATION_LOCK } from "./writer.ts";
|
|
8
|
+
import { loadRegistry } from "./registry.ts";
|
|
9
|
+
import { logBinding, matchesBinding } from "./vault-log.ts";
|
|
10
|
+
import { isDenied } from "./platform.ts";
|
|
11
|
+
import { notePath } from "./write-operations.ts";
|
|
12
|
+
import { screenMetadata } from "./write-validation.ts";
|
|
13
|
+
import { VaultIndex, MAX_NOTE_BYTES } from "./vault-index.ts";
|
|
14
|
+
import { readEnrichmentStore, writeEnrichmentRecord, validateEnrichmentRecord, type EnrichmentKind, type EnrichmentProducer, type EnrichmentRecord } from "./enrichment-store.ts";
|
|
15
|
+
import type { VaultSpec } from "./config.ts";
|
|
16
|
+
import { appendAssertions, readJournal } from "./assertions.ts";
|
|
17
|
+
import { submitProposal, showProposal } from "./proposal-provider.ts";
|
|
18
|
+
|
|
19
|
+
const sha256 = (value: string | Uint8Array) => createHash("sha256").update(value).digest("hex");
|
|
20
|
+
const kinds: readonly EnrichmentKind[] = ["embedding", "summary", "community-label", "inferred-link", "fact"];
|
|
21
|
+
function fail(message: string): never { throw new Error(`enrichment: ${message}`); }
|
|
22
|
+
function context(home: string, id: string, channel?: "cli" | "mcp") {
|
|
23
|
+
const config = loadRegistry(home), vault = config.vaults.find(v => v.id === id);
|
|
24
|
+
if (!vault) return fail("vault is not registered; choose an id from vmem vaults list");
|
|
25
|
+
if (!vault.enrichment?.enabled) return fail("enrichment is disabled; enable it for this vault in settings");
|
|
26
|
+
if (channel && (vault.write === "none" || (channel === "mcp" && vault.write !== "mcp"))) return fail("vault write policy refuses this channel; change the policy in settings or use an allowed channel");
|
|
27
|
+
let binding: ReturnType<typeof logBinding>;
|
|
28
|
+
try { binding = logBinding(vault, config); } catch { return fail("vault root is unavailable; inspect the registered vault and retry"); }
|
|
29
|
+
const check = () => {
|
|
30
|
+
let current: ReturnType<typeof loadRegistry>;
|
|
31
|
+
try { current = loadRegistry(home); } catch { return fail("registry is unavailable after the read; inspect the configuration and retry"); }
|
|
32
|
+
if (!matchesBinding(current, vault.id, binding)) fail("vault binding or policy changed during the read; retry with the current registry");
|
|
33
|
+
};
|
|
34
|
+
return { config, vault, check };
|
|
35
|
+
}
|
|
36
|
+
function sourcePath(path: string): string {
|
|
37
|
+
if (typeof path !== "string") return fail("source path is missing; choose a note from vmem list");
|
|
38
|
+
const result = path.startsWith("notes/") ? path : `notes/${path.endsWith(".md") ? path : `${path}.md`}`;
|
|
39
|
+
if (!/^notes\/(?!\.)[^/\\\p{Cc}]+\.md$/u.test(result) || Buffer.byteLength(result) > 1024) return fail("source path is not a flat note path; choose a note from vmem list");
|
|
40
|
+
return result;
|
|
41
|
+
}
|
|
42
|
+
function noteBytes(vault: VaultSpec, path: string, denied: string[]): Buffer {
|
|
43
|
+
const normalized = sourcePath(path);
|
|
44
|
+
let root: string;
|
|
45
|
+
try { root = realpathSync(vault.path); } catch { return fail("vault root is unavailable; restore the registered vault and retry"); }
|
|
46
|
+
const notes = join(root, "notes"), file = join(root, normalized);
|
|
47
|
+
let plainNotes = false; try { plainNotes = realpathSync(notes) === notes; } catch { /* Missing notes are refused below. */ }
|
|
48
|
+
if (root !== vault.path || !plainNotes || isDenied(file, denied)) fail("note path is denied or vault identity changed; inspect the registered vault and policy");
|
|
49
|
+
let fd: number;
|
|
50
|
+
try { fd = openSync(file, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); }
|
|
51
|
+
catch { return fail("source note is unavailable or linked; inspect its path and retry"); }
|
|
52
|
+
try {
|
|
53
|
+
const before = fstatSync(fd);
|
|
54
|
+
if (!before.isFile() || before.size > MAX_NOTE_BYTES) fail("source note is not a bounded regular file; inspect or split the note");
|
|
55
|
+
const bytes = Buffer.alloc(before.size + 1); let length = 0;
|
|
56
|
+
while (length < bytes.length) { const n = readSync(fd, bytes, length, bytes.length - length, null); if (!n) break; length += n; }
|
|
57
|
+
const after = fstatSync(fd);
|
|
58
|
+
let named: ReturnType<typeof lstatSync>;
|
|
59
|
+
try { named = lstatSync(file); } catch { return fail("source note changed during read; reread it and retry"); }
|
|
60
|
+
if (named.isSymbolicLink() || named.dev !== after.dev || named.ino !== after.ino || length !== before.size || before.size !== after.size || before.ino !== after.ino || before.dev !== after.dev || before.mtimeMs !== after.mtimeMs || before.ctimeMs !== after.ctimeMs) fail("source note changed while reading; reread it and retry");
|
|
61
|
+
return bytes.subarray(0, length);
|
|
62
|
+
} finally { closeSync(fd); }
|
|
63
|
+
}
|
|
64
|
+
function withMutationLock<T>(vault: VaultSpec, denied: string[], work: (fd: number) => T): T {
|
|
65
|
+
const file = join(vault.path, MUTATION_LOCK);
|
|
66
|
+
if (isDenied(file, denied)) fail("vault mutation lock is denied; inspect configured deny paths");
|
|
67
|
+
let fd: number;
|
|
68
|
+
try { fd = openSync(file, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600); }
|
|
69
|
+
catch { return fail("vault mutation lock is unavailable; stop concurrent writers and inspect the lock"); }
|
|
70
|
+
try { writeFileSync(fd, JSON.stringify({ pid: process.pid, host: hostname(), created: new Date().toISOString() }) + "\n"); fsyncSync(fd); return work(fd); }
|
|
71
|
+
finally { try { cleanupOwnedFile(file, fd); } finally { closeSync(fd); } }
|
|
72
|
+
}
|
|
73
|
+
export interface EnrichmentWorkRequest { vault: string; kind: EnrichmentKind; limit?: number; after?: string }
|
|
74
|
+
export function listEnrichmentWork(home: string, request: EnrichmentWorkRequest) {
|
|
75
|
+
if (!request || typeof request !== "object" || Object.keys(request).some(k => !["vault", "kind", "limit", "after"].includes(k))) fail("invalid work request; supply a vault and enrichment kind");
|
|
76
|
+
if (request.after !== undefined && (typeof request.after !== "string" || !/^notes\/(?!\.)[^/\\\p{Cc}]+\.md$/u.test(request.after) || Buffer.byteLength(request.after) > 1024)) fail("work cursor is invalid; restart without a cursor");
|
|
77
|
+
if (!kinds.includes(request.kind)) fail("unknown work kind; choose embedding, summary, community-label, inferred-link or fact");
|
|
78
|
+
const limit = request.limit ?? 50;
|
|
79
|
+
if (!Number.isInteger(limit) || limit < 1 || limit > 100) fail("work limit is outside 1..100; choose a smaller page");
|
|
80
|
+
const { config, vault, check } = context(home, request.vault), store = readEnrichmentStore(home, vault, config.vaults, config.denyPaths ?? []);
|
|
81
|
+
const index = new VaultIndex(home, vault, config.vaults.map(v => v.path), { memory: true });
|
|
82
|
+
try {
|
|
83
|
+
index.scan(false, { captureInvalidFrontmatter: true, denyPaths: config.denyPaths });
|
|
84
|
+
const snapshot = index.analysisSnapshot({ today: "", logPath: "", log: null, registryIds: null, reports: {} });
|
|
85
|
+
const records = new Map<string, EnrichmentRecord[]>();
|
|
86
|
+
for (const record of store.records) if (record.kind === request.kind) {
|
|
87
|
+
const group = records.get(record.path) ?? []; group.push(record); records.set(record.path, group);
|
|
88
|
+
}
|
|
89
|
+
const pending = snapshot.notes.map(n => {
|
|
90
|
+
const path = `notes/${n.path.replace(/^notes\//, "")}`, contentHash = sha256(n.source), relevant = records.get(path) ?? [];
|
|
91
|
+
const uncertain = relevant.some(r => (r.value as { state?: unknown } | undefined)?.state === "pending");
|
|
92
|
+
const current = relevant.some(r => r.contentHash === contentHash && (r.value as { state?: unknown } | undefined)?.state !== "released");
|
|
93
|
+
return { path, contentHash, current, ...(uncertain ? { status: "needs-reconciliation" as const } : {}) };
|
|
94
|
+
}).filter(n => (n.status || !n.current) && (!request.after || n.path > request.after)).sort((a, b) => a.path < b.path ? -1 : a.path > b.path ? 1 : 0).map(({ current: _, ...item }) => item);
|
|
95
|
+
const items = pending.slice(0, limit);
|
|
96
|
+
check();
|
|
97
|
+
return { vault: vault.id, kind: request.kind, items, ...(pending.length > limit ? { next: items.at(-1)!.path } : {}) };
|
|
98
|
+
} finally { index.close(); }
|
|
99
|
+
}
|
|
100
|
+
export interface EnrichmentIntakeRequest { vault: string; path: string; contentHash: string; kind: EnrichmentKind; producer: EnrichmentProducer; createdAt: string; value: unknown; channel: "cli" | "mcp" }
|
|
101
|
+
export function intakeEnrichment(home: string, request: EnrichmentIntakeRequest, options: { env?: NodeJS.ProcessEnv; expectedVault?: VaultSpec } = {}) {
|
|
102
|
+
if (!request || typeof request !== "object" || Object.keys(request).some(k => !["vault", "path", "contentHash", "kind", "producer", "createdAt", "value", "channel"].includes(k)) || !kinds.includes(request.kind) || (request.channel !== "cli" && request.channel !== "mcp") || typeof request.contentHash !== "string" || !/^[a-f0-9]{64}$/.test(request.contentHash)) fail("invalid intake request; supply a vault, source path, SHA-256, kind, producer, creation time and value");
|
|
103
|
+
const { config, vault, check } = context(home, request.vault, request.channel), path = sourcePath(request.path);
|
|
104
|
+
if (options.expectedVault && JSON.stringify(options.expectedVault) !== JSON.stringify(vault)) fail("vault binding changed since MCP startup; restart the MCP server");
|
|
105
|
+
const candidate: EnrichmentRecord = { path, contentHash: request.contentHash, kind: request.kind, producer: request.producer, createdAt: request.createdAt, value: request.value };
|
|
106
|
+
validateEnrichmentRecord(candidate);
|
|
107
|
+
const issues = screenMetadata({ producer: request.producer, ...(request.kind === "embedding" ? {} : { value: request.value }) }, { vault, config }, path);
|
|
108
|
+
if (issues.some(issue => issue.severity === "error")) fail("intake text is refused by the secret or PII policy; remove prohibited content or correct the vault policy");
|
|
109
|
+
if (sha256(noteBytes(vault, path, config.denyPaths ?? [])) !== request.contentHash) fail("source content hash is stale; reread the note and regenerate the result");
|
|
110
|
+
if (request.kind === "fact" || request.kind === "inferred-link") {
|
|
111
|
+
const bound = { vault: vault.id, path, contentHash: request.contentHash, producer: request.producer, createdAt: request.createdAt };
|
|
112
|
+
let operation: unknown;
|
|
113
|
+
const origin = { path, sha256: request.contentHash, producer: request.producer, createdAt: request.createdAt };
|
|
114
|
+
const checkSource = () => { if (sha256(noteBytes(vault, path, config.denyPaths ?? [])) !== request.contentHash) fail("source content hash is stale; reread the note and regenerate the result"); };
|
|
115
|
+
if (request.kind === "fact") {
|
|
116
|
+
if (!Array.isArray(request.value) || !request.value.some(op => op && typeof op === "object" && ["assert", "revise"].includes(String((op as { op?: unknown }).op)))) fail("fact intake requires assertion operations; include an assert or revise operation with evidence");
|
|
117
|
+
const dry = appendAssertions(home, { vault: vault.id, ops: request.value, channel: request.channel, dryRun: true }, { env: options.env, origin });
|
|
118
|
+
if (!dry.ok) { check(); return { ...bound, ok: false, applied: false, kind: request.kind, receipt: dry }; }
|
|
119
|
+
} else {
|
|
120
|
+
try { notePath(path); } catch { fail("proposal targets require a lowercase ASCII note name; rename the note before proposing a link"); }
|
|
121
|
+
const target = request.value && typeof request.value === "object" && !Array.isArray(request.value) && Object.keys(request.value).join(",") === "target" ? (request.value as { target?: unknown }).target : undefined;
|
|
122
|
+
if (typeof target !== "string" || !/^[a-z0-9][a-z0-9-]{0,190}$/.test(target) || `notes/${target}.md` === path) fail("inferred link target is invalid; provide a different flat note stem");
|
|
123
|
+
operation = { op: "infer-link", path, target, expectedSha: request.contentHash };
|
|
124
|
+
const dry = applyWrite(home, { vault: vault.id, operation: operation as import("./write-types.ts").WriteOp, channel: request.channel, dryRun: true }, { env: options.env });
|
|
125
|
+
if (!dry.ok) { check(); return { ...bound, ok: false, applied: false, kind: request.kind, receipt: dry }; }
|
|
126
|
+
}
|
|
127
|
+
// Reserve a durable intent before calling a second store. A crash or lost response can never auto-replay it.
|
|
128
|
+
const token = randomUUID(), pending: EnrichmentRecord = { path, contentHash: request.contentHash, kind: request.kind, producer: request.producer, createdAt: request.createdAt, value: { state: "pending", token } };
|
|
129
|
+
const reserved = writeEnrichmentRecord(home, vault, pending, checkSource, store => {
|
|
130
|
+
const previous = store.records.filter(r => r.path === path && r.kind === request.kind);
|
|
131
|
+
if (previous.some(r => (r.value as { state?: unknown } | undefined)?.state === "pending")) fail("a prior intake has an unresolved outcome; inspect the journal or proposals and reconcile it before retrying");
|
|
132
|
+
if (previous.some(r => r.contentHash === request.contentHash && (r.value as { state?: unknown } | undefined)?.state === "complete")) fail("this note version already has an intake record; inspect the existing result before retrying");
|
|
133
|
+
});
|
|
134
|
+
if (reserved.issues.length) return { ...bound, ok: false, applied: false, kind: request.kind, issues: reserved.issues, issue: "intake reservation needs inspection; inspect the sidecar before retrying" };
|
|
135
|
+
const releaseReservation = () => {
|
|
136
|
+
const released: EnrichmentRecord = { ...pending, value: { state: "released", token } };
|
|
137
|
+
return writeEnrichmentRecord(home, vault, released, () => {}, store => {
|
|
138
|
+
const prior = store.records.find(r => r.path === path && r.kind === request.kind && r.contentHash === request.contentHash)?.value as { state?: unknown; token?: unknown } | undefined;
|
|
139
|
+
if (prior?.state !== "pending" || prior.token !== token) fail("intake reservation changed; inspect it before retrying");
|
|
140
|
+
});
|
|
141
|
+
};
|
|
142
|
+
let started = false, sourceRefusal: unknown;
|
|
143
|
+
try {
|
|
144
|
+
const result = withMutationLock(vault, config.denyPaths ?? [], mutationFd => {
|
|
145
|
+
checkSource();
|
|
146
|
+
const currentConfig = loadRegistry(home), currentVault = currentConfig.vaults.find(v => v.id === vault.id);
|
|
147
|
+
if (!currentVault || JSON.stringify(currentVault) !== JSON.stringify(vault)) fail("vault binding changed; reload settings and retry");
|
|
148
|
+
const activeStore = readEnrichmentStore(home, currentVault, currentConfig.vaults, currentConfig.denyPaths ?? []);
|
|
149
|
+
const active = activeStore.records.find(r => r.path === path && r.kind === request.kind && r.contentHash === request.contentHash)?.value as { state?: unknown; token?: unknown } | undefined;
|
|
150
|
+
if (active?.state !== "pending" || active.token !== token) fail("intake reservation changed; inspect its outcome and request new work");
|
|
151
|
+
started = true;
|
|
152
|
+
const receipt = request.kind === "fact"
|
|
153
|
+
? appendAssertions(home, { vault: vault.id, ops: request.value, channel: request.channel }, { env: options.env, origin: { ...origin, intakeId: token } })
|
|
154
|
+
: submitProposal(home, { vault: vault.id, operation, agentLabel: `enrichment:${token}` }, { channel: request.channel, env: options.env, expectedVault: options.expectedVault ?? vault, proposalId: token, validateBeforeStore: () => { try { checkSource(); } catch (error) { sourceRefusal = error; throw error; } } });
|
|
155
|
+
const applied = "stored" in receipt ? receipt.stored === true : receipt.applied === true;
|
|
156
|
+
if (!applied) return { receipt, applied: "stored" in receipt && receipt.stored === null ? null : false, trackingIssues: [] as string[] };
|
|
157
|
+
const complete: EnrichmentRecord = { ...pending, value: { state: "complete", token, receipt } };
|
|
158
|
+
try {
|
|
159
|
+
const stored = writeEnrichmentRecord(home, vault, complete, checkSource, store => {
|
|
160
|
+
const prior = store.records.find(r => r.path === path && r.kind === request.kind && r.contentHash === request.contentHash)?.value as { state?: unknown; token?: unknown } | undefined;
|
|
161
|
+
if (prior?.state !== "pending" || prior.token !== token) fail("intake reservation changed; inspect the journal or proposals and sidecar before retrying");
|
|
162
|
+
}, mutationFd);
|
|
163
|
+
return { receipt, applied: true, trackingIssues: stored.issues };
|
|
164
|
+
} catch { return { receipt, applied: true, trackingIssues: ["result was published but local intake tracking failed; inspect the journal or proposals and reconcile before retrying"] }; }
|
|
165
|
+
});
|
|
166
|
+
const { receipt, applied, trackingIssues } = result;
|
|
167
|
+
if (!applied) {
|
|
168
|
+
if (applied === false) {
|
|
169
|
+
try { const released = releaseReservation(); if (!released.issues.length) return { ...bound, ok: false, applied: false, kind: request.kind, receipt }; } catch { /* Keep the reservation when cleanup cannot be confirmed. */ }
|
|
170
|
+
}
|
|
171
|
+
return { ...bound, ok: false, applied, kind: request.kind, receipt, issue: "intake reservation remains unresolved; inspect the journal or proposals and reconcile before retrying" };
|
|
172
|
+
}
|
|
173
|
+
return { ...bound, ok: receipt.ok && trackingIssues.length === 0, applied: true, kind: request.kind, receipt, ...(trackingIssues.length ? { issues: trackingIssues } : {}) };
|
|
174
|
+
} catch (error) {
|
|
175
|
+
if (!started || sourceRefusal) {
|
|
176
|
+
try { const released = releaseReservation(); if (!released.issues.length) throw (sourceRefusal ?? error); } catch (cleanup) { if (cleanup === (sourceRefusal ?? error)) throw cleanup; }
|
|
177
|
+
return { ...bound, ok: false, applied: false, kind: request.kind, issue: `${error instanceof Error ? error.message : "intake failed before publication; inspect the source and retry"}; reservation cleanup failed, inspect and reconcile it before retrying` };
|
|
178
|
+
}
|
|
179
|
+
return { ...bound, ok: false, applied: null, kind: request.kind, issue: "intake outcome is uncertain; inspect the journal or proposals and reconcile before retrying" };
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
if (request.kind !== "embedding" && (typeof request.value !== "string" || !request.value.trim() || Buffer.byteLength(request.value) > 8192)) fail("summary or community label must be nonempty text of at most 8192 bytes; shorten the result");
|
|
183
|
+
const record: EnrichmentRecord = { path, contentHash: request.contentHash, kind: request.kind, producer: request.producer, createdAt: request.createdAt, value: request.value };
|
|
184
|
+
const stored = writeEnrichmentRecord(home, vault, record, () => {
|
|
185
|
+
if (sha256(noteBytes(vault, path, config.denyPaths ?? [])) !== request.contentHash) fail("source content hash is stale; reread the note and regenerate the result");
|
|
186
|
+
});
|
|
187
|
+
return { ok: stored.issues.length === 0, applied: true, vault: vault.id, path, contentHash: request.contentHash, kind: request.kind, producer: request.producer, createdAt: request.createdAt, issues: stored.issues };
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
/** Return one current record after rechecking path and disclosure policy. */
|
|
191
|
+
export function readEnrichment(home: string, request: { vault: string; path: string; kind: EnrichmentKind }) {
|
|
192
|
+
if (!request || typeof request !== "object" || Object.keys(request).some(k => !["vault", "path", "kind"].includes(k)) || !kinds.includes(request.kind)) fail("invalid record request; supply a vault, note path and enrichment kind");
|
|
193
|
+
const { config, vault, check } = context(home, request.vault), path = sourcePath(request.path);
|
|
194
|
+
const current = sha256(noteBytes(vault, path, config.denyPaths ?? []));
|
|
195
|
+
const store = readEnrichmentStore(home, vault, config.vaults, config.denyPaths ?? []);
|
|
196
|
+
const candidates = store.records.filter(r => r.path === path && r.kind === request.kind);
|
|
197
|
+
const uncertain = candidates.find(r => (r.value as { state?: unknown } | undefined)?.state === "pending");
|
|
198
|
+
if (uncertain) { check(); return { vault: vault.id, path, kind: request.kind, status: "needs-reconciliation" as const }; }
|
|
199
|
+
const record = candidates.find(r => r.contentHash === current) ?? candidates.at(-1);
|
|
200
|
+
if (!record) { check(); return { vault: vault.id, path, kind: request.kind, status: "missing" as const }; }
|
|
201
|
+
if (record.contentHash !== current) { check(); return { vault: vault.id, path, kind: request.kind, status: "stale" as const }; }
|
|
202
|
+
const issues = screenMetadata({ producer: record.producer, ...(record.kind === "embedding" ? {} : { value: record.value }) }, { vault, config }, path);
|
|
203
|
+
if (issues.some(issue => issue.severity === "error")) fail("current secret or PII policy refuses the stored result; inspect the vault policy");
|
|
204
|
+
check();
|
|
205
|
+
return { vault: vault.id, path, kind: request.kind, status: "current" as const, producer: record.producer, createdAt: record.createdAt, contentHash: record.contentHash, value: record.value };
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
/** Explicitly release an uncertain intake only after the caller has inspected its journal or proposal. */
|
|
209
|
+
export function reconcileEnrichment(home: string, request: { vault: string; path: string; kind: "fact" | "inferred-link"; resolution: "published" | "not_published"; confirm?: boolean; fingerprint?: string; channel: "cli" | "mcp" }, options: { expectedVault?: VaultSpec } = {}) {
|
|
210
|
+
if (!request || typeof request !== "object" || Object.keys(request).some(k => !["vault", "path", "kind", "resolution", "confirm", "fingerprint", "channel"].includes(k)) || !["fact", "inferred-link"].includes(request.kind) || !["published", "not_published"].includes(request.resolution) || !["cli", "mcp"].includes(request.channel) || (request.confirm !== undefined && typeof request.confirm !== "boolean")) fail("invalid reconciliation request; choose fact or inferred-link and preview first");
|
|
211
|
+
const { config, vault, check } = context(home, request.vault, request.channel), path = sourcePath(request.path);
|
|
212
|
+
if (options.expectedVault && JSON.stringify(options.expectedVault) !== JSON.stringify(vault)) fail("vault binding changed since MCP startup; restart the MCP server");
|
|
213
|
+
if (isDenied(path, config.denyPaths ?? []) || isDenied(join(vault.path, path), config.denyPaths ?? [])) fail("source path is denied; inspect the vault policy before reconciling");
|
|
214
|
+
const store = readEnrichmentStore(home, vault, config.vaults, config.denyPaths ?? []), prior = store.records.find(r => r.path === path && r.kind === request.kind && (r.value as { state?: unknown } | undefined)?.state === "pending");
|
|
215
|
+
const value = prior?.value as { state?: unknown; token?: unknown } | undefined;
|
|
216
|
+
if (!prior || value?.state !== "pending" || typeof value.token !== "string") fail("no unresolved intake reservation exists; inspect the work list before retrying");
|
|
217
|
+
const fingerprint = sha256(JSON.stringify([config, prior, request.resolution]));
|
|
218
|
+
if (request.confirm !== true) {
|
|
219
|
+
if (request.fingerprint !== undefined) fail("fingerprint requires confirmation; request a fresh preview first");
|
|
220
|
+
check();
|
|
221
|
+
return { ok: true, applied: false, fingerprint, path, kind: request.kind, resolution: request.resolution, intakeId: value.token, warning: "Inspect the journal or proposals first. An incorrect resolution can cause a duplicate or suppress needed work." };
|
|
222
|
+
}
|
|
223
|
+
if (request.fingerprint !== fingerprint) fail("reconciliation fingerprint is missing or stale; inspect the outcome and request a fresh preview");
|
|
224
|
+
const released: EnrichmentRecord = { ...prior, value: { state: request.resolution === "published" ? "complete" : "released", token: value.token, reconciled: request.resolution } };
|
|
225
|
+
const stored = writeEnrichmentRecord(home, vault, released, () => {}, current => {
|
|
226
|
+
const now = current.records.find(r => r.path === path && r.kind === request.kind && r.contentHash === prior.contentHash);
|
|
227
|
+
if (!now || sha256(JSON.stringify([loadRegistry(home), now, request.resolution])) !== fingerprint) fail("intake reservation changed; inspect the outcome and request a fresh preview");
|
|
228
|
+
if (request.resolution === "not_published") {
|
|
229
|
+
if (request.kind === "fact") {
|
|
230
|
+
const journal = readJournal(vault.path, config.denyPaths ?? []);
|
|
231
|
+
if (journal.ignored.length || journal.segments.some(segment => segment.untrusted || segment.unsupported || segment.tornTail || segment.partialBatch || segment.entries.some(entry => entry.purged))) fail("journal coverage cannot prove non-publication; inspect and repair the journal before reconciling");
|
|
232
|
+
if (journal.segments.some(segment => segment.entries.some(entry => (entry.payload?.origin as { intakeId?: unknown } | undefined)?.intakeId === value.token))) fail("the intake was published to the journal; reconcile it as published");
|
|
233
|
+
} else {
|
|
234
|
+
try {
|
|
235
|
+
showProposal(home, { vault: vault.id, id: value.token as string }, { channel: request.channel });
|
|
236
|
+
fail("the intake proposal was published; reconcile it as published");
|
|
237
|
+
} catch (error) {
|
|
238
|
+
const message = error instanceof Error ? error.message : "";
|
|
239
|
+
if (!((error as NodeJS.ErrnoException).code === "ENOENT" || message === "proposal: no proposal store for this vault; use proposal list to find an id")) throw error;
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
});
|
|
244
|
+
return { ok: stored.issues.length === 0, applied: true, path, kind: request.kind, issues: stored.issues };
|
|
245
|
+
}
|
package/src/graph-evidence.ts
CHANGED
|
@@ -41,7 +41,7 @@ export interface GraphHop {
|
|
|
41
41
|
from: string;
|
|
42
42
|
to: string;
|
|
43
43
|
/** The deduplicated graph edge this hop used: structural metadata only, never a quote or a probability. */
|
|
44
|
-
edge: { kind: "structural"; relation: GraphEdge["relation"]; rel?: string; field: "body" };
|
|
44
|
+
edge: { kind: "structural"; relation: GraphEdge["relation"]; rel?: string; field: "body"; confidence_tag?: "INFERRED" };
|
|
45
45
|
occurrences: GraphOccurrence[];
|
|
46
46
|
/** Occurrences found for this pair that the per-hop or byte bound left out. */
|
|
47
47
|
omitted: number;
|
|
@@ -149,7 +149,7 @@ export function explainGraphPath(snapshot: AnalysisSnapshot, from: string, to: s
|
|
|
149
149
|
const snapshotHash = sha(JSON.stringify({ algorithm: ALGORITHM, notes: identity, options: { maxHops, relation, viaHubs } }));
|
|
150
150
|
const hops: GraphHop[] = path.edges.map((edge, index) => {
|
|
151
151
|
const found = candidates(notes, edge.source, edge.target, edge.source);
|
|
152
|
-
return { index, from: edge.source, to: edge.target, edge: { kind: "structural", relation: edge.relation, ...(edge.rel ? { rel: edge.rel } : {}), field: "body" },
|
|
152
|
+
return { index, from: edge.source, to: edge.target, edge: { kind: "structural", relation: edge.relation, ...(edge.rel ? { rel: edge.rel } : {}), field: "body", ...(edge.confidence_tag === "INFERRED" ? { confidence_tag: "INFERRED" as const } : {}) },
|
|
153
153
|
occurrences: locate(notes, found.slice(0, EXPLAIN_OCCURRENCES_PER_HOP), edge.source), omitted: Math.max(0, found.length - EXPLAIN_OCCURRENCES_PER_HOP) };
|
|
154
154
|
});
|
|
155
155
|
const result: GraphExplanation = { schemaVersion: 1, snapshotHash,
|