@staix/vault-mem 0.11.0 → 0.12.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/README.md +17 -2
- package/bin/vmem.js +5 -1
- package/package.json +2 -1
- package/skills/_shared/vault-adapter.md +131 -0
- package/skills/skill-mine/SKILL.md +52 -0
- package/skills/skill-mine/vocab/actions.yaml +110 -0
- package/skills/skill-mine/vocab/objects.yaml +123 -0
- package/skills/vault-connect/SKILL.md +75 -0
- package/skills/vault-enrich/SKILL.md +31 -0
- package/skills/vault-extract/README.md +20 -0
- package/skills/vault-extract/SKILL.md +139 -0
- package/skills/vault-graph/SKILL.md +120 -0
- package/skills/vault-learn/SKILL.md +111 -0
- package/skills/vault-lint/README.md +32 -0
- package/skills/vault-lint/SKILL.md +137 -0
- package/skills/vault-lint/references/lint-output.md +97 -0
- package/skills/vault-lint/references/vault-scope-rules.md +26 -0
- package/skills/vault-lint/references/work-scope-checks.md +71 -0
- package/skills/vault-refactor/SKILL.md +112 -0
- package/skills/vault-remember/SKILL.md +38 -0
- package/skills/vault-rethink/README.md +15 -0
- package/skills/vault-rethink/SKILL.md +58 -0
- package/skills/vault-stats/SKILL.md +49 -0
- package/skills/vault-sync/README.md +15 -0
- package/skills/vault-sync/SKILL.md +195 -0
- package/skills/vault-sync/references/r4-method-evidence.md +12 -0
- package/skills/vault-update/SKILL.md +68 -0
- package/skills/vmem-status/SKILL.md +30 -0
- package/src/amem/host-api.ts +5 -1
- package/src/analysis-input.ts +15 -0
- package/src/analysis-types.ts +3 -1
- package/src/assertion-policy.ts +4 -2
- package/src/assertions.ts +48 -8
- package/src/cli-output.ts +10 -3
- package/src/cli.ts +29 -3
- package/src/commands-table.ts +3 -1
- package/src/config.ts +21 -7
- package/src/enrichment-query.ts +65 -0
- package/src/enrichment-search.ts +69 -0
- package/src/enrichment-store.ts +197 -0
- package/src/enrichment.ts +245 -0
- package/src/graph-evidence.ts +2 -2
- package/src/index.ts +9 -0
- package/src/mcp-vault.ts +46 -8
- package/src/proposal-provider.ts +3 -2
- package/src/proposals.ts +10 -4
- package/src/read-output.ts +1 -1
- package/src/registry.ts +57 -2
- package/src/setup.ts +12 -3
- package/src/skills-install.ts +216 -0
- package/src/ui/panel-types.ts +55 -0
- package/src/ui/panel.js +76 -0
- package/src/ui/plugin.ts +95 -5
- package/src/vault-graph.ts +15 -7
- package/src/vault-index.ts +17 -0
- package/src/vault-lint.ts +12 -4
- package/src/write-operations.ts +38 -1
- package/src/write-types.ts +1 -0
- package/src/write-validation.ts +2 -1
- package/src/writer.ts +1 -1
package/src/cli.ts
CHANGED
|
@@ -18,11 +18,11 @@ import { VMEM_TOOL_NAMES, vaultTools } from "./mcp-vault.ts";
|
|
|
18
18
|
import { CONFIG_VERSION, effectiveDomains } from "./config.ts";
|
|
19
19
|
import { runSetup, type SetupMode } from "./setup.ts";
|
|
20
20
|
import { resolveHome } from "./home.ts";
|
|
21
|
-
import { addVault, addSpec, removeSpec, specStatus, effectivePii, loadRegistry, recoverRegistry, registryStatus, removeVault, setVaultPii } from "./registry.ts";
|
|
21
|
+
import { changeSettings, settingsSnapshot, addVault, addSpec, removeSpec, specStatus, effectivePii, loadRegistry, recoverRegistry, registryStatus, removeVault, setVaultPii } from "./registry.ts";
|
|
22
22
|
import { piiPolicy } from "./pii.ts";
|
|
23
23
|
import { vaultStats } from "./vault-stats.ts";
|
|
24
24
|
import { PLAIN, UsageError, errorLine, shellWord, takeColorFlag, textOptions, type TextOptions } from "./text-output.ts";
|
|
25
|
-
import { renderDoctor, renderManifest, renderRecover, renderRemove, renderSetup, renderSpecAdd, renderSpecsList, renderStats, renderStatus, renderVaultAdd, renderVaultPii, renderVaultsList, type StatusResult } from "./cli-output.ts";
|
|
25
|
+
import { renderDoctor, renderManifest, renderRecover, renderSkillsExport, renderRemove, renderSetup, renderSpecAdd, renderSpecsList, renderStats, renderStatus, renderVaultAdd, renderVaultPii, renderVaultsList, type StatusResult } from "./cli-output.ts";
|
|
26
26
|
function parse(args: string[], allowed: string[], values: string[]) {
|
|
27
27
|
const flags: Record<string, string | boolean> = {}, positional: string[] = [];
|
|
28
28
|
for (let i = 0; i < args.length; i++) {
|
|
@@ -78,12 +78,29 @@ function runWith(args: string[], env: NodeJS.ProcessEnv, text: TextOptions = PLA
|
|
|
78
78
|
const mode: SetupMode = flags["--apply"] ? "apply" : flags["--undo"] ? "undo" : "dry-run";
|
|
79
79
|
const report = runSetup(positional[0]!, mode, { home: resolveHome(env).home, env, ...(typeof flags["--root"] === "string" ? { root: flags["--root"] } : {}) });
|
|
80
80
|
const apply = ["vmem", "setup", positional[0]!, "--apply", ...(typeof flags["--root"] === "string" ? ["--root", flags["--root"]] : [])].map(shellWord).join(" ");
|
|
81
|
-
|
|
81
|
+
const failed = [...report.actions, report.console].some(a => a?.change === "refused") || (report.skills ?? []).some(s => s.refused || s.report?.actions.some(a => ["refused", "foreign", "modified"].includes(a.change)));
|
|
82
|
+
const progress = report.applied || (mode === "dry-run" && (report.actions.some(a => ["add", "replace"].includes(a.change)) || (report.skills ?? []).some(s => s.report?.actions.some(a => ["install", "replace"].includes(a.change)))));
|
|
83
|
+
return { text: flags["--json"] ? JSON.stringify(report) : renderSetup(report, text, apply), code: failed && !progress ? 1 : 0 };
|
|
82
84
|
}
|
|
83
85
|
if (args[0] === "completion") {
|
|
84
86
|
if (args.length !== 2) throw new UsageError("completion: requires bash, zsh or fish");
|
|
85
87
|
return { text: completionScript(args[1]!).trimEnd(), code: 0 };
|
|
86
88
|
}
|
|
89
|
+
if (args[0] === "settings") {
|
|
90
|
+
const { flags, positional } = parse(args.slice(1), ["--set", "--confirm", "--fingerprint", "--json"], ["--set", "--fingerprint"]);
|
|
91
|
+
if (positional.length !== 1) throw new UsageError("settings: supply one vault id; use vmem settings <vault> [--set JSON]");
|
|
92
|
+
const home = resolveHome(env).home, value = flags["--set"];
|
|
93
|
+
if (value === undefined) {
|
|
94
|
+
if (flags["--confirm"] || flags["--fingerprint"]) throw new UsageError("settings: confirmation requires changes; use --set JSON with a preview fingerprint");
|
|
95
|
+
return { text: JSON.stringify(settingsSnapshot(home, positional[0]!), null, flags["--json"] ? undefined : 2), code: 0 };
|
|
96
|
+
}
|
|
97
|
+
let changes: unknown;
|
|
98
|
+
try { changes = JSON.parse(String(value)); } catch { throw new UsageError("settings: invalid JSON; supply an object to --set"); }
|
|
99
|
+
if (Boolean(flags["--confirm"]) !== Boolean(flags["--fingerprint"])) throw new UsageError("settings: --confirm and --fingerprint must be supplied together; preview first");
|
|
100
|
+
const result = changeSettings(home, { vault: positional[0]!, changes: changes as import("./registry.ts").SettingsChanges, confirm: flags["--confirm"] === true, ...(typeof flags["--fingerprint"] === "string" ? { fingerprint: flags["--fingerprint"] } : {}) });
|
|
101
|
+
const next = result.applied ? undefined : ["vmem", "settings", positional[0]!, "--set", String(value), "--confirm", "--fingerprint", result.fingerprint].map(shellWord).join(" ");
|
|
102
|
+
return { text: JSON.stringify({ ...result, ...(next ? { next } : {}) }, null, flags["--json"] ? undefined : 2), code: 0 };
|
|
103
|
+
}
|
|
87
104
|
if (args[0] === "spec") return runSpecCommand(args.slice(1), env, text);
|
|
88
105
|
if (args[0] === "proposal") return runProposalCommand(args.slice(1), resolveHome(env).home, env, text);
|
|
89
106
|
if (args[0] === "lint" || args[0] === "graph") return runAnalysisCommand(args[0],args.slice(1),resolveHome(env).home,env,text);
|
|
@@ -228,6 +245,15 @@ export async function runAsync(argv: string[], env: NodeJS.ProcessEnv = process.
|
|
|
228
245
|
const taken = takeColorFlag(argv), args = taken.args, text = textOptions(taken.color, env, io.stdout ?? {});
|
|
229
246
|
const usage = commandHelp(args, text);
|
|
230
247
|
if (usage) return { text: usage, code: 0 };
|
|
248
|
+
if (args[0] === "skills" && args[1] === "export") {
|
|
249
|
+
const { flags, positional } = parse(args.slice(2), ["--json"], []);
|
|
250
|
+
if (positional.length !== 1) throw new UsageError("skills export: provide one destination directory; use vmem skills export <dir>");
|
|
251
|
+
const { exportSkills } = await import("./skills-install.ts");
|
|
252
|
+
const report = exportSkills(positional[0]!);
|
|
253
|
+
const failed = report.actions.some(a => ["refused", "foreign", "modified"].includes(a.change));
|
|
254
|
+
const changed = report.actions.some(a => ["install", "replace"].includes(a.change));
|
|
255
|
+
return { text: flags["--json"] ? JSON.stringify(report) : renderSkillsExport(report, text), code: failed && !changed ? 1 : 0 };
|
|
256
|
+
}
|
|
231
257
|
if (args[0] === "skills") {
|
|
232
258
|
const { runSkillsCommand } = await import("./skills-mine.ts");
|
|
233
259
|
return runSkillsCommand(args.slice(1), resolveHome(env).home, env, text);
|
package/src/commands-table.ts
CHANGED
|
@@ -27,6 +27,7 @@ export const COMMANDS = [
|
|
|
27
27
|
{ name: "graph neighbors", usage: "graph neighbors <vault> <stem> [--hops 1..2] [--limit 1..500] [--relation REL] [--json]", mutates: false, section: "Analysis", description: "read a note's graph neighborhood up to two hops, optionally over one link relation" },
|
|
28
28
|
{ name: "graph path", usage: "graph path <vault> <from> <to> [--max-hops 1..6] [--relation REL] [--via-hubs] [--json]", mutates: false, section: "Analysis", description: "read the shortest link path between two notes, skipping MOC/bu hubs unless --via-hubs" },
|
|
29
29
|
{ name: "graph explain", usage: "graph explain <vault> <from> <to> [--max-hops 1..6] [--relation REL] [--via-hubs] [--json]", mutates: false, section: "Analysis", description: "explain each hop of the shortest path by its source link occurrences; structural only" },
|
|
30
|
+
{ name: "skills export", usage: "skills export <dir> [--json]", mutates: true, section: "Setup", description: "export self-contained vmem skills to a directory" },
|
|
30
31
|
{ name: "skills mine", usage: "skills mine <vault> [--days N] [--project KEY|--all-projects] [--vocab DIR] [--skills DIR] [--min-size N] [--min-novelty X] [--json]", mutates: false, section: "Analysis", description: "rank repeated work patterns in agent memory as new-skill candidates" },
|
|
31
32
|
{ name: "rebuild", usage: "rebuild <vault> [--json]", mutates: "cache", section: "Read", description: "rebuild the derived search index from the vault files" },
|
|
32
33
|
{ name: "brief", usage: "brief <vault> --for INTENT [--paths PATH]... [--bytes 256..4096] [--json]", mutates: "cache", section: "Read", description: "rank screened note pointers and one-hop neighbors within a byte cap" },
|
|
@@ -40,6 +41,7 @@ export const COMMANDS = [
|
|
|
40
41
|
{ name: "history", usage: "history <vault> <note> [--limit 1..100] [--cursor C] [--json]", mutates: false, section: "Read", description: "list one note path's Git snapshot revisions newest first, pinned to the HEAD the page started from" },
|
|
41
42
|
{ name: "changes", usage: "changes <vault> --from OID --to OID [--limit 1..500] [--cursor C] [--json]", mutates: false, section: "Read", description: "compare two pinned Git snapshots over notes/ with before/after blob ids and status" },
|
|
42
43
|
{ name: "status", usage: "status [--json]", mutates: false, section: "Registry", description: "show the home, registered vaults and spec sources, and vault health" },
|
|
44
|
+
{ name: "settings", usage: "settings <vault> [--set JSON] [--confirm --fingerprint SHA] [--json]", mutates: "confirm", section: "Registry", description: "read settings or preview and confirm a settings change" },
|
|
43
45
|
{ name: "doctor", usage: "doctor [--json]", mutates: false, section: "Registry", description: "check vault paths, remotes, project registries, pending tidy and spec sources; exits 1 when unhealthy" },
|
|
44
46
|
{ name: "recover", usage: "recover [--confirm] [--json]", mutates: "confirm", section: "Registry", description: "preview, then remove with --confirm, a stale registry lock and leftover temporary files" },
|
|
45
47
|
{ name: "vaults list", usage: "vaults list [--json]", mutates: false, section: "Registry", description: "list registered vaults with kind, write policy and effective PII policy" },
|
|
@@ -116,7 +118,7 @@ export function completionScript(shell: string): string {
|
|
|
116
118
|
cur="\${COMP_WORDS[COMP_CWORD]}"
|
|
117
119
|
if [ "$COMP_CWORD" -eq 1 ]; then COMPREPLY=($(compgen -W ${quote(top.join(" "))} -- "$cur")); return; fi
|
|
118
120
|
key="\${COMP_WORDS[1]}"
|
|
119
|
-
case "$key" in spec) if [ "$COMP_CWORD" -eq 2 ]; then COMPREPLY=($(compgen -W '${SPEC_COMMANDS.map(item => item.command).join(" ")}' -- "$cur")); return; fi; key="$key \${COMP_WORDS[2]}" ;; graph) if [ "$COMP_CWORD" -eq 2 ]; then COMPREPLY=($(compgen -W 'build neighbors path explain' -- "$cur")); return; fi; key="$key \${COMP_WORDS[2]}" ;; skills) if [ "$COMP_CWORD" -eq 2 ]; then COMPREPLY=($(compgen -W 'mine' -- "$cur")); return; fi; key="$key \${COMP_WORDS[2]}" ;; specs) if [ "$COMP_CWORD" -eq 2 ]; then COMPREPLY=($(compgen -W 'list add remove' -- "$cur")); return; fi; key="$key \${COMP_WORDS[2]}" ;; vaults|log) if [ "$COMP_CWORD" -eq 2 ]; then COMPREPLY=($(compgen -W "$( [ "$key" = vaults ] && echo 'list add remove' || echo 'tail grep append')" -- "$cur")); return; fi; key="$key \${COMP_WORDS[2]}" ;; esac
|
|
121
|
+
case "$key" in spec) if [ "$COMP_CWORD" -eq 2 ]; then COMPREPLY=($(compgen -W '${SPEC_COMMANDS.map(item => item.command).join(" ")}' -- "$cur")); return; fi; key="$key \${COMP_WORDS[2]}" ;; graph) if [ "$COMP_CWORD" -eq 2 ]; then COMPREPLY=($(compgen -W 'build neighbors path explain' -- "$cur")); return; fi; key="$key \${COMP_WORDS[2]}" ;; skills) if [ "$COMP_CWORD" -eq 2 ]; then COMPREPLY=($(compgen -W 'mine export' -- "$cur")); return; fi; key="$key \${COMP_WORDS[2]}" ;; specs) if [ "$COMP_CWORD" -eq 2 ]; then COMPREPLY=($(compgen -W 'list add remove' -- "$cur")); return; fi; key="$key \${COMP_WORDS[2]}" ;; vaults|log) if [ "$COMP_CWORD" -eq 2 ]; then COMPREPLY=($(compgen -W "$( [ "$key" = vaults ] && echo 'list add remove' || echo 'tail grep append')" -- "$cur")); return; fi; key="$key \${COMP_WORDS[2]}" ;; esac
|
|
120
122
|
case "$key" in
|
|
121
123
|
${cases.join("\n")}
|
|
122
124
|
*) words='' ;;
|
package/src/config.ts
CHANGED
|
@@ -15,6 +15,8 @@ export interface VaultSpec {
|
|
|
15
15
|
registry?: string;
|
|
16
16
|
domains?: string[];
|
|
17
17
|
canonTables?: CanonTableSpec[];
|
|
18
|
+
graph?: { resolution?: number };
|
|
19
|
+
enrichment?: { enabled: boolean };
|
|
18
20
|
}
|
|
19
21
|
/** Advisory canonical tables do not participate in mutation identity or policy. */
|
|
20
22
|
export function mutationVault(vault: VaultSpec): Omit<VaultSpec, "canonTables"> {
|
|
@@ -44,11 +46,11 @@ export interface PiiSpec { email?: PiiCell; phone?: PiiCell; custom?: PiiCell; p
|
|
|
44
46
|
* The config format this build writes. A build that adds a key bumps it. A config whose `version` is newer
|
|
45
47
|
* than this build is read tolerantly: unknown keys are kept as they are and written back unchanged, so an
|
|
46
48
|
* older vmem can share a config with a newer one. At or below this version unknown keys stay errors (typos).
|
|
47
|
-
* An absent `version` means 1. Registry writes add format 2 for `specs`, 3 for vault `domains`,
|
|
49
|
+
* An absent `version` means 1. Registry writes add format 2 for `specs`, 3 for vault `domains`, 4 for vault `canonTables`, and 5 for default vault or graph options, and 6 for enrichment; otherwise they keep the existing version field.
|
|
48
50
|
*/
|
|
49
|
-
export const CONFIG_VERSION =
|
|
51
|
+
export const CONFIG_VERSION = 6;
|
|
50
52
|
export interface SpecSourceSpec { id: string; path: string; write: "mcp" | "cli" | "none"; pii: "allow" | "deny" }
|
|
51
|
-
export interface VmemConfig { version?: number; home?: string; vaults: VaultSpec[]; specs?: SpecSourceSpec[]; denyPaths?: string[]; piiPatterns?: string[] }
|
|
53
|
+
export interface VmemConfig { version?: number; home?: string; vaults: VaultSpec[]; specs?: SpecSourceSpec[]; denyPaths?: string[]; piiPatterns?: string[]; defaultVault?: string }
|
|
52
54
|
const controls = /[\x00-\x1f\x7f]/;
|
|
53
55
|
function object(value: unknown, keys: string[], label: string, tolerant = false): Record<string, unknown> {
|
|
54
56
|
if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error(`${label}: expected object`);
|
|
@@ -155,7 +157,7 @@ function parseCanonTables(value: unknown, tolerant: boolean): CanonTableSpec[] {
|
|
|
155
157
|
}
|
|
156
158
|
}
|
|
157
159
|
export function parseVault(value: unknown, tolerant = false): VaultSpec {
|
|
158
|
-
const keys = ["id", "path", "kind", "write", "pii", "remote", "registry", "domains", "canonTables"], v = object(value, keys, "vault", tolerant);
|
|
160
|
+
const keys = ["id", "path", "kind", "write", "pii", "remote", "registry", "domains", "canonTables", "graph", "enrichment"], v = object(value, keys, "vault", tolerant);
|
|
159
161
|
const id = string(v.id, "vault.id");
|
|
160
162
|
if (!/^[a-z][a-z0-9-]{0,31}$/.test(id)) throw new Error("vault.id: invalid id");
|
|
161
163
|
if (v.kind !== "personal" && v.kind !== "org") throw new Error("vault.kind: expected personal or org");
|
|
@@ -171,6 +173,16 @@ export function parseVault(value: unknown, tolerant = false): VaultSpec {
|
|
|
171
173
|
result.domains = [...v.domains];
|
|
172
174
|
}
|
|
173
175
|
if (v.canonTables !== undefined) result.canonTables = parseCanonTables(v.canonTables, tolerant);
|
|
176
|
+
if (v.enrichment !== undefined) {
|
|
177
|
+
const enrichment = object(v.enrichment, ["enabled"], "vault.enrichment", tolerant);
|
|
178
|
+
if (typeof enrichment.enabled !== "boolean") throw new Error("vault.enrichment.enabled: expected true or false; correct the vault setting");
|
|
179
|
+
result.enrichment = { ...unknown(enrichment, ["enabled"], tolerant), enabled: enrichment.enabled };
|
|
180
|
+
}
|
|
181
|
+
if (v.graph !== undefined) {
|
|
182
|
+
const graph = object(v.graph, ["resolution"], "vault.graph", tolerant);
|
|
183
|
+
if (graph.resolution !== undefined && (typeof graph.resolution !== "number" || !Number.isFinite(graph.resolution) || graph.resolution <= 0 || graph.resolution > 100)) throw new Error("vault.graph.resolution: expected a number above 0 and at most 100; set a valid graph resolution");
|
|
184
|
+
result.graph = { ...unknown(graph, ["resolution"], tolerant), ...(graph.resolution !== undefined ? { resolution: graph.resolution as number } : {}) };
|
|
185
|
+
}
|
|
174
186
|
return result;
|
|
175
187
|
}
|
|
176
188
|
/** Spec sources have their own registry shape; they are not vaults. */
|
|
@@ -188,13 +200,13 @@ export function parseSpec(value: unknown, tolerant = false): SpecSourceSpec {
|
|
|
188
200
|
}
|
|
189
201
|
}
|
|
190
202
|
export function parseConfig(value: unknown): VmemConfig {
|
|
191
|
-
const keys = ["version", "home", "vaults", "specs", "denyPaths", "piiPatterns"];
|
|
203
|
+
const keys = ["version", "home", "vaults", "specs", "denyPaths", "piiPatterns", "defaultVault"];
|
|
192
204
|
const version = (value as Record<string, unknown> | null)?.version;
|
|
193
205
|
if (version !== undefined && (!Number.isSafeInteger(version) || (version as number) < 1)) throw new Error("config.version: expected a positive integer");
|
|
194
206
|
const tolerant = newerFormat(value), raw = object(value, keys, "config", tolerant);
|
|
195
207
|
const stamp = version === undefined ? {} : { version: version as number };
|
|
196
208
|
if (raw.home !== undefined) {
|
|
197
|
-
if (["vaults", "specs", "denyPaths", "piiPatterns"].some(key => raw[key] !== undefined)) throw new Error("config: home pointer must not contain vaults, specs or policy; put source entries in the pointed-to home config");
|
|
209
|
+
if (["vaults", "specs", "denyPaths", "piiPatterns", "defaultVault"].some(key => raw[key] !== undefined)) throw new Error("config: home pointer must not contain vaults, specs or policy; put source entries in the pointed-to home config");
|
|
198
210
|
return { ...unknown(raw, keys, tolerant), ...stamp, home: absolute(raw.home, "config.home"), vaults: [] };
|
|
199
211
|
}
|
|
200
212
|
const vaultInput = raw.vaults === undefined && raw.specs !== undefined ? [] : raw.vaults;
|
|
@@ -206,11 +218,13 @@ export function parseConfig(value: unknown): VmemConfig {
|
|
|
206
218
|
if (specs && new Set(specs.map(source => source.path)).size !== specs.length) throw new Error("config: duplicate spec source path; keep only one entry for each repository root");
|
|
207
219
|
if (new Set(vaults.map(v => v.id)).size !== vaults.length) throw new Error("config: duplicate vault id");
|
|
208
220
|
if (new Set(vaults.map(v => v.path)).size !== vaults.length) throw new Error("config: duplicate vault path");
|
|
221
|
+
const defaultVault = raw.defaultVault === undefined ? undefined : string(raw.defaultVault, "config.defaultVault");
|
|
222
|
+
if (defaultVault !== undefined && !/^[a-z][a-z0-9-]{0,31}$/.test(defaultVault)) throw new Error("config.defaultVault: invalid vault id; choose an id from vmem vaults list");
|
|
209
223
|
const denyPaths = strings(raw.denyPaths, "denyPaths"), piiPatterns = strings(raw.piiPatterns, "piiPatterns");
|
|
210
224
|
for (const pattern of piiPatterns ?? []) {
|
|
211
225
|
try { new RegExp(pattern); } catch { throw new Error("piiPatterns: invalid regular expression"); }
|
|
212
226
|
}
|
|
213
|
-
return { ...unknown(raw, keys, tolerant), ...stamp, vaults, ...(specs !== undefined ? { specs } : {}), ...(denyPaths ? { denyPaths } : {}), ...(piiPatterns ? { piiPatterns } : {}) };
|
|
227
|
+
return { ...unknown(raw, keys, tolerant), ...stamp, vaults, ...(defaultVault !== undefined ? { defaultVault } : {}), ...(specs !== undefined ? { specs } : {}), ...(denyPaths ? { denyPaths } : {}), ...(piiPatterns ? { piiPatterns } : {}) };
|
|
214
228
|
}
|
|
215
229
|
/** Bounded descriptor read: reject links, devices and oversized/growing files. */
|
|
216
230
|
export function readConfig(home: string): VmemConfig {
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
/** Bounded hybrid ranking over current indexed notes and local stored vectors. */
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { loadRegistry } from "./registry.ts";
|
|
4
|
+
import { logBinding, matchesBinding } from "./vault-log.ts";
|
|
5
|
+
import { canonicalPath, isDenied } from "./platform.ts";
|
|
6
|
+
import { readEnrichmentStore } from "./enrichment-store.ts";
|
|
7
|
+
import { rankHybridCandidates, vectorSimilarity } from "./enrichment-search.ts";
|
|
8
|
+
import { resultLimit, type VaultIndex, type SearchOptions, type SearchResult } from "./vault-index.ts";
|
|
9
|
+
|
|
10
|
+
export interface QueryVector { model: string; values: number[] }
|
|
11
|
+
function queryContext(home: string, vaultId: string) {
|
|
12
|
+
const config = loadRegistry(home), vault = config.vaults.find(v => v.id === vaultId);
|
|
13
|
+
if (!vault) throw new Error("enrichment: vault is not registered; select a registered vault");
|
|
14
|
+
let binding: ReturnType<typeof logBinding>;
|
|
15
|
+
try { binding = logBinding(vault, config); } catch { throw new Error("enrichment: vault root is unavailable; restore the registered vault and retry"); }
|
|
16
|
+
const check = () => {
|
|
17
|
+
let current: ReturnType<typeof loadRegistry>;
|
|
18
|
+
try { current = loadRegistry(home); } catch { throw new Error("enrichment: registry is unavailable after the read; inspect the configuration and retry"); }
|
|
19
|
+
if (!matchesBinding(current, vault.id, binding)) throw new Error("enrichment: vault binding or policy changed during the read; retry with the current registry");
|
|
20
|
+
};
|
|
21
|
+
const store = vault.enrichment?.enabled ? readEnrichmentStore(home, vault, config.vaults, config.denyPaths ?? []) : undefined;
|
|
22
|
+
return { config, vault, store, check };
|
|
23
|
+
}
|
|
24
|
+
function allowed(ctx: ReturnType<typeof queryContext>, path: string): boolean {
|
|
25
|
+
const file = join(ctx.vault.path, "notes", path);
|
|
26
|
+
return !isDenied(file, ctx.config.denyPaths ?? []) && !isDenied(canonicalPath(file), ctx.config.denyPaths ?? []);
|
|
27
|
+
}
|
|
28
|
+
function candidates(ctx: ReturnType<typeof queryContext>, index: VaultIndex, query: QueryVector) {
|
|
29
|
+
const { store } = ctx;
|
|
30
|
+
if (!store) return [];
|
|
31
|
+
const vectors = store.records.filter(r => r.kind === "embedding");
|
|
32
|
+
if (!vectors.length) return [];
|
|
33
|
+
if (!query || query.model !== store.model || !Array.isArray(query.values) || query.values.length !== store.dimension) throw new Error("enrichment: query vector model or dimension differs from stored vectors; use the recorded model and dimension");
|
|
34
|
+
try { vectorSimilarity(query.values, query.values); } catch { throw new Error("enrichment: query vector is invalid or zero; provide finite nonzero values"); }
|
|
35
|
+
const metadata = new Map(index.enrichmentPointers().filter(p => allowed(ctx, p.path)).map(p => [p.path, p]));
|
|
36
|
+
return vectors.flatMap(r => {
|
|
37
|
+
const id = r.path.slice(6), note = metadata.get(id);
|
|
38
|
+
if (!note || note.hash !== r.contentHash || !Array.isArray(r.value)) return [];
|
|
39
|
+
return [{ id, score: vectorSimilarity(query.values, r.value as number[]), note }];
|
|
40
|
+
}).sort((a, b) => b.score - a.score || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
|
|
41
|
+
}
|
|
42
|
+
export function hybridSearch(home: string, vault: string, index: VaultIndex, query: string, options: SearchOptions, vector: QueryVector): SearchResult[] {
|
|
43
|
+
const limit = resultLimit(options.limit), ctx = queryContext(home, vault);
|
|
44
|
+
if (index.vault.id !== ctx.vault.id || index.vault.path !== ctx.vault.path || !index.sourceIdentityCurrent()) throw new Error("enrichment: index is unscanned or belongs to another vault; reopen and scan the selected vault before retrying");
|
|
45
|
+
const lexical = index.search(query, { ...options, limit: 500 }).filter(note => allowed(ctx, note.path));
|
|
46
|
+
const hits = candidates(ctx, index, vector).filter(h => (!options.type || h.note.type === options.type) && (!options.domain || h.note.domain === options.domain));
|
|
47
|
+
if (!hits.length) { ctx.check(); if (!index.sourceIdentityCurrent()) throw new Error("enrichment: index source changed during the read; reopen the vault and retry"); return lexical.slice(0, limit); }
|
|
48
|
+
const vectorHits = hits.slice(0, 9500), ranked = rankHybridCandidates(lexical.map((v, i) => ({ id: v.path, rank: i + 1 })), vectorHits.map((v, i) => ({ id: v.id, rank: i + 1 })));
|
|
49
|
+
const notes = new Map(vectorHits.map(h => [h.id, h.note]));
|
|
50
|
+
for (const note of lexical) notes.set(note.path, { ...note, hash: "" });
|
|
51
|
+
const result = ranked.slice(0, limit).map(r => { const { hash: _, ...note } = notes.get(r.id)!; return { ...note, score: r.score }; });
|
|
52
|
+
ctx.check(); if (!index.sourceIdentityCurrent()) throw new Error("enrichment: index source changed during the read; reopen the vault and retry"); return result;
|
|
53
|
+
}
|
|
54
|
+
export function similarNotes(home: string, vault: string, index: VaultIndex, path: string, limit = 20) {
|
|
55
|
+
if (!Number.isSafeInteger(limit) || limit < 1 || limit > 100) throw new Error("enrichment: similar limit must be 1..100; choose a smaller page");
|
|
56
|
+
const ctx = queryContext(home, vault), { vault: spec, store } = ctx;
|
|
57
|
+
if (index.vault.id !== spec.id || index.vault.path !== spec.path || !index.sourceIdentityCurrent()) throw new Error("enrichment: index is unscanned or belongs to another vault; reopen and scan the selected vault before retrying");
|
|
58
|
+
if (!store) throw new Error("enrichment: enrichment is disabled; enable it in settings");
|
|
59
|
+
const id = index.resolve(path), pointer = index.enrichmentPointers().find(p => p.path === id);
|
|
60
|
+
if (!allowed(ctx, id)) throw new Error("enrichment: source note is denied; select an allowed note");
|
|
61
|
+
const source = store.records.find(r => r.kind === "embedding" && r.path === `notes/${id}` && r.contentHash === pointer?.hash);
|
|
62
|
+
if (!source || !Array.isArray(source.value) || !store.model) throw new Error("enrichment: no current vector exists for this note; run enrichment for the current content");
|
|
63
|
+
const result = candidates(ctx, index, { model: store.model, values: source.value as number[] }).filter(hit => hit.id !== id).slice(0, limit).map(({ note, score }) => ({ path: note.path, title: note.title, score }));
|
|
64
|
+
ctx.check(); if (!index.sourceIdentityCurrent()) throw new Error("enrichment: index source changed during the read; reopen the vault and retry"); return result;
|
|
65
|
+
}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
/** Pure, bounded helpers for combining lexical and vector search results. */
|
|
2
|
+
|
|
3
|
+
export const MAX_VECTOR_DIMENSIONS = 4096;
|
|
4
|
+
export const MAX_RANK_CANDIDATES = 10_000;
|
|
5
|
+
const RRF_K = 60;
|
|
6
|
+
|
|
7
|
+
/** Cosine similarity for two finite, non-zero vectors of equal bounded size. */
|
|
8
|
+
export function vectorSimilarity(left: readonly number[], right: readonly number[]): number {
|
|
9
|
+
if (!Array.isArray(left) || !Array.isArray(right) || left.length === 0 || right.length === 0) {
|
|
10
|
+
throw new TypeError("vectors must be nonempty arrays");
|
|
11
|
+
}
|
|
12
|
+
if (left.length > MAX_VECTOR_DIMENSIONS || right.length > MAX_VECTOR_DIMENSIONS) {
|
|
13
|
+
throw new RangeError(`vector dimensions must not exceed ${MAX_VECTOR_DIMENSIONS}`);
|
|
14
|
+
}
|
|
15
|
+
if (left.length !== right.length) throw new RangeError("vector dimensions do not match");
|
|
16
|
+
let leftScale = 0, rightScale = 0;
|
|
17
|
+
for (let i = 0; i < left.length; i++) {
|
|
18
|
+
const a = left[i], b = right[i];
|
|
19
|
+
if (!Number.isFinite(a) || !Number.isFinite(b)) throw new TypeError("vector values must be finite numbers");
|
|
20
|
+
leftScale = Math.max(leftScale, Math.abs(a));
|
|
21
|
+
rightScale = Math.max(rightScale, Math.abs(b));
|
|
22
|
+
}
|
|
23
|
+
if (leftScale === 0 || rightScale === 0) throw new RangeError("zero vectors are not supported");
|
|
24
|
+
let dot = 0, leftNorm = 0, rightNorm = 0;
|
|
25
|
+
for (let i = 0; i < left.length; i++) {
|
|
26
|
+
const a = left[i] / leftScale, b = right[i] / rightScale;
|
|
27
|
+
dot += a * b;
|
|
28
|
+
leftNorm += a * a;
|
|
29
|
+
rightNorm += b * b;
|
|
30
|
+
}
|
|
31
|
+
return dot / Math.sqrt(leftNorm * rightNorm);
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export interface RankedCandidate { id: string; rank: number }
|
|
35
|
+
export interface HybridRankedCandidate { id: string; score: number; lexicalRank?: number; vectorRank?: number }
|
|
36
|
+
|
|
37
|
+
/** Reciprocal-rank fusion with lexical then id tie-breaking for stable output. */
|
|
38
|
+
export function rankHybridCandidates(
|
|
39
|
+
lexical: readonly RankedCandidate[],
|
|
40
|
+
vector: readonly RankedCandidate[],
|
|
41
|
+
): HybridRankedCandidate[] {
|
|
42
|
+
if (!Array.isArray(lexical) || !Array.isArray(vector)) throw new TypeError("candidate lists must be arrays");
|
|
43
|
+
if (lexical.length > MAX_RANK_CANDIDATES || vector.length > MAX_RANK_CANDIDATES || lexical.length + vector.length > MAX_RANK_CANDIDATES) {
|
|
44
|
+
throw new RangeError(`candidate count must not exceed ${MAX_RANK_CANDIDATES}`);
|
|
45
|
+
}
|
|
46
|
+
const merged = new Map<string, { lexicalRank?: number; vectorRank?: number }>();
|
|
47
|
+
const add = (items: readonly RankedCandidate[], field: "lexicalRank" | "vectorRank") => {
|
|
48
|
+
for (const item of items) {
|
|
49
|
+
if (!item || typeof item.id !== "string" || item.id.length === 0 || !Number.isSafeInteger(item.rank) || item.rank < 1 || item.rank > MAX_RANK_CANDIDATES) {
|
|
50
|
+
throw new TypeError(`each candidate needs a nonempty id and rank in 1..${MAX_RANK_CANDIDATES}`);
|
|
51
|
+
}
|
|
52
|
+
let entry = merged.get(item.id);
|
|
53
|
+
if (!entry) {
|
|
54
|
+
entry = {};
|
|
55
|
+
merged.set(item.id, entry);
|
|
56
|
+
}
|
|
57
|
+
if (entry[field] !== undefined) throw new TypeError(`duplicate candidate id in ${field === "lexicalRank" ? "lexical" : "vector"} results`);
|
|
58
|
+
entry[field] = item.rank;
|
|
59
|
+
}
|
|
60
|
+
};
|
|
61
|
+
add(lexical, "lexicalRank");
|
|
62
|
+
add(vector, "vectorRank");
|
|
63
|
+
return Array.from(merged, ([id, ranks]) => ({
|
|
64
|
+
id,
|
|
65
|
+
score: (ranks.lexicalRank === undefined ? 0 : 1 / (RRF_K + ranks.lexicalRank)) +
|
|
66
|
+
(ranks.vectorRank === undefined ? 0 : 1 / (RRF_K + ranks.vectorRank)),
|
|
67
|
+
...ranks,
|
|
68
|
+
})).sort((a, b) => b.score - a.score || (a.lexicalRank ?? Infinity) - (b.lexicalRank ?? Infinity) || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
|
|
69
|
+
}
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
/** Bounded, vault-bound local enrichment storage. No model is called here. */
|
|
2
|
+
import { closeSync, constants, fstatSync, fsyncSync, lstatSync, mkdirSync, openSync, readSync, realpathSync, writeFileSync } from "node:fs";
|
|
3
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
4
|
+
import { hostname } from "node:os";
|
|
5
|
+
import { isAbsolute, join, relative, resolve } from "node:path";
|
|
6
|
+
import { isDenied } from "./platform.ts";
|
|
7
|
+
import { loadRegistry, LOCK_FILE, syncDirectory } from "./registry.ts";
|
|
8
|
+
import { MUTATION_LOCK } from "./writer.ts";
|
|
9
|
+
import { logBinding, matchesBinding } from "./vault-log.ts";
|
|
10
|
+
import { capturePublicationPath, cleanupOwnedFile, publishFile } from "./write-files.ts";
|
|
11
|
+
import type { VaultSpec } from "./config.ts";
|
|
12
|
+
|
|
13
|
+
export const MAX_ENRICHMENT_STORE_BYTES = 128 * 1024 * 1024;
|
|
14
|
+
export const MAX_ENRICHMENT_RECORDS = 50000;
|
|
15
|
+
export const MAX_EMBEDDING_DIMENSION = 4096;
|
|
16
|
+
export type EnrichmentKind = "embedding" | "summary" | "community-label" | "inferred-link" | "fact";
|
|
17
|
+
export interface EnrichmentProducer { skill: string; version: string; model: string }
|
|
18
|
+
export interface EnrichmentRecord { path: string; contentHash: string; kind: EnrichmentKind; producer: EnrichmentProducer; createdAt: string; value: unknown }
|
|
19
|
+
export interface EnrichmentStore { schemaVersion: 1; binding: string; model?: string; dimension?: number; records: EnrichmentRecord[] }
|
|
20
|
+
function fail(message: string): never { throw new Error(`enrichment: ${message}`); }
|
|
21
|
+
const hash = (value: string) => createHash("sha256").update(value).digest("hex");
|
|
22
|
+
const within = (root: string, target: string) => { const r = relative(root, target); return r === "" || (!isAbsolute(r) && r !== ".." && !r.startsWith(`..${process.platform === "win32" ? "\\" : "/"}`)); };
|
|
23
|
+
const digest = /^[a-f0-9]{64}$/;
|
|
24
|
+
const kinds = new Set(["embedding", "summary", "community-label", "inferred-link", "fact"]);
|
|
25
|
+
const object = (v: unknown): v is Record<string, unknown> => !!v && typeof v === "object" && !Array.isArray(v) && Object.getPrototypeOf(v) === Object.prototype;
|
|
26
|
+
const text = (v: unknown, maximum: number) => typeof v === "string" && !!v.trim() && Buffer.byteLength(v) <= maximum && !/[\p{Cc}\p{Zl}\p{Zp}]/u.test(v);
|
|
27
|
+
function validVector(value: unknown, dimension: number): value is number[] {
|
|
28
|
+
return Array.isArray(value) && value.length === dimension && value.every(n => typeof n === "number" && Number.isFinite(n)) && value.some(n => n !== 0);
|
|
29
|
+
}
|
|
30
|
+
/** Float32 encoding keeps a 1536-dimensional vector to exactly 8192 base64 bytes. */
|
|
31
|
+
function encodeVector(value: unknown, dimension: number): { encoding: "float32le"; data: string } {
|
|
32
|
+
if (!validVector(value, dimension)) fail("embedding vector is invalid; supply finite nonzero values within the dimension limit");
|
|
33
|
+
const bytes = Buffer.alloc(dimension * 4); let nonzero = false;
|
|
34
|
+
for (let i = 0; i < dimension; i++) {
|
|
35
|
+
const rounded = Math.fround(value[i]!);
|
|
36
|
+
if (!Number.isFinite(rounded)) fail("embedding value overflows Float32; use finite Float32-range values");
|
|
37
|
+
if (rounded !== 0) nonzero = true;
|
|
38
|
+
bytes.writeFloatLE(rounded, i * 4);
|
|
39
|
+
}
|
|
40
|
+
if (!nonzero) fail("embedding becomes zero in Float32; provide a nonzero representable vector");
|
|
41
|
+
return { encoding: "float32le", data: bytes.toString("base64") };
|
|
42
|
+
}
|
|
43
|
+
function decodeVector(value: unknown, dimension: number): number[] {
|
|
44
|
+
if (Array.isArray(value)) { encodeVector(value, dimension); return value; } // unpublished v1 data
|
|
45
|
+
if (!object(value) || Object.keys(value).sort().join(",") !== "data,encoding" || value.encoding !== "float32le" || typeof value.data !== "string" || value.data.length !== Math.ceil(dimension * 4 / 3) * 4) fail("sidecar vector encoding is invalid; inspect the local enrichment store");
|
|
46
|
+
const bytes = Buffer.from(value.data, "base64");
|
|
47
|
+
if (bytes.length !== dimension * 4 || bytes.toString("base64") !== value.data) fail("sidecar vector encoding is invalid; inspect the local enrichment store");
|
|
48
|
+
const vector = Array.from({ length: dimension }, (_, i) => bytes.readFloatLE(i * 4));
|
|
49
|
+
if (!validVector(vector, dimension)) fail("sidecar vector is invalid; inspect the local enrichment store");
|
|
50
|
+
return vector;
|
|
51
|
+
}
|
|
52
|
+
function parseStore(value: unknown, binding: string): EnrichmentStore {
|
|
53
|
+
if (!object(value) || Object.keys(value).some(k => !["schemaVersion", "binding", "model", "dimension", "records"].includes(k)) || value.schemaVersion !== 1 || value.binding !== binding || !Array.isArray(value.records) || value.records.length > MAX_ENRICHMENT_RECORDS) fail("sidecar metadata is invalid or bound to another vault; inspect the local enrichment store");
|
|
54
|
+
if ((value.model === undefined) !== (value.dimension === undefined) || (value.model !== undefined && (!text(value.model, 256) || !Number.isSafeInteger(value.dimension) || (value.dimension as number) < 1 || (value.dimension as number) > MAX_EMBEDDING_DIMENSION))) fail("sidecar model metadata is invalid; inspect the local enrichment store");
|
|
55
|
+
const seen = new Set<string>();
|
|
56
|
+
for (const item of value.records) {
|
|
57
|
+
if (!object(item) || Object.keys(item).some(k => !["path", "contentHash", "kind", "producer", "createdAt", "value"].includes(k)) || typeof item.path !== "string" || !/^notes\/(?!\.)[^/\\\p{Cc}]+\.md$/u.test(item.path) || Buffer.byteLength(item.path) > 1024 || typeof item.contentHash !== "string" || !digest.test(item.contentHash) || typeof item.kind !== "string" || !kinds.has(item.kind) || !object(item.producer) || Object.keys(item.producer).sort().join(",") !== "model,skill,version" || !text(item.producer.skill, 128) || !text(item.producer.version, 128) || !text(item.producer.model, 256) || typeof item.createdAt !== "string" || Number.isNaN(Date.parse(item.createdAt)) || new Date(item.createdAt).toISOString() !== item.createdAt || !("value" in item)) fail("sidecar record is invalid; inspect the local enrichment store");
|
|
58
|
+
const key = JSON.stringify([item.path, item.kind, item.contentHash]);
|
|
59
|
+
if (seen.has(key)) fail("sidecar contains duplicate records; inspect the local enrichment store");
|
|
60
|
+
seen.add(key);
|
|
61
|
+
if (item.kind === "embedding") {
|
|
62
|
+
if (item.producer.model !== value.model || typeof value.dimension !== "number") fail("sidecar vector has a different model or dimension; inspect the local enrichment store");
|
|
63
|
+
item.value = decodeVector(item.value, value.dimension);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
return value as unknown as EnrichmentStore;
|
|
67
|
+
}
|
|
68
|
+
function location(home: string, vault: VaultSpec, roots: VaultSpec[], denyPaths: string[], create: boolean) {
|
|
69
|
+
const configured = resolve(home);
|
|
70
|
+
if (isDenied(configured, denyPaths)) fail("home is denied; choose an allowed VMEM_HOME");
|
|
71
|
+
let root: string;
|
|
72
|
+
try { root = realpathSync(configured); } catch { return fail("home is unavailable; restore the configured VMEM_HOME and retry"); }
|
|
73
|
+
let protectedRoots: string[];
|
|
74
|
+
try { protectedRoots = roots.map(v => realpathSync(v.path)); } catch { return fail("a registered vault root is unavailable; restore it or unregister it before using enrichment"); }
|
|
75
|
+
if (isDenied(root, denyPaths) || protectedRoots.some(path => within(path, root))) fail("home is denied or inside a vault; choose an allowed VMEM_HOME outside all vaults");
|
|
76
|
+
let stat: ReturnType<typeof lstatSync>;
|
|
77
|
+
try { stat = lstatSync(vault.path); } catch { return fail("vault root is unavailable; inspect the registry and restore the vault"); }
|
|
78
|
+
if (!stat.isDirectory() || stat.isSymbolicLink() || realpathSync(vault.path) !== vault.path || isDenied(vault.path, denyPaths)) fail("vault root is linked, changed or denied; inspect the registry and policy");
|
|
79
|
+
const binding = hash(JSON.stringify([vault.id, vault.path, stat.dev, stat.ino]));
|
|
80
|
+
const dir = join(root, "enrichment"), file = join(dir, `${binding}.json`);
|
|
81
|
+
if (isDenied(dir, denyPaths) || isDenied(file, denyPaths)) fail("sidecar is denied; inspect configured deny paths");
|
|
82
|
+
if (create) {
|
|
83
|
+
try { mkdirSync(dir, { mode: 0o700 }); syncDirectory(root); }
|
|
84
|
+
catch (error) { if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; }
|
|
85
|
+
}
|
|
86
|
+
try {
|
|
87
|
+
const d = lstatSync(dir);
|
|
88
|
+
if (!d.isDirectory() || d.isSymbolicLink() || (d.mode & 0o077) !== 0 || (typeof process.getuid === "function" && d.uid !== process.getuid())) fail("sidecar directory is linked or not private; inspect VMEM_HOME/enrichment");
|
|
89
|
+
} catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT" || create) throw error; }
|
|
90
|
+
return { root, dir, file, binding };
|
|
91
|
+
}
|
|
92
|
+
function readFile(loc: ReturnType<typeof location>): EnrichmentStore {
|
|
93
|
+
const empty: EnrichmentStore = { schemaVersion: 1, binding: loc.binding, records: [] };
|
|
94
|
+
let fd: number;
|
|
95
|
+
try { fd = openSync(loc.file, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); }
|
|
96
|
+
catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return empty; throw new Error("enrichment: sidecar cannot be read; inspect VMEM_HOME/enrichment", { cause: error }); }
|
|
97
|
+
try {
|
|
98
|
+
const captured = capturePublicationPath(loc.root, loc.file);
|
|
99
|
+
const before = fstatSync(fd);
|
|
100
|
+
if (!before.isFile() || before.nlink !== 1 || (before.mode & 0o077) !== 0 || (typeof process.getuid === "function" && before.uid !== process.getuid()) || before.size > MAX_ENRICHMENT_STORE_BYTES) fail("sidecar is linked, not private or exceeds its byte limit; inspect the local enrichment store");
|
|
101
|
+
const bytes = Buffer.alloc(before.size + 1); let count = 0;
|
|
102
|
+
while (count < bytes.length) { const n = readSync(fd, bytes, count, bytes.length - count, null); if (!n) break; count += n; }
|
|
103
|
+
const after = fstatSync(fd), named = lstatSync(loc.file);
|
|
104
|
+
if (count !== before.size || after.size !== before.size || before.ino !== after.ino || before.dev !== after.dev || after.ino !== named.ino || after.dev !== named.dev || before.mtimeMs !== after.mtimeMs || before.ctimeMs !== after.ctimeMs || JSON.stringify(captured.parents) !== JSON.stringify(capturePublicationPath(loc.root, loc.file).parents)) fail("sidecar changed during read; stop concurrent edits and retry");
|
|
105
|
+
const raw = bytes.subarray(0, count), content = raw.toString("utf8");
|
|
106
|
+
if (!Buffer.from(content).equals(raw)) fail("sidecar is not UTF-8; inspect the local enrichment store");
|
|
107
|
+
let parsed: unknown; try { parsed = JSON.parse(content); } catch { return fail("sidecar is not valid JSON; inspect the local enrichment store"); }
|
|
108
|
+
return parseStore(parsed, loc.binding);
|
|
109
|
+
} finally { closeSync(fd); }
|
|
110
|
+
}
|
|
111
|
+
export function readEnrichmentStore(home: string, vault: VaultSpec, roots: VaultSpec[], denyPaths: string[]): EnrichmentStore {
|
|
112
|
+
return readFile(location(home, vault, roots, denyPaths, false));
|
|
113
|
+
}
|
|
114
|
+
/** Validate complete metadata and vector shape before any side effects. */
|
|
115
|
+
export function validateEnrichmentRecord(record: EnrichmentRecord): void {
|
|
116
|
+
if (!record || typeof record !== "object") fail("intake record is missing; supply a complete result");
|
|
117
|
+
const provisional: EnrichmentStore = { schemaVersion: 1, binding: "0".repeat(64), records: [record] };
|
|
118
|
+
if (record.kind === "embedding") {
|
|
119
|
+
if (!object(record.producer) || !Array.isArray(record.value) || !validVector(record.value, record.value.length) || record.value.length < 1 || record.value.length > MAX_EMBEDDING_DIMENSION) fail("embedding is invalid; supply a producer and a finite nonzero vector");
|
|
120
|
+
encodeVector(record.value, record.value.length);
|
|
121
|
+
provisional.model = record.producer.model; provisional.dimension = record.value.length;
|
|
122
|
+
}
|
|
123
|
+
parseStore(provisional, provisional.binding);
|
|
124
|
+
}
|
|
125
|
+
/** Serialize under the registry lock; callers must recheck the note hash before returning their receipt. */
|
|
126
|
+
export function writeEnrichmentRecord(home: string, vault: VaultSpec, record: EnrichmentRecord, validateCurrent: () => void, validateStore: (store: EnrichmentStore) => void = () => {}, heldMutation?: number): { applied: true; issues: string[] } {
|
|
127
|
+
validateEnrichmentRecord(record);
|
|
128
|
+
const config = loadRegistry(home), currentVault = config.vaults.find(v => v.id === vault.id);
|
|
129
|
+
if (!currentVault || JSON.stringify(currentVault) !== JSON.stringify(vault)) fail("vault registration changed; reload settings and retry");
|
|
130
|
+
if (!vault.enrichment?.enabled) fail("enrichment is disabled for this vault; enable it in settings first");
|
|
131
|
+
location(home, vault, config.vaults, config.denyPaths ?? [], false);
|
|
132
|
+
const binding = logBinding(vault, config), lock = join(resolve(home), LOCK_FILE);
|
|
133
|
+
if (isDenied(lock, config.denyPaths ?? [])) fail("registry lock is denied; inspect configured deny paths");
|
|
134
|
+
const mutation = join(vault.path, MUTATION_LOCK);
|
|
135
|
+
if (isDenied(mutation, config.denyPaths ?? [])) fail("vault mutation lock is denied; inspect configured deny paths");
|
|
136
|
+
let mutationFd: number;
|
|
137
|
+
if (heldMutation !== undefined) mutationFd = heldMutation;
|
|
138
|
+
else {
|
|
139
|
+
try { mutationFd = openSync(mutation, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600); }
|
|
140
|
+
catch (error) { throw new Error("enrichment: vault mutation lock is unavailable; stop concurrent writers and inspect the lock", { cause: error }); }
|
|
141
|
+
}
|
|
142
|
+
const verifyMutation = () => {
|
|
143
|
+
const held = fstatSync(mutationFd), named = lstatSync(mutation);
|
|
144
|
+
if (!held.isFile() || named.isSymbolicLink() || held.dev !== named.dev || held.ino !== named.ino) fail("vault mutation lock ownership changed; inspect the lock before retrying");
|
|
145
|
+
};
|
|
146
|
+
let fd: number | undefined, published = false, failure: unknown;
|
|
147
|
+
const issues: string[] = [];
|
|
148
|
+
try {
|
|
149
|
+
if (heldMutation === undefined) { writeFileSync(mutationFd, JSON.stringify({ pid: process.pid, host: hostname(), created: new Date().toISOString() }) + "\n"); fsyncSync(mutationFd); }
|
|
150
|
+
verifyMutation();
|
|
151
|
+
try { fd = openSync(lock, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600); }
|
|
152
|
+
catch (error) { throw new Error("enrichment: registry lock is unavailable; stop concurrent writers and inspect the lock", { cause: error }); }
|
|
153
|
+
writeFileSync(fd, JSON.stringify({ pid: process.pid, host: hostname(), created: new Date().toISOString() }) + "\n"); fsyncSync(fd);
|
|
154
|
+
const current = loadRegistry(home);
|
|
155
|
+
if (!matchesBinding(current, vault.id, binding)) fail("vault binding or policy changed; reload settings and retry");
|
|
156
|
+
validateCurrent();
|
|
157
|
+
const loc = location(home, vault, current.vaults, current.denyPaths ?? [], true), store = readFile(loc);
|
|
158
|
+
validateStore(store);
|
|
159
|
+
if (record.kind === "embedding") {
|
|
160
|
+
if (!Array.isArray(record.value) || record.value.length < 1 || record.value.length > MAX_EMBEDDING_DIMENSION || !validVector(record.value, record.value.length)) fail("embedding vector is invalid; supply finite nonzero values within the dimension limit");
|
|
161
|
+
if (store.model !== undefined && (store.model !== record.producer.model || store.dimension !== record.value.length)) fail("embedding model or dimension differs from this vault's sidecar; use the recorded model and dimension");
|
|
162
|
+
store.model = record.producer.model; store.dimension = record.value.length;
|
|
163
|
+
}
|
|
164
|
+
const replaceLatest = ["embedding", "summary", "community-label"].includes(record.kind);
|
|
165
|
+
store.records = store.records.filter(r => r.path !== record.path || r.kind !== record.kind || (!replaceLatest && r.contentHash !== record.contentHash));
|
|
166
|
+
store.records.push(record);
|
|
167
|
+
parseStore(store, loc.binding);
|
|
168
|
+
const persisted = { ...store, records: store.records.map(r => r.kind === "embedding" ? { ...r, value: encodeVector(r.value, store.dimension!) } : r) };
|
|
169
|
+
const bytes = Buffer.from(JSON.stringify(persisted));
|
|
170
|
+
if (bytes.length > MAX_ENRICHMENT_STORE_BYTES) fail("sidecar exceeds the aggregate byte limit; reduce stored records before retrying");
|
|
171
|
+
validateCurrent(); verifyMutation();
|
|
172
|
+
const captured = capturePublicationPath(loc.root, loc.file), temp = join(loc.dir, `.enrichment-${randomUUID()}.tmp`);
|
|
173
|
+
if (isDenied(temp, current.denyPaths ?? [])) fail("temporary sidecar is denied; inspect configured deny paths");
|
|
174
|
+
const out = openSync(temp, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600);
|
|
175
|
+
try { writeFileSync(out, bytes); fsyncSync(out); publishFile(captured, { kind: "rename", temp }, () => { published = true; }); syncDirectory(loc.dir); }
|
|
176
|
+
finally { try { cleanupOwnedFile(temp, out); } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; } finally { closeSync(out); } }
|
|
177
|
+
try { validateCurrent(); } catch { issues.push("source changed after the sidecar was published; inspect the note and regenerate the result"); }
|
|
178
|
+
if (!matchesBinding(loadRegistry(home), vault.id, binding)) issues.push("vault binding or policy changed after publication; inspect the registry and sidecar before retrying");
|
|
179
|
+
} catch (error) { failure = error; }
|
|
180
|
+
finally {
|
|
181
|
+
if (fd !== undefined) {
|
|
182
|
+
try { cleanupOwnedFile(lock, fd); } catch { issues.push("registry lock cleanup failed; inspect the lock before retrying"); }
|
|
183
|
+
try { closeSync(fd); } catch { issues.push("registry lock close failed; inspect the lock before retrying"); }
|
|
184
|
+
}
|
|
185
|
+
if (heldMutation === undefined) {
|
|
186
|
+
try { cleanupOwnedFile(mutation, mutationFd); } catch { issues.push("vault mutation lock cleanup failed; inspect the lock before retrying"); }
|
|
187
|
+
try { closeSync(mutationFd); } catch { issues.push("vault mutation lock close failed; inspect the lock before retrying"); }
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
if (!published && failure) {
|
|
191
|
+
if (issues.length) throw new Error(`enrichment: sidecar was not published; ${issues.join("; ")}`, { cause: failure });
|
|
192
|
+
throw failure;
|
|
193
|
+
}
|
|
194
|
+
if (!published) fail("sidecar was not published; inspect the local store before retrying");
|
|
195
|
+
if (failure) issues.push("sidecar was published but finalization failed; inspect the local store before retrying");
|
|
196
|
+
return { applied: true, issues };
|
|
197
|
+
}
|