@gtrabanco/agentic-workflow 0.2.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +50 -0
- package/bin/agentic-workflow.mjs +109 -4
- package/package.json +3 -3
- package/src/doc/chunks.mjs +5 -1
- package/src/doc/config.mjs +63 -19
- package/src/doc/query.mjs +57 -9
- package/src/doc/retrieval.mjs +15 -2
- package/src/doc/store.mjs +7 -9
- package/src/doc/sync.mjs +34 -17
- package/src/edit/Roadmap.mjs +135 -2
- package/src/edit/UnitDoc.mjs +199 -11
- package/src/edit/schema.mjs +45 -1
- package/src/memory/probe.mjs +388 -0
package/README.md
CHANGED
|
@@ -21,3 +21,53 @@ repository's `scripts/` tree (feature 37 ED1). Feature 38's
|
|
|
21
21
|
crate existed, so re-homing it here is a recorded follow-up (feature 37
|
|
22
22
|
ED8.4c) with no unit assigned yet; feature 42 is the next candidate to add the
|
|
23
23
|
first crate subcommand.
|
|
24
|
+
|
|
25
|
+
## `memory probe` — bounded memory-substrate health (feature 66)
|
|
26
|
+
|
|
27
|
+
One command answers "can this session trust the Engram memory layer?" before
|
|
28
|
+
its first memory read. It writes nothing, issues no MCP tool call (only
|
|
29
|
+
`initialize`), and **always exits 0** — a `degraded` verdict is an answer to
|
|
30
|
+
record, never a failure that blocks a unit.
|
|
31
|
+
|
|
32
|
+
```sh
|
|
33
|
+
agentic-workflow memory probe # human block
|
|
34
|
+
agentic-workflow memory probe --json # memory-probe@1 verdict
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
Five checks, fixed order. `ok` requires all five; any failure makes the whole
|
|
38
|
+
verdict `degraded` and names the failing checks:
|
|
39
|
+
|
|
40
|
+
| check | source | bound |
|
|
41
|
+
|---|---|---|
|
|
42
|
+
| `http-health` | `GET http://127.0.0.1:7437/health`, body `status` must be `ok` | 4 s |
|
|
43
|
+
| `mcp-runner` | one JSON-RPC `initialize` to `engram mcp --tools=agent` | 4 s |
|
|
44
|
+
| `binary-version` | `engram --version` | 4 s |
|
|
45
|
+
| `sync-staleness` | `engram sync --status`, `Pending import` must be 0 | 10 s |
|
|
46
|
+
| `session-ambiguity` | `engram doctor --json --check` on `ambiguous_active_runtime_sessions` + `sync_target_closed_space`, every non-`ok` answer reported | 10 s |
|
|
47
|
+
|
|
48
|
+
The checks are independent, so they run together: the probe costs the slowest
|
|
49
|
+
bound (10 s), never the sum of five (32 s). Whole-probe ceiling: 30 s, which is
|
|
50
|
+
the backstop for a dependency that ignored its own bound. A bound hit fails that
|
|
51
|
+
check and degrades the verdict — never a hang.
|
|
52
|
+
|
|
53
|
+
Three rules are load-bearing and measured, not assumed:
|
|
54
|
+
|
|
55
|
+
- **Exit code is not a health signal.** `engram doctor` exits 0 while its JSON
|
|
56
|
+
reports `status: "error"`, so every check parses the answer instead of
|
|
57
|
+
trusting the status code.
|
|
58
|
+
- **A responding server that has not synced is not `ok`.** That is why
|
|
59
|
+
`sync-staleness` is a check, and why an unreadable status (e.g. `ambiguous
|
|
60
|
+
project` outside a repo) is a degradation rather than a pass.
|
|
61
|
+
- **One answer never hides another.** The session check asks both scoped
|
|
62
|
+
`--check` ids and reports every non-`ok` finding. Stopping at the first would
|
|
63
|
+
silence the sync-target signal permanently on a machine whose session
|
|
64
|
+
attribution is ambiguous — which is the state this unit's own SPEC records.
|
|
65
|
+
- **`degraded` is reportable, not fatal.** The caller records the verdict in
|
|
66
|
+
the unit's Evidence and keeps working.
|
|
67
|
+
|
|
68
|
+
The full `engram doctor --json` sweep is deliberately excluded: it costs 12–23 s
|
|
69
|
+
at this repository and the two scoped `--check` calls answer the same question
|
|
70
|
+
in under half a second each. `runProbe({ fetchImpl, runImpl, timeoutMs,
|
|
71
|
+
totalTimeoutMs })` takes its side effects by injection, so the suite drives
|
|
72
|
+
every degraded path with fakes and millisecond bounds — no network, no PATH, no
|
|
73
|
+
real engram process.
|
package/bin/agentic-workflow.mjs
CHANGED
|
@@ -9,24 +9,30 @@
|
|
|
9
9
|
* agentic-workflow budgets <op> [args...]
|
|
10
10
|
* agentic-workflow manifest <op> [args...]
|
|
11
11
|
* agentic-workflow doc (--sync | --query <term> | --status | --rebuild) [--json-only]
|
|
12
|
+
* agentic-workflow memory probe [--json]
|
|
12
13
|
*
|
|
13
14
|
* --json prints the operation receipt as JSON.
|
|
14
15
|
* Every operation prints a fixed receipt block to stdout.
|
|
15
16
|
*
|
|
16
17
|
* Refusals exit 2 with "EDIT REFUSAL — <reason>" before writing anything.
|
|
18
|
+
*
|
|
19
|
+
* `memory probe` is a sensor, not an editor: it writes nothing and always
|
|
20
|
+
* exits 0. Its `degraded` verdict is an answer to report, never a failure
|
|
21
|
+
* that blocks the caller (feature 66, AC2).
|
|
17
22
|
*/
|
|
18
23
|
|
|
19
|
-
import { readFileSync } from "node:fs";
|
|
20
|
-
import { join, resolve, basename } from "node:path";
|
|
24
|
+
import { readFileSync, existsSync } from "node:fs";
|
|
25
|
+
import { join, resolve, basename, dirname } from "node:path";
|
|
21
26
|
|
|
22
27
|
// Import SDK modules
|
|
23
28
|
import { create, setSection, evidence_addRow, progress_logEntry, validate } from "../src/edit/UnitDoc.mjs";
|
|
24
|
-
import { rowUpsert, annotate as roadmapAnnotate } from "../src/edit/Roadmap.mjs";
|
|
29
|
+
import { rowUpsert, annotate as roadmapAnnotate, checkUnitDoc, ROADMAP_CHECK_CODES } from "../src/edit/Roadmap.mjs";
|
|
25
30
|
import { rowAdd } from "../src/edit/Changelog.mjs";
|
|
26
31
|
import { ceilingRebase } from "../src/edit/Budgets.mjs";
|
|
27
32
|
import { skillAdd, skillRemove } from "../src/edit/Manifest.mjs";
|
|
28
33
|
import { parseDocArgs } from "../src/doc/grammar.mjs";
|
|
29
34
|
import { runDocOp } from "../src/doc/retrieval.mjs";
|
|
35
|
+
import { runProbe } from "../src/memory/probe.mjs";
|
|
30
36
|
|
|
31
37
|
// ── Helpers ──────────────────────────────────────────────────────────────
|
|
32
38
|
|
|
@@ -62,11 +68,26 @@ function refuse(reason) {
|
|
|
62
68
|
process.exit(2);
|
|
63
69
|
}
|
|
64
70
|
|
|
71
|
+
/**
|
|
72
|
+
* Walk up from a unit doc to the repo root until the default index (e.g.
|
|
73
|
+
* `docs/features/ROADMAP.md`) exists; null when no ancestor carries it.
|
|
74
|
+
*/
|
|
75
|
+
function findUpIndex(docPath, relativeIndex) {
|
|
76
|
+
let dir = dirname(docPath);
|
|
77
|
+
for (;;) {
|
|
78
|
+
const candidate = join(dir, relativeIndex);
|
|
79
|
+
if (existsSync(candidate)) return candidate;
|
|
80
|
+
const parent = dirname(dir);
|
|
81
|
+
if (parent === dir) return null;
|
|
82
|
+
dir = parent;
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
65
86
|
// ── Router ───────────────────────────────────────────────────────────────
|
|
66
87
|
|
|
67
88
|
const command = process.argv[2];
|
|
68
89
|
if (!command) {
|
|
69
|
-
console.error("usage: agentic-workflow <unit-doc|roadmap|changelog|budgets|manifest|doc> ...");
|
|
90
|
+
console.error("usage: agentic-workflow <unit-doc|roadmap|changelog|budgets|manifest|doc|memory> ...");
|
|
70
91
|
process.exit(1);
|
|
71
92
|
}
|
|
72
93
|
|
|
@@ -114,6 +135,28 @@ switch (command) {
|
|
|
114
135
|
break;
|
|
115
136
|
}
|
|
116
137
|
|
|
138
|
+
// ── memory — bounded memory-substrate health probe (unit 66, P1) ────
|
|
139
|
+
case "memory": {
|
|
140
|
+
const op = rawArgs[0];
|
|
141
|
+
if (op !== "probe") refuse(`memory requires the \`probe\` operation, got "${op ?? ""}"`);
|
|
142
|
+
const report = await runProbe();
|
|
143
|
+
if (parsed.json) {
|
|
144
|
+
process.stdout.write(JSON.stringify(report) + "\n");
|
|
145
|
+
} else {
|
|
146
|
+
console.log(`MEMORY PROBE — ${report.verdict}`);
|
|
147
|
+
for (const check of report.checks) {
|
|
148
|
+
const mark = check.status === "ok" ? "ok" : `failed:${check.reason}`;
|
|
149
|
+
console.log(` ${check.id} · ${mark} · ${check.detail}`);
|
|
150
|
+
}
|
|
151
|
+
if (report.degraded.length > 0) {
|
|
152
|
+
console.log(`Degraded: ${report.degraded.join(", ")} — record this in Evidence, keep working`);
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
// A sensor that exits non-zero for a healthy machine's degraded layer
|
|
156
|
+
// would block the very units it exists to protect (AC2 non-goal).
|
|
157
|
+
break;
|
|
158
|
+
}
|
|
159
|
+
|
|
117
160
|
// ── unit-doc ─────────────────────────────────────────────────────────
|
|
118
161
|
case "unit-doc": {
|
|
119
162
|
const slug = rawArgs[0];
|
|
@@ -182,6 +225,14 @@ switch (command) {
|
|
|
182
225
|
|
|
183
226
|
// ── roadmap ──────────────────────────────────────────────────────────
|
|
184
227
|
case "roadmap": {
|
|
228
|
+
// `rowUpsert`/`annotate` take a table path first: a bare first positional is
|
|
229
|
+
// a missing path, never a filename. Resolving the subcommand word would make
|
|
230
|
+
// `roadmap rowUpsert …` open a file literally named `rowUpsert` and refuse
|
|
231
|
+
// with ENOENT, which reads as a missing table instead of a usage error.
|
|
232
|
+
if (rawArgs[0] === "rowUpsert" || rawArgs[0] === "annotate") {
|
|
233
|
+
process.stderr.write(`usage: agentic-workflow roadmap <table-path> ${rawArgs[0]} <json>\n`);
|
|
234
|
+
process.exit(2);
|
|
235
|
+
}
|
|
185
236
|
const filePath = rawArgs[0] ? resolve(rawArgs[0]) : join(process.cwd(), "ROADMAP.md");
|
|
186
237
|
|
|
187
238
|
if (rawArgs[0] === "rowUpsert" || (rawArgs[0] && rawArgs[1] === "rowUpsert")) {
|
|
@@ -204,6 +255,60 @@ switch (command) {
|
|
|
204
255
|
} catch (e) {
|
|
205
256
|
refuse(e.message);
|
|
206
257
|
}
|
|
258
|
+
} else if (rawArgs[0] === "check" || (rawArgs[0] && rawArgs[1] === "check")) {
|
|
259
|
+
// roadmap check <unit-doc-path> [--roadmap <file>] [--index <file>] [--json]
|
|
260
|
+
// Cross-document check (issue #316): the unit doc's own facts (NN,
|
|
261
|
+
// slug, PR URL) against its index row. Exit contract: 0 consistent,
|
|
262
|
+
// 1 violations (named codes), 2 usage.
|
|
263
|
+
const docArg = rawArgs[1] === "check" ? rawArgs[2] : rawArgs[1];
|
|
264
|
+
if (!docArg) {
|
|
265
|
+
process.stderr.write("usage: agentic-workflow roadmap check <unit-doc-path> [--roadmap <file>] [--index <file>] [--json]\n");
|
|
266
|
+
process.exit(2);
|
|
267
|
+
}
|
|
268
|
+
const flag = (name) => {
|
|
269
|
+
const i = restArgs.indexOf(name);
|
|
270
|
+
return i !== -1 ? restArgs[i + 1] : undefined;
|
|
271
|
+
};
|
|
272
|
+
try {
|
|
273
|
+
const docPath = resolve(docArg);
|
|
274
|
+
const docText = readFileSync(docPath, "utf8");
|
|
275
|
+
// Kind and index defaults follow the doc's own tree position:
|
|
276
|
+
// docs/features → the roadmap, docs/fix → the fix index (its rows are
|
|
277
|
+
// removed once merged, so a missing row there is convention, not
|
|
278
|
+
// contradiction).
|
|
279
|
+
const parts = docPath.split("/");
|
|
280
|
+
const isFix = parts.includes("docs") && parts[parts.indexOf("docs") + 1] === "fix";
|
|
281
|
+
const defaultIndex = isFix ? "docs/fix/README.md" : "docs/features/ROADMAP.md";
|
|
282
|
+
const indexArg = flag("--roadmap") ?? flag("--index");
|
|
283
|
+
const indexPath = indexArg
|
|
284
|
+
? resolve(indexArg)
|
|
285
|
+
: findUpIndex(docPath, defaultIndex.split("/").pop());
|
|
286
|
+
if (!indexPath) {
|
|
287
|
+
refuse(`index not found for ${docArg} — pass --roadmap <file>`);
|
|
288
|
+
}
|
|
289
|
+
const dirName = basename(dirname(docPath));
|
|
290
|
+
const nn = /^(\d+)/.exec(dirName)?.[1] ?? "";
|
|
291
|
+
const slug = dirName.replace(/^\d+-/, "");
|
|
292
|
+
const result = checkUnitDoc({ nn, slug, kind: isFix ? "fix" : "feature", docText, indexText: readFileSync(indexPath, "utf8") });
|
|
293
|
+
if (parsed.json) {
|
|
294
|
+
console.log(
|
|
295
|
+
JSON.stringify(
|
|
296
|
+
{ service: "roadmap", op: "check", path: docPath, schema: "roadmap-table@1", codes: ROADMAP_CHECK_CODES, ok: result.ok, violations: result.violations },
|
|
297
|
+
null,
|
|
298
|
+
2,
|
|
299
|
+
),
|
|
300
|
+
);
|
|
301
|
+
} else {
|
|
302
|
+
for (const v of result.violations) {
|
|
303
|
+
console.log(`roadmap-check: ${docPath}:${v.line} ${v.code} — ${v.detail}`);
|
|
304
|
+
}
|
|
305
|
+
console.log(`verdict: ${result.ok ? "PASS" : "FAIL"}`);
|
|
306
|
+
console.log(`violations: ${result.violations.length}`);
|
|
307
|
+
}
|
|
308
|
+
process.exit(result.ok ? 0 : 1);
|
|
309
|
+
} catch (e) {
|
|
310
|
+
refuse(e.message);
|
|
311
|
+
}
|
|
207
312
|
} else {
|
|
208
313
|
refuse(`Unknown roadmap operation`);
|
|
209
314
|
}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@gtrabanco/agentic-workflow",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"description": "Deterministic runner for the agentic-workflow lane: file services (agwo edit), sensors, guards and receipts.
|
|
3
|
+
"version": "0.5.0",
|
|
4
|
+
"description": "Deterministic runner for the agentic-workflow lane: file services (agwo edit), sensors, guards and receipts. One pinned pure-JS dependency, no native deps, no build step.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "Gabriel Trabanco <1969593+gtrabanco@users.noreply.github.com>",
|
|
7
7
|
"repository": {
|
|
@@ -27,6 +27,6 @@
|
|
|
27
27
|
"README.md"
|
|
28
28
|
],
|
|
29
29
|
"dependencies": {
|
|
30
|
-
"typebox": "1.3
|
|
30
|
+
"typebox": "1.4.3"
|
|
31
31
|
}
|
|
32
32
|
}
|
package/src/doc/chunks.mjs
CHANGED
|
@@ -58,7 +58,11 @@ function parseFrontmatter(lines) {
|
|
|
58
58
|
* document order. Deterministic: identical input ⇒ identical output.
|
|
59
59
|
*/
|
|
60
60
|
export function scanChunks(markdown, relativePath) {
|
|
61
|
-
const
|
|
61
|
+
const content = String(markdown);
|
|
62
|
+
const lines = content.split(/\r?\n/);
|
|
63
|
+
// A file ending in a newline splits into one phantom trailing element; it is
|
|
64
|
+
// not a line, and counting it overshoots every last chunk's line_end (F47).
|
|
65
|
+
if (lines.length > 1 && lines[lines.length - 1] === "" && /\r?\n$/.test(content)) lines.pop();
|
|
62
66
|
|
|
63
67
|
// Frontmatter block, only when it opens the file.
|
|
64
68
|
let contentStart = 0;
|
package/src/doc/config.mjs
CHANGED
|
@@ -9,8 +9,8 @@
|
|
|
9
9
|
* an unknown version fails closed — the reader never guesses.
|
|
10
10
|
*/
|
|
11
11
|
|
|
12
|
-
import { existsSync, readFileSync } from "node:fs";
|
|
13
|
-
import { join, resolve, sep } from "node:path";
|
|
12
|
+
import { existsSync, readFileSync, realpathSync } from "node:fs";
|
|
13
|
+
import { dirname, join, resolve, sep } from "node:path";
|
|
14
14
|
|
|
15
15
|
export const CONFIG_VERSION = 1;
|
|
16
16
|
|
|
@@ -27,14 +27,63 @@ export function configPath(root) {
|
|
|
27
27
|
return join(root, ".agentic-workflow", "index.json");
|
|
28
28
|
}
|
|
29
29
|
|
|
30
|
+
/**
|
|
31
|
+
* Realpath of the deepest EXISTING ancestor of `p`, with the not-yet-existing
|
|
32
|
+
* tail re-joined lexically (F36): the configured store leaf usually does not
|
|
33
|
+
* exist yet, but its intermediate directories do — and an intermediate
|
|
34
|
+
* symlink escaping the repository must be caught before any write.
|
|
35
|
+
*/
|
|
36
|
+
function realpathDeepest(p) {
|
|
37
|
+
const tail = [];
|
|
38
|
+
let cur = p;
|
|
39
|
+
while (!existsSync(cur)) {
|
|
40
|
+
const parent = dirname(cur);
|
|
41
|
+
if (parent === cur) return cur; // filesystem root — nothing to resolve
|
|
42
|
+
tail.unshift(cur.slice(parent.length + 1));
|
|
43
|
+
cur = parent;
|
|
44
|
+
}
|
|
45
|
+
return join(realpathSync(cur), ...tail);
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Contain a configured `store.path` inside the repository's gitignored
|
|
50
|
+
* `.agentic-workflow/` directory (F36/F37, decision D12):
|
|
51
|
+
* - F37: a store outside `.agentic-workflow/` leaves `git status` dirty
|
|
52
|
+
* after `--sync` (AC7) — the configured store must stay inside the only
|
|
53
|
+
* directory the committed ignore rules cover;
|
|
54
|
+
* - F36: containment is anchored at `realpath(root)` and the candidate is
|
|
55
|
+
* resolved through the realpath of its deepest existing ancestor, so a
|
|
56
|
+
* symlinked intermediate directory (or a symlinked `.agentic-workflow`
|
|
57
|
+
* itself) fails closed — `--rebuild`'s `rmSync` targets this path, so a
|
|
58
|
+
* misconfigured store path is a destructive-write footgun.
|
|
59
|
+
*/
|
|
60
|
+
function containStorePath(root, candidate) {
|
|
61
|
+
const anchoredRoot = realpathDeepest(resolve(root));
|
|
62
|
+
const workflowRoot = join(anchoredRoot, ".agentic-workflow");
|
|
63
|
+
const resolvedCandidate = realpathDeepest(resolve(root, candidate));
|
|
64
|
+
if (!resolvedCandidate.startsWith(workflowRoot + sep)) {
|
|
65
|
+
throw new Error(
|
|
66
|
+
`index.json store.path "${candidate}" must stay inside .agentic-workflow/ (got "${resolvedCandidate}")`,
|
|
67
|
+
);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
30
71
|
/**
|
|
31
72
|
* Load the config for the project at `root`: defaults merged under the
|
|
32
|
-
* file's overrides. Throws on unparseable JSON (message names the file)
|
|
33
|
-
* on an unknown `version` (fail closed).
|
|
73
|
+
* file's overrides. Throws on unparseable JSON (message names the file),
|
|
74
|
+
* on an unknown `version` (fail closed), and on a `store.path` outside the
|
|
75
|
+
* gitignored `.agentic-workflow/` directory (F36/F37 containment).
|
|
34
76
|
*/
|
|
35
77
|
export function loadConfig(root) {
|
|
36
78
|
const path = configPath(root);
|
|
37
|
-
if (!existsSync(path))
|
|
79
|
+
if (!existsSync(path)) {
|
|
80
|
+
// F50: the absent-config branch is NOT a containment bypass — the merged
|
|
81
|
+
// default store path runs through the same check, so a symlinked
|
|
82
|
+
// `.agentic-workflow` fails closed even with no index.json on disk.
|
|
83
|
+
const merged = structuredClone(DEFAULT_CONFIG);
|
|
84
|
+
containStorePath(root, merged.store.path);
|
|
85
|
+
return merged;
|
|
86
|
+
}
|
|
38
87
|
let parsed;
|
|
39
88
|
try {
|
|
40
89
|
parsed = JSON.parse(readFileSync(path, "utf8"));
|
|
@@ -47,18 +96,13 @@ export function loadConfig(root) {
|
|
|
47
96
|
if (parsed.version !== undefined && parsed.version !== CONFIG_VERSION) {
|
|
48
97
|
throw new Error(`index.json version ${parsed.version} is not supported (expected ${CONFIG_VERSION})`);
|
|
49
98
|
}
|
|
50
|
-
// Contain store
|
|
51
|
-
//
|
|
52
|
-
//
|
|
53
|
-
//
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
`index.json store.path "${candidate}" escapes the repository root (${root})`,
|
|
60
|
-
);
|
|
61
|
-
}
|
|
62
|
-
}
|
|
63
|
-
return { ...structuredClone(DEFAULT_CONFIG), ...parsed };
|
|
99
|
+
// Contain the MERGED store path inside the repository's gitignored
|
|
100
|
+
// `.agentic-workflow/` directory (F36/F37/F50, decision D12) — see
|
|
101
|
+
// containStorePath. Runs on every branch: an explicit `store.path`, a
|
|
102
|
+
// config without a `store` key, and the absent-config default alike.
|
|
103
|
+
const merged = { ...structuredClone(DEFAULT_CONFIG), ...parsed };
|
|
104
|
+
// storePath() falls back to the default when `store.path` is falsy —
|
|
105
|
+
// contain exactly the path that will be used.
|
|
106
|
+
containStorePath(root, merged.store?.path || DEFAULT_CONFIG.store.path);
|
|
107
|
+
return merged;
|
|
64
108
|
}
|
package/src/doc/query.mjs
CHANGED
|
@@ -153,11 +153,34 @@ export async function queryIndex(root, term, filters = {}, opts = {}) {
|
|
|
153
153
|
store,
|
|
154
154
|
});
|
|
155
155
|
}
|
|
156
|
+
// F39: the store-wide pin is not enough — a row embedded under a foreign
|
|
157
|
+
// model (or left unmodelled by an older schema) must fail closed too.
|
|
158
|
+
const foreignRows = db
|
|
159
|
+
.prepare(
|
|
160
|
+
"select count(*) as n from chunks where embedding is not null and (emb_model is null or emb_model is not ?)",
|
|
161
|
+
)
|
|
162
|
+
.get(provider.model).n;
|
|
163
|
+
if (foreignRows > 0) {
|
|
164
|
+
db.close();
|
|
165
|
+
return buildEnvelope({
|
|
166
|
+
command: "query",
|
|
167
|
+
results: toResults(keywordRows, hasLogFilters, filters),
|
|
168
|
+
degradations: ["embeddings-model-mismatch"],
|
|
169
|
+
store,
|
|
170
|
+
});
|
|
171
|
+
}
|
|
156
172
|
try {
|
|
157
173
|
const [queryVector] = await provider.embed([term]);
|
|
158
174
|
const vectorRows = vectorTopK(db, queryVector, MATCH_POOL);
|
|
159
|
-
db.close();
|
|
160
175
|
const fused = fuseRrf(keywordRows, vectorRows);
|
|
176
|
+
// F46: chunk bodies are huge (≈9 MB across this repo's corpus) and are
|
|
177
|
+
// needed only for log-entry rows (the AC14 metadata). Fetch them for the
|
|
178
|
+
// fused candidates that can reach a result row — never for the whole scan.
|
|
179
|
+
for (const r of fused) {
|
|
180
|
+
if (r.path === LOGS_PATH && r.body === undefined) {
|
|
181
|
+
r.body = db.prepare("select body from chunks where chunk_id = ?").get(r.chunk_id)?.body ?? "";
|
|
182
|
+
}
|
|
183
|
+
}
|
|
161
184
|
const results = toResults(
|
|
162
185
|
fused.map((r) => ({ ...r, rank: -r.fusedScore })),
|
|
163
186
|
hasLogFilters,
|
|
@@ -165,7 +188,16 @@ export async function queryIndex(root, term, filters = {}, opts = {}) {
|
|
|
165
188
|
);
|
|
166
189
|
return buildEnvelope({ command: "query", results, degradations: [], store });
|
|
167
190
|
} catch (e) {
|
|
168
|
-
|
|
191
|
+
// F40: a stored vector whose dimensionality differs from the query
|
|
192
|
+
// vector is corrupt or foreign-shaped — degrade, never truncate-and-rank.
|
|
193
|
+
if (e instanceof VectorShapeMismatch) {
|
|
194
|
+
return buildEnvelope({
|
|
195
|
+
command: "query",
|
|
196
|
+
results: toResults(keywordRows, hasLogFilters, filters),
|
|
197
|
+
degradations: ["embeddings-model-mismatch"],
|
|
198
|
+
store,
|
|
199
|
+
});
|
|
200
|
+
}
|
|
169
201
|
// AC17: provider down ⇒ declared cause, keyword results survive.
|
|
170
202
|
return buildEnvelope({
|
|
171
203
|
command: "query",
|
|
@@ -173,6 +205,8 @@ export async function queryIndex(root, term, filters = {}, opts = {}) {
|
|
|
173
205
|
degradations: ["unavailable-embeddings-provider-down"],
|
|
174
206
|
store,
|
|
175
207
|
});
|
|
208
|
+
} finally {
|
|
209
|
+
db.close();
|
|
176
210
|
}
|
|
177
211
|
}
|
|
178
212
|
|
|
@@ -191,11 +225,14 @@ function toResults(rows, hasLogFilters, filters) {
|
|
|
191
225
|
meta = {};
|
|
192
226
|
}
|
|
193
227
|
const isLog = r.path === LOGS_PATH;
|
|
194
|
-
if (isLog
|
|
195
|
-
const entry = parseLogEntry(r.body);
|
|
196
|
-
if (
|
|
197
|
-
|
|
198
|
-
|
|
228
|
+
if (isLog) {
|
|
229
|
+
const entry = r.body === undefined ? null : parseLogEntry(r.body); // F46: parsed once per row
|
|
230
|
+
if (hasLogFilters) {
|
|
231
|
+
if (entry === null || !logEntryMatches(entry, filters)) continue;
|
|
232
|
+
meta = entry;
|
|
233
|
+
} else if (entry !== null) {
|
|
234
|
+
meta = entry;
|
|
235
|
+
}
|
|
199
236
|
} else if (hasLogFilters) {
|
|
200
237
|
continue; // structured filters narrow to log entries by definition
|
|
201
238
|
}
|
|
@@ -224,11 +261,19 @@ export async function statusIndex(root) {
|
|
|
224
261
|
|
|
225
262
|
const RRF_K = 60;
|
|
226
263
|
|
|
264
|
+
/** A stored vector's dimensionality differs from the query vector (F40). */
|
|
265
|
+
class VectorShapeMismatch extends Error {}
|
|
266
|
+
|
|
227
267
|
function cosine(a, b) {
|
|
268
|
+
if (a.length !== b.length) {
|
|
269
|
+
// F40: truncating to the shorter vector ranked confidently on garbage;
|
|
270
|
+
// a shape mismatch is a corrupt or foreign-model vector — fail closed.
|
|
271
|
+
throw new VectorShapeMismatch(`vector dimension mismatch: ${a.length} vs ${b.length}`);
|
|
272
|
+
}
|
|
228
273
|
let dot = 0;
|
|
229
274
|
let na = 0;
|
|
230
275
|
let nb = 0;
|
|
231
|
-
const n =
|
|
276
|
+
const n = a.length;
|
|
232
277
|
for (let i = 0; i < n; i++) {
|
|
233
278
|
dot += a[i] * b[i];
|
|
234
279
|
na += a[i] * a[i];
|
|
@@ -240,9 +285,12 @@ function cosine(a, b) {
|
|
|
240
285
|
|
|
241
286
|
/** JS-cosine kNN over D4's float32-LE BLOBs (decision D11 — no second vec0 store). */
|
|
242
287
|
function vectorTopK(db, queryVector, pool) {
|
|
288
|
+
// F46: the scan scores vectors — it must NOT drag every chunk's body
|
|
289
|
+
// (megabytes) out of SQLite per query; bodies are fetched lazily for the
|
|
290
|
+
// few fused candidates that need them.
|
|
243
291
|
const rows = db
|
|
244
292
|
.prepare(
|
|
245
|
-
"select chunk_id, path, section, line_start, line_end, meta,
|
|
293
|
+
"select chunk_id, path, section, line_start, line_end, meta, embedding from chunks where embedding is not null",
|
|
246
294
|
)
|
|
247
295
|
.all();
|
|
248
296
|
const scored = rows.map((r) => ({ ...r, sim: cosine(decodeVector(r.embedding), queryVector) }));
|
package/src/doc/retrieval.mjs
CHANGED
|
@@ -33,13 +33,26 @@ export async function runDocOp(grammar, ctx = {}) {
|
|
|
33
33
|
// same pass; otherwise the store answers keyword-only (AC17's root).
|
|
34
34
|
const cfg = resolveEmbedConfig({ config: loadConfig(root), env: process.env });
|
|
35
35
|
let embedded = null;
|
|
36
|
+
let degradations = [];
|
|
36
37
|
if (cfg.configured) {
|
|
37
|
-
|
|
38
|
+
try {
|
|
39
|
+
embedded = await embedPendingChunks(root, makeOpenAIProvider(cfg));
|
|
40
|
+
if (embedded.skipped === true) {
|
|
41
|
+
// Model mismatch: keep the frozen key and declare the cause.
|
|
42
|
+
// (AC18: embeddings-model-mismatch; D3 frozen key set.)
|
|
43
|
+
degradations = [embedded.reason || "embeddings-model-mismatch"];
|
|
44
|
+
embedded = { embedded: 0 };
|
|
45
|
+
}
|
|
46
|
+
} catch (e) {
|
|
47
|
+
// AC17: provider-down ⇒ declared cause, ok:true, keyword results survive.
|
|
48
|
+
degradations = ["unavailable-embeddings-provider-down"];
|
|
49
|
+
embedded = { embedded: 0 };
|
|
50
|
+
}
|
|
38
51
|
}
|
|
39
52
|
return buildEnvelope({
|
|
40
53
|
command: "sync",
|
|
41
54
|
results: [],
|
|
42
|
-
degradations
|
|
55
|
+
degradations,
|
|
43
56
|
store: {
|
|
44
57
|
path: storePath(root),
|
|
45
58
|
present: true,
|
package/src/doc/store.mjs
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
*/
|
|
10
10
|
|
|
11
11
|
import { existsSync } from "node:fs";
|
|
12
|
-
import { join } from "node:path";
|
|
12
|
+
import { dirname, join } from "node:path";
|
|
13
13
|
import { openDatabase } from "./sqlite.mjs";
|
|
14
14
|
import { loadConfig } from "./config.mjs";
|
|
15
15
|
|
|
@@ -18,11 +18,6 @@ export const MANIFEST_EXPORT_NAME = "manifest.json";
|
|
|
18
18
|
/** Default store location: `.agentic-workflow/index/index.db`. */
|
|
19
19
|
const DEFAULT_STORE_DIR = join(".agentic-workflow", "index");
|
|
20
20
|
|
|
21
|
-
/** Resolve the store's parent directory (the gitignored index dir). */
|
|
22
|
-
function storeDir(root) {
|
|
23
|
-
return join(root, DEFAULT_STORE_DIR);
|
|
24
|
-
}
|
|
25
|
-
|
|
26
21
|
/**
|
|
27
22
|
* Resolve the store file path. Honours `config.store.path` when present;
|
|
28
23
|
* falls back to the default (`.agentic-workflow/index/index.db`).
|
|
@@ -34,11 +29,14 @@ export function storePath(root) {
|
|
|
34
29
|
}
|
|
35
30
|
|
|
36
31
|
/**
|
|
37
|
-
* Canonical manifest export: the sidecar
|
|
38
|
-
*
|
|
32
|
+
* Canonical manifest export: the sidecar `manifest.json` beside the resolved
|
|
33
|
+
* store — the configured `store.path`'s directory when set (F18: the sidecar
|
|
34
|
+
* must travel with the store, or a custom store path ENOENTs on sync), else
|
|
35
|
+
* the default gitignored index dir. AC12's byte-for-byte assertion runs over
|
|
36
|
+
* that export, never over SQLite page bytes.
|
|
39
37
|
*/
|
|
40
38
|
export function manifestExportPath(root) {
|
|
41
|
-
return join(
|
|
39
|
+
return join(dirname(storePath(root)), MANIFEST_EXPORT_NAME);
|
|
42
40
|
}
|
|
43
41
|
|
|
44
42
|
export function storePresent(root) {
|
package/src/doc/sync.mjs
CHANGED
|
@@ -89,25 +89,42 @@ export async function syncIndex(root) {
|
|
|
89
89
|
const insert = db.prepare(
|
|
90
90
|
"INSERT INTO chunks (chunk_id, path, section, line_start, line_end, meta, body) VALUES (?, ?, ?, ?, ?, ?, ?)",
|
|
91
91
|
);
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
//
|
|
106
|
-
// (AC11: treat as deletion).
|
|
92
|
+
// F51: one transaction for the whole reap/insert pass. Per-row autocommit
|
|
93
|
+
// paid one fsync per chunk on the cold path (~9.6x measured); a single
|
|
94
|
+
// BEGIN/COMMIT — with rollback on throw so a mid-pass failure leaves the
|
|
95
|
+
// previous store state intact — is the fix, never a weakening of the
|
|
96
|
+
// per-file error semantics (the ENOENT-as-deletion branch still throws
|
|
97
|
+
// non-ENOENT errors out of the pass).
|
|
98
|
+
db.exec("BEGIN");
|
|
99
|
+
try {
|
|
100
|
+
for (const path of deleted) reap.run(path);
|
|
101
|
+
for (const path of changed) {
|
|
102
|
+
const src = join(root, path);
|
|
103
|
+
try {
|
|
104
|
+
// Reap existing chunks for this path before re-inserting (original
|
|
105
|
+
// reap.run(path) before the read — preserved for the non-ENOENT path).
|
|
107
106
|
reap.run(path);
|
|
108
|
-
|
|
109
|
-
|
|
107
|
+
const rows = scanChunks(readFileSync(src, "utf8"), path);
|
|
108
|
+
for (const c of rows) {
|
|
109
|
+
insert.run(c.id, c.path, c.section, c.lines[0], c.lines[1], JSON.stringify(c.meta), c.body);
|
|
110
|
+
}
|
|
111
|
+
} catch (e) {
|
|
112
|
+
if (e.code === "ENOENT") {
|
|
113
|
+
// File deleted in working tree but not yet committed → reap
|
|
114
|
+
// (AC11: treat as deletion).
|
|
115
|
+
reap.run(path);
|
|
116
|
+
deleted.push(path);
|
|
117
|
+
} else throw e;
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
db.exec("COMMIT");
|
|
121
|
+
} catch (e) {
|
|
122
|
+
try {
|
|
123
|
+
db.exec("ROLLBACK");
|
|
124
|
+
} catch {
|
|
125
|
+
// the transaction may already be rolled back by the driver
|
|
110
126
|
}
|
|
127
|
+
throw e;
|
|
111
128
|
}
|
|
112
129
|
|
|
113
130
|
// Persist + export the canonical manifest (AC12's compared artifact).
|