@gtrabanco/agentic-workflow 0.1.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +56 -2
- package/bin/agentic-workflow.mjs +71 -1
- package/package.json +7 -4
- package/src/doc/chunks.mjs +154 -0
- package/src/doc/config.mjs +108 -0
- package/src/doc/embeddings.mjs +162 -0
- package/src/doc/envelope.mjs +120 -0
- package/src/doc/grammar.mjs +109 -0
- package/src/doc/manifest.mjs +84 -0
- package/src/doc/query.mjs +327 -0
- package/src/doc/retrieval.mjs +95 -0
- package/src/doc/sqlite.mjs +34 -0
- package/src/doc/store.mjs +55 -0
- package/src/doc/sync.mjs +176 -0
- package/src/edit/UnitDoc.mjs +199 -11
- package/src/edit/schema.mjs +45 -1
- package/src/memory/probe.mjs +388 -0
package/README.md
CHANGED
|
@@ -5,8 +5,12 @@ created it as the vehicle the roadmap mandated (rows 37/38/42/45, declined 43):
|
|
|
5
5
|
the first producer feature creates the crate, later producers land their scripts
|
|
6
6
|
as subcommands of it.
|
|
7
7
|
|
|
8
|
-
- **
|
|
9
|
-
bun first and node as the guaranteed fallback
|
|
8
|
+
- **Shipped via `@gtrabanco/pi-agentic-workflow` (the pi package), one pinned pure-JS dependency, no native deps, no build step.**
|
|
9
|
+
Repository tooling runs with bun first and node as the guaranteed fallback
|
|
10
|
+
(`AGENTS.md` §Verification); `typebox` is exact-pinned and carries the `doc`
|
|
11
|
+
entry point's envelope validation. The retrieval store lives in a gitignored
|
|
12
|
+
SQLite database under `.agentic-workflow/index/` — disposable build output,
|
|
13
|
+
regenerated, never committed.
|
|
10
14
|
- **Scratch convention:** `.agentic-workflow/tmp/` at the repository root holds
|
|
11
15
|
throwaway producer output. It is committed as a directory (`.gitkeep`) so the
|
|
12
16
|
convention exists on a fresh clone.
|
|
@@ -17,3 +21,53 @@ repository's `scripts/` tree (feature 37 ED1). Feature 38's
|
|
|
17
21
|
crate existed, so re-homing it here is a recorded follow-up (feature 37
|
|
18
22
|
ED8.4c) with no unit assigned yet; feature 42 is the next candidate to add the
|
|
19
23
|
first crate subcommand.
|
|
24
|
+
|
|
25
|
+
## `memory probe` — bounded memory-substrate health (feature 66)
|
|
26
|
+
|
|
27
|
+
One command answers "can this session trust the Engram memory layer?" before
|
|
28
|
+
its first memory read. It writes nothing, issues no MCP tool call (only
|
|
29
|
+
`initialize`), and **always exits 0** — a `degraded` verdict is an answer to
|
|
30
|
+
record, never a failure that blocks a unit.
|
|
31
|
+
|
|
32
|
+
```sh
|
|
33
|
+
agentic-workflow memory probe # human block
|
|
34
|
+
agentic-workflow memory probe --json # memory-probe@1 verdict
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
Five checks, fixed order. `ok` requires all five; any failure makes the whole
|
|
38
|
+
verdict `degraded` and names the failing checks:
|
|
39
|
+
|
|
40
|
+
| check | source | bound |
|
|
41
|
+
|---|---|---|
|
|
42
|
+
| `http-health` | `GET http://127.0.0.1:7437/health`, body `status` must be `ok` | 4 s |
|
|
43
|
+
| `mcp-runner` | one JSON-RPC `initialize` to `engram mcp --tools=agent` | 4 s |
|
|
44
|
+
| `binary-version` | `engram --version` | 4 s |
|
|
45
|
+
| `sync-staleness` | `engram sync --status`, `Pending import` must be 0 | 10 s |
|
|
46
|
+
| `session-ambiguity` | `engram doctor --json --check` on `ambiguous_active_runtime_sessions` + `sync_target_closed_space`, every non-`ok` answer reported | 10 s |
|
|
47
|
+
|
|
48
|
+
The checks are independent, so they run together: the probe costs the slowest
|
|
49
|
+
bound (10 s), never the sum of five (32 s). Whole-probe ceiling: 30 s, which is
|
|
50
|
+
the backstop for a dependency that ignored its own bound. A bound hit fails that
|
|
51
|
+
check and degrades the verdict — never a hang.
|
|
52
|
+
|
|
53
|
+
Three rules are load-bearing and measured, not assumed:
|
|
54
|
+
|
|
55
|
+
- **Exit code is not a health signal.** `engram doctor` exits 0 while its JSON
|
|
56
|
+
reports `status: "error"`, so every check parses the answer instead of
|
|
57
|
+
trusting the status code.
|
|
58
|
+
- **A responding server that has not synced is not `ok`.** That is why
|
|
59
|
+
`sync-staleness` is a check, and why an unreadable status (e.g. `ambiguous
|
|
60
|
+
project` outside a repo) is a degradation rather than a pass.
|
|
61
|
+
- **One answer never hides another.** The session check asks both scoped
|
|
62
|
+
`--check` ids and reports every non-`ok` finding. Stopping at the first would
|
|
63
|
+
silence the sync-target signal permanently on a machine whose session
|
|
64
|
+
attribution is ambiguous — which is the state this unit's own SPEC records.
|
|
65
|
+
- **`degraded` is reportable, not fatal.** The caller records the verdict in
|
|
66
|
+
the unit's Evidence and keeps working.
|
|
67
|
+
|
|
68
|
+
The full `engram doctor --json` sweep is deliberately excluded: it costs 12–23 s
|
|
69
|
+
at this repository and the two scoped `--check` calls answer the same question
|
|
70
|
+
in under half a second each. `runProbe({ fetchImpl, runImpl, timeoutMs,
|
|
71
|
+
totalTimeoutMs })` takes its side effects by injection, so the suite drives
|
|
72
|
+
every degraded path with fakes and millisecond bounds — no network, no PATH, no
|
|
73
|
+
real engram process.
|
package/bin/agentic-workflow.mjs
CHANGED
|
@@ -8,11 +8,17 @@
|
|
|
8
8
|
* agentic-workflow changelog <op> [args...]
|
|
9
9
|
* agentic-workflow budgets <op> [args...]
|
|
10
10
|
* agentic-workflow manifest <op> [args...]
|
|
11
|
+
* agentic-workflow doc (--sync | --query <term> | --status | --rebuild) [--json-only]
|
|
12
|
+
* agentic-workflow memory probe [--json]
|
|
11
13
|
*
|
|
12
14
|
* --json prints the operation receipt as JSON.
|
|
13
15
|
* Every operation prints a fixed receipt block to stdout.
|
|
14
16
|
*
|
|
15
17
|
* Refusals exit 2 with "EDIT REFUSAL — <reason>" before writing anything.
|
|
18
|
+
*
|
|
19
|
+
* `memory probe` is a sensor, not an editor: it writes nothing and always
|
|
20
|
+
* exits 0. Its `degraded` verdict is an answer to report, never a failure
|
|
21
|
+
* that blocks the caller (feature 66, AC2).
|
|
16
22
|
*/
|
|
17
23
|
|
|
18
24
|
import { readFileSync } from "node:fs";
|
|
@@ -24,6 +30,9 @@ import { rowUpsert, annotate as roadmapAnnotate } from "../src/edit/Roadmap.mjs"
|
|
|
24
30
|
import { rowAdd } from "../src/edit/Changelog.mjs";
|
|
25
31
|
import { ceilingRebase } from "../src/edit/Budgets.mjs";
|
|
26
32
|
import { skillAdd, skillRemove } from "../src/edit/Manifest.mjs";
|
|
33
|
+
import { parseDocArgs } from "../src/doc/grammar.mjs";
|
|
34
|
+
import { runDocOp } from "../src/doc/retrieval.mjs";
|
|
35
|
+
import { runProbe } from "../src/memory/probe.mjs";
|
|
27
36
|
|
|
28
37
|
// ── Helpers ──────────────────────────────────────────────────────────────
|
|
29
38
|
|
|
@@ -63,7 +72,7 @@ function refuse(reason) {
|
|
|
63
72
|
|
|
64
73
|
const command = process.argv[2];
|
|
65
74
|
if (!command) {
|
|
66
|
-
console.error("usage: agentic-workflow <unit-doc|roadmap|changelog|budgets|manifest> ...");
|
|
75
|
+
console.error("usage: agentic-workflow <unit-doc|roadmap|changelog|budgets|manifest|doc|memory> ...");
|
|
67
76
|
process.exit(1);
|
|
68
77
|
}
|
|
69
78
|
|
|
@@ -72,6 +81,67 @@ const parsed = parseArgs(restArgs);
|
|
|
72
81
|
const rawArgs = parsed.args;
|
|
73
82
|
|
|
74
83
|
switch (command) {
|
|
84
|
+
// ── doc — retrieval entry point (unit 65, D2) ────────────────────────
|
|
85
|
+
case "doc": {
|
|
86
|
+
let grammar;
|
|
87
|
+
try {
|
|
88
|
+
grammar = parseDocArgs(restArgs);
|
|
89
|
+
} catch (e) {
|
|
90
|
+
process.stderr.write(`doc: ${e.message}\n`);
|
|
91
|
+
process.exit(1);
|
|
92
|
+
}
|
|
93
|
+
let envelope;
|
|
94
|
+
try {
|
|
95
|
+
envelope = await runDocOp(grammar, { rootDir: process.cwd() });
|
|
96
|
+
} catch (e) {
|
|
97
|
+
// IO/store failures are usage/IO territory (exit 1, D3); the message
|
|
98
|
+
// goes to stderr and stdout stays empty.
|
|
99
|
+
process.stderr.write(`doc: ${e.message}\n`);
|
|
100
|
+
process.exit(1);
|
|
101
|
+
}
|
|
102
|
+
if (grammar.jsonOnly) {
|
|
103
|
+
// AC8: exactly one JSON document on stdout; diagnostics on stderr.
|
|
104
|
+
process.stdout.write(JSON.stringify(envelope) + "\n");
|
|
105
|
+
} else if (grammar.quiet) {
|
|
106
|
+
// AC23's hook spelling: --sync --quiet writes nothing on success;
|
|
107
|
+
// diagnostics stay on stderr. Exit code still reports failure.
|
|
108
|
+
if (envelope.ok !== true) process.stderr.write(`doc ${envelope.command}: failed\n`);
|
|
109
|
+
} else {
|
|
110
|
+
for (const d of envelope.degradations) process.stderr.write(`doc: degradation: ${d}\n`);
|
|
111
|
+
for (const r of envelope.results) {
|
|
112
|
+
console.log(`${r.path} · ${r.section ?? "-"} · L${r.lines[0]}-${r.lines[1]} · ${r.score}`);
|
|
113
|
+
}
|
|
114
|
+
if (envelope.store?.lastSync) {
|
|
115
|
+
const s = envelope.store.lastSync;
|
|
116
|
+
console.log(`doc sync: files_scanned=${s.files_scanned} files_changed=${s.files_changed} files_deleted=${s.files_deleted} chunks=${envelope.store.chunks}`);
|
|
117
|
+
}
|
|
118
|
+
console.log(`doc ${envelope.command}: ${envelope.results.length} result(s)`);
|
|
119
|
+
}
|
|
120
|
+
break;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
// ── memory — bounded memory-substrate health probe (unit 66, P1) ────
|
|
124
|
+
case "memory": {
|
|
125
|
+
const op = rawArgs[0];
|
|
126
|
+
if (op !== "probe") refuse(`memory requires the \`probe\` operation, got "${op ?? ""}"`);
|
|
127
|
+
const report = await runProbe();
|
|
128
|
+
if (parsed.json) {
|
|
129
|
+
process.stdout.write(JSON.stringify(report) + "\n");
|
|
130
|
+
} else {
|
|
131
|
+
console.log(`MEMORY PROBE — ${report.verdict}`);
|
|
132
|
+
for (const check of report.checks) {
|
|
133
|
+
const mark = check.status === "ok" ? "ok" : `failed:${check.reason}`;
|
|
134
|
+
console.log(` ${check.id} · ${mark} · ${check.detail}`);
|
|
135
|
+
}
|
|
136
|
+
if (report.degraded.length > 0) {
|
|
137
|
+
console.log(`Degraded: ${report.degraded.join(", ")} — record this in Evidence, keep working`);
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
// A sensor that exits non-zero for a healthy machine's degraded layer
|
|
141
|
+
// would block the very units it exists to protect (AC2 non-goal).
|
|
142
|
+
break;
|
|
143
|
+
}
|
|
144
|
+
|
|
75
145
|
// ── unit-doc ─────────────────────────────────────────────────────────
|
|
76
146
|
case "unit-doc": {
|
|
77
147
|
const slug = rawArgs[0];
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@gtrabanco/agentic-workflow",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"description": "Deterministic runner for the agentic-workflow lane: file services (agwo edit), sensors, guards and receipts.
|
|
3
|
+
"version": "0.4.0",
|
|
4
|
+
"description": "Deterministic runner for the agentic-workflow lane: file services (agwo edit), sensors, guards and receipts. One pinned pure-JS dependency, no native deps, no build step.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "Gabriel Trabanco <1969593+gtrabanco@users.noreply.github.com>",
|
|
7
7
|
"repository": {
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
"agentic-workflow": "./bin/agentic-workflow.mjs"
|
|
15
15
|
},
|
|
16
16
|
"engines": {
|
|
17
|
-
"node": ">=
|
|
17
|
+
"node": ">=24",
|
|
18
18
|
"bun": ">=1.4"
|
|
19
19
|
},
|
|
20
20
|
"scripts": {
|
|
@@ -25,5 +25,8 @@
|
|
|
25
25
|
"bin",
|
|
26
26
|
"src",
|
|
27
27
|
"README.md"
|
|
28
|
-
]
|
|
28
|
+
],
|
|
29
|
+
"dependencies": {
|
|
30
|
+
"typebox": "1.4.3"
|
|
31
|
+
}
|
|
29
32
|
}
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Fence-aware section chunker (unit 65, D5).
|
|
3
|
+
*
|
|
4
|
+
* A chunk is one ATX-heading-delimited section found by a scanner where a `#`
|
|
5
|
+
* inside a fenced code block is never a heading. The id is
|
|
6
|
+
* `<relative-path>#<slug of the heading path>` with `-2` style dedupe,
|
|
7
|
+
* derived only from content — no timestamps, no mtimes — so ids are stable
|
|
8
|
+
* across sync runs (AC2/AC12) and half A's structure index shares the same
|
|
9
|
+
* function (no drift between the halves). `lines` is 1-based inclusive.
|
|
10
|
+
*
|
|
11
|
+
* File-level frontmatter (`---` block) is parsed once and attached to every
|
|
12
|
+
* chunk as `meta`; frontmatter lines never enter a chunk body. The preamble
|
|
13
|
+
* (content before the first heading) is a chunk with `section: null` and id
|
|
14
|
+
* `<path>#` — emitted only when that range is non-empty. A file with no
|
|
15
|
+
* headings at all yields exactly the preamble chunk.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
const FENCE_RE = /^(`{3,}|~{3,})/;
|
|
19
|
+
const HEADING_RE = /^(#{1,6})\s+(.*)$/;
|
|
20
|
+
|
|
21
|
+
function slugify(headingPath) {
|
|
22
|
+
return headingPath
|
|
23
|
+
.toLowerCase()
|
|
24
|
+
.replace(/\s*>\s*/g, "-")
|
|
25
|
+
.replace(/[^a-z0-9-]+/g, "-")
|
|
26
|
+
.replace(/-{2,}/g, "-")
|
|
27
|
+
.replace(/^-+|-+$/g, "");
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** Minimal frontmatter: top-level `key: value` scalars, comma lists, and `>`/`|` folded blocks. */
|
|
31
|
+
function parseFrontmatter(lines) {
|
|
32
|
+
const meta = {};
|
|
33
|
+
for (let i = 0; i < lines.length; i++) {
|
|
34
|
+
const m = lines[i].match(/^([A-Za-z0-9_-]+):\s*(.*)$/);
|
|
35
|
+
if (!m) continue;
|
|
36
|
+
const key = m[1];
|
|
37
|
+
let value = m[2].trim();
|
|
38
|
+
if (value === ">" || value === "|" || value === ">-" || value === "|-") {
|
|
39
|
+
const block = [];
|
|
40
|
+
while (i + 1 < lines.length && /^\s/.test(lines[i + 1])) {
|
|
41
|
+
block.push(lines[i + 1].trim());
|
|
42
|
+
i++;
|
|
43
|
+
}
|
|
44
|
+
value = block.join(" ");
|
|
45
|
+
} else if (value.includes(",")) {
|
|
46
|
+
value = value.split(",").map((s) => s.trim()).filter((s) => s !== "");
|
|
47
|
+
} else {
|
|
48
|
+
value = value.replace(/^["']|["']$/g, "");
|
|
49
|
+
}
|
|
50
|
+
meta[key] = value;
|
|
51
|
+
}
|
|
52
|
+
return meta;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Scan `markdown` into chunks attributed to `relativePath`.
|
|
57
|
+
* Returns rows `{id, path, section, lines: [start, end], meta, body}` in
|
|
58
|
+
* document order. Deterministic: identical input ⇒ identical output.
|
|
59
|
+
*/
|
|
60
|
+
export function scanChunks(markdown, relativePath) {
|
|
61
|
+
const content = String(markdown);
|
|
62
|
+
const lines = content.split(/\r?\n/);
|
|
63
|
+
// A file ending in a newline splits into one phantom trailing element; it is
|
|
64
|
+
// not a line, and counting it overshoots every last chunk's line_end (F47).
|
|
65
|
+
if (lines.length > 1 && lines[lines.length - 1] === "" && /\r?\n$/.test(content)) lines.pop();
|
|
66
|
+
|
|
67
|
+
// Frontmatter block, only when it opens the file.
|
|
68
|
+
let contentStart = 0;
|
|
69
|
+
let meta = {};
|
|
70
|
+
if (lines.length > 0 && lines[0].trim() === "---") {
|
|
71
|
+
let end = -1;
|
|
72
|
+
for (let i = 1; i < lines.length; i++) {
|
|
73
|
+
if (lines[i].trim() === "---") {
|
|
74
|
+
end = i;
|
|
75
|
+
break;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
if (end !== -1) {
|
|
79
|
+
meta = parseFrontmatter(lines.slice(1, end));
|
|
80
|
+
contentStart = end + 1;
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
// Headings outside fenced code blocks.
|
|
85
|
+
const headings = []; // {level, title, line (1-based)}
|
|
86
|
+
let fence = null;
|
|
87
|
+
for (let i = contentStart; i < lines.length; i++) {
|
|
88
|
+
const trimmed = lines[i].trim();
|
|
89
|
+
if (fence !== null) {
|
|
90
|
+
if (trimmed.startsWith(fence)) fence = null;
|
|
91
|
+
continue;
|
|
92
|
+
}
|
|
93
|
+
const f = trimmed.match(FENCE_RE);
|
|
94
|
+
if (f) {
|
|
95
|
+
fence = f[1][0].repeat(3);
|
|
96
|
+
continue;
|
|
97
|
+
}
|
|
98
|
+
const h = trimmed.match(HEADING_RE);
|
|
99
|
+
if (h) headings.push({ level: h[1].length, title: h[2].trim(), line: i + 1 });
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
const chunks = [];
|
|
103
|
+
const usedIds = new Set();
|
|
104
|
+
const nextId = (headingPath) => {
|
|
105
|
+
const slug = slugify(headingPath);
|
|
106
|
+
let id = `${relativePath}#${slug}`;
|
|
107
|
+
let n = 2;
|
|
108
|
+
while (usedIds.has(id)) id = `${relativePath}#${slug}-${n++}`;
|
|
109
|
+
usedIds.add(id);
|
|
110
|
+
return id;
|
|
111
|
+
};
|
|
112
|
+
|
|
113
|
+
// Preamble chunk — only when its range is non-empty and body is non-blank
|
|
114
|
+
// (AC10: skip frontmatter-only preamble to avoid empty-body noise).
|
|
115
|
+
const firstHeadingLine = headings.length > 0 ? headings[0].line : lines.length + 1;
|
|
116
|
+
if (firstHeadingLine > 1 || headings.length === 0) {
|
|
117
|
+
const to = headings.length > 0 ? firstHeadingLine - 1 : lines.length;
|
|
118
|
+
const body = lines
|
|
119
|
+
.slice(0, to)
|
|
120
|
+
.filter((_, idx) => !(idx < contentStart))
|
|
121
|
+
.join("\n");
|
|
122
|
+
if (body.trim() !== "") {
|
|
123
|
+
chunks.push({
|
|
124
|
+
id: `${relativePath}#`,
|
|
125
|
+
path: relativePath,
|
|
126
|
+
section: null,
|
|
127
|
+
lines: [contentStart + 1, to],
|
|
128
|
+
meta,
|
|
129
|
+
body,
|
|
130
|
+
});
|
|
131
|
+
usedIds.add(`${relativePath}#`);
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
// Heading sections with their heading-path chain.
|
|
136
|
+
const stack = []; // {level, title}
|
|
137
|
+
for (let s = 0; s < headings.length; s++) {
|
|
138
|
+
const heading = headings[s];
|
|
139
|
+
while (stack.length > 0 && stack[stack.length - 1].level >= heading.level) stack.pop();
|
|
140
|
+
stack.push({ level: heading.level, title: heading.title });
|
|
141
|
+
const headingPath = stack.map((e) => e.title).join(" > ");
|
|
142
|
+
const to = s + 1 < headings.length ? headings[s + 1].line - 1 : lines.length;
|
|
143
|
+
chunks.push({
|
|
144
|
+
id: nextId(headingPath),
|
|
145
|
+
path: relativePath,
|
|
146
|
+
section: headingPath,
|
|
147
|
+
lines: [heading.line, to],
|
|
148
|
+
meta,
|
|
149
|
+
body: lines.slice(heading.line - 1, to).join("\n"),
|
|
150
|
+
});
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
return chunks;
|
|
154
|
+
}
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Committed config for the `doc` retrieval index (unit 65, D4/D10).
|
|
3
|
+
*
|
|
4
|
+
* `.agentic-workflow/index.json` is the committed decision surface (shipped
|
|
5
|
+
* by init-workspace in target projects). It defaults when absent — a missing
|
|
6
|
+
* config is never a hard failure; the index answers keyword-only until a
|
|
7
|
+
* provider is configured (AC17's `unavailable-embeddings-not-configured`).
|
|
8
|
+
* A present but unparseable config is a usage/IO error (exit 1 territory);
|
|
9
|
+
* an unknown version fails closed — the reader never guesses.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
import { existsSync, readFileSync, realpathSync } from "node:fs";
|
|
13
|
+
import { dirname, join, resolve, sep } from "node:path";
|
|
14
|
+
|
|
15
|
+
export const CONFIG_VERSION = 1;
|
|
16
|
+
|
|
17
|
+
export const DEFAULT_CONFIG = Object.freeze({
|
|
18
|
+
version: CONFIG_VERSION,
|
|
19
|
+
store: Object.freeze({ path: ".agentic-workflow/index/index.db" }),
|
|
20
|
+
// D10 provider surface: null ⇒ keyword-only. When configured: an
|
|
21
|
+
// OpenAI-compatible {baseUrl, model, envVar} — the env var is read at
|
|
22
|
+
// runtime in pi's precedence (CLI flag > environment > config > default).
|
|
23
|
+
provider: null,
|
|
24
|
+
});
|
|
25
|
+
|
|
26
|
+
export function configPath(root) {
|
|
27
|
+
return join(root, ".agentic-workflow", "index.json");
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Realpath of the deepest EXISTING ancestor of `p`, with the not-yet-existing
|
|
32
|
+
* tail re-joined lexically (F36): the configured store leaf usually does not
|
|
33
|
+
* exist yet, but its intermediate directories do — and an intermediate
|
|
34
|
+
* symlink escaping the repository must be caught before any write.
|
|
35
|
+
*/
|
|
36
|
+
function realpathDeepest(p) {
|
|
37
|
+
const tail = [];
|
|
38
|
+
let cur = p;
|
|
39
|
+
while (!existsSync(cur)) {
|
|
40
|
+
const parent = dirname(cur);
|
|
41
|
+
if (parent === cur) return cur; // filesystem root — nothing to resolve
|
|
42
|
+
tail.unshift(cur.slice(parent.length + 1));
|
|
43
|
+
cur = parent;
|
|
44
|
+
}
|
|
45
|
+
return join(realpathSync(cur), ...tail);
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Contain a configured `store.path` inside the repository's gitignored
|
|
50
|
+
* `.agentic-workflow/` directory (F36/F37, decision D12):
|
|
51
|
+
* - F37: a store outside `.agentic-workflow/` leaves `git status` dirty
|
|
52
|
+
* after `--sync` (AC7) — the configured store must stay inside the only
|
|
53
|
+
* directory the committed ignore rules cover;
|
|
54
|
+
* - F36: containment is anchored at `realpath(root)` and the candidate is
|
|
55
|
+
* resolved through the realpath of its deepest existing ancestor, so a
|
|
56
|
+
* symlinked intermediate directory (or a symlinked `.agentic-workflow`
|
|
57
|
+
* itself) fails closed — `--rebuild`'s `rmSync` targets this path, so a
|
|
58
|
+
* misconfigured store path is a destructive-write footgun.
|
|
59
|
+
*/
|
|
60
|
+
function containStorePath(root, candidate) {
|
|
61
|
+
const anchoredRoot = realpathDeepest(resolve(root));
|
|
62
|
+
const workflowRoot = join(anchoredRoot, ".agentic-workflow");
|
|
63
|
+
const resolvedCandidate = realpathDeepest(resolve(root, candidate));
|
|
64
|
+
if (!resolvedCandidate.startsWith(workflowRoot + sep)) {
|
|
65
|
+
throw new Error(
|
|
66
|
+
`index.json store.path "${candidate}" must stay inside .agentic-workflow/ (got "${resolvedCandidate}")`,
|
|
67
|
+
);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Load the config for the project at `root`: defaults merged under the
|
|
73
|
+
* file's overrides. Throws on unparseable JSON (message names the file),
|
|
74
|
+
* on an unknown `version` (fail closed), and on a `store.path` outside the
|
|
75
|
+
* gitignored `.agentic-workflow/` directory (F36/F37 containment).
|
|
76
|
+
*/
|
|
77
|
+
export function loadConfig(root) {
|
|
78
|
+
const path = configPath(root);
|
|
79
|
+
if (!existsSync(path)) {
|
|
80
|
+
// F50: the absent-config branch is NOT a containment bypass — the merged
|
|
81
|
+
// default store path runs through the same check, so a symlinked
|
|
82
|
+
// `.agentic-workflow` fails closed even with no index.json on disk.
|
|
83
|
+
const merged = structuredClone(DEFAULT_CONFIG);
|
|
84
|
+
containStorePath(root, merged.store.path);
|
|
85
|
+
return merged;
|
|
86
|
+
}
|
|
87
|
+
let parsed;
|
|
88
|
+
try {
|
|
89
|
+
parsed = JSON.parse(readFileSync(path, "utf8"));
|
|
90
|
+
} catch (error) {
|
|
91
|
+
throw new Error(`index.json is not valid JSON (${path}): ${error.message}`);
|
|
92
|
+
}
|
|
93
|
+
if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) {
|
|
94
|
+
throw new Error(`index.json must be a JSON object (${path})`);
|
|
95
|
+
}
|
|
96
|
+
if (parsed.version !== undefined && parsed.version !== CONFIG_VERSION) {
|
|
97
|
+
throw new Error(`index.json version ${parsed.version} is not supported (expected ${CONFIG_VERSION})`);
|
|
98
|
+
}
|
|
99
|
+
// Contain the MERGED store path inside the repository's gitignored
|
|
100
|
+
// `.agentic-workflow/` directory (F36/F37/F50, decision D12) — see
|
|
101
|
+
// containStorePath. Runs on every branch: an explicit `store.path`, a
|
|
102
|
+
// config without a `store` key, and the absent-config default alike.
|
|
103
|
+
const merged = { ...structuredClone(DEFAULT_CONFIG), ...parsed };
|
|
104
|
+
// storePath() falls back to the default when `store.path` is falsy —
|
|
105
|
+
// contain exactly the path that will be used.
|
|
106
|
+
containStorePath(root, merged.store?.path || DEFAULT_CONFIG.store.path);
|
|
107
|
+
return merged;
|
|
108
|
+
}
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Embeddings write path for the `doc` retrieval index (unit 65, P6).
|
|
3
|
+
*
|
|
4
|
+
* D10 provider resolution: CLI flag > direct pi-style environment variables
|
|
5
|
+
* > the config-named env var + config file > default (none ⇒ keyword-only,
|
|
6
|
+
* never a hard failure). Vectors are float32-LE blobs behind a format tag
|
|
7
|
+
* (D4) with the producing model pinned per row and in the store meta — a
|
|
8
|
+
* model mismatch fails closed: nothing re-embeds and no cross-model store
|
|
9
|
+
* can form (AC18); `--rebuild` under the new model is the documented path.
|
|
10
|
+
*
|
|
11
|
+
* Incrementalism (AC16): embedding runs only over chunks with no stored
|
|
12
|
+
* vector, batched per file, so a warm sync makes 0 API calls (AC19's write
|
|
13
|
+
* half) and changing one file re-embeds exactly that file's chunks.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
export const VECTOR_TAG = "awkvec-f32le1";
|
|
17
|
+
|
|
18
|
+
import { openDatabase } from "./sqlite.mjs";
|
|
19
|
+
import { storePath } from "./store.mjs";
|
|
20
|
+
|
|
21
|
+
const DIRECT_ENV = {
|
|
22
|
+
baseUrl: "AGENTIC_WORKFLOW_DOC_EMBEDDINGS_BASE_URL",
|
|
23
|
+
model: "AGENTIC_WORKFLOW_DOC_EMBEDDINGS_MODEL",
|
|
24
|
+
apiKey: "AGENTIC_WORKFLOW_DOC_EMBEDDINGS_API_KEY",
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
/** Encode a float vector as the tagged little-endian float32 blob. */
|
|
28
|
+
export function encodeVector(vector) {
|
|
29
|
+
const header = Buffer.from(VECTOR_TAG, "binary");
|
|
30
|
+
const payload = Buffer.alloc(vector.length * 4);
|
|
31
|
+
for (let i = 0; i < vector.length; i++) payload.writeFloatLE(vector[i], i * 4);
|
|
32
|
+
return Buffer.concat([header, payload]);
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** Decode a tagged float32-LE blob; refuses blobs without the format tag. */
|
|
36
|
+
export function decodeVector(blob) {
|
|
37
|
+
const bytes = Buffer.isBuffer(blob) ? blob : Buffer.from(blob);
|
|
38
|
+
if (bytes.length < VECTOR_TAG.length || bytes.subarray(0, VECTOR_TAG.length).toString("binary") !== VECTOR_TAG) {
|
|
39
|
+
throw new Error("embedding blob is missing the format tag — refusing to guess the layout");
|
|
40
|
+
}
|
|
41
|
+
const payload = bytes.subarray(VECTOR_TAG.length);
|
|
42
|
+
const out = new Array(payload.length / 4);
|
|
43
|
+
for (let i = 0; i < out.length; i++) out[i] = payload.readFloatLE(i * 4);
|
|
44
|
+
return out;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Resolve the embedding provider config (D10):
|
|
49
|
+
* `cli` > direct pi-style env vars > the config file (with the key read
|
|
50
|
+
* from the config-named env var) > default (none). Returns
|
|
51
|
+
* `{configured: true, baseUrl, model, apiKey}` or `{configured: false}`.
|
|
52
|
+
*/
|
|
53
|
+
export function resolveEmbedConfig({ config = {}, env = {}, cli = {} } = {}) {
|
|
54
|
+
const provider = config.provider ?? {};
|
|
55
|
+
const pick = (cliValue, envName, configValue) =>
|
|
56
|
+
cliValue ?? (envName ? env[envName] : undefined) ?? configValue ?? null;
|
|
57
|
+
const baseUrl = pick(cli.baseUrl, DIRECT_ENV.baseUrl, provider.baseUrl);
|
|
58
|
+
const model = pick(cli.model, DIRECT_ENV.model, provider.model);
|
|
59
|
+
const apiKey = pick(cli.apiKey, DIRECT_ENV.apiKey, provider.envVar ? env[provider.envVar] : undefined);
|
|
60
|
+
if (!baseUrl || !model || !apiKey) return { configured: false };
|
|
61
|
+
return { configured: true, baseUrl, model, apiKey };
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* OpenAI-compatible embeddings provider. One POST per batch; `fetchImpl`
|
|
66
|
+
* is injectable for tests. A failed call throws — it never silently
|
|
67
|
+
* returns junk vectors.
|
|
68
|
+
*/
|
|
69
|
+
export function makeOpenAIProvider({ baseUrl, apiKey, model, fetchImpl } = {}) {
|
|
70
|
+
const doFetch = fetchImpl ?? ((url, init) => fetch(url, init));
|
|
71
|
+
return {
|
|
72
|
+
model,
|
|
73
|
+
async embed(texts) {
|
|
74
|
+
const response = await doFetch(`${baseUrl.replace(/\/$/, "")}/embeddings`, {
|
|
75
|
+
method: "POST",
|
|
76
|
+
headers: { "content-type": "application/json", authorization: `Bearer ${apiKey}` },
|
|
77
|
+
body: JSON.stringify({ model, input: texts }),
|
|
78
|
+
});
|
|
79
|
+
if (!response.ok) {
|
|
80
|
+
throw new Error(`embeddings provider ${response.status} ${response.statusText}`);
|
|
81
|
+
}
|
|
82
|
+
const data = await response.json();
|
|
83
|
+
return data.data.map((d) => d.embedding);
|
|
84
|
+
},
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* Embed every chunk that has no stored vector, batched per file. The
|
|
90
|
+
* store's pinned model (meta `emb_model`) fails closed on mismatch (AC18):
|
|
91
|
+
* the new provider is never called, the old vectors stay consistent, and
|
|
92
|
+
* the caller degrades — `--rebuild` re-embeds under the new model.
|
|
93
|
+
* Returns `{embedded, calls, model}` or `{skipped: true, reason}`.
|
|
94
|
+
*/
|
|
95
|
+
export async function embedPendingChunks(root, provider) {
|
|
96
|
+
const { db } = await openDatabase(storePath(root));
|
|
97
|
+
try {
|
|
98
|
+
ensureEmbeddingColumns(db);
|
|
99
|
+
const pinned = db.prepare("select value from meta where key = 'emb_model'").get();
|
|
100
|
+
if (pinned && pinned.value !== provider.model) {
|
|
101
|
+
return { skipped: true, reason: "embeddings-model-mismatch", model: provider.model, stored: pinned.value };
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
const pending = db
|
|
105
|
+
.prepare(
|
|
106
|
+
"select rowid, path, body from chunks where embedding is null order by path, rowid",
|
|
107
|
+
)
|
|
108
|
+
.all();
|
|
109
|
+
if (pending.length === 0) {
|
|
110
|
+
db.prepare("INSERT OR REPLACE INTO meta (key, value) VALUES ('emb_model', ?)").run(provider.model);
|
|
111
|
+
return { embedded: 0, calls: 0, model: provider.model };
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
let calls = 0;
|
|
115
|
+
let embedded = 0;
|
|
116
|
+
let batch = [];
|
|
117
|
+
const flush = async () => {
|
|
118
|
+
if (batch.length === 0) return;
|
|
119
|
+
const vectors = await provider.embed(batch.map((c) => c.body));
|
|
120
|
+
calls += 1;
|
|
121
|
+
for (let i = 0; i < batch.length; i++) {
|
|
122
|
+
db.prepare("update chunks set embedding = ?, emb_model = ? where rowid = ?").run(
|
|
123
|
+
encodeVector(vectors[i]),
|
|
124
|
+
provider.model,
|
|
125
|
+
batch[i].rowid,
|
|
126
|
+
);
|
|
127
|
+
embedded += 1;
|
|
128
|
+
}
|
|
129
|
+
batch = [];
|
|
130
|
+
};
|
|
131
|
+
|
|
132
|
+
let currentPath = null;
|
|
133
|
+
for (const chunk of pending) {
|
|
134
|
+
if (currentPath !== null && chunk.path !== currentPath) await flush();
|
|
135
|
+
currentPath = chunk.path;
|
|
136
|
+
batch.push(chunk);
|
|
137
|
+
}
|
|
138
|
+
await flush();
|
|
139
|
+
|
|
140
|
+
db.prepare("INSERT OR REPLACE INTO meta (key, value) VALUES ('emb_model', ?)").run(provider.model);
|
|
141
|
+
return { embedded, calls, model: provider.model };
|
|
142
|
+
} finally {
|
|
143
|
+
db.close();
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/** The store's pinned embedding model (null before the first embed). */
|
|
148
|
+
export async function storedEmbeddingModel(root) {
|
|
149
|
+
const { db } = await openDatabase(storePath(root));
|
|
150
|
+
try {
|
|
151
|
+
const row = db.prepare("select value from meta where key = 'emb_model'").get();
|
|
152
|
+
return row ? row.value : null;
|
|
153
|
+
} finally {
|
|
154
|
+
db.close();
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
function ensureEmbeddingColumns(db) {
|
|
159
|
+
const cols = db.prepare("pragma table_info(chunks)").all().map((c) => c.name);
|
|
160
|
+
if (!cols.includes("embedding")) db.exec("ALTER TABLE chunks ADD COLUMN embedding BLOB");
|
|
161
|
+
if (!cols.includes("emb_model")) db.exec("ALTER TABLE chunks ADD COLUMN emb_model TEXT");
|
|
162
|
+
}
|