@gtrabanco/agentic-workflow 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -2
- package/bin/{agwo.mjs → agentic-workflow.mjs} +49 -7
- package/package.json +17 -5
- package/src/doc/chunks.mjs +150 -0
- package/src/doc/config.mjs +64 -0
- package/src/doc/embeddings.mjs +162 -0
- package/src/doc/envelope.mjs +120 -0
- package/src/doc/grammar.mjs +109 -0
- package/src/doc/manifest.mjs +84 -0
- package/src/doc/query.mjs +279 -0
- package/src/doc/retrieval.mjs +82 -0
- package/src/doc/sqlite.mjs +34 -0
- package/src/doc/store.mjs +57 -0
- package/src/doc/sync.mjs +159 -0
- package/src/edit/schema.mjs +1 -1
- package/test/edit.test.mjs +0 -358
- package/test/fixtures.mjs +0 -326
- package/test/path-guard.engine.test.mjs +0 -304
- package/test/turn-contract.engine.test.mjs +0 -103
- package/test/turn-contract.parity.test.mjs +0 -20
package/README.md
CHANGED
|
@@ -5,8 +5,12 @@ created it as the vehicle the roadmap mandated (rows 37/38/42/45, declined 43):
|
|
|
5
5
|
the first producer feature creates the crate, later producers land their scripts
|
|
6
6
|
as subcommands of it.
|
|
7
7
|
|
|
8
|
-
- **
|
|
9
|
-
bun first and node as the guaranteed fallback
|
|
8
|
+
- **Shipped via `@gtrabanco/pi-agentic-workflow` (the pi package), one pinned pure-JS dependency, no native deps, no build step.**
|
|
9
|
+
Repository tooling runs with bun first and node as the guaranteed fallback
|
|
10
|
+
(`AGENTS.md` §Verification); `typebox` is exact-pinned and carries the `doc`
|
|
11
|
+
entry point's envelope validation. The retrieval store lives in a gitignored
|
|
12
|
+
SQLite database under `.agentic-workflow/index/` — disposable build output,
|
|
13
|
+
regenerated, never committed.
|
|
10
14
|
- **Scratch convention:** `.agentic-workflow/tmp/` at the repository root holds
|
|
11
15
|
throwaway producer output. It is committed as a directory (`.gitkeep`) so the
|
|
12
16
|
convention exists on a fresh clone.
|
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
/**
|
|
3
|
-
*
|
|
3
|
+
* agentic-workflow — CLI router for agentic-workflow typed file services.
|
|
4
4
|
*
|
|
5
5
|
* Usage:
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
6
|
+
* agentic-workflow unit-doc <NN-slug> [--dir <root>] <op> [args...]
|
|
7
|
+
* agentic-workflow roadmap <op> [args...]
|
|
8
|
+
* agentic-workflow changelog <op> [args...]
|
|
9
|
+
* agentic-workflow budgets <op> [args...]
|
|
10
|
+
* agentic-workflow manifest <op> [args...]
|
|
11
|
+
* agentic-workflow doc (--sync | --query <term> | --status | --rebuild) [--json-only]
|
|
11
12
|
*
|
|
12
13
|
* --json prints the operation receipt as JSON.
|
|
13
14
|
* Every operation prints a fixed receipt block to stdout.
|
|
@@ -24,6 +25,8 @@ import { rowUpsert, annotate as roadmapAnnotate } from "../src/edit/Roadmap.mjs"
|
|
|
24
25
|
import { rowAdd } from "../src/edit/Changelog.mjs";
|
|
25
26
|
import { ceilingRebase } from "../src/edit/Budgets.mjs";
|
|
26
27
|
import { skillAdd, skillRemove } from "../src/edit/Manifest.mjs";
|
|
28
|
+
import { parseDocArgs } from "../src/doc/grammar.mjs";
|
|
29
|
+
import { runDocOp } from "../src/doc/retrieval.mjs";
|
|
27
30
|
|
|
28
31
|
// ── Helpers ──────────────────────────────────────────────────────────────
|
|
29
32
|
|
|
@@ -63,7 +66,7 @@ function refuse(reason) {
|
|
|
63
66
|
|
|
64
67
|
const command = process.argv[2];
|
|
65
68
|
if (!command) {
|
|
66
|
-
console.error("usage:
|
|
69
|
+
console.error("usage: agentic-workflow <unit-doc|roadmap|changelog|budgets|manifest|doc> ...");
|
|
67
70
|
process.exit(1);
|
|
68
71
|
}
|
|
69
72
|
|
|
@@ -72,6 +75,45 @@ const parsed = parseArgs(restArgs);
|
|
|
72
75
|
const rawArgs = parsed.args;
|
|
73
76
|
|
|
74
77
|
switch (command) {
|
|
78
|
+
// ── doc — retrieval entry point (unit 65, D2) ────────────────────────
|
|
79
|
+
case "doc": {
|
|
80
|
+
let grammar;
|
|
81
|
+
try {
|
|
82
|
+
grammar = parseDocArgs(restArgs);
|
|
83
|
+
} catch (e) {
|
|
84
|
+
process.stderr.write(`doc: ${e.message}\n`);
|
|
85
|
+
process.exit(1);
|
|
86
|
+
}
|
|
87
|
+
let envelope;
|
|
88
|
+
try {
|
|
89
|
+
envelope = await runDocOp(grammar, { rootDir: process.cwd() });
|
|
90
|
+
} catch (e) {
|
|
91
|
+
// IO/store failures are usage/IO territory (exit 1, D3); the message
|
|
92
|
+
// goes to stderr and stdout stays empty.
|
|
93
|
+
process.stderr.write(`doc: ${e.message}\n`);
|
|
94
|
+
process.exit(1);
|
|
95
|
+
}
|
|
96
|
+
if (grammar.jsonOnly) {
|
|
97
|
+
// AC8: exactly one JSON document on stdout; diagnostics on stderr.
|
|
98
|
+
process.stdout.write(JSON.stringify(envelope) + "\n");
|
|
99
|
+
} else if (grammar.quiet) {
|
|
100
|
+
// AC23's hook spelling: --sync --quiet writes nothing on success;
|
|
101
|
+
// diagnostics stay on stderr. Exit code still reports failure.
|
|
102
|
+
if (envelope.ok !== true) process.stderr.write(`doc ${envelope.command}: failed\n`);
|
|
103
|
+
} else {
|
|
104
|
+
for (const d of envelope.degradations) process.stderr.write(`doc: degradation: ${d}\n`);
|
|
105
|
+
for (const r of envelope.results) {
|
|
106
|
+
console.log(`${r.path} · ${r.section ?? "-"} · L${r.lines[0]}-${r.lines[1]} · ${r.score}`);
|
|
107
|
+
}
|
|
108
|
+
if (envelope.store?.lastSync) {
|
|
109
|
+
const s = envelope.store.lastSync;
|
|
110
|
+
console.log(`doc sync: files_scanned=${s.files_scanned} files_changed=${s.files_changed} files_deleted=${s.files_deleted} chunks=${envelope.store.chunks}`);
|
|
111
|
+
}
|
|
112
|
+
console.log(`doc ${envelope.command}: ${envelope.results.length} result(s)`);
|
|
113
|
+
}
|
|
114
|
+
break;
|
|
115
|
+
}
|
|
116
|
+
|
|
75
117
|
// ── unit-doc ─────────────────────────────────────────────────────────
|
|
76
118
|
case "unit-doc": {
|
|
77
119
|
const slug = rawArgs[0];
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@gtrabanco/agentic-workflow",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"description": "
|
|
3
|
+
"version": "0.2.0",
|
|
4
|
+
"description": "Deterministic runner for the agentic-workflow lane: file services (agwo edit), sensors, guards and receipts. No dependencies.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "Gabriel Trabanco <1969593+gtrabanco@users.noreply.github.com>",
|
|
7
7
|
"repository": {
|
|
@@ -11,10 +11,22 @@
|
|
|
11
11
|
},
|
|
12
12
|
"type": "module",
|
|
13
13
|
"bin": {
|
|
14
|
-
"
|
|
14
|
+
"agentic-workflow": "./bin/agentic-workflow.mjs"
|
|
15
15
|
},
|
|
16
16
|
"engines": {
|
|
17
|
-
"node": ">=
|
|
17
|
+
"node": ">=24",
|
|
18
18
|
"bun": ">=1.4"
|
|
19
|
+
},
|
|
20
|
+
"scripts": {
|
|
21
|
+
"test": "bun test test/",
|
|
22
|
+
"test:node": "node --test"
|
|
23
|
+
},
|
|
24
|
+
"files": [
|
|
25
|
+
"bin",
|
|
26
|
+
"src",
|
|
27
|
+
"README.md"
|
|
28
|
+
],
|
|
29
|
+
"dependencies": {
|
|
30
|
+
"typebox": "1.3.7"
|
|
19
31
|
}
|
|
20
|
-
}
|
|
32
|
+
}
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Fence-aware section chunker (unit 65, D5).
|
|
3
|
+
*
|
|
4
|
+
* A chunk is one ATX-heading-delimited section found by a scanner where a `#`
|
|
5
|
+
* inside a fenced code block is never a heading. The id is
|
|
6
|
+
* `<relative-path>#<slug of the heading path>` with `-2` style dedupe,
|
|
7
|
+
* derived only from content — no timestamps, no mtimes — so ids are stable
|
|
8
|
+
* across sync runs (AC2/AC12) and half A's structure index shares the same
|
|
9
|
+
* function (no drift between the halves). `lines` is 1-based inclusive.
|
|
10
|
+
*
|
|
11
|
+
* File-level frontmatter (`---` block) is parsed once and attached to every
|
|
12
|
+
* chunk as `meta`; frontmatter lines never enter a chunk body. The preamble
|
|
13
|
+
* (content before the first heading) is a chunk with `section: null` and id
|
|
14
|
+
* `<path>#` — emitted only when that range is non-empty. A file with no
|
|
15
|
+
* headings at all yields exactly the preamble chunk.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
const FENCE_RE = /^(`{3,}|~{3,})/;
|
|
19
|
+
const HEADING_RE = /^(#{1,6})\s+(.*)$/;
|
|
20
|
+
|
|
21
|
+
function slugify(headingPath) {
|
|
22
|
+
return headingPath
|
|
23
|
+
.toLowerCase()
|
|
24
|
+
.replace(/\s*>\s*/g, "-")
|
|
25
|
+
.replace(/[^a-z0-9-]+/g, "-")
|
|
26
|
+
.replace(/-{2,}/g, "-")
|
|
27
|
+
.replace(/^-+|-+$/g, "");
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** Minimal frontmatter: top-level `key: value` scalars, comma lists, and `>`/`|` folded blocks. */
|
|
31
|
+
function parseFrontmatter(lines) {
|
|
32
|
+
const meta = {};
|
|
33
|
+
for (let i = 0; i < lines.length; i++) {
|
|
34
|
+
const m = lines[i].match(/^([A-Za-z0-9_-]+):\s*(.*)$/);
|
|
35
|
+
if (!m) continue;
|
|
36
|
+
const key = m[1];
|
|
37
|
+
let value = m[2].trim();
|
|
38
|
+
if (value === ">" || value === "|" || value === ">-" || value === "|-") {
|
|
39
|
+
const block = [];
|
|
40
|
+
while (i + 1 < lines.length && /^\s/.test(lines[i + 1])) {
|
|
41
|
+
block.push(lines[i + 1].trim());
|
|
42
|
+
i++;
|
|
43
|
+
}
|
|
44
|
+
value = block.join(" ");
|
|
45
|
+
} else if (value.includes(",")) {
|
|
46
|
+
value = value.split(",").map((s) => s.trim()).filter((s) => s !== "");
|
|
47
|
+
} else {
|
|
48
|
+
value = value.replace(/^["']|["']$/g, "");
|
|
49
|
+
}
|
|
50
|
+
meta[key] = value;
|
|
51
|
+
}
|
|
52
|
+
return meta;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Scan `markdown` into chunks attributed to `relativePath`.
|
|
57
|
+
* Returns rows `{id, path, section, lines: [start, end], meta, body}` in
|
|
58
|
+
* document order. Deterministic: identical input ⇒ identical output.
|
|
59
|
+
*/
|
|
60
|
+
export function scanChunks(markdown, relativePath) {
|
|
61
|
+
const lines = String(markdown).split(/\r?\n/);
|
|
62
|
+
|
|
63
|
+
// Frontmatter block, only when it opens the file.
|
|
64
|
+
let contentStart = 0;
|
|
65
|
+
let meta = {};
|
|
66
|
+
if (lines.length > 0 && lines[0].trim() === "---") {
|
|
67
|
+
let end = -1;
|
|
68
|
+
for (let i = 1; i < lines.length; i++) {
|
|
69
|
+
if (lines[i].trim() === "---") {
|
|
70
|
+
end = i;
|
|
71
|
+
break;
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
if (end !== -1) {
|
|
75
|
+
meta = parseFrontmatter(lines.slice(1, end));
|
|
76
|
+
contentStart = end + 1;
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
// Headings outside fenced code blocks.
|
|
81
|
+
const headings = []; // {level, title, line (1-based)}
|
|
82
|
+
let fence = null;
|
|
83
|
+
for (let i = contentStart; i < lines.length; i++) {
|
|
84
|
+
const trimmed = lines[i].trim();
|
|
85
|
+
if (fence !== null) {
|
|
86
|
+
if (trimmed.startsWith(fence)) fence = null;
|
|
87
|
+
continue;
|
|
88
|
+
}
|
|
89
|
+
const f = trimmed.match(FENCE_RE);
|
|
90
|
+
if (f) {
|
|
91
|
+
fence = f[1][0].repeat(3);
|
|
92
|
+
continue;
|
|
93
|
+
}
|
|
94
|
+
const h = trimmed.match(HEADING_RE);
|
|
95
|
+
if (h) headings.push({ level: h[1].length, title: h[2].trim(), line: i + 1 });
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
const chunks = [];
|
|
99
|
+
const usedIds = new Set();
|
|
100
|
+
const nextId = (headingPath) => {
|
|
101
|
+
const slug = slugify(headingPath);
|
|
102
|
+
let id = `${relativePath}#${slug}`;
|
|
103
|
+
let n = 2;
|
|
104
|
+
while (usedIds.has(id)) id = `${relativePath}#${slug}-${n++}`;
|
|
105
|
+
usedIds.add(id);
|
|
106
|
+
return id;
|
|
107
|
+
};
|
|
108
|
+
|
|
109
|
+
// Preamble chunk — only when its range is non-empty and body is non-blank
|
|
110
|
+
// (AC10: skip frontmatter-only preamble to avoid empty-body noise).
|
|
111
|
+
const firstHeadingLine = headings.length > 0 ? headings[0].line : lines.length + 1;
|
|
112
|
+
if (firstHeadingLine > 1 || headings.length === 0) {
|
|
113
|
+
const to = headings.length > 0 ? firstHeadingLine - 1 : lines.length;
|
|
114
|
+
const body = lines
|
|
115
|
+
.slice(0, to)
|
|
116
|
+
.filter((_, idx) => !(idx < contentStart))
|
|
117
|
+
.join("\n");
|
|
118
|
+
if (body.trim() !== "") {
|
|
119
|
+
chunks.push({
|
|
120
|
+
id: `${relativePath}#`,
|
|
121
|
+
path: relativePath,
|
|
122
|
+
section: null,
|
|
123
|
+
lines: [contentStart + 1, to],
|
|
124
|
+
meta,
|
|
125
|
+
body,
|
|
126
|
+
});
|
|
127
|
+
usedIds.add(`${relativePath}#`);
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
// Heading sections with their heading-path chain.
|
|
132
|
+
const stack = []; // {level, title}
|
|
133
|
+
for (let s = 0; s < headings.length; s++) {
|
|
134
|
+
const heading = headings[s];
|
|
135
|
+
while (stack.length > 0 && stack[stack.length - 1].level >= heading.level) stack.pop();
|
|
136
|
+
stack.push({ level: heading.level, title: heading.title });
|
|
137
|
+
const headingPath = stack.map((e) => e.title).join(" > ");
|
|
138
|
+
const to = s + 1 < headings.length ? headings[s + 1].line - 1 : lines.length;
|
|
139
|
+
chunks.push({
|
|
140
|
+
id: nextId(headingPath),
|
|
141
|
+
path: relativePath,
|
|
142
|
+
section: headingPath,
|
|
143
|
+
lines: [heading.line, to],
|
|
144
|
+
meta,
|
|
145
|
+
body: lines.slice(heading.line - 1, to).join("\n"),
|
|
146
|
+
});
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
return chunks;
|
|
150
|
+
}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Committed config for the `doc` retrieval index (unit 65, D4/D10).
|
|
3
|
+
*
|
|
4
|
+
* `.agentic-workflow/index.json` is the committed decision surface (shipped
|
|
5
|
+
* by init-workspace in target projects). It defaults when absent — a missing
|
|
6
|
+
* config is never a hard failure; the index answers keyword-only until a
|
|
7
|
+
* provider is configured (AC17's `unavailable-embeddings-not-configured`).
|
|
8
|
+
* A present but unparseable config is a usage/IO error (exit 1 territory);
|
|
9
|
+
* an unknown version fails closed — the reader never guesses.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
13
|
+
import { join, resolve, sep } from "node:path";
|
|
14
|
+
|
|
15
|
+
export const CONFIG_VERSION = 1;
|
|
16
|
+
|
|
17
|
+
export const DEFAULT_CONFIG = Object.freeze({
|
|
18
|
+
version: CONFIG_VERSION,
|
|
19
|
+
store: Object.freeze({ path: ".agentic-workflow/index/index.db" }),
|
|
20
|
+
// D10 provider surface: null ⇒ keyword-only. When configured: an
|
|
21
|
+
// OpenAI-compatible {baseUrl, model, envVar} — the env var is read at
|
|
22
|
+
// runtime in pi's precedence (CLI flag > environment > config > default).
|
|
23
|
+
provider: null,
|
|
24
|
+
});
|
|
25
|
+
|
|
26
|
+
export function configPath(root) {
|
|
27
|
+
return join(root, ".agentic-workflow", "index.json");
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Load the config for the project at `root`: defaults merged under the
|
|
32
|
+
* file's overrides. Throws on unparseable JSON (message names the file) and
|
|
33
|
+
* on an unknown `version` (fail closed).
|
|
34
|
+
*/
|
|
35
|
+
export function loadConfig(root) {
|
|
36
|
+
const path = configPath(root);
|
|
37
|
+
if (!existsSync(path)) return structuredClone(DEFAULT_CONFIG);
|
|
38
|
+
let parsed;
|
|
39
|
+
try {
|
|
40
|
+
parsed = JSON.parse(readFileSync(path, "utf8"));
|
|
41
|
+
} catch (error) {
|
|
42
|
+
throw new Error(`index.json is not valid JSON (${path}): ${error.message}`);
|
|
43
|
+
}
|
|
44
|
+
if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) {
|
|
45
|
+
throw new Error(`index.json must be a JSON object (${path})`);
|
|
46
|
+
}
|
|
47
|
+
if (parsed.version !== undefined && parsed.version !== CONFIG_VERSION) {
|
|
48
|
+
throw new Error(`index.json version ${parsed.version} is not supported (expected ${CONFIG_VERSION})`);
|
|
49
|
+
}
|
|
50
|
+
// Contain store.path inside the repository root (AC7: gitignored cache must not
|
|
51
|
+
// escape). An absolute path that is outside the root, or a relative path that
|
|
52
|
+
// resolves outside (contains `..`), is rejected — the `rmSync` in --rebuild
|
|
53
|
+
// targets this path, so a misconfigured store path is a destructive-write
|
|
54
|
+
// footgun.
|
|
55
|
+
if (parsed.store?.path) {
|
|
56
|
+
const candidate = parsed.store.path;
|
|
57
|
+
if (!resolve(root, candidate).startsWith(resolve(root) + sep)) {
|
|
58
|
+
throw new Error(
|
|
59
|
+
`index.json store.path "${candidate}" escapes the repository root (${root})`,
|
|
60
|
+
);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
return { ...structuredClone(DEFAULT_CONFIG), ...parsed };
|
|
64
|
+
}
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Embeddings write path for the `doc` retrieval index (unit 65, P6).
|
|
3
|
+
*
|
|
4
|
+
* D10 provider resolution: CLI flag > direct pi-style environment variables
|
|
5
|
+
* > the config-named env var + config file > default (none ⇒ keyword-only,
|
|
6
|
+
* never a hard failure). Vectors are float32-LE blobs behind a format tag
|
|
7
|
+
* (D4) with the producing model pinned per row and in the store meta — a
|
|
8
|
+
* model mismatch fails closed: nothing re-embeds and no cross-model store
|
|
9
|
+
* can form (AC18); `--rebuild` under the new model is the documented path.
|
|
10
|
+
*
|
|
11
|
+
* Incrementalism (AC16): embedding runs only over chunks with no stored
|
|
12
|
+
* vector, batched per file, so a warm sync makes 0 API calls (AC19's write
|
|
13
|
+
* half) and changing one file re-embeds exactly that file's chunks.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
export const VECTOR_TAG = "awkvec-f32le1";
|
|
17
|
+
|
|
18
|
+
import { openDatabase } from "./sqlite.mjs";
|
|
19
|
+
import { storePath } from "./store.mjs";
|
|
20
|
+
|
|
21
|
+
const DIRECT_ENV = {
|
|
22
|
+
baseUrl: "AGENTIC_WORKFLOW_DOC_EMBEDDINGS_BASE_URL",
|
|
23
|
+
model: "AGENTIC_WORKFLOW_DOC_EMBEDDINGS_MODEL",
|
|
24
|
+
apiKey: "AGENTIC_WORKFLOW_DOC_EMBEDDINGS_API_KEY",
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
/** Encode a float vector as the tagged little-endian float32 blob. */
|
|
28
|
+
export function encodeVector(vector) {
|
|
29
|
+
const header = Buffer.from(VECTOR_TAG, "binary");
|
|
30
|
+
const payload = Buffer.alloc(vector.length * 4);
|
|
31
|
+
for (let i = 0; i < vector.length; i++) payload.writeFloatLE(vector[i], i * 4);
|
|
32
|
+
return Buffer.concat([header, payload]);
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** Decode a tagged float32-LE blob; refuses blobs without the format tag. */
|
|
36
|
+
export function decodeVector(blob) {
|
|
37
|
+
const bytes = Buffer.isBuffer(blob) ? blob : Buffer.from(blob);
|
|
38
|
+
if (bytes.length < VECTOR_TAG.length || bytes.subarray(0, VECTOR_TAG.length).toString("binary") !== VECTOR_TAG) {
|
|
39
|
+
throw new Error("embedding blob is missing the format tag — refusing to guess the layout");
|
|
40
|
+
}
|
|
41
|
+
const payload = bytes.subarray(VECTOR_TAG.length);
|
|
42
|
+
const out = new Array(payload.length / 4);
|
|
43
|
+
for (let i = 0; i < out.length; i++) out[i] = payload.readFloatLE(i * 4);
|
|
44
|
+
return out;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Resolve the embedding provider config (D10):
|
|
49
|
+
* `cli` > direct pi-style env vars > the config file (with the key read
|
|
50
|
+
* from the config-named env var) > default (none). Returns
|
|
51
|
+
* `{configured: true, baseUrl, model, apiKey}` or `{configured: false}`.
|
|
52
|
+
*/
|
|
53
|
+
export function resolveEmbedConfig({ config = {}, env = {}, cli = {} } = {}) {
|
|
54
|
+
const provider = config.provider ?? {};
|
|
55
|
+
const pick = (cliValue, envName, configValue) =>
|
|
56
|
+
cliValue ?? (envName ? env[envName] : undefined) ?? configValue ?? null;
|
|
57
|
+
const baseUrl = pick(cli.baseUrl, DIRECT_ENV.baseUrl, provider.baseUrl);
|
|
58
|
+
const model = pick(cli.model, DIRECT_ENV.model, provider.model);
|
|
59
|
+
const apiKey = pick(cli.apiKey, DIRECT_ENV.apiKey, provider.envVar ? env[provider.envVar] : undefined);
|
|
60
|
+
if (!baseUrl || !model || !apiKey) return { configured: false };
|
|
61
|
+
return { configured: true, baseUrl, model, apiKey };
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* OpenAI-compatible embeddings provider. One POST per batch; `fetchImpl`
|
|
66
|
+
* is injectable for tests. A failed call throws — it never silently
|
|
67
|
+
* returns junk vectors.
|
|
68
|
+
*/
|
|
69
|
+
export function makeOpenAIProvider({ baseUrl, apiKey, model, fetchImpl } = {}) {
|
|
70
|
+
const doFetch = fetchImpl ?? ((url, init) => fetch(url, init));
|
|
71
|
+
return {
|
|
72
|
+
model,
|
|
73
|
+
async embed(texts) {
|
|
74
|
+
const response = await doFetch(`${baseUrl.replace(/\/$/, "")}/embeddings`, {
|
|
75
|
+
method: "POST",
|
|
76
|
+
headers: { "content-type": "application/json", authorization: `Bearer ${apiKey}` },
|
|
77
|
+
body: JSON.stringify({ model, input: texts }),
|
|
78
|
+
});
|
|
79
|
+
if (!response.ok) {
|
|
80
|
+
throw new Error(`embeddings provider ${response.status} ${response.statusText}`);
|
|
81
|
+
}
|
|
82
|
+
const data = await response.json();
|
|
83
|
+
return data.data.map((d) => d.embedding);
|
|
84
|
+
},
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* Embed every chunk that has no stored vector, batched per file. The
|
|
90
|
+
* store's pinned model (meta `emb_model`) fails closed on mismatch (AC18):
|
|
91
|
+
* the new provider is never called, the old vectors stay consistent, and
|
|
92
|
+
* the caller degrades — `--rebuild` re-embeds under the new model.
|
|
93
|
+
* Returns `{embedded, calls, model}` or `{skipped: true, reason}`.
|
|
94
|
+
*/
|
|
95
|
+
export async function embedPendingChunks(root, provider) {
|
|
96
|
+
const { db } = await openDatabase(storePath(root));
|
|
97
|
+
try {
|
|
98
|
+
ensureEmbeddingColumns(db);
|
|
99
|
+
const pinned = db.prepare("select value from meta where key = 'emb_model'").get();
|
|
100
|
+
if (pinned && pinned.value !== provider.model) {
|
|
101
|
+
return { skipped: true, reason: "embeddings-model-mismatch", model: provider.model, stored: pinned.value };
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
const pending = db
|
|
105
|
+
.prepare(
|
|
106
|
+
"select rowid, path, body from chunks where embedding is null order by path, rowid",
|
|
107
|
+
)
|
|
108
|
+
.all();
|
|
109
|
+
if (pending.length === 0) {
|
|
110
|
+
db.prepare("INSERT OR REPLACE INTO meta (key, value) VALUES ('emb_model', ?)").run(provider.model);
|
|
111
|
+
return { embedded: 0, calls: 0, model: provider.model };
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
let calls = 0;
|
|
115
|
+
let embedded = 0;
|
|
116
|
+
let batch = [];
|
|
117
|
+
const flush = async () => {
|
|
118
|
+
if (batch.length === 0) return;
|
|
119
|
+
const vectors = await provider.embed(batch.map((c) => c.body));
|
|
120
|
+
calls += 1;
|
|
121
|
+
for (let i = 0; i < batch.length; i++) {
|
|
122
|
+
db.prepare("update chunks set embedding = ?, emb_model = ? where rowid = ?").run(
|
|
123
|
+
encodeVector(vectors[i]),
|
|
124
|
+
provider.model,
|
|
125
|
+
batch[i].rowid,
|
|
126
|
+
);
|
|
127
|
+
embedded += 1;
|
|
128
|
+
}
|
|
129
|
+
batch = [];
|
|
130
|
+
};
|
|
131
|
+
|
|
132
|
+
let currentPath = null;
|
|
133
|
+
for (const chunk of pending) {
|
|
134
|
+
if (currentPath !== null && chunk.path !== currentPath) await flush();
|
|
135
|
+
currentPath = chunk.path;
|
|
136
|
+
batch.push(chunk);
|
|
137
|
+
}
|
|
138
|
+
await flush();
|
|
139
|
+
|
|
140
|
+
db.prepare("INSERT OR REPLACE INTO meta (key, value) VALUES ('emb_model', ?)").run(provider.model);
|
|
141
|
+
return { embedded, calls, model: provider.model };
|
|
142
|
+
} finally {
|
|
143
|
+
db.close();
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/** The store's pinned embedding model (null before the first embed). */
|
|
148
|
+
export async function storedEmbeddingModel(root) {
|
|
149
|
+
const { db } = await openDatabase(storePath(root));
|
|
150
|
+
try {
|
|
151
|
+
const row = db.prepare("select value from meta where key = 'emb_model'").get();
|
|
152
|
+
return row ? row.value : null;
|
|
153
|
+
} finally {
|
|
154
|
+
db.close();
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
function ensureEmbeddingColumns(db) {
|
|
159
|
+
const cols = db.prepare("pragma table_info(chunks)").all().map((c) => c.name);
|
|
160
|
+
if (!cols.includes("embedding")) db.exec("ALTER TABLE chunks ADD COLUMN embedding BLOB");
|
|
161
|
+
if (!cols.includes("emb_model")) db.exec("ALTER TABLE chunks ADD COLUMN emb_model TEXT");
|
|
162
|
+
}
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Canonical envelope for the `doc` retrieval entry point (unit 65, D3).
|
|
3
|
+
*
|
|
4
|
+
* Under `--json-only` stdout carries exactly one JSON document:
|
|
5
|
+
* {ok, command, results[], degradations[], store}
|
|
6
|
+
* with the frozen key order above and result rows
|
|
7
|
+
* {path, section, lines, score, meta}
|
|
8
|
+
* in that order. `lines` is a 1-based inclusive [start, end] pair (D5).
|
|
9
|
+
* No timestamps anywhere (AC2/AC12): the envelope is byte-stable for an
|
|
10
|
+
* unchanged corpus and answer.
|
|
11
|
+
*
|
|
12
|
+
* `degradations[]` is a CLOSED vocabulary frozen here and asserted by test:
|
|
13
|
+
* unavailable-sqlite-<runtime> (runtime ∈ {node, bun})
|
|
14
|
+
* unavailable-embeddings-not-configured
|
|
15
|
+
* unavailable-embeddings-<cause> (kebab-case cause)
|
|
16
|
+
* embeddings-model-mismatch (AC18 fails closed)
|
|
17
|
+
* An unknown value is rejected at construction time — it fails, never ships.
|
|
18
|
+
*
|
|
19
|
+
* Exit codes (D3): 0 success including degraded · 1 usage/IO · 2 path
|
|
20
|
+
* refusal (edit services only).
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
// Lazy TypeBox import — typebox is only loaded when validateEnvelope is called.
|
|
24
|
+
// The bin/hook path that builds envelopes (buildEnvelope, buildResultRow)
|
|
25
|
+
// never calls validateEnvelope, so the cold path stays typebox-free.
|
|
26
|
+
|
|
27
|
+
let _Type = null;
|
|
28
|
+
let _Compile = null;
|
|
29
|
+
|
|
30
|
+
async function ensureTypes() {
|
|
31
|
+
if (_Type === null) {
|
|
32
|
+
const mod = await import("typebox");
|
|
33
|
+
_Type = mod.Type;
|
|
34
|
+
const compileMod = await import("typebox/compile");
|
|
35
|
+
_Compile = compileMod.Compile;
|
|
36
|
+
}
|
|
37
|
+
return { Type: _Type, Compile: _Compile };
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
let _ResultRowSchema = null;
|
|
41
|
+
let _EnvelopeSchema = null;
|
|
42
|
+
|
|
43
|
+
async function ensureSchemas() {
|
|
44
|
+
if (_ResultRowSchema === null) {
|
|
45
|
+
const { Type, Compile } = await ensureTypes();
|
|
46
|
+
_ResultRowSchema = Type.Object({
|
|
47
|
+
path: Type.String(),
|
|
48
|
+
section: Type.Union([Type.String(), Type.Null()]),
|
|
49
|
+
lines: Type.Tuple([Type.Integer({ minimum: 1 }), Type.Integer({ minimum: 1 })]),
|
|
50
|
+
score: Type.Number(),
|
|
51
|
+
meta: Type.Record(Type.String(), Type.Unknown()),
|
|
52
|
+
});
|
|
53
|
+
_EnvelopeSchema = Type.Object({
|
|
54
|
+
ok: Type.Boolean(),
|
|
55
|
+
command: Type.Union(DOC_COMMANDS.map((c) => Type.Literal(c))),
|
|
56
|
+
results: Type.Array(_ResultRowSchema),
|
|
57
|
+
degradations: Type.Array(Type.String()),
|
|
58
|
+
store: Type.Union([Type.Null(), Type.Record(Type.String(), Type.Unknown())]),
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
return { _ResultRowSchema, _EnvelopeSchema };
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
export const DOC_COMMANDS = ["sync", "query", "status", "rebuild"];
|
|
65
|
+
|
|
66
|
+
const DEGRADATION_PATTERNS = [
|
|
67
|
+
/^unavailable-sqlite-(node|bun)$/,
|
|
68
|
+
/^unavailable-embeddings-not-configured$/,
|
|
69
|
+
/^unavailable-embeddings-[a-z0-9]+(?:-[a-z0-9]+)*$/,
|
|
70
|
+
/^embeddings-model-mismatch$/,
|
|
71
|
+
];
|
|
72
|
+
|
|
73
|
+
/** True only for values in the closed degradation vocabulary (D3). */
|
|
74
|
+
export function degradationIsKnown(degradation) {
|
|
75
|
+
return DEGRADATION_PATTERNS.some((re) => re.test(degradation));
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
let _compiled = null;
|
|
80
|
+
|
|
81
|
+
async function getCompiled() {
|
|
82
|
+
if (_compiled === null) {
|
|
83
|
+
const { _EnvelopeSchema: EnvelopeSchema } = await ensureSchemas();
|
|
84
|
+
const { Compile } = await ensureTypes();
|
|
85
|
+
_compiled = Compile(EnvelopeSchema);
|
|
86
|
+
}
|
|
87
|
+
return _compiled;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/** TypeBox validation of a full envelope. Returns `{ok, errors[]}`. */
|
|
91
|
+
export async function validateEnvelope(value) {
|
|
92
|
+
const Compiled = await getCompiled();
|
|
93
|
+
const ok = Compiled.Check(value);
|
|
94
|
+
const errors = ok
|
|
95
|
+
? []
|
|
96
|
+
: [...Compiled.Errors(value)].map((e) => `${e.path || "/"}: ${e.message}`);
|
|
97
|
+
return { ok, errors };
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* Build the canonical envelope. Key order is the contract — constructed as a
|
|
102
|
+
* literal in the frozen order. Unknown commands and unknown degradations
|
|
103
|
+
* throw (fail closed, D3).
|
|
104
|
+
*/
|
|
105
|
+
export function buildEnvelope({ command, results = [], degradations = [], store = null, ok = true }) {
|
|
106
|
+
if (!DOC_COMMANDS.includes(command)) {
|
|
107
|
+
throw new Error(`usage: unknown doc command "${command}" (expected one of ${DOC_COMMANDS.join("|")})`);
|
|
108
|
+
}
|
|
109
|
+
for (const d of degradations) {
|
|
110
|
+
if (!degradationIsKnown(d)) {
|
|
111
|
+
throw new Error(`unknown degradation "${d}" — the vocabulary is closed (D3)`);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
return { ok, command, results, degradations: [...degradations], store };
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/** Build one result row in the frozen key order. */
|
|
118
|
+
export function buildResultRow({ path, section = null, lines, score, meta = {} }) {
|
|
119
|
+
return { path, section, lines: [lines[0], lines[1]], score, meta };
|
|
120
|
+
}
|