@gtrabanco/agentic-workflow 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -2
- package/bin/{agwo.mjs → agentic-workflow.mjs} +49 -7
- package/package.json +17 -5
- package/src/doc/chunks.mjs +150 -0
- package/src/doc/config.mjs +64 -0
- package/src/doc/embeddings.mjs +162 -0
- package/src/doc/envelope.mjs +120 -0
- package/src/doc/grammar.mjs +109 -0
- package/src/doc/manifest.mjs +84 -0
- package/src/doc/query.mjs +279 -0
- package/src/doc/retrieval.mjs +82 -0
- package/src/doc/sqlite.mjs +34 -0
- package/src/doc/store.mjs +57 -0
- package/src/doc/sync.mjs +159 -0
- package/src/edit/schema.mjs +1 -1
- package/test/edit.test.mjs +0 -358
- package/test/fixtures.mjs +0 -326
- package/test/path-guard.engine.test.mjs +0 -304
- package/test/turn-contract.engine.test.mjs +0 -103
- package/test/turn-contract.parity.test.mjs +0 -20
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Argument grammar for the `doc` retrieval entry point (unit 65, D2).
|
|
3
|
+
*
|
|
4
|
+
* One grammar branch: the first token after `doc` starting with `-` selects
|
|
5
|
+
* a retrieval operation (`--sync` / `--query <term>` / `--status` /
|
|
6
|
+
* `--rebuild`); a positional first token is a half-A doc operation that ships
|
|
7
|
+
* with unit 71 and is refused here. `--json-only` may appear anywhere and
|
|
8
|
+
* switches stdout to exactly one JSON document (AC8).
|
|
9
|
+
*
|
|
10
|
+
* Exit codes (D3): 0 success including degraded · 1 usage/IO · 2 path
|
|
11
|
+
* refusal (owned by the crate's edit services, not this grammar).
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
export const RETRIEVAL_FLAGS = {
|
|
15
|
+
"--sync": "sync",
|
|
16
|
+
"--query": "query",
|
|
17
|
+
"--status": "status",
|
|
18
|
+
"--rebuild": "rebuild",
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
function usageError(msg) {
|
|
22
|
+
const e = new Error(
|
|
23
|
+
`usage: agentic-workflow doc (--sync [--quiet] | --query <term> [--since <ISO>] [--until <ISO>] [--file <path>] [--mode keyword|hybrid] | --status | --rebuild) [--json-only] — ${msg}`,
|
|
24
|
+
);
|
|
25
|
+
e.code = "USAGE";
|
|
26
|
+
return e;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Parse the arguments after `doc`. Returns
|
|
31
|
+
* `{op: "sync"|"query"|"status"|"rebuild", query: string|null, jsonOnly: boolean}`.
|
|
32
|
+
* Throws a usage error (exit 1 territory) on any malformed invocation.
|
|
33
|
+
*/
|
|
34
|
+
export function parseDocArgs(argv) {
|
|
35
|
+
const out = { op: null, query: null, jsonOnly: false, quiet: false };
|
|
36
|
+
const filters = { since: null, until: null, file: null };
|
|
37
|
+
const FILTER_FLAGS = { "--since": "since", "--until": "until", "--file": "file" };
|
|
38
|
+
const MODES = new Set(["keyword", "hybrid"]);
|
|
39
|
+
let sawFilter = false;
|
|
40
|
+
let sawMode = false;
|
|
41
|
+
if (!Array.isArray(argv) || argv.length === 0) throw usageError("no operation given");
|
|
42
|
+
|
|
43
|
+
// D2: a positional first token is the half-A branch — not shipped in this unit.
|
|
44
|
+
if (!argv[0].startsWith("-")) {
|
|
45
|
+
throw usageError(
|
|
46
|
+
`positional doc operations (read/edit/index) ship with unit 71; retrieval ops: ${Object.keys(RETRIEVAL_FLAGS).join(" | ")}`,
|
|
47
|
+
);
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
let i = 0;
|
|
51
|
+
while (i < argv.length) {
|
|
52
|
+
const arg = argv[i];
|
|
53
|
+
if (arg === "--json-only") {
|
|
54
|
+
out.jsonOnly = true;
|
|
55
|
+
i++;
|
|
56
|
+
continue;
|
|
57
|
+
}
|
|
58
|
+
if (arg === "--quiet") {
|
|
59
|
+
out.quiet = true;
|
|
60
|
+
i++;
|
|
61
|
+
continue;
|
|
62
|
+
}
|
|
63
|
+
if (arg === "--mode") {
|
|
64
|
+
const value = argv[i + 1];
|
|
65
|
+
if (value === undefined || value === "") throw usageError("--mode requires a value (keyword | hybrid)");
|
|
66
|
+
if (!MODES.has(value)) throw usageError(`unknown mode "${value}" (expected keyword | hybrid)`);
|
|
67
|
+
out.mode = value;
|
|
68
|
+
sawMode = true;
|
|
69
|
+
i += 2;
|
|
70
|
+
continue;
|
|
71
|
+
}
|
|
72
|
+
if (arg in FILTER_FLAGS) {
|
|
73
|
+
const value = argv[i + 1];
|
|
74
|
+
if (value === undefined || value === "") {
|
|
75
|
+
throw usageError(`${arg} requires a non-empty value`);
|
|
76
|
+
}
|
|
77
|
+
filters[FILTER_FLAGS[arg]] = value;
|
|
78
|
+
sawFilter = true;
|
|
79
|
+
i += 2;
|
|
80
|
+
continue;
|
|
81
|
+
}
|
|
82
|
+
const op = RETRIEVAL_FLAGS[arg];
|
|
83
|
+
if (!op) throw usageError(`unknown flag "${arg}"`);
|
|
84
|
+
if (out.op) throw usageError("only one retrieval operation allowed");
|
|
85
|
+
out.op = op;
|
|
86
|
+
if (op === "query") {
|
|
87
|
+
const term = argv[i + 1];
|
|
88
|
+
if (term === undefined || term === "" || term.startsWith("-")) {
|
|
89
|
+
throw usageError("--query requires a non-empty term");
|
|
90
|
+
}
|
|
91
|
+
out.query = term;
|
|
92
|
+
i += 2;
|
|
93
|
+
continue;
|
|
94
|
+
}
|
|
95
|
+
i++;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
if (!out.op) throw usageError("no retrieval operation given");
|
|
99
|
+
// AC14's structured filters and the P7 mode flag belong to queries only.
|
|
100
|
+
if (out.op === "query") {
|
|
101
|
+
out.since = filters.since;
|
|
102
|
+
out.until = filters.until;
|
|
103
|
+
out.file = filters.file;
|
|
104
|
+
out.mode = out.mode ?? "keyword";
|
|
105
|
+
} else if (sawFilter || sawMode) {
|
|
106
|
+
throw usageError("--since/--until/--file/--mode are query flags and require --query");
|
|
107
|
+
}
|
|
108
|
+
return out;
|
|
109
|
+
}
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Canonical manifest for the `doc` retrieval index (unit 65, D4/D5).
|
|
3
|
+
*
|
|
4
|
+
* The manifest is the freshness surface: per-file sha256 over the tracked
|
|
5
|
+
* markdown corpus plus the git HEAD. It is a canonical JSON document —
|
|
6
|
+
* sorted keys at every level, no timestamps — so AC12's byte-for-byte
|
|
7
|
+
* determinism assertion compares two `collectManifest` runs' exports
|
|
8
|
+
* directly. `manifestHash` is the store's freshness fingerprint.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import { createHash } from "node:crypto";
|
|
12
|
+
import { execFileSync } from "node:child_process";
|
|
13
|
+
import { readFileSync } from "node:fs";
|
|
14
|
+
import { join } from "node:path";
|
|
15
|
+
|
|
16
|
+
/** Recursively JSON-stringify with object keys sorted — the canonical form. */
|
|
17
|
+
export function canonicalJson(value) {
|
|
18
|
+
if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`;
|
|
19
|
+
if (value !== null && typeof value === "object") {
|
|
20
|
+
const keys = Object.keys(value).sort();
|
|
21
|
+
return `{${keys.map((k) => `${JSON.stringify(k)}:${canonicalJson(value[k])}`).join(",")}}`;
|
|
22
|
+
}
|
|
23
|
+
return JSON.stringify(value);
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export function manifestHash(manifest) {
|
|
27
|
+
return createHash("sha256").update(canonicalJson(manifest)).digest("hex");
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Build the manifest document. `files` rows arrive in any order; the output
|
|
32
|
+
* is key-sorted. Rows carry exactly `{sha256, bytes}` — no mtimes, no
|
|
33
|
+
* timestamps anywhere (AC2/AC12).
|
|
34
|
+
*/
|
|
35
|
+
export function buildManifest({ head, files }) {
|
|
36
|
+
const sorted = [...files].sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0));
|
|
37
|
+
const entries = Object.fromEntries(
|
|
38
|
+
sorted.map((f) => [f.path, { sha256: f.sha256, bytes: f.bytes }]),
|
|
39
|
+
);
|
|
40
|
+
return { files: entries, head, version: 1 };
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/** Tracked markdown corpus: `git ls-files '*.md'` under `root`. */
|
|
44
|
+
export function corpusFiles(root) {
|
|
45
|
+
const out = execFileSync("git", ["ls-files", "--", "*.md"], {
|
|
46
|
+
cwd: root,
|
|
47
|
+
encoding: "utf8",
|
|
48
|
+
maxBuffer: 64 * 1024 * 1024,
|
|
49
|
+
});
|
|
50
|
+
return out.split("\n").map((s) => s.trim()).filter((s) => s !== "");
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Collect the manifest over the live repo at `root`: git HEAD plus one
|
|
55
|
+
* sha256 per tracked markdown file. Deterministic for an unchanged corpus
|
|
56
|
+
* (AC12): same files, same bytes, same HEAD ⇒ byte-identical canonical form.
|
|
57
|
+
*/
|
|
58
|
+
export function collectManifest(root) {
|
|
59
|
+
const head = execFileSync("git", ["rev-parse", "HEAD"], {
|
|
60
|
+
cwd: root,
|
|
61
|
+
encoding: "utf8",
|
|
62
|
+
maxBuffer: 1024 * 1024,
|
|
63
|
+
}).trim();
|
|
64
|
+
const files = corpusFiles(root).map((rel) => {
|
|
65
|
+
const abs = join(root, rel);
|
|
66
|
+
let bytes;
|
|
67
|
+
try {
|
|
68
|
+
bytes = readFileSync(abs);
|
|
69
|
+
} catch (e) {
|
|
70
|
+
if (e.code === "ENOENT") {
|
|
71
|
+
// File deleted in working tree but not yet committed — skip it
|
|
72
|
+
// (AC11: treat as deletion, sync.mjs will reap it)
|
|
73
|
+
return null;
|
|
74
|
+
}
|
|
75
|
+
throw e;
|
|
76
|
+
}
|
|
77
|
+
return {
|
|
78
|
+
path: rel,
|
|
79
|
+
sha256: createHash("sha256").update(bytes).digest("hex"),
|
|
80
|
+
bytes: bytes.length,
|
|
81
|
+
};
|
|
82
|
+
}).filter((f) => f !== null); // drop missing files
|
|
83
|
+
return buildManifest({ head, files });
|
|
84
|
+
}
|
|
@@ -0,0 +1,279 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Keyword query core for the `doc` retrieval index (unit 65, P5).
|
|
3
|
+
*
|
|
4
|
+
* Every query answers after the freshness invariant (AC9: `ensureFresh`),
|
|
5
|
+
* fully offline — no network, no API key (AC13). FTS5 keyword search with
|
|
6
|
+
* result rows `{path, section, lines, score, meta}` in the frozen D3
|
|
7
|
+
* ordering (score desc → path asc → section asc → lines[0] asc; `score` is
|
|
8
|
+
* the negated FTS5 bm25 rank, higher is better).
|
|
9
|
+
*
|
|
10
|
+
* The structured session-log filter (AC14) narrows to `docs/LOGS.md`
|
|
11
|
+
* entries — `log-session` owns that entry shape and the index consumes it
|
|
12
|
+
* as-is: the `## <ISO timestamp> — <branch> — mode` heading and the
|
|
13
|
+
* `- **Files:**` cell become queryable metadata, never an index-side edit.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import { join } from "node:path";
|
|
17
|
+
import { decodeVector } from "./embeddings.mjs";
|
|
18
|
+
import { openDatabase } from "./sqlite.mjs";
|
|
19
|
+
import { storePath, storePresent } from "./store.mjs";
|
|
20
|
+
import { ensureFresh } from "./sync.mjs";
|
|
21
|
+
import { storedEmbeddingModel, resolveEmbedConfig, makeOpenAIProvider } from "./embeddings.mjs";
|
|
22
|
+
import { buildEnvelope, buildResultRow } from "./envelope.mjs";
|
|
23
|
+
import { loadConfig } from "./config.mjs";
|
|
24
|
+
|
|
25
|
+
const LOGS_PATH = "docs/LOGS.md";
|
|
26
|
+
const MAX_RESULTS = 20;
|
|
27
|
+
const MATCH_POOL = 200; // candidate pool before filters and the top-K cut
|
|
28
|
+
|
|
29
|
+
/** Quote every token as an FTS5 phrase and AND them — hyphens stay literal. */
|
|
30
|
+
export function ftsQueryFor(term) {
|
|
31
|
+
const tokens = String(term).split(/\s+/).map((t) => t.trim()).filter((t) => t !== "");
|
|
32
|
+
if (tokens.length === 0) return null;
|
|
33
|
+
return tokens.map((t) => `"${t.replace(/"/g, '""')}"`).join(" AND ");
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Parse a `docs/LOGS.md` entry chunk body into queryable metadata
|
|
38
|
+
* `{timestamp, branch, mode, files}` — null when the chunk is not a log
|
|
39
|
+
* entry (the shape is log-session's; consumed as-is, D-log).
|
|
40
|
+
*/
|
|
41
|
+
export function parseLogEntry(body) {
|
|
42
|
+
const lines = String(body).split(/\r?\n/);
|
|
43
|
+
const heading = lines[0]?.match(/^##\s+(\S+)\s+—\s+(\S+)\s+—\s+(.+)$/);
|
|
44
|
+
if (!heading) return null;
|
|
45
|
+
let files = null;
|
|
46
|
+
for (const line of lines.slice(1)) {
|
|
47
|
+
const f = line.match(/^-\s+\*\*Files:\*\*\s+(.+)$/);
|
|
48
|
+
if (f) {
|
|
49
|
+
files = f[1].trim();
|
|
50
|
+
break;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
return { timestamp: heading[1], branch: heading[2], mode: heading[3].trim(), files };
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function logEntryMatches(entry, filters) {
|
|
57
|
+
if (filters.since !== undefined && filters.since !== null) {
|
|
58
|
+
if (entry.timestamp < filters.since) return false;
|
|
59
|
+
}
|
|
60
|
+
if (filters.until !== undefined && filters.until !== null) {
|
|
61
|
+
if (entry.timestamp > filters.until) return false;
|
|
62
|
+
}
|
|
63
|
+
if (filters.file !== undefined && filters.file !== null) {
|
|
64
|
+
const cell = (entry.files ?? "").replace(/`/g, "");
|
|
65
|
+
if (!cell.includes(filters.file)) return false;
|
|
66
|
+
}
|
|
67
|
+
return true;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
async function storeValue(root) {
|
|
71
|
+
if (!storePresent(root)) {
|
|
72
|
+
return { path: storePath(root), present: false, files: 0, chunks: 0 };
|
|
73
|
+
}
|
|
74
|
+
const { db } = await openDatabase(storePath(root));
|
|
75
|
+
// Check if the chunks table exists (AC11: schema-less store is not an error —
|
|
76
|
+
// it just means sync hasn't run yet, so the table is missing).
|
|
77
|
+
const tableExists = db
|
|
78
|
+
.prepare(
|
|
79
|
+
"select count(*) as n from sqlite_master where type='table' and name='chunks'",
|
|
80
|
+
)
|
|
81
|
+
.get().n;
|
|
82
|
+
if (tableExists === 0) {
|
|
83
|
+
db.close();
|
|
84
|
+
return { path: storePath(root), present: true, files: 0, chunks: 0 };
|
|
85
|
+
}
|
|
86
|
+
const files = db.prepare("select count(distinct path) as n from chunks").get().n;
|
|
87
|
+
const chunks = db.prepare("select count(*) as n from chunks").get().n;
|
|
88
|
+
db.close();
|
|
89
|
+
return { path: storePath(root), present: true, files, chunks };
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* Run one query. `filters` carries the optional AC14 structured filter
|
|
94
|
+
* `{since, until, file}`; when any is present, results narrow to matching
|
|
95
|
+
* `docs/LOGS.md` entries. `opts.mode` is `"keyword"` (default) or
|
|
96
|
+
* `"hybrid"` (P7); `opts.embedder` overrides the resolved provider for
|
|
97
|
+
* tests. Returns the canonical query envelope — degraded hybrid answers
|
|
98
|
+
* keyword-only with the cause declared (AC17/AC18), exit-0 ok throughout.
|
|
99
|
+
*/
|
|
100
|
+
export async function queryIndex(root, term, filters = {}, opts = {}) {
|
|
101
|
+
const mode = opts.mode ?? "keyword";
|
|
102
|
+
await ensureFresh(root);
|
|
103
|
+
const store = await storeValue(root);
|
|
104
|
+
|
|
105
|
+
const match = ftsQueryFor(term);
|
|
106
|
+
const keywordRows = [];
|
|
107
|
+
if (match !== null) {
|
|
108
|
+
const { db } = await openDatabase(storePath(root));
|
|
109
|
+
const rows = db
|
|
110
|
+
.prepare(
|
|
111
|
+
`select c.chunk_id, c.path, c.section, c.line_start, c.line_end, c.meta, c.body, bm25(chunks_fts) as rank
|
|
112
|
+
from chunks_fts join chunks c on c.rowid = chunks_fts.rowid
|
|
113
|
+
where chunks_fts match ?
|
|
114
|
+
order by rank, c.path, c.section, c.line_start
|
|
115
|
+
limit ${MATCH_POOL}`,
|
|
116
|
+
)
|
|
117
|
+
.all(match);
|
|
118
|
+
db.close();
|
|
119
|
+
keywordRows.push(...rows);
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
const hasLogFilters = filters.since != null || filters.until != null || filters.file != null;
|
|
123
|
+
|
|
124
|
+
if (mode === "hybrid") {
|
|
125
|
+
const provider = opts.embedder !== undefined ? opts.embedder : realProviderOrNull(root);
|
|
126
|
+
if (provider === null) {
|
|
127
|
+
// AC17: no key ⇒ declared degradation, keyword still answers.
|
|
128
|
+
return buildEnvelope({
|
|
129
|
+
command: "query",
|
|
130
|
+
results: toResults(keywordRows, hasLogFilters, filters),
|
|
131
|
+
degradations: ["unavailable-embeddings-not-configured"],
|
|
132
|
+
store,
|
|
133
|
+
});
|
|
134
|
+
}
|
|
135
|
+
const pinned = await storedEmbeddingModel(root);
|
|
136
|
+
if (pinned !== null && pinned !== provider.model) {
|
|
137
|
+
// AC18: never rank across models.
|
|
138
|
+
return buildEnvelope({
|
|
139
|
+
command: "query",
|
|
140
|
+
results: toResults(keywordRows, hasLogFilters, filters),
|
|
141
|
+
degradations: ["embeddings-model-mismatch"],
|
|
142
|
+
store,
|
|
143
|
+
});
|
|
144
|
+
}
|
|
145
|
+
const { db } = await openDatabase(storePath(root));
|
|
146
|
+
const embeddedCount = db.prepare("select count(*) as n from chunks where embedding is not null").get().n;
|
|
147
|
+
if (embeddedCount === 0) {
|
|
148
|
+
db.close();
|
|
149
|
+
return buildEnvelope({
|
|
150
|
+
command: "query",
|
|
151
|
+
results: toResults(keywordRows, hasLogFilters, filters),
|
|
152
|
+
degradations: ["unavailable-embeddings-not-indexed"],
|
|
153
|
+
store,
|
|
154
|
+
});
|
|
155
|
+
}
|
|
156
|
+
try {
|
|
157
|
+
const [queryVector] = await provider.embed([term]);
|
|
158
|
+
const vectorRows = vectorTopK(db, queryVector, MATCH_POOL);
|
|
159
|
+
db.close();
|
|
160
|
+
const fused = fuseRrf(keywordRows, vectorRows);
|
|
161
|
+
const results = toResults(
|
|
162
|
+
fused.map((r) => ({ ...r, rank: -r.fusedScore })),
|
|
163
|
+
hasLogFilters,
|
|
164
|
+
filters,
|
|
165
|
+
);
|
|
166
|
+
return buildEnvelope({ command: "query", results, degradations: [], store });
|
|
167
|
+
} catch (e) {
|
|
168
|
+
db.close();
|
|
169
|
+
// AC17: provider down ⇒ declared cause, keyword results survive.
|
|
170
|
+
return buildEnvelope({
|
|
171
|
+
command: "query",
|
|
172
|
+
results: toResults(keywordRows, hasLogFilters, filters),
|
|
173
|
+
degradations: ["unavailable-embeddings-provider-down"],
|
|
174
|
+
store,
|
|
175
|
+
});
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
const results = toResults(keywordRows, hasLogFilters, filters);
|
|
180
|
+
return buildEnvelope({ command: "query", results, degradations: [], store });
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
/** Map ranked store rows to result rows, applying the AC14 log filters. */
|
|
184
|
+
function toResults(rows, hasLogFilters, filters) {
|
|
185
|
+
const results = [];
|
|
186
|
+
for (const r of rows) {
|
|
187
|
+
let meta = {};
|
|
188
|
+
try {
|
|
189
|
+
meta = JSON.parse(r.meta);
|
|
190
|
+
} catch {
|
|
191
|
+
meta = {};
|
|
192
|
+
}
|
|
193
|
+
const isLog = r.path === LOGS_PATH;
|
|
194
|
+
if (isLog && (hasLogFilters || parseLogEntry(r.body) !== null)) {
|
|
195
|
+
const entry = parseLogEntry(r.body);
|
|
196
|
+
if (entry === null) continue;
|
|
197
|
+
if (hasLogFilters && !logEntryMatches(entry, filters)) continue;
|
|
198
|
+
meta = entry;
|
|
199
|
+
} else if (hasLogFilters) {
|
|
200
|
+
continue; // structured filters narrow to log entries by definition
|
|
201
|
+
}
|
|
202
|
+
const score = r.fusedScore !== undefined ? r.fusedScore : -r.rank;
|
|
203
|
+
results.push(
|
|
204
|
+
buildResultRow({
|
|
205
|
+
path: r.path,
|
|
206
|
+
section: r.section,
|
|
207
|
+
lines: [r.line_start, r.line_end],
|
|
208
|
+
score,
|
|
209
|
+
meta,
|
|
210
|
+
}),
|
|
211
|
+
);
|
|
212
|
+
if (results.length >= MAX_RESULTS) break;
|
|
213
|
+
}
|
|
214
|
+
return results;
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
/** Status reporting — informational, never syncs. */
|
|
218
|
+
export async function statusIndex(root) {
|
|
219
|
+
const store = await storeValue(root);
|
|
220
|
+
return buildEnvelope({ command: "status", results: [], store });
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
// ── Hybrid mode (P7) ───────────────────────────────────────────────────
|
|
224
|
+
|
|
225
|
+
const RRF_K = 60;
|
|
226
|
+
|
|
227
|
+
function cosine(a, b) {
|
|
228
|
+
let dot = 0;
|
|
229
|
+
let na = 0;
|
|
230
|
+
let nb = 0;
|
|
231
|
+
const n = Math.min(a.length, b.length);
|
|
232
|
+
for (let i = 0; i < n; i++) {
|
|
233
|
+
dot += a[i] * b[i];
|
|
234
|
+
na += a[i] * a[i];
|
|
235
|
+
nb += b[i] * b[i];
|
|
236
|
+
}
|
|
237
|
+
const denom = Math.sqrt(na) * Math.sqrt(nb);
|
|
238
|
+
return denom === 0 ? 0 : dot / denom;
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
/** JS-cosine kNN over D4's float32-LE BLOBs (decision D11 — no second vec0 store). */
|
|
242
|
+
function vectorTopK(db, queryVector, pool) {
|
|
243
|
+
const rows = db
|
|
244
|
+
.prepare(
|
|
245
|
+
"select chunk_id, path, section, line_start, line_end, meta, body, embedding from chunks where embedding is not null",
|
|
246
|
+
)
|
|
247
|
+
.all();
|
|
248
|
+
const scored = rows.map((r) => ({ ...r, sim: cosine(decodeVector(r.embedding), queryVector) }));
|
|
249
|
+
scored.sort((a, b) => b.sim - a.sim || (a.path < b.path ? -1 : a.path > b.path ? 1 : 0));
|
|
250
|
+
return scored.slice(0, pool);
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
/** Reciprocal-rank fusion over the keyword and vector lists. */
|
|
254
|
+
function fuseRrf(keywordRows, vectorRows) {
|
|
255
|
+
const fused = new Map();
|
|
256
|
+
keywordRows.forEach((r, i) => {
|
|
257
|
+
fused.set(r.chunk_id, { row: r, score: 1 / (RRF_K + i + 1) });
|
|
258
|
+
});
|
|
259
|
+
vectorRows.forEach((r, i) => {
|
|
260
|
+
const contribution = 1 / (RRF_K + i + 1);
|
|
261
|
+
const existing = fused.get(r.chunk_id);
|
|
262
|
+
if (existing) existing.score += contribution;
|
|
263
|
+
else fused.set(r.chunk_id, { row: r, score: contribution });
|
|
264
|
+
});
|
|
265
|
+
return [...fused.values()]
|
|
266
|
+
.sort(
|
|
267
|
+
(a, b) =>
|
|
268
|
+
b.score - a.score ||
|
|
269
|
+
(a.row.path < b.row.path ? -1 : a.row.path > b.row.path ? 1 : 0) ||
|
|
270
|
+
(a.row.section ?? "").localeCompare(b.row.section ?? "") ||
|
|
271
|
+
a.row.line_start - b.row.line_start,
|
|
272
|
+
)
|
|
273
|
+
.map((e) => ({ ...e.row, fusedScore: e.score }));
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
function realProviderOrNull(root) {
|
|
277
|
+
const cfg = resolveEmbedConfig({ config: loadConfig(root), env: process.env });
|
|
278
|
+
return cfg.configured ? makeOpenAIProvider(cfg) : null;
|
|
279
|
+
}
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Retrieval operations for `agentic-workflow doc` (unit 65).
|
|
3
|
+
*
|
|
4
|
+
* Each operation keeps the envelope contract fixed while its engine grows:
|
|
5
|
+
* sync — incremental sync engine: runs syncIndex and reports
|
|
6
|
+
* files_scanned / files_changed / files_deleted in the store value
|
|
7
|
+
* query — FTS5 keyword core behind ensureFresh (AC9), offline (AC13),
|
|
8
|
+
* with the AC14 structured session-log filters
|
|
9
|
+
* status — informational store report; never syncs
|
|
10
|
+
* rebuild — discard the disposable cache and resync from scratch
|
|
11
|
+
* Degradations come only from D3's closed vocabulary, as engines grow into
|
|
12
|
+
* the hybrid mode (P6/P7).
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import { buildEnvelope } from "./envelope.mjs";
|
|
16
|
+
import { storePath } from "./store.mjs";
|
|
17
|
+
import { syncIndex, rebuildIndex } from "./sync.mjs";
|
|
18
|
+
import { queryIndex, statusIndex } from "./query.mjs";
|
|
19
|
+
import { loadConfig } from "./config.mjs";
|
|
20
|
+
import { resolveEmbedConfig, makeOpenAIProvider, embedPendingChunks } from "./embeddings.mjs";
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Run one retrieval operation. `grammar` is parseDocArgs' output;
|
|
24
|
+
* `ctx.rootDir` is the project root the store and config resolve against.
|
|
25
|
+
* Returns the canonical envelope (async: the engines are async).
|
|
26
|
+
*/
|
|
27
|
+
export async function runDocOp(grammar, ctx = {}) {
|
|
28
|
+
const root = ctx.rootDir ?? process.cwd();
|
|
29
|
+
|
|
30
|
+
if (grammar.op === "sync") {
|
|
31
|
+
const r = await syncIndex(root);
|
|
32
|
+
// P6: when a provider is configured (D10), embed pending chunks in the
|
|
33
|
+
// same pass; otherwise the store answers keyword-only (AC17's root).
|
|
34
|
+
const cfg = resolveEmbedConfig({ config: loadConfig(root), env: process.env });
|
|
35
|
+
let embedded = null;
|
|
36
|
+
if (cfg.configured) {
|
|
37
|
+
embedded = await embedPendingChunks(root, makeOpenAIProvider(cfg));
|
|
38
|
+
}
|
|
39
|
+
return buildEnvelope({
|
|
40
|
+
command: "sync",
|
|
41
|
+
results: [],
|
|
42
|
+
degradations: [],
|
|
43
|
+
store: {
|
|
44
|
+
path: storePath(root),
|
|
45
|
+
present: true,
|
|
46
|
+
files: r.filesScanned,
|
|
47
|
+
chunks: r.chunks,
|
|
48
|
+
lastSync: {
|
|
49
|
+
files_scanned: r.filesScanned,
|
|
50
|
+
files_changed: r.filesChanged,
|
|
51
|
+
files_deleted: r.filesDeleted,
|
|
52
|
+
embedded: embedded === null ? 0 : embedded.embedded,
|
|
53
|
+
},
|
|
54
|
+
},
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
if (grammar.op === "query") {
|
|
59
|
+
return queryIndex(
|
|
60
|
+
root,
|
|
61
|
+
grammar.query,
|
|
62
|
+
{ since: grammar.since, until: grammar.until, file: grammar.file },
|
|
63
|
+
{ mode: grammar.mode ?? "keyword" },
|
|
64
|
+
);
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
if (grammar.op === "status") {
|
|
68
|
+
return statusIndex(root);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
if (grammar.op === "rebuild") {
|
|
72
|
+
const r = await rebuildIndex(root);
|
|
73
|
+
return buildEnvelope({
|
|
74
|
+
command: "rebuild",
|
|
75
|
+
results: [],
|
|
76
|
+
degradations: [],
|
|
77
|
+
store: { path: storePath(root), present: true, files: r.filesScanned, chunks: r.chunks },
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
throw new Error(`usage: unknown doc op "${grammar.op}"`);
|
|
82
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Runtime SQLite adapter for the `doc` retrieval index (unit 65, D7).
|
|
3
|
+
*
|
|
4
|
+
* bun is the primary runtime (`bun:sqlite`); node ≥ 24 is the supported floor
|
|
5
|
+
* (`node:sqlite`). Both ship FTS5 here (P1 verified). The adapter returns the
|
|
6
|
+
* raw database plus the runtime name so callers can paper over the few API
|
|
7
|
+
* differences; a runtime without a usable SQLite module surfaces the closed
|
|
8
|
+
* `unavailable-sqlite-<runtime>` degradation (D3) — it never crashes.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import { mkdirSync } from "node:fs";
|
|
12
|
+
import { dirname } from "node:path";
|
|
13
|
+
|
|
14
|
+
export function runtimeName() {
|
|
15
|
+
return typeof Bun !== "undefined" ? "bun" : "node";
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Open (creating if needed) the SQLite database at `path`. For `:memory:`
|
|
20
|
+
* no directory creation happens. Returns `{ db, kind }` where `kind` is
|
|
21
|
+
* `"bun" | "node"` and `db` is the raw runtime database handle.
|
|
22
|
+
*/
|
|
23
|
+
export async function openDatabase(path) {
|
|
24
|
+
const kind = runtimeName();
|
|
25
|
+
if (path !== ":memory:") {
|
|
26
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
27
|
+
}
|
|
28
|
+
if (kind === "bun") {
|
|
29
|
+
const { Database } = await import("bun:sqlite");
|
|
30
|
+
return { db: new Database(path), kind };
|
|
31
|
+
}
|
|
32
|
+
const { DatabaseSync } = await import("node:sqlite");
|
|
33
|
+
return { db: new DatabaseSync(path), kind };
|
|
34
|
+
}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Store location for the `doc` retrieval index (unit 65, D4).
|
|
3
|
+
*
|
|
4
|
+
* The store is `.agentic-workflow/index/index.db` under the project root —
|
|
5
|
+
* a gitignored, disposable cache over git (AC7: `git status` stays clean
|
|
6
|
+
* after first creation). The canonical manifest export is the sidecar
|
|
7
|
+
* `manifest.json` in the same gitignored directory; AC12's byte-for-byte
|
|
8
|
+
* assertion runs over that export, never over SQLite page bytes.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import { existsSync } from "node:fs";
|
|
12
|
+
import { join } from "node:path";
|
|
13
|
+
import { openDatabase } from "./sqlite.mjs";
|
|
14
|
+
import { loadConfig } from "./config.mjs";
|
|
15
|
+
|
|
16
|
+
export const MANIFEST_EXPORT_NAME = "manifest.json";
|
|
17
|
+
|
|
18
|
+
/** Default store location: `.agentic-workflow/index/index.db`. */
|
|
19
|
+
const DEFAULT_STORE_DIR = join(".agentic-workflow", "index");
|
|
20
|
+
|
|
21
|
+
/** Resolve the store's parent directory (the gitignored index dir). */
|
|
22
|
+
function storeDir(root) {
|
|
23
|
+
return join(root, DEFAULT_STORE_DIR);
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Resolve the store file path. Honours `config.store.path` when present;
|
|
28
|
+
* falls back to the default (`.agentic-workflow/index/index.db`).
|
|
29
|
+
*/
|
|
30
|
+
export function storePath(root) {
|
|
31
|
+
const cfg = loadConfig(root);
|
|
32
|
+
if (cfg.store?.path) return join(root, cfg.store.path);
|
|
33
|
+
return join(root, DEFAULT_STORE_DIR, "index.db");
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Canonical manifest export: the sidecar `.agentic-workflow/index/manifest.json`
|
|
38
|
+
* in the same gitignored directory as the store (AC12).
|
|
39
|
+
*/
|
|
40
|
+
export function manifestExportPath(root) {
|
|
41
|
+
return join(storeDir(root), MANIFEST_EXPORT_NAME);
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export function storePresent(root) {
|
|
45
|
+
return existsSync(storePath(root));
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Create the store if absent. Idempotent: an existing store is left exactly
|
|
50
|
+
* as it is (P4's sync engine owns its schema migration).
|
|
51
|
+
*/
|
|
52
|
+
export async function ensureStore(root) {
|
|
53
|
+
if (storePresent(root)) return storePath(root);
|
|
54
|
+
const { db } = await openDatabase(storePath(root));
|
|
55
|
+
db.close();
|
|
56
|
+
return storePath(root);
|
|
57
|
+
}
|