@gtrabanco/agentic-workflow 0.1.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -2
- package/bin/agentic-workflow.mjs +43 -1
- package/package.json +6 -3
- package/src/doc/chunks.mjs +150 -0
- package/src/doc/config.mjs +64 -0
- package/src/doc/embeddings.mjs +162 -0
- package/src/doc/envelope.mjs +120 -0
- package/src/doc/grammar.mjs +109 -0
- package/src/doc/manifest.mjs +84 -0
- package/src/doc/query.mjs +279 -0
- package/src/doc/retrieval.mjs +82 -0
- package/src/doc/sqlite.mjs +34 -0
- package/src/doc/store.mjs +57 -0
- package/src/doc/sync.mjs +159 -0
package/src/doc/sync.mjs
ADDED
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Incremental sync engine for the `doc` retrieval index (unit 65, P4).
|
|
3
|
+
*
|
|
4
|
+
* Change detection is hash-based (per-file sha256 from the canonical
|
|
5
|
+
* manifest, AC10): only files whose hash changed — or that disappeared —
|
|
6
|
+
* are reprocessed. Deletions reap every chunk of the removed file, and the
|
|
7
|
+
* external-content FTS5 table is kept in step by triggers, so no orphan
|
|
8
|
+
* chunk or ghost row survives (AC11). Determinism (AC12): the canonical
|
|
9
|
+
* manifest export written per sync is byte-identical for an unchanged
|
|
10
|
+
* corpus — sorted keys, git HEAD, no timestamps.
|
|
11
|
+
*
|
|
12
|
+
* Freshness invariant (AC9): `ensureFresh` compares the stored manifest
|
|
13
|
+
* (hashes + git HEAD) against the live corpus on every call; a stale store
|
|
14
|
+
* gets an inline incremental sync before the caller answers.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import { readFileSync, writeFileSync, rmSync } from "node:fs";
|
|
18
|
+
import { join } from "node:path";
|
|
19
|
+
import { openDatabase } from "./sqlite.mjs";
|
|
20
|
+
import { storePath, storePresent, manifestExportPath } from "./store.mjs";
|
|
21
|
+
import { loadConfig } from "./config.mjs";
|
|
22
|
+
import { collectManifest, canonicalJson } from "./manifest.mjs";
|
|
23
|
+
import { scanChunks } from "./chunks.mjs";
|
|
24
|
+
|
|
25
|
+
const SCHEMA = `
|
|
26
|
+
CREATE TABLE IF NOT EXISTS chunks (
|
|
27
|
+
chunk_id TEXT UNIQUE NOT NULL,
|
|
28
|
+
path TEXT NOT NULL,
|
|
29
|
+
section TEXT,
|
|
30
|
+
line_start INTEGER NOT NULL,
|
|
31
|
+
line_end INTEGER NOT NULL,
|
|
32
|
+
meta TEXT NOT NULL,
|
|
33
|
+
body TEXT NOT NULL,
|
|
34
|
+
embedding BLOB,
|
|
35
|
+
emb_model TEXT
|
|
36
|
+
);
|
|
37
|
+
CREATE INDEX IF NOT EXISTS chunks_path ON chunks(path);
|
|
38
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(body, content='chunks', content_rowid='rowid');
|
|
39
|
+
CREATE TRIGGER IF NOT EXISTS chunks_ai AFTER INSERT ON chunks BEGIN
|
|
40
|
+
INSERT INTO chunks_fts(rowid, body) VALUES (new.rowid, new.body);
|
|
41
|
+
END;
|
|
42
|
+
CREATE TRIGGER IF NOT EXISTS chunks_ad AFTER DELETE ON chunks BEGIN
|
|
43
|
+
INSERT INTO chunks_fts(chunks_fts, rowid, body) VALUES ('delete', old.rowid, old.body);
|
|
44
|
+
END;
|
|
45
|
+
CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
|
|
46
|
+
`;
|
|
47
|
+
|
|
48
|
+
/** Read the manifest persisted in the store's meta table (null if absent). */
|
|
49
|
+
export function storedManifest(db) {
|
|
50
|
+
try {
|
|
51
|
+
const row = db.prepare("select value from meta where key = 'manifest'").get();
|
|
52
|
+
return row ? JSON.parse(row.value) : null;
|
|
53
|
+
} catch {
|
|
54
|
+
return null;
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** Counts for `--status`: `{files, chunks}` over the store at `root`. */
|
|
59
|
+
export async function storeStats(root) {
|
|
60
|
+
const { db } = await openDatabase(storePath(root));
|
|
61
|
+
const files = db.prepare("select count(distinct path) as n from chunks").get().n;
|
|
62
|
+
const chunks = db.prepare("select count(*) as n from chunks").get().n;
|
|
63
|
+
db.close();
|
|
64
|
+
return { files, chunks };
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* One incremental sync pass over the tracked markdown corpus at `root`.
|
|
69
|
+
* Returns `{filesScanned, filesChanged, filesDeleted, chunks}`.
|
|
70
|
+
*/
|
|
71
|
+
export async function syncIndex(root) {
|
|
72
|
+
loadConfig(root); // config errors surface here too (usage/IO), even in sync
|
|
73
|
+
const manifest = collectManifest(root);
|
|
74
|
+
const { db } = await openDatabase(storePath(root));
|
|
75
|
+
db.exec(SCHEMA);
|
|
76
|
+
|
|
77
|
+
const stored = storedManifest(db);
|
|
78
|
+
const oldFiles = stored?.files ?? {};
|
|
79
|
+
const changed = [];
|
|
80
|
+
const deleted = [];
|
|
81
|
+
for (const [path, info] of Object.entries(manifest.files)) {
|
|
82
|
+
if (oldFiles[path] === undefined || oldFiles[path].sha256 !== info.sha256) changed.push(path);
|
|
83
|
+
}
|
|
84
|
+
for (const path of Object.keys(oldFiles)) {
|
|
85
|
+
if (manifest.files[path] === undefined) deleted.push(path);
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
const reap = db.prepare("DELETE FROM chunks WHERE path = ?");
|
|
89
|
+
const insert = db.prepare(
|
|
90
|
+
"INSERT INTO chunks (chunk_id, path, section, line_start, line_end, meta, body) VALUES (?, ?, ?, ?, ?, ?, ?)",
|
|
91
|
+
);
|
|
92
|
+
for (const path of deleted) reap.run(path);
|
|
93
|
+
for (const path of changed) {
|
|
94
|
+
const src = join(root, path);
|
|
95
|
+
try {
|
|
96
|
+
// Reap existing chunks for this path before re-inserting (original
|
|
97
|
+
// reap.run(path) before the read — preserved for the non-ENOENT path).
|
|
98
|
+
reap.run(path);
|
|
99
|
+
const rows = scanChunks(readFileSync(src, "utf8"), path);
|
|
100
|
+
for (const c of rows) {
|
|
101
|
+
insert.run(c.id, c.path, c.section, c.lines[0], c.lines[1], JSON.stringify(c.meta), c.body);
|
|
102
|
+
}
|
|
103
|
+
} catch (e) {
|
|
104
|
+
if (e.code === "ENOENT") {
|
|
105
|
+
// File deleted in working tree but not yet committed → reap
|
|
106
|
+
// (AC11: treat as deletion).
|
|
107
|
+
reap.run(path);
|
|
108
|
+
deleted.push(path);
|
|
109
|
+
} else throw e;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
// Persist + export the canonical manifest (AC12's compared artifact).
|
|
114
|
+
db.prepare("INSERT OR REPLACE INTO meta (key, value) VALUES ('manifest', ?)").run(canonicalJson(manifest));
|
|
115
|
+
db.close();
|
|
116
|
+
writeFileSync(manifestExportPath(root), `${canonicalJson(manifest)}\n`);
|
|
117
|
+
|
|
118
|
+
const stats = await storeStats(root);
|
|
119
|
+
return {
|
|
120
|
+
filesScanned: Object.keys(manifest.files).length,
|
|
121
|
+
filesChanged: changed.length,
|
|
122
|
+
filesDeleted: deleted.length,
|
|
123
|
+
chunks: stats.chunks,
|
|
124
|
+
};
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/**
|
|
128
|
+
* Freshness invariant (AC9): manifest hashes + git HEAD checked against the
|
|
129
|
+
* store; stale or missing ⇒ inline incremental sync before answering.
|
|
130
|
+
* Returns `{fresh, synced}`.
|
|
131
|
+
*/
|
|
132
|
+
export async function ensureFresh(root) {
|
|
133
|
+
const current = collectManifest(root);
|
|
134
|
+
if (storePresent(root)) {
|
|
135
|
+
const { db } = await openDatabase(storePath(root));
|
|
136
|
+
const stored = storedManifest(db);
|
|
137
|
+
db.close();
|
|
138
|
+
if (stored !== null && canonicalJson(stored) === canonicalJson(current)) {
|
|
139
|
+
return { fresh: true, synced: false };
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
await syncIndex(root);
|
|
143
|
+
return { fresh: false, synced: true };
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/**
|
|
147
|
+
* `--rebuild`: discard the disposable cache entirely (db + WAL/SHM + the
|
|
148
|
+
* manifest export) and resync from scratch. Returns the fresh sync counts
|
|
149
|
+
* with `rebuilt: true`.
|
|
150
|
+
*/
|
|
151
|
+
export async function rebuildIndex(root) {
|
|
152
|
+
const base = storePath(root);
|
|
153
|
+
rmSync(base, { force: true });
|
|
154
|
+
rmSync(`${base}-wal`, { force: true });
|
|
155
|
+
rmSync(`${base}-shm`, { force: true });
|
|
156
|
+
rmSync(manifestExportPath(root), { force: true });
|
|
157
|
+
const r = await syncIndex(root);
|
|
158
|
+
return { rebuilt: true, ...r };
|
|
159
|
+
}
|