agent-rag-mcp 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +92 -2
- package/bin/agent-rag-mcp.mjs +28 -0
- package/bin/rag.mjs +59 -0
- package/dist/src/adapters/docx.mjs +30 -0
- package/dist/src/adapters/image-ocr.mjs +102 -0
- package/dist/src/adapters/pdf.mjs +94 -0
- package/dist/src/adapters/plain-text.mjs +72 -0
- package/dist/src/adapters/pptx.mjs +214 -0
- package/dist/src/adapters/registry.mjs +45 -0
- package/dist/src/adapters/structured-text.mjs +150 -0
- package/dist/src/adapters/xlsx.mjs +506 -0
- package/dist/src/cli/index.mjs +677 -0
- package/dist/src/config/discovery.mjs +81 -0
- package/dist/src/ingest/import.mjs +71 -0
- package/dist/src/ingest/parse-child.mjs +112 -0
- package/dist/src/ingest/parse-host.mjs +270 -0
- package/dist/src/ingest/pipeline.mjs +223 -0
- package/dist/src/limits.mjs +52 -0
- package/dist/src/ports/extractor.mjs +11 -0
- package/dist/src/retrieve/find.mjs +101 -0
- package/dist/src/retrieve/read.mjs +92 -0
- package/dist/src/server.mjs +68 -0
- package/dist/src/store/backup.mjs +267 -0
- package/dist/src/store/db.mjs +54 -0
- package/dist/src/store/identity.mjs +90 -0
- package/dist/src/store/lifecycle.mjs +161 -0
- package/dist/src/store/manifest.mjs +121 -0
- package/dist/src/store/objects.mjs +361 -0
- package/dist/src/store/schema.mjs +178 -0
- package/dist/src/store/similarity.mjs +43 -0
- package/dist/src/tools/schema.mjs +150 -0
- package/extensions/agent-rag.ts +89 -0
- package/extensions/pi-host.d.ts +55 -0
- package/package.json +61 -4
|
@@ -0,0 +1,267 @@
|
|
|
1
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
2
|
+
import * as fs from "node:fs/promises";
|
|
3
|
+
import * as path from "node:path";
|
|
4
|
+
import { clientDbPath, openDatabase } from "./db.mjs";
|
|
5
|
+
import { buildManifest, importManifest, ManifestError, serializeManifest, validateManifest } from "./manifest.mjs";
|
|
6
|
+
import { objectRelPath } from "./objects.mjs";
|
|
7
|
+
function counts(manifest) {
|
|
8
|
+
return {
|
|
9
|
+
objects: manifest.objects.length,
|
|
10
|
+
bytes: manifest.objects.reduce((total, object) => total + object.byte_size, 0),
|
|
11
|
+
documents: manifest.documents.length,
|
|
12
|
+
versions: manifest.versions.length
|
|
13
|
+
};
|
|
14
|
+
}
|
|
15
|
+
function ioError(code, action) {
|
|
16
|
+
return new ManifestError(code, action);
|
|
17
|
+
}
|
|
18
|
+
function errorDetails(error) {
|
|
19
|
+
const code = error !== null && typeof error === "object" ? error.code : void 0;
|
|
20
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
21
|
+
return code && !message.startsWith(`${code}:`) ? `${code}: ${message}` : message;
|
|
22
|
+
}
|
|
23
|
+
async function renameWithRetry(source, destination) {
|
|
24
|
+
const delays = [25, 50, 100];
|
|
25
|
+
for (let attempt = 0; ; attempt++) {
|
|
26
|
+
try {
|
|
27
|
+
await fs.rename(source, destination);
|
|
28
|
+
return;
|
|
29
|
+
} catch (error) {
|
|
30
|
+
const code = error.code;
|
|
31
|
+
if (attempt >= delays.length || !["EPERM", "EBUSY", "EACCES"].includes(code ?? "")) throw error;
|
|
32
|
+
await new Promise((resolve) => setTimeout(resolve, delays[attempt]));
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
async function writeRootManifest(root, db) {
|
|
37
|
+
const manifest = buildManifest(db);
|
|
38
|
+
validateManifest(manifest);
|
|
39
|
+
const destination = path.join(root, "manifest.json");
|
|
40
|
+
const temporary = path.join(root, `.manifest.${randomUUID()}.tmp`);
|
|
41
|
+
try {
|
|
42
|
+
await fs.mkdir(root, { recursive: true });
|
|
43
|
+
await fs.writeFile(temporary, serializeManifest(manifest), { encoding: "utf8", flag: "wx" });
|
|
44
|
+
await renameWithRetry(temporary, destination);
|
|
45
|
+
return manifest;
|
|
46
|
+
} catch {
|
|
47
|
+
let cleanupFailed = false;
|
|
48
|
+
try {
|
|
49
|
+
await fs.rm(temporary, { force: true });
|
|
50
|
+
} catch {
|
|
51
|
+
cleanupFailed = true;
|
|
52
|
+
}
|
|
53
|
+
throw ioError("manifest_write_failed", `Unable to atomically write manifest under ${root}.${cleanupFailed ? " Temporary manifest cleanup failed." : ""}`);
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
function verifyObjectBytes(bytes, object) {
|
|
57
|
+
if (bytes.byteLength !== object.byte_size || createHash("sha256").update(bytes).digest("hex") !== object.sha256) {
|
|
58
|
+
throw ioError("object_corrupt", `Object ${object.sha256} does not match its recorded hash and size.`);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
async function readAndVerifyObject(root, object) {
|
|
62
|
+
let bytes;
|
|
63
|
+
try {
|
|
64
|
+
bytes = await fs.readFile(path.join(root, objectRelPath(object.sha256)));
|
|
65
|
+
} catch (error) {
|
|
66
|
+
if (error.code === "ENOENT") {
|
|
67
|
+
throw ioError("object_missing", `Object ${object.sha256} is missing.`);
|
|
68
|
+
}
|
|
69
|
+
throw ioError("object_read_failed", `Unable to read object ${object.sha256}: ${errorDetails(error)}`);
|
|
70
|
+
}
|
|
71
|
+
verifyObjectBytes(bytes, object);
|
|
72
|
+
return bytes;
|
|
73
|
+
}
|
|
74
|
+
async function copyVerifiedObjects(manifest, sourceRoot, destinationRoot) {
|
|
75
|
+
for (const object of manifest.objects) {
|
|
76
|
+
const bytes = await readAndVerifyObject(sourceRoot, object);
|
|
77
|
+
const destination = path.join(destinationRoot, objectRelPath(object.sha256));
|
|
78
|
+
try {
|
|
79
|
+
await fs.mkdir(path.dirname(destination), { recursive: true });
|
|
80
|
+
await fs.writeFile(destination, bytes, { flag: "wx" });
|
|
81
|
+
} catch (error) {
|
|
82
|
+
throw ioError("object_copy_failed", `Unable to copy object ${object.sha256}: ${errorDetails(error)}`);
|
|
83
|
+
}
|
|
84
|
+
let copied;
|
|
85
|
+
try {
|
|
86
|
+
copied = await fs.readFile(destination);
|
|
87
|
+
} catch (error) {
|
|
88
|
+
throw ioError("object_copy_failed", `Unable to verify copied object ${object.sha256}: ${errorDetails(error)}`);
|
|
89
|
+
}
|
|
90
|
+
verifyObjectBytes(copied, object);
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
async function readRootManifest(root) {
|
|
94
|
+
let contents;
|
|
95
|
+
try {
|
|
96
|
+
contents = await fs.readFile(path.join(root, "manifest.json"), "utf8");
|
|
97
|
+
} catch (error) {
|
|
98
|
+
if (error.code === "ENOENT") throw ioError("manifest_missing", `Manifest not found under ${root}.`);
|
|
99
|
+
throw ioError("manifest_read_failed", `Unable to read manifest under ${root}.`);
|
|
100
|
+
}
|
|
101
|
+
let value;
|
|
102
|
+
try {
|
|
103
|
+
value = JSON.parse(contents);
|
|
104
|
+
} catch {
|
|
105
|
+
throw ioError("manifest_invalid", `Manifest under ${root} is not valid JSON.`);
|
|
106
|
+
}
|
|
107
|
+
validateManifest(value);
|
|
108
|
+
return value;
|
|
109
|
+
}
|
|
110
|
+
function validateBackupObjectInventory(manifest) {
|
|
111
|
+
const hashes = new Set(manifest.objects.map((object) => object.sha256));
|
|
112
|
+
for (const version of manifest.versions) {
|
|
113
|
+
if (!hashes.has(version.sha256)) {
|
|
114
|
+
throw ioError("object_missing", `Object ${version.sha256} referenced by a version is absent from the catalog inventory.`);
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
async function backupRoot(sourceRoot, db, destinationRoot) {
|
|
119
|
+
const manifest = buildManifest(db);
|
|
120
|
+
validateBackupObjectInventory(manifest);
|
|
121
|
+
validateManifest(manifest);
|
|
122
|
+
const destination = path.resolve(destinationRoot);
|
|
123
|
+
const parent = path.dirname(destination);
|
|
124
|
+
let exists = false;
|
|
125
|
+
try {
|
|
126
|
+
await fs.access(destination);
|
|
127
|
+
exists = true;
|
|
128
|
+
} catch (error) {
|
|
129
|
+
if (error.code !== "ENOENT") {
|
|
130
|
+
throw ioError("backup_failed", `Unable to inspect backup destination: ${errorDetails(error)}`);
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
if (exists) throw ioError("destination_exists", `Backup destination already exists: ${destination}`);
|
|
134
|
+
const temporary = path.join(parent, `.${path.basename(destination)}.${randomUUID()}.tmp`);
|
|
135
|
+
try {
|
|
136
|
+
await fs.mkdir(parent, { recursive: true });
|
|
137
|
+
await fs.mkdir(temporary);
|
|
138
|
+
await fs.writeFile(path.join(temporary, "manifest.json"), serializeManifest(manifest), "utf8");
|
|
139
|
+
await copyVerifiedObjects(manifest, sourceRoot, temporary);
|
|
140
|
+
await renameWithRetry(temporary, destination);
|
|
141
|
+
return counts(manifest);
|
|
142
|
+
} catch (error) {
|
|
143
|
+
let cleanupFailed = false;
|
|
144
|
+
let cleanupDetails = "";
|
|
145
|
+
try {
|
|
146
|
+
await fs.rm(temporary, { recursive: true, force: true });
|
|
147
|
+
} catch (cleanupError) {
|
|
148
|
+
cleanupFailed = true;
|
|
149
|
+
cleanupDetails = ` Temporary backup cleanup failed: ${errorDetails(cleanupError)}`;
|
|
150
|
+
}
|
|
151
|
+
if (error instanceof ManifestError) {
|
|
152
|
+
if (!cleanupFailed) throw error;
|
|
153
|
+
throw ioError(error.code, `${error.message}${cleanupDetails}`);
|
|
154
|
+
}
|
|
155
|
+
throw ioError("backup_failed", `Backup could not be published: ${errorDetails(error)}${cleanupDetails}`);
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
async function restoreRoot(sourceRoot, destinationRoot, client) {
|
|
159
|
+
const manifest = await readRootManifest(sourceRoot);
|
|
160
|
+
const foreignClient = manifest.clients.find((row) => row.name !== client);
|
|
161
|
+
if (foreignClient) {
|
|
162
|
+
throw ioError("foreign_client_in_manifest", `Manifest contains foreign client ${foreignClient.name}.`);
|
|
163
|
+
}
|
|
164
|
+
if (!manifest.clients.some((row) => row.name === client)) {
|
|
165
|
+
throw ioError("namespace_mismatch", `Manifest does not contain client ${client}.`);
|
|
166
|
+
}
|
|
167
|
+
const destination = path.resolve(destinationRoot);
|
|
168
|
+
const parent = path.dirname(destination);
|
|
169
|
+
let destinationExists = false;
|
|
170
|
+
try {
|
|
171
|
+
await fs.access(destination);
|
|
172
|
+
destinationExists = true;
|
|
173
|
+
if ((await fs.readdir(destination)).length > 0) {
|
|
174
|
+
throw ioError("restore_destination_not_empty", `Restore destination is not empty: ${destination}`);
|
|
175
|
+
}
|
|
176
|
+
} catch (error) {
|
|
177
|
+
if (error instanceof ManifestError) throw error;
|
|
178
|
+
if (error.code !== "ENOENT") {
|
|
179
|
+
throw ioError("restore_failed", `Unable to inspect restore destination: ${errorDetails(error)}`);
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
const temporary = path.join(parent, `.${path.basename(destination)}.${randomUUID()}.tmp`);
|
|
183
|
+
let db;
|
|
184
|
+
try {
|
|
185
|
+
await fs.mkdir(parent, { recursive: true });
|
|
186
|
+
await fs.mkdir(temporary);
|
|
187
|
+
await copyVerifiedObjects(manifest, sourceRoot, temporary);
|
|
188
|
+
db = openDatabase(clientDbPath(temporary, client));
|
|
189
|
+
importManifest(db, manifest);
|
|
190
|
+
await writeRootManifest(temporary, db);
|
|
191
|
+
db.close();
|
|
192
|
+
db = void 0;
|
|
193
|
+
if (destinationExists) await fs.rmdir(destination);
|
|
194
|
+
await renameWithRetry(temporary, destination);
|
|
195
|
+
return counts(manifest);
|
|
196
|
+
} catch (error) {
|
|
197
|
+
try {
|
|
198
|
+
db?.close();
|
|
199
|
+
} catch {
|
|
200
|
+
}
|
|
201
|
+
let cleanupDetails = "";
|
|
202
|
+
try {
|
|
203
|
+
await fs.rm(temporary, { recursive: true, force: true });
|
|
204
|
+
} catch (cleanupError) {
|
|
205
|
+
cleanupDetails = ` Temporary restore cleanup failed: ${errorDetails(cleanupError)}`;
|
|
206
|
+
}
|
|
207
|
+
if (error instanceof ManifestError) {
|
|
208
|
+
if (!cleanupDetails) throw error;
|
|
209
|
+
throw ioError(error.code, `${error.message}${cleanupDetails}`);
|
|
210
|
+
}
|
|
211
|
+
throw ioError("restore_failed", `Restore could not be published: ${errorDetails(error)}${cleanupDetails}`);
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
async function removeDatabaseFiles(dbPath) {
|
|
215
|
+
const failures = [];
|
|
216
|
+
for (const filePath of [dbPath, `${dbPath}-wal`, `${dbPath}-shm`]) {
|
|
217
|
+
try {
|
|
218
|
+
await fs.rm(filePath, { force: true });
|
|
219
|
+
} catch (error) {
|
|
220
|
+
failures.push(errorDetails(error));
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
return failures.length > 0 ? ` Database cleanup failed: ${failures.join("; ")}` : "";
|
|
224
|
+
}
|
|
225
|
+
async function reindexRoot(root, client) {
|
|
226
|
+
const manifest = await readRootManifest(root);
|
|
227
|
+
if (!manifest.clients.some((row) => row.name === client)) {
|
|
228
|
+
throw ioError("namespace_mismatch", `Manifest does not contain client ${client}.`);
|
|
229
|
+
}
|
|
230
|
+
for (const object of manifest.objects) await readAndVerifyObject(root, object);
|
|
231
|
+
const dbPath = clientDbPath(root, client);
|
|
232
|
+
try {
|
|
233
|
+
await fs.access(dbPath);
|
|
234
|
+
throw ioError("database_exists", `Database already exists: ${dbPath}`);
|
|
235
|
+
} catch (error) {
|
|
236
|
+
if (error instanceof ManifestError) throw error;
|
|
237
|
+
if (error.code !== "ENOENT") throw ioError("reindex_failed", `Unable to inspect database path: ${errorDetails(error)}`);
|
|
238
|
+
}
|
|
239
|
+
let db;
|
|
240
|
+
try {
|
|
241
|
+
db = openDatabase(dbPath);
|
|
242
|
+
importManifest(db, manifest);
|
|
243
|
+
const relations = db.prepare("SELECT COUNT(*) AS count FROM doc_relation").get()?.count;
|
|
244
|
+
await writeRootManifest(root, db);
|
|
245
|
+
db.close();
|
|
246
|
+
db = void 0;
|
|
247
|
+
return { ...counts(manifest), relations };
|
|
248
|
+
} catch (error) {
|
|
249
|
+
try {
|
|
250
|
+
db?.close();
|
|
251
|
+
} catch {
|
|
252
|
+
}
|
|
253
|
+
const cleanupDetails = await removeDatabaseFiles(dbPath);
|
|
254
|
+
if (error instanceof ManifestError) {
|
|
255
|
+
if (!cleanupDetails) throw error;
|
|
256
|
+
throw ioError(error.code, `${error.message}${cleanupDetails}`);
|
|
257
|
+
}
|
|
258
|
+
throw ioError("reindex_failed", `Manifest could not be loaded into the database: ${errorDetails(error)}${cleanupDetails}`);
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
export {
|
|
262
|
+
backupRoot,
|
|
263
|
+
readRootManifest,
|
|
264
|
+
reindexRoot,
|
|
265
|
+
restoreRoot,
|
|
266
|
+
writeRootManifest
|
|
267
|
+
};
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import * as fs from "node:fs";
|
|
2
|
+
import * as path from "node:path";
|
|
3
|
+
import { DatabaseSync } from "node:sqlite";
|
|
4
|
+
import { MIGRATIONS } from "./schema.mjs";
|
|
5
|
+
class InvalidClientNameError extends Error {
|
|
6
|
+
code = "invalid_client_name";
|
|
7
|
+
constructor(client) {
|
|
8
|
+
super(`Invalid client name "${client}". Client names must match [A-Za-z0-9][A-Za-z0-9._-]* and cannot be "." or "..".`);
|
|
9
|
+
this.name = "InvalidClientNameError";
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
function validateClientName(client) {
|
|
13
|
+
if (client === "." || client === ".." || !/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(client)) {
|
|
14
|
+
throw new InvalidClientNameError(client);
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
function clientDbPath(root, client) {
|
|
18
|
+
validateClientName(client);
|
|
19
|
+
return path.join(root, "clients", client, "rag.db");
|
|
20
|
+
}
|
|
21
|
+
function openDatabase(dbPath) {
|
|
22
|
+
fs.mkdirSync(path.dirname(dbPath), { recursive: true });
|
|
23
|
+
const db = new DatabaseSync(dbPath);
|
|
24
|
+
db.exec("PRAGMA busy_timeout = 1000;");
|
|
25
|
+
db.exec("PRAGMA journal_mode = WAL;");
|
|
26
|
+
db.exec("PRAGMA foreign_keys = ON;");
|
|
27
|
+
db.exec("CREATE TABLE IF NOT EXISTS schema_migration (version INTEGER PRIMARY KEY, applied_at TEXT NOT NULL);");
|
|
28
|
+
const hasMigration = db.prepare("SELECT 1 AS applied FROM schema_migration WHERE version = ?");
|
|
29
|
+
const recordMigration = db.prepare("INSERT INTO schema_migration (version, applied_at) VALUES (?, ?)");
|
|
30
|
+
for (const migration of [...MIGRATIONS].sort((left, right) => left.version - right.version)) {
|
|
31
|
+
if (hasMigration.get(migration.version)) continue;
|
|
32
|
+
const migrationTable = /CREATE TABLE schema_migration\s*\(\s*version INTEGER PRIMARY KEY,\s*applied_at TEXT NOT NULL\s*\);/;
|
|
33
|
+
const sql = migration.sql.replace(migrationTable, "");
|
|
34
|
+
if (sql === migration.sql) {
|
|
35
|
+
throw new Error(`Migration ${migration.version} does not define schema_migration`);
|
|
36
|
+
}
|
|
37
|
+
db.exec("BEGIN;");
|
|
38
|
+
try {
|
|
39
|
+
db.exec(sql);
|
|
40
|
+
recordMigration.run(migration.version, (/* @__PURE__ */ new Date()).toISOString());
|
|
41
|
+
db.exec("COMMIT;");
|
|
42
|
+
} catch (error) {
|
|
43
|
+
db.exec("ROLLBACK;");
|
|
44
|
+
throw error;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
return db;
|
|
48
|
+
}
|
|
49
|
+
export {
|
|
50
|
+
InvalidClientNameError,
|
|
51
|
+
clientDbPath,
|
|
52
|
+
openDatabase,
|
|
53
|
+
validateClientName
|
|
54
|
+
};
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
function slugify(label) {
|
|
3
|
+
return label.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "");
|
|
4
|
+
}
|
|
5
|
+
function normalizeEntityKey(label) {
|
|
6
|
+
const compact = label.toUpperCase().replace(/[-_\s]+/g, "");
|
|
7
|
+
const recognized = compact.match(/^(HU|US|BR|REQ|CT)(\d+)/);
|
|
8
|
+
if (!recognized) return slugify(label);
|
|
9
|
+
const digits = recognized[2].replace(/^0+(?=\d)/, "");
|
|
10
|
+
return `${recognized[1]}${digits}`;
|
|
11
|
+
}
|
|
12
|
+
function parseVersionLabel(label) {
|
|
13
|
+
const match = label.match(/^(.*?)[-_ ]v(\d+(?:\.\d+)*)(\.[^./\\]+)?$/);
|
|
14
|
+
if (!match) return null;
|
|
15
|
+
return {
|
|
16
|
+
prefix: match[1].replace(/[-_ ]+$/, ""),
|
|
17
|
+
version: match[2],
|
|
18
|
+
suffix: match[3] ?? null
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
function compareVersions(left, right) {
|
|
22
|
+
const leftParts = left.split(".").map(Number);
|
|
23
|
+
const rightParts = right.split(".").map(Number);
|
|
24
|
+
const partCount = Math.max(leftParts.length, rightParts.length);
|
|
25
|
+
for (let index = 0; index < partCount; index += 1) {
|
|
26
|
+
const leftPart = leftParts[index] ?? 0;
|
|
27
|
+
const rightPart = rightParts[index] ?? 0;
|
|
28
|
+
if (leftPart < rightPart) return -1;
|
|
29
|
+
if (leftPart > rightPart) return 1;
|
|
30
|
+
}
|
|
31
|
+
return 0;
|
|
32
|
+
}
|
|
33
|
+
const DECIDED_BY_VALUES = ["tier0", "tier1", "tier2", "human"];
|
|
34
|
+
const TIER0_RELATION_ORDER = {
|
|
35
|
+
same_bytes: 0,
|
|
36
|
+
same_content: 1,
|
|
37
|
+
same_entity: 2,
|
|
38
|
+
newer_version_of: 3
|
|
39
|
+
};
|
|
40
|
+
function relationId(fromDocVersionId, toDocVersionId, type) {
|
|
41
|
+
const digest = createHash("sha256").update(JSON.stringify([fromDocVersionId, toDocVersionId, type])).digest("hex");
|
|
42
|
+
return `rel_${digest}`;
|
|
43
|
+
}
|
|
44
|
+
function makeRelation(fromDocVersionId, toDocVersionId, type, evidence) {
|
|
45
|
+
return {
|
|
46
|
+
relationId: relationId(fromDocVersionId, toDocVersionId, type),
|
|
47
|
+
fromDocVersionId,
|
|
48
|
+
toDocVersionId,
|
|
49
|
+
type,
|
|
50
|
+
decidedBy: "tier0",
|
|
51
|
+
score: null,
|
|
52
|
+
evidence
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
function deriveTier0Relations(a, b) {
|
|
56
|
+
if (a.projectId !== b.projectId || a.docVersionId === b.docVersionId) return [];
|
|
57
|
+
const relations = [];
|
|
58
|
+
if (a.sha256 === b.sha256) {
|
|
59
|
+
relations.push(makeRelation(a.docVersionId, b.docVersionId, "same_bytes", { sha256: a.sha256 }));
|
|
60
|
+
} else if (a.contentHash !== null && b.contentHash !== null && a.contentHash === b.contentHash) {
|
|
61
|
+
relations.push(makeRelation(a.docVersionId, b.docVersionId, "same_content", { contentHash: a.contentHash }));
|
|
62
|
+
}
|
|
63
|
+
const sameEntity = normalizeEntityKey(a.entityKey) === normalizeEntityKey(b.entityKey);
|
|
64
|
+
if (sameEntity) {
|
|
65
|
+
relations.push(makeRelation(a.docVersionId, b.docVersionId, "same_entity", { entityKey: normalizeEntityKey(a.entityKey) }));
|
|
66
|
+
if (a.versionLabel !== null && b.versionLabel !== null) {
|
|
67
|
+
const aVersion = parseVersionLabel(a.versionLabel)?.version;
|
|
68
|
+
const bVersion = parseVersionLabel(b.versionLabel)?.version;
|
|
69
|
+
if (aVersion && bVersion) {
|
|
70
|
+
const comparison = compareVersions(aVersion, bVersion);
|
|
71
|
+
if (comparison < 0) {
|
|
72
|
+
relations.push(makeRelation(a.docVersionId, b.docVersionId, "newer_version_of", { fromVersion: aVersion, toVersion: bVersion }));
|
|
73
|
+
} else if (comparison > 0) {
|
|
74
|
+
relations.push(makeRelation(b.docVersionId, a.docVersionId, "newer_version_of", { fromVersion: bVersion, toVersion: aVersion }));
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
return relations.sort(
|
|
80
|
+
(left, right) => TIER0_RELATION_ORDER[left.type] - TIER0_RELATION_ORDER[right.type] || left.relationId.localeCompare(right.relationId)
|
|
81
|
+
);
|
|
82
|
+
}
|
|
83
|
+
export {
|
|
84
|
+
DECIDED_BY_VALUES,
|
|
85
|
+
compareVersions,
|
|
86
|
+
deriveTier0Relations,
|
|
87
|
+
normalizeEntityKey,
|
|
88
|
+
parseVersionLabel,
|
|
89
|
+
slugify
|
|
90
|
+
};
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
class LifecycleError extends Error {
|
|
3
|
+
code;
|
|
4
|
+
constructor(code, message) {
|
|
5
|
+
super(message);
|
|
6
|
+
this.name = "LifecycleError";
|
|
7
|
+
this.code = code;
|
|
8
|
+
}
|
|
9
|
+
}
|
|
10
|
+
function retireDocument(db, projectId, documentId) {
|
|
11
|
+
const row = db.prepare("SELECT deleted_at FROM document WHERE document_id = ? AND project_id = ?").get(documentId, projectId);
|
|
12
|
+
if (!row) throw new LifecycleError("document_not_found", `Document not found in this project: ${documentId}.`);
|
|
13
|
+
if (row.deleted_at !== null) return { documentId, status: "already_retired", deletedAt: row.deleted_at };
|
|
14
|
+
const deletedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
15
|
+
db.prepare("UPDATE document SET deleted_at = ? WHERE document_id = ? AND project_id = ? AND deleted_at IS NULL").run(deletedAt, documentId, projectId);
|
|
16
|
+
const updated = db.prepare("SELECT deleted_at FROM document WHERE document_id = ? AND project_id = ?").get(documentId, projectId);
|
|
17
|
+
if (!updated || updated.deleted_at === null) {
|
|
18
|
+
throw new LifecycleError("document_not_found", `Document not found in this project: ${documentId}.`);
|
|
19
|
+
}
|
|
20
|
+
return { documentId, status: updated.deleted_at === deletedAt ? "retired" : "already_retired", deletedAt: updated.deleted_at };
|
|
21
|
+
}
|
|
22
|
+
function relabelDocument(db, projectId, documentId, label) {
|
|
23
|
+
db.exec("BEGIN IMMEDIATE;");
|
|
24
|
+
try {
|
|
25
|
+
const update = db.prepare("UPDATE document SET label = ? WHERE document_id = ? AND project_id = ?").run(label, documentId, projectId);
|
|
26
|
+
if (Number(update.changes) !== 1) {
|
|
27
|
+
throw new LifecycleError("document_not_found", `Document not found in this project: ${documentId}.`);
|
|
28
|
+
}
|
|
29
|
+
db.exec("COMMIT;");
|
|
30
|
+
} catch (error) {
|
|
31
|
+
db.exec("ROLLBACK;");
|
|
32
|
+
throw error;
|
|
33
|
+
}
|
|
34
|
+
return { documentId, label, status: "relabeled" };
|
|
35
|
+
}
|
|
36
|
+
function renameEntity(db, projectId, previousEntityKey, entityKey) {
|
|
37
|
+
const entity = db.prepare("SELECT entity_id FROM entity WHERE project_id = ? AND entity_key = ?").get(projectId, previousEntityKey);
|
|
38
|
+
if (!entity) throw new LifecycleError("entity_not_found", `Entity key not found in this project: ${previousEntityKey}.`);
|
|
39
|
+
const conflictMessage = `Entity key "${entityKey}" already exists in this project.`;
|
|
40
|
+
db.exec("BEGIN IMMEDIATE;");
|
|
41
|
+
try {
|
|
42
|
+
const conflict = db.prepare("SELECT 1 AS present FROM entity WHERE project_id = ? AND entity_key = ? AND entity_id <> ?").get(projectId, entityKey, entity.entity_id);
|
|
43
|
+
if (conflict) throw new LifecycleError("entity_key_conflict", conflictMessage);
|
|
44
|
+
const update = db.prepare("UPDATE entity SET entity_key = ?, updated_at = ? WHERE entity_id = ? AND entity_key = ?").run(entityKey, (/* @__PURE__ */ new Date()).toISOString(), entity.entity_id, previousEntityKey);
|
|
45
|
+
if (Number(update.changes) === 0) throw new LifecycleError("entity_key_conflict", conflictMessage);
|
|
46
|
+
db.exec("COMMIT;");
|
|
47
|
+
} catch (error) {
|
|
48
|
+
db.exec("ROLLBACK;");
|
|
49
|
+
throw error;
|
|
50
|
+
}
|
|
51
|
+
return { entityKey, previousEntityKey, status: "renamed" };
|
|
52
|
+
}
|
|
53
|
+
function listReviewCandidates(db, projectId) {
|
|
54
|
+
return db.prepare(`WITH candidate_pairs AS (
|
|
55
|
+
SELECT CASE WHEN left_version.doc_version_id COLLATE BINARY < right_version.doc_version_id COLLATE BINARY
|
|
56
|
+
THEN left_version.doc_version_id ELSE right_version.doc_version_id END AS fromDocVersionId,
|
|
57
|
+
CASE WHEN left_version.doc_version_id COLLATE BINARY < right_version.doc_version_id COLLATE BINARY
|
|
58
|
+
THEN right_version.doc_version_id ELSE left_version.doc_version_id END AS toDocVersionId
|
|
59
|
+
FROM doc_version AS left_version
|
|
60
|
+
JOIN document AS left_document ON left_document.document_id = left_version.document_id
|
|
61
|
+
JOIN entity AS left_entity ON left_entity.entity_id = left_document.entity_id
|
|
62
|
+
JOIN doc_version AS right_version ON left_version.doc_version_id < right_version.doc_version_id
|
|
63
|
+
JOIN document AS right_document ON right_document.document_id = right_version.document_id
|
|
64
|
+
JOIN entity AS right_entity ON right_entity.entity_id = right_document.entity_id
|
|
65
|
+
WHERE left_document.project_id = ? AND right_document.project_id = left_document.project_id
|
|
66
|
+
AND left_version.content_hash IS NOT NULL AND left_version.content_hash = right_version.content_hash
|
|
67
|
+
AND left_document.deleted_at IS NULL AND right_document.deleted_at IS NULL
|
|
68
|
+
AND left_entity.deleted_at IS NULL AND right_entity.deleted_at IS NULL
|
|
69
|
+
AND NOT EXISTS (
|
|
70
|
+
SELECT 1 FROM doc_relation AS human_relation
|
|
71
|
+
WHERE human_relation.decided_by = 'human'
|
|
72
|
+
AND ((human_relation.from_doc_version_id = left_version.doc_version_id AND human_relation.to_doc_version_id = right_version.doc_version_id)
|
|
73
|
+
OR (human_relation.from_doc_version_id = right_version.doc_version_id AND human_relation.to_doc_version_id = left_version.doc_version_id))
|
|
74
|
+
)
|
|
75
|
+
UNION
|
|
76
|
+
SELECT CASE WHEN near_relation.from_doc_version_id COLLATE BINARY < near_relation.to_doc_version_id COLLATE BINARY
|
|
77
|
+
THEN near_relation.from_doc_version_id ELSE near_relation.to_doc_version_id END AS fromDocVersionId,
|
|
78
|
+
CASE WHEN near_relation.from_doc_version_id COLLATE BINARY < near_relation.to_doc_version_id COLLATE BINARY
|
|
79
|
+
THEN near_relation.to_doc_version_id ELSE near_relation.from_doc_version_id END AS toDocVersionId
|
|
80
|
+
FROM doc_relation AS near_relation
|
|
81
|
+
JOIN doc_version AS from_version ON from_version.doc_version_id = near_relation.from_doc_version_id
|
|
82
|
+
JOIN document AS from_document ON from_document.document_id = from_version.document_id
|
|
83
|
+
JOIN entity AS from_entity ON from_entity.entity_id = from_document.entity_id
|
|
84
|
+
JOIN doc_version AS to_version ON to_version.doc_version_id = near_relation.to_doc_version_id
|
|
85
|
+
JOIN document AS to_document ON to_document.document_id = to_version.document_id
|
|
86
|
+
JOIN entity AS to_entity ON to_entity.entity_id = to_document.entity_id
|
|
87
|
+
WHERE near_relation.relation_type = 'near_duplicate' AND near_relation.decided_by <> 'human'
|
|
88
|
+
AND from_document.project_id = ? AND to_document.project_id = from_document.project_id
|
|
89
|
+
AND from_document.deleted_at IS NULL AND to_document.deleted_at IS NULL
|
|
90
|
+
AND from_entity.deleted_at IS NULL AND to_entity.deleted_at IS NULL
|
|
91
|
+
AND NOT EXISTS (
|
|
92
|
+
SELECT 1 FROM doc_relation AS human_relation
|
|
93
|
+
WHERE human_relation.decided_by = 'human'
|
|
94
|
+
AND ((human_relation.from_doc_version_id = near_relation.from_doc_version_id AND human_relation.to_doc_version_id = near_relation.to_doc_version_id)
|
|
95
|
+
OR (human_relation.from_doc_version_id = near_relation.to_doc_version_id AND human_relation.to_doc_version_id = near_relation.from_doc_version_id))
|
|
96
|
+
)
|
|
97
|
+
)
|
|
98
|
+
SELECT fromDocVersionId, toDocVersionId FROM candidate_pairs
|
|
99
|
+
ORDER BY fromDocVersionId COLLATE BINARY, toDocVersionId COLLATE BINARY`).all(projectId, projectId);
|
|
100
|
+
}
|
|
101
|
+
const HUMAN_LINK_NOTE = "Human-reviewed link.";
|
|
102
|
+
function linkReviewVersions(db, projectId, fromDocVersionId, toDocVersionId, type) {
|
|
103
|
+
if (fromDocVersionId === toDocVersionId) {
|
|
104
|
+
throw new LifecycleError("invalid_relation", "A review link must refer to two different versions.");
|
|
105
|
+
}
|
|
106
|
+
const versions = db.prepare(`SELECT COUNT(*) AS count FROM doc_version AS version
|
|
107
|
+
JOIN document ON document.document_id = version.document_id
|
|
108
|
+
WHERE document.project_id = ? AND document.deleted_at IS NULL
|
|
109
|
+
AND version.doc_version_id IN (?, ?)`).get(projectId, fromDocVersionId, toDocVersionId);
|
|
110
|
+
if (Number(versions.count) !== 2) {
|
|
111
|
+
throw new LifecycleError("doc_version_not_found", "Both document versions must exist in this project.");
|
|
112
|
+
}
|
|
113
|
+
const existing = db.prepare(`SELECT relation_id, decided_by FROM doc_relation
|
|
114
|
+
WHERE from_doc_version_id = ? AND to_doc_version_id = ? AND relation_type = ?`).get(fromDocVersionId, toDocVersionId, type);
|
|
115
|
+
const decidedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
116
|
+
const evidence = JSON.stringify({ note: HUMAN_LINK_NOTE });
|
|
117
|
+
if (existing?.decided_by === "human") {
|
|
118
|
+
return {
|
|
119
|
+
relationId: existing.relation_id,
|
|
120
|
+
fromDocVersionId,
|
|
121
|
+
toDocVersionId,
|
|
122
|
+
type,
|
|
123
|
+
decidedBy: "human",
|
|
124
|
+
status: "already_present",
|
|
125
|
+
note: HUMAN_LINK_NOTE
|
|
126
|
+
};
|
|
127
|
+
}
|
|
128
|
+
if (existing) {
|
|
129
|
+
db.prepare("UPDATE doc_relation SET decided_by = 'human', decided_at = ?, evidence = ? WHERE relation_id = ?").run(decidedAt, evidence, existing.relation_id);
|
|
130
|
+
return {
|
|
131
|
+
relationId: existing.relation_id,
|
|
132
|
+
fromDocVersionId,
|
|
133
|
+
toDocVersionId,
|
|
134
|
+
type,
|
|
135
|
+
decidedBy: "human",
|
|
136
|
+
status: "linked",
|
|
137
|
+
note: HUMAN_LINK_NOTE
|
|
138
|
+
};
|
|
139
|
+
}
|
|
140
|
+
const relationId = randomUUID();
|
|
141
|
+
db.prepare(`INSERT INTO doc_relation
|
|
142
|
+
(relation_id, from_doc_version_id, to_doc_version_id, relation_type, score, decided_by, decided_at, evidence)
|
|
143
|
+
VALUES (?, ?, ?, ?, NULL, 'human', ?, ?)`).run(relationId, fromDocVersionId, toDocVersionId, type, decidedAt, evidence);
|
|
144
|
+
return {
|
|
145
|
+
relationId,
|
|
146
|
+
fromDocVersionId,
|
|
147
|
+
toDocVersionId,
|
|
148
|
+
type,
|
|
149
|
+
decidedBy: "human",
|
|
150
|
+
status: "linked",
|
|
151
|
+
note: HUMAN_LINK_NOTE
|
|
152
|
+
};
|
|
153
|
+
}
|
|
154
|
+
export {
|
|
155
|
+
LifecycleError,
|
|
156
|
+
linkReviewVersions,
|
|
157
|
+
listReviewCandidates,
|
|
158
|
+
relabelDocument,
|
|
159
|
+
renameEntity,
|
|
160
|
+
retireDocument
|
|
161
|
+
};
|