agent-rag-mcp 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -2
- package/dist/src/adapters/registry.mjs +6 -1
- package/dist/src/adapters/structured-text.mjs +150 -0
- package/dist/src/cli/format.mjs +106 -0
- package/dist/src/cli/index.mjs +78 -12
- package/dist/src/ingest/parse-child.mjs +5 -0
- package/dist/src/ingest/parse-host.mjs +4 -0
- package/dist/src/ingest/pipeline.mjs +3 -2
- package/dist/src/store/lifecycle.mjs +15 -0
- package/package.json +15 -3
package/README.md
CHANGED
|
@@ -23,6 +23,7 @@ The `rag` commands available today are:
|
|
|
23
23
|
- `rag import <dir>` — Ingests files from a directory.
|
|
24
24
|
- `rag ls` — Lists active documents in the selected namespace.
|
|
25
25
|
- `rag rm <document-id>` — Logically retires a document and preserves its history.
|
|
26
|
+
- `rag relabel <document-id> --label <text>` — Replaces one document's stored label.
|
|
26
27
|
- `rag entity rename <from> <to>` — Corrects an entity key within the project.
|
|
27
28
|
- `rag review list` — Lists version pairs awaiting a human decision.
|
|
28
29
|
- `rag review link <from> <to> --type <conflicts_with|same_entity>` — Records a human link between versions with a note.
|
|
@@ -32,9 +33,22 @@ The `rag` commands available today are:
|
|
|
32
33
|
- `rag restore --from <dir>` — Restores a backup under this root.
|
|
33
34
|
- `rag reindex` — Rebuilds the database from the manifest and objects.
|
|
34
35
|
|
|
36
|
+
### CLI output
|
|
37
|
+
|
|
38
|
+
Interactive terminals render document lists as aligned columns and other supported results as concise summaries. When output is piped, commands keep the single-line JSON envelope. Set `RAG_FORMAT=json` or `RAG_FORMAT=pretty` to override terminal detection; an invalid value falls back to automatic detection. `rag --help` defaults to raw usage text on a terminal, while `RAG_FORMAT=json` restores its JSON envelope for scripts that parse the legacy help output. Set `NO_COLOR` (any value, including empty) to disable ANSI styling without changing the output layout.
|
|
39
|
+
|
|
40
|
+
`rag ls` includes each document's `documentId`. Copy that value to relabel the same document:
|
|
41
|
+
|
|
42
|
+
```sh
|
|
43
|
+
rag ls
|
|
44
|
+
rag relabel <document-id> --label "replacement label"
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
Structured files can use `.yaml`, `.yml`, `.json`, or `.jsonl`; each structured file yields one section per top-level key.
|
|
48
|
+
|
|
35
49
|
The default knowledge-base root is `%USERPROFILE%\.rag\`. Each client has its own `rag.db` under `.rag\clients\<client>\`; projects are logical within that client and are identified by `project_id`. The namespace is the pair `(client, project)`. By default, `failClosed` is `true`; without a namespace, reads are allowed, and when `failClosed` is set to `false`, writes go to the reserved `default/default` namespace. The Tier 3 human review queue is available through `rag review list`.
|
|
36
50
|
|
|
37
|
-
`rag add` is idempotent by `sha256`: adding the same bytes again returns the `{namespace, result}` envelope with `result.status` set to `"unchanged"` and creates no new version. A different hash creates a new `doc_version` without deleting the previous one. Deletion is a logical tombstone; version history is preserved.
|
|
51
|
+
`rag add` is idempotent by `sha256`: adding the same bytes again returns the `{namespace, result}` envelope with `result.status` set to `"unchanged"` and creates no new version. A different hash creates a new `doc_version` without deleting the previous one. Deletion is a logical tombstone; version history is preserved. `label` is what re-ingestion matches on, so relabeling detaches the document from future imports of that same source path, which will then appear as a new document.
|
|
38
52
|
|
|
39
53
|
## MCP
|
|
40
54
|
|
|
@@ -75,7 +89,7 @@ The namespace travels in every `rag_find` or `rag_read` call; it does not come f
|
|
|
75
89
|
- `npm test` runs the tests with Node's `node:test`.
|
|
76
90
|
- `npm run typecheck` runs the TypeScript typecheck without emitting files.
|
|
77
91
|
|
|
78
|
-
## Known limits
|
|
92
|
+
## Known limits
|
|
79
93
|
|
|
80
94
|
Version 0.1.0 has no embeddings or Tier 2; scanned PDFs go through OCR. Document parsing runs in a child process with a 512 MB V8 heap cap.
|
|
81
95
|
|
|
@@ -3,6 +3,7 @@ import { docxExtractor } from "./docx.mjs";
|
|
|
3
3
|
import { imageOcrExtractor } from "./image-ocr.mjs";
|
|
4
4
|
import { pdfExtractor } from "./pdf.mjs";
|
|
5
5
|
import { plainTextExtractor } from "./plain-text.mjs";
|
|
6
|
+
import { jsonExtractor, jsonlExtractor, yamlExtractor } from "./structured-text.mjs";
|
|
6
7
|
import { pptxExtractor } from "./pptx.mjs";
|
|
7
8
|
import { xlsxExtractor } from "./xlsx.mjs";
|
|
8
9
|
class UnsupportedMediaTypeError extends Error {
|
|
@@ -24,7 +25,11 @@ const extractorsByExtension = /* @__PURE__ */ new Map([
|
|
|
24
25
|
[".xlsx", xlsxExtractor],
|
|
25
26
|
[".pptx", pptxExtractor],
|
|
26
27
|
[".txt", plainTextExtractor],
|
|
27
|
-
[".md", plainTextExtractor]
|
|
28
|
+
[".md", plainTextExtractor],
|
|
29
|
+
[".yaml", yamlExtractor],
|
|
30
|
+
[".yml", yamlExtractor],
|
|
31
|
+
[".json", jsonExtractor],
|
|
32
|
+
[".jsonl", jsonlExtractor]
|
|
28
33
|
]);
|
|
29
34
|
function getExtractorForPath(filePath) {
|
|
30
35
|
const extension = path.extname(filePath).toLowerCase();
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
class StructuredTextExtractionError extends Error {
|
|
2
|
+
code;
|
|
3
|
+
constructor(code) {
|
|
4
|
+
super(code === "extraction_invalid_json" ? "Structured JSON is invalid." : "Structured YAML is invalid.");
|
|
5
|
+
this.name = "StructuredTextExtractionError";
|
|
6
|
+
this.code = code;
|
|
7
|
+
}
|
|
8
|
+
}
|
|
9
|
+
function getReferenceField(ref, names) {
|
|
10
|
+
if (typeof ref === "string") return ref;
|
|
11
|
+
if (typeof ref !== "object" || ref === null) return null;
|
|
12
|
+
const value = ref;
|
|
13
|
+
for (const name of names) {
|
|
14
|
+
if (typeof value[name] === "string") return value[name];
|
|
15
|
+
}
|
|
16
|
+
return null;
|
|
17
|
+
}
|
|
18
|
+
function supportsReference(ref, options) {
|
|
19
|
+
if (typeof ref !== "string") {
|
|
20
|
+
const mediaType = getReferenceField(ref, ["mediaType", "mimeType", "contentType"]);
|
|
21
|
+
if (mediaType !== null) {
|
|
22
|
+
return options.mediaTypes.includes(mediaType.split(";")[0].trim().toLowerCase());
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
const fileName = typeof ref === "string" ? ref : getReferenceField(ref, ["filename", "name", "path", "extension"]);
|
|
26
|
+
if (fileName === null) return false;
|
|
27
|
+
const normalizedFileName = fileName.toLowerCase();
|
|
28
|
+
return options.extensions.some((extension) => normalizedFileName.endsWith(extension));
|
|
29
|
+
}
|
|
30
|
+
function renderJsonValue(value) {
|
|
31
|
+
if (typeof value === "string") return value;
|
|
32
|
+
return JSON.stringify(value, null, 2) ?? "null";
|
|
33
|
+
}
|
|
34
|
+
function extractJson(content) {
|
|
35
|
+
let parsed;
|
|
36
|
+
try {
|
|
37
|
+
parsed = JSON.parse(content);
|
|
38
|
+
} catch {
|
|
39
|
+
throw new StructuredTextExtractionError("extraction_invalid_json");
|
|
40
|
+
}
|
|
41
|
+
if (typeof parsed === "object" && parsed !== null && !Array.isArray(parsed)) {
|
|
42
|
+
return Object.entries(parsed).map(([heading, value], ordinal) => ({
|
|
43
|
+
ordinal,
|
|
44
|
+
heading,
|
|
45
|
+
level: 1,
|
|
46
|
+
text: renderJsonValue(value)
|
|
47
|
+
}));
|
|
48
|
+
}
|
|
49
|
+
return [{ ordinal: 0, heading: null, level: null, text: renderJsonValue(parsed) }];
|
|
50
|
+
}
|
|
51
|
+
function extractJsonl(content) {
|
|
52
|
+
const sections = [];
|
|
53
|
+
for (const line of content.split(/\r?\n/)) {
|
|
54
|
+
const text = line.trim();
|
|
55
|
+
if (text.length === 0) continue;
|
|
56
|
+
try {
|
|
57
|
+
JSON.parse(text);
|
|
58
|
+
} catch {
|
|
59
|
+
throw new StructuredTextExtractionError("extraction_invalid_json");
|
|
60
|
+
}
|
|
61
|
+
sections.push({ ordinal: sections.length, heading: null, level: null, text });
|
|
62
|
+
}
|
|
63
|
+
return sections;
|
|
64
|
+
}
|
|
65
|
+
function extractYaml(content) {
|
|
66
|
+
const lines = content.split("\n").map((line) => line.replace(/\r$/, ""));
|
|
67
|
+
const topLevelKey = /^([A-Za-z0-9_.-]+):/;
|
|
68
|
+
const starts = [];
|
|
69
|
+
for (let index = 0; index < lines.length; index += 1) {
|
|
70
|
+
const line = lines[index];
|
|
71
|
+
if (line === void 0) continue;
|
|
72
|
+
if (/^[ \t]*\t/.test(line)) throw new StructuredTextExtractionError("extraction_invalid_yaml");
|
|
73
|
+
const match = topLevelKey.exec(line);
|
|
74
|
+
if (match?.[1] !== void 0) starts.push({ index, heading: match[1] });
|
|
75
|
+
}
|
|
76
|
+
if (starts.length === 0) {
|
|
77
|
+
return [{ ordinal: 0, heading: null, level: null, text: content }];
|
|
78
|
+
}
|
|
79
|
+
let hasCurrentBlock = false;
|
|
80
|
+
for (const line of lines) {
|
|
81
|
+
const trimmed = line.trim();
|
|
82
|
+
if (trimmed.length === 0 || trimmed.startsWith("#") || trimmed === "---" || trimmed === "...") continue;
|
|
83
|
+
const match = topLevelKey.exec(line);
|
|
84
|
+
if (match?.[1] !== void 0) {
|
|
85
|
+
hasCurrentBlock = true;
|
|
86
|
+
continue;
|
|
87
|
+
}
|
|
88
|
+
const indentation = line.match(/^ */)?.[0].length ?? 0;
|
|
89
|
+
if (indentation === 0 || !hasCurrentBlock) throw new StructuredTextExtractionError("extraction_invalid_yaml");
|
|
90
|
+
}
|
|
91
|
+
return starts.map((start, ordinal) => {
|
|
92
|
+
const nextStart = starts[ordinal + 1];
|
|
93
|
+
const block = lines.slice(start.index, nextStart?.index ?? lines.length).join("\n").trim();
|
|
94
|
+
return { ordinal, heading: start.heading, level: 1, text: block };
|
|
95
|
+
});
|
|
96
|
+
}
|
|
97
|
+
function createStructuredTextExtractor(options) {
|
|
98
|
+
return {
|
|
99
|
+
name: options.name,
|
|
100
|
+
version: "1.0.0",
|
|
101
|
+
mediaTypes: options.mediaTypes,
|
|
102
|
+
supports(ref) {
|
|
103
|
+
return supportsReference(ref, options);
|
|
104
|
+
},
|
|
105
|
+
async extract(bytes) {
|
|
106
|
+
const decoded = new TextDecoder("utf-8", { fatal: false, ignoreBOM: true }).decode(bytes);
|
|
107
|
+
const content = decoded.startsWith("\uFEFF") ? decoded.slice(1) : decoded;
|
|
108
|
+
if (content.length === 0) {
|
|
109
|
+
return { sections: [], versionLabel: null, mediaType: null };
|
|
110
|
+
}
|
|
111
|
+
let sections;
|
|
112
|
+
switch (options.kind) {
|
|
113
|
+
case "json":
|
|
114
|
+
sections = extractJson(content);
|
|
115
|
+
break;
|
|
116
|
+
case "jsonl":
|
|
117
|
+
sections = extractJsonl(content);
|
|
118
|
+
break;
|
|
119
|
+
case "yaml":
|
|
120
|
+
sections = extractYaml(content);
|
|
121
|
+
break;
|
|
122
|
+
}
|
|
123
|
+
return { sections, versionLabel: null, mediaType: options.mediaTypes[0] ?? null };
|
|
124
|
+
}
|
|
125
|
+
};
|
|
126
|
+
}
|
|
127
|
+
const jsonExtractor = createStructuredTextExtractor({
|
|
128
|
+
kind: "json",
|
|
129
|
+
name: "json",
|
|
130
|
+
extensions: [".json"],
|
|
131
|
+
mediaTypes: ["application/json"]
|
|
132
|
+
});
|
|
133
|
+
const jsonlExtractor = createStructuredTextExtractor({
|
|
134
|
+
kind: "jsonl",
|
|
135
|
+
name: "jsonl",
|
|
136
|
+
extensions: [".jsonl"],
|
|
137
|
+
mediaTypes: ["application/x-ndjson"]
|
|
138
|
+
});
|
|
139
|
+
const yamlExtractor = createStructuredTextExtractor({
|
|
140
|
+
kind: "yaml",
|
|
141
|
+
name: "yaml",
|
|
142
|
+
extensions: [".yaml", ".yml"],
|
|
143
|
+
mediaTypes: ["application/yaml"]
|
|
144
|
+
});
|
|
145
|
+
export {
|
|
146
|
+
StructuredTextExtractionError,
|
|
147
|
+
jsonExtractor,
|
|
148
|
+
jsonlExtractor,
|
|
149
|
+
yamlExtractor
|
|
150
|
+
};
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
const ANSI_ESCAPE = /\u001B(?:\[[0-?]*[ -/]*[@-~]|\][^\u0007]*(?:\u0007|\u001B\\))/g;
|
|
2
|
+
function record(value) {
|
|
3
|
+
return value !== null && typeof value === "object" && !Array.isArray(value) ? value : null;
|
|
4
|
+
}
|
|
5
|
+
function oneLine(value) {
|
|
6
|
+
return String(value ?? "").replace(/[\r\n]+/g, " ");
|
|
7
|
+
}
|
|
8
|
+
function styled(value, code, enabled) {
|
|
9
|
+
return enabled ? `\x1B[${code}m${value}\x1B[0m` : value;
|
|
10
|
+
}
|
|
11
|
+
function resolveOutputFormat(env, isTTY) {
|
|
12
|
+
const requested = env.RAG_FORMAT?.trim().toLowerCase();
|
|
13
|
+
if (requested === "json" || requested === "pretty") return requested;
|
|
14
|
+
return isTTY ? "pretty" : "json";
|
|
15
|
+
}
|
|
16
|
+
function useColor(env) {
|
|
17
|
+
return !Object.hasOwn(env, "NO_COLOR");
|
|
18
|
+
}
|
|
19
|
+
function stripAnsi(value) {
|
|
20
|
+
return value.replace(ANSI_ESCAPE, "");
|
|
21
|
+
}
|
|
22
|
+
function padVisible(value, width) {
|
|
23
|
+
return value.padEnd(width + value.length - stripAnsi(value).length);
|
|
24
|
+
}
|
|
25
|
+
function renderDocuments(documents, color) {
|
|
26
|
+
if (documents.length === 0) return "No documents.";
|
|
27
|
+
const columns = [
|
|
28
|
+
{ header: "DOCUMENT ID", value: (row) => row.documentId },
|
|
29
|
+
{ header: "ENTITY", value: (row) => row.entityKey },
|
|
30
|
+
{ header: "LABEL", value: (row) => row.label },
|
|
31
|
+
{ header: "VERSION", value: (row) => row.versionLabel }
|
|
32
|
+
];
|
|
33
|
+
const rows = documents.map((item) => record(item) ?? {});
|
|
34
|
+
const widths = columns.map((column) => Math.max(
|
|
35
|
+
column.header.length,
|
|
36
|
+
...rows.map((row) => stripAnsi(oneLine(column.value(row))).length)
|
|
37
|
+
));
|
|
38
|
+
const header = columns.map((column, index) => padVisible(
|
|
39
|
+
styled(column.header, "1", color),
|
|
40
|
+
widths[index]
|
|
41
|
+
)).join(" ").trimEnd();
|
|
42
|
+
const body = rows.map((row) => columns.map((column, index) => padVisible(
|
|
43
|
+
oneLine(column.value(row)),
|
|
44
|
+
widths[index]
|
|
45
|
+
)).join(" ").trimEnd());
|
|
46
|
+
return [header, ...body].join("\n");
|
|
47
|
+
}
|
|
48
|
+
function renderDoctor(result) {
|
|
49
|
+
const checked = Number(result.checked ?? 0);
|
|
50
|
+
if (result.ok === true) return `Doctor: healthy (${checked} objects checked)`;
|
|
51
|
+
const missing = Array.isArray(result.missing) ? result.missing.length : 0;
|
|
52
|
+
const corrupt = Array.isArray(result.corrupt) ? result.corrupt.length : 0;
|
|
53
|
+
const details = [`${missing} missing`, `${corrupt} corrupt`];
|
|
54
|
+
if (typeof result.chunks === "number" && typeof result.indexedChunks === "number" && result.chunks !== result.indexedChunks) {
|
|
55
|
+
details.push(`index ${result.indexedChunks}/${result.chunks}`);
|
|
56
|
+
}
|
|
57
|
+
if (typeof result.unreferencedObjects === "number" && result.unreferencedObjects > 0) {
|
|
58
|
+
details.push(`${result.unreferencedObjects} orphan objects`);
|
|
59
|
+
}
|
|
60
|
+
if (typeof result.leftoverTemporaries === "number" && result.leftoverTemporaries > 0) {
|
|
61
|
+
details.push(`${result.leftoverTemporaries} leftover temporaries`);
|
|
62
|
+
}
|
|
63
|
+
return `Doctor: issues found (${checked} checked; ${details.join(", ")})`;
|
|
64
|
+
}
|
|
65
|
+
function renderImport(result) {
|
|
66
|
+
const summary = record(result.summary) ?? {};
|
|
67
|
+
const count = (key) => typeof summary[key] === "number" ? summary[key] : 0;
|
|
68
|
+
return `Import: ${count("inserted")} inserted, ${count("unchanged")} unchanged, ${count("skipped")} skipped, ${count("failed")} failed (${count("total")} total)`;
|
|
69
|
+
}
|
|
70
|
+
function renderDocumentAction(result) {
|
|
71
|
+
const status = oneLine(result.status);
|
|
72
|
+
const documentId = oneLine(result.documentId);
|
|
73
|
+
if (status === "relabeled") return `Relabeled document ${documentId}: ${oneLine(result.label)}`;
|
|
74
|
+
if (status === "retired") return `Retired document ${documentId}`;
|
|
75
|
+
const action = status === "inserted" ? "Added" : status === "unchanged" ? "Unchanged" : "Updated";
|
|
76
|
+
const label = typeof result.label === "string" ? ` ${oneLine(result.label)}` : " document";
|
|
77
|
+
return `${action}${label} (${status}; documentId: ${documentId})`;
|
|
78
|
+
}
|
|
79
|
+
function renderError(error) {
|
|
80
|
+
const code = oneLine(error.code || "error");
|
|
81
|
+
return `Error [${code}]: ${oneLine(error.message || "Unknown error")}`;
|
|
82
|
+
}
|
|
83
|
+
function renderOutput(output, env) {
|
|
84
|
+
const envelope = record(output);
|
|
85
|
+
if (!envelope) return JSON.stringify(output) ?? "null";
|
|
86
|
+
const error = record(envelope.error);
|
|
87
|
+
if (error) return renderError(error);
|
|
88
|
+
const result = record(envelope.result);
|
|
89
|
+
if (!result) return JSON.stringify(output) ?? "null";
|
|
90
|
+
const color = useColor(env);
|
|
91
|
+
if (Array.isArray(result.documents)) return renderDocuments(result.documents, color);
|
|
92
|
+
if (typeof result.documents === "number" && typeof result.versions === "number" && typeof result.objects === "number") {
|
|
93
|
+
return `Documents: ${result.documents} | Versions: ${result.versions} | Objects: ${result.objects}`;
|
|
94
|
+
}
|
|
95
|
+
if (typeof result.ok === "boolean" && typeof result.checked === "number") return renderDoctor(result);
|
|
96
|
+
if (Array.isArray(result.files) && record(result.summary)) return renderImport(result);
|
|
97
|
+
if (typeof result.status === "string" && typeof result.documentId === "string") return renderDocumentAction(result);
|
|
98
|
+
return JSON.stringify(output) ?? "null";
|
|
99
|
+
}
|
|
100
|
+
export {
|
|
101
|
+
padVisible,
|
|
102
|
+
renderOutput,
|
|
103
|
+
resolveOutputFormat,
|
|
104
|
+
stripAnsi,
|
|
105
|
+
useColor
|
|
106
|
+
};
|
package/dist/src/cli/index.mjs
CHANGED
|
@@ -11,9 +11,11 @@ import { clientDbPath, openDatabase, validateClientName } from "../store/db.mjs"
|
|
|
11
11
|
import { backupRoot, reindexRoot, restoreRoot, writeRootManifest } from "../store/backup.mjs";
|
|
12
12
|
import { readObject } from "../store/objects.mjs";
|
|
13
13
|
import { getActiveLimits } from "../limits.mjs";
|
|
14
|
+
import { renderOutput, resolveOutputFormat } from "./format.mjs";
|
|
14
15
|
import {
|
|
15
16
|
linkReviewVersions,
|
|
16
17
|
listReviewCandidates,
|
|
18
|
+
relabelDocument,
|
|
17
19
|
renameEntity,
|
|
18
20
|
retireDocument
|
|
19
21
|
} from "../store/lifecycle.mjs";
|
|
@@ -25,6 +27,7 @@ Commands:
|
|
|
25
27
|
import <dir> Ingest files from a directory
|
|
26
28
|
ls List current documents
|
|
27
29
|
rm <document-id> Retire a document without deleting its history
|
|
30
|
+
relabel <document-id> --label <text> Repair a document label
|
|
28
31
|
entity rename <from> <to> Correct an entity key within the project
|
|
29
32
|
review list List human review candidates
|
|
30
33
|
review link <from> <to> --type <conflicts_with|same_entity>
|
|
@@ -42,7 +45,16 @@ Options:
|
|
|
42
45
|
--to <dir> Backup destination
|
|
43
46
|
--from <dir> Backup source
|
|
44
47
|
--type <type> Human relation type for rag review link
|
|
48
|
+
--label <text> Replacement label for rag relabel
|
|
45
49
|
--help Show this help`;
|
|
50
|
+
const HELP = `${USAGE}
|
|
51
|
+
|
|
52
|
+
Output format:
|
|
53
|
+
Interactive terminals render human-readable output; pipes use JSON.
|
|
54
|
+
RAG_FORMAT=json|pretty overrides auto-detection; NO_COLOR disables ANSI styling.
|
|
55
|
+
--help defaults to raw usage on a terminal; RAG_FORMAT=json restores its JSON envelope.
|
|
56
|
+
|
|
57
|
+
Use the documentId shown by rag ls with rag relabel <document-id> --label <text>.`;
|
|
46
58
|
const NAMED_ERROR_CODES = /* @__PURE__ */ new Set([
|
|
47
59
|
"usage",
|
|
48
60
|
"ingest_failed",
|
|
@@ -87,8 +99,10 @@ const NAMED_ERROR_CODES = /* @__PURE__ */ new Set([
|
|
|
87
99
|
"entity_key_conflict",
|
|
88
100
|
"project_not_found",
|
|
89
101
|
"doc_version_not_found",
|
|
90
|
-
"invalid_relation"
|
|
102
|
+
"invalid_relation",
|
|
103
|
+
"invalid_label"
|
|
91
104
|
]);
|
|
105
|
+
const USAGE_ERROR_CODES = /* @__PURE__ */ new Set(["usage", "invalid_label"]);
|
|
92
106
|
function stripErrorPrefix(message) {
|
|
93
107
|
return message.replace(/^(?:(?:ENOENT|EACCES|EPERM|EBUSY):|UNIQUE constraint failed:)\s*/, "");
|
|
94
108
|
}
|
|
@@ -98,14 +112,15 @@ function publicErrorMessage(error) {
|
|
|
98
112
|
function errorOutput(namespace, error, exitCode) {
|
|
99
113
|
const rawCode = error !== null && typeof error === "object" ? error.code : void 0;
|
|
100
114
|
const code = error instanceof IngestError ? "ingest_failed" : rawCode && NAMED_ERROR_CODES.has(rawCode) ? rawCode : rawCode && /^E[A-Z0-9]+$/.test(rawCode) ? "io_error" : "internal_error";
|
|
115
|
+
const isUsageError = USAGE_ERROR_CODES.has(code);
|
|
101
116
|
return {
|
|
102
|
-
exitCode: exitCode ?? (
|
|
117
|
+
exitCode: exitCode ?? (isUsageError ? 2 : 1),
|
|
103
118
|
output: {
|
|
104
119
|
namespace,
|
|
105
120
|
error: {
|
|
106
121
|
code,
|
|
107
122
|
message: publicErrorMessage(error),
|
|
108
|
-
...
|
|
123
|
+
...isUsageError ? { usage: USAGE } : {}
|
|
109
124
|
}
|
|
110
125
|
}
|
|
111
126
|
};
|
|
@@ -113,8 +128,8 @@ function errorOutput(namespace, error, exitCode) {
|
|
|
113
128
|
function commandErrorOutput(namespace, error) {
|
|
114
129
|
return errorOutput(namespace, error);
|
|
115
130
|
}
|
|
116
|
-
function usageError(message) {
|
|
117
|
-
return Object.assign(new Error(message), { code
|
|
131
|
+
function usageError(message, code = "usage") {
|
|
132
|
+
return Object.assign(new Error(message), { code });
|
|
118
133
|
}
|
|
119
134
|
function codedError(code, message) {
|
|
120
135
|
return Object.assign(new Error(message), { code });
|
|
@@ -150,15 +165,17 @@ function parseArgs(argv) {
|
|
|
150
165
|
help = true;
|
|
151
166
|
continue;
|
|
152
167
|
}
|
|
153
|
-
const assignment = /^--(to|from|type)=(.*)$/.exec(argument);
|
|
168
|
+
const assignment = /^--(to|from|type|label)=(.*)$/.exec(argument);
|
|
154
169
|
if (assignment) {
|
|
155
170
|
values[assignment[1]] = assignment[2];
|
|
156
171
|
continue;
|
|
157
172
|
}
|
|
158
173
|
if (/^--(root|client|project)=/.test(argument)) continue;
|
|
159
|
-
if (["--to", "--from", "--type"].includes(argument)) {
|
|
174
|
+
if (["--to", "--from", "--type", "--label"].includes(argument)) {
|
|
160
175
|
if (argv[index + 1] === void 0 || argv[index + 1].startsWith("--")) {
|
|
161
|
-
|
|
176
|
+
const code = argument === "--label" ? "invalid_label" : "usage";
|
|
177
|
+
const message = argument === "--label" ? "Option --label requires a non-blank value." : `Option ${argument} requires a value.`;
|
|
178
|
+
throw usageError(message, code);
|
|
162
179
|
}
|
|
163
180
|
values[argument.slice(2)] = argv[index + 1];
|
|
164
181
|
index += 1;
|
|
@@ -215,11 +232,16 @@ function queryDocuments(root, namespace) {
|
|
|
215
232
|
try {
|
|
216
233
|
const project = db.prepare(`SELECT p.project_id FROM project AS p
|
|
217
234
|
JOIN client AS c ON c.client_id = p.client_id WHERE c.name = ? AND p.slug = ?`).get(namespace.client, namespace.project);
|
|
218
|
-
const rows = project ? db.prepare(`SELECT e.entity_key AS entityKey, d.label AS label, dv.version_label AS versionLabel
|
|
235
|
+
const rows = project ? db.prepare(`SELECT d.document_id AS documentId, e.entity_key AS entityKey, d.label AS label, dv.version_label AS versionLabel
|
|
219
236
|
FROM document AS d JOIN entity AS e ON e.entity_id = d.entity_id
|
|
220
237
|
JOIN doc_version AS dv ON dv.document_id = d.document_id AND dv.is_current = 1
|
|
221
238
|
WHERE d.project_id = ? AND d.deleted_at IS NULL ORDER BY e.entity_key, d.label`).all(project.project_id) : [];
|
|
222
|
-
const documents = rows.map((row) => ({
|
|
239
|
+
const documents = rows.map((row) => ({
|
|
240
|
+
documentId: row.documentId,
|
|
241
|
+
entityKey: row.entityKey,
|
|
242
|
+
label: row.label,
|
|
243
|
+
versionLabel: row.versionLabel
|
|
244
|
+
}));
|
|
223
245
|
return { namespace, result: { documents } };
|
|
224
246
|
} finally {
|
|
225
247
|
db.close();
|
|
@@ -374,6 +396,12 @@ async function runCli(options) {
|
|
|
374
396
|
if (parsed.values.type !== void 0 && (parsed.command !== "review" || parsed.positional[0] !== "link")) {
|
|
375
397
|
return errorOutput(config.namespace, usageError('Option --type is only accepted by "rag review link".'));
|
|
376
398
|
}
|
|
399
|
+
if (parsed.values.label !== void 0 && parsed.command !== "relabel") {
|
|
400
|
+
return errorOutput(config.namespace, usageError('Option --label is only accepted by "rag relabel".'));
|
|
401
|
+
}
|
|
402
|
+
if (parsed.command === "relabel" && parsed.values.label === void 0) {
|
|
403
|
+
return errorOutput(config.namespace, usageError("Option --label requires a non-blank value.", "invalid_label"));
|
|
404
|
+
}
|
|
377
405
|
if (parsed.help) return { exitCode: 0, output: { namespace: config.namespace, result: USAGE } };
|
|
378
406
|
if (parsed.command === "init") {
|
|
379
407
|
if (parsed.positional.length !== 0) return errorOutput(config.namespace, usageError("init does not accept positional arguments."));
|
|
@@ -443,6 +471,28 @@ async function runCli(options) {
|
|
|
443
471
|
return errorOutput(config.namespace, error);
|
|
444
472
|
}
|
|
445
473
|
}
|
|
474
|
+
if (parsed.command === "relabel") {
|
|
475
|
+
if (parsed.positional.length !== 1) return errorOutput(config.namespace, usageError("relabel requires exactly one document ID."));
|
|
476
|
+
const label = parsed.values.label;
|
|
477
|
+
if (label === void 0 || label.trim().length === 0) {
|
|
478
|
+
return errorOutput(config.namespace, usageError("Option --label requires a non-blank value.", "invalid_label"));
|
|
479
|
+
}
|
|
480
|
+
try {
|
|
481
|
+
const namespace = requireNamespace(config, "write");
|
|
482
|
+
if (!namespace) throw new NamespaceRequiredError();
|
|
483
|
+
const root = path.resolve(options.cwd, config.root);
|
|
484
|
+
const db = openDatabase(clientDbPath(root, namespace.client));
|
|
485
|
+
try {
|
|
486
|
+
const result = relabelDocument(db, projectIdForNamespace(db, namespace), parsed.positional[0], label);
|
|
487
|
+
await writeRootManifest(root, db);
|
|
488
|
+
return { exitCode: 0, output: { namespace, result } };
|
|
489
|
+
} finally {
|
|
490
|
+
db.close();
|
|
491
|
+
}
|
|
492
|
+
} catch (error) {
|
|
493
|
+
return commandErrorOutput(config.namespace, error);
|
|
494
|
+
}
|
|
495
|
+
}
|
|
446
496
|
if (parsed.command === "rm") {
|
|
447
497
|
if (parsed.positional.length !== 1) return errorOutput(config.namespace, usageError("rm requires exactly one document ID."));
|
|
448
498
|
try {
|
|
@@ -611,9 +661,11 @@ async function runCli(options) {
|
|
|
611
661
|
}
|
|
612
662
|
async function main(dependencies = {}) {
|
|
613
663
|
const writeStdout = dependencies.writeStdout ?? ((line) => process.stdout.write(line));
|
|
664
|
+
const writeStderr = dependencies.writeStderr ?? ((line) => process.stderr.write(line));
|
|
614
665
|
const setExitCode = dependencies.setExitCode ?? ((exitCode) => {
|
|
615
666
|
process.exitCode = exitCode;
|
|
616
667
|
});
|
|
668
|
+
const format = resolveOutputFormat(process.env, process.stdout.isTTY === true);
|
|
617
669
|
try {
|
|
618
670
|
const result = await (dependencies.runCli ?? runCli)({
|
|
619
671
|
argv: process.argv.slice(2),
|
|
@@ -621,12 +673,25 @@ async function main(dependencies = {}) {
|
|
|
621
673
|
cwd: process.cwd(),
|
|
622
674
|
homeDir: process.env.RAG_HOME ?? homedir()
|
|
623
675
|
});
|
|
624
|
-
|
|
676
|
+
const output = result.output;
|
|
677
|
+
if (format === "pretty" && result.exitCode === 0 && typeof output === "object" && output !== null && "result" in output && output.result === USAGE) {
|
|
678
|
+
writeStdout(`${HELP}
|
|
625
679
|
`);
|
|
680
|
+
} else if (format === "pretty") {
|
|
681
|
+
const rendered = `${renderOutput(output, process.env)}
|
|
682
|
+
`;
|
|
683
|
+
if (typeof output === "object" && output !== null && "error" in output) writeStderr(rendered);
|
|
684
|
+
else writeStdout(rendered);
|
|
685
|
+
} else {
|
|
686
|
+
writeStdout(`${JSON.stringify(output)}
|
|
687
|
+
`);
|
|
688
|
+
}
|
|
626
689
|
setExitCode(result.exitCode);
|
|
627
690
|
} catch (error) {
|
|
628
691
|
const message = publicErrorMessage(error);
|
|
629
|
-
|
|
692
|
+
if (format === "pretty") writeStderr(`${renderOutput({ error: { code: "internal_error", message } }, process.env)}
|
|
693
|
+
`);
|
|
694
|
+
else writeStdout(`${JSON.stringify({ namespace: null, error: { code: "internal_error", message } })}
|
|
630
695
|
`);
|
|
631
696
|
setExitCode(1);
|
|
632
697
|
}
|
|
@@ -635,6 +700,7 @@ if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.me
|
|
|
635
700
|
await main();
|
|
636
701
|
}
|
|
637
702
|
export {
|
|
703
|
+
HELP,
|
|
638
704
|
USAGE,
|
|
639
705
|
main,
|
|
640
706
|
runCli
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import * as fs from "node:fs/promises";
|
|
2
2
|
import { performance } from "node:perf_hooks";
|
|
3
3
|
import { getExtractorForPath, UnsupportedMediaTypeError } from "../adapters/registry.mjs";
|
|
4
|
+
import { StructuredTextExtractionError } from "../adapters/structured-text.mjs";
|
|
4
5
|
import { MAX_SOURCE_FILE_BYTES } from "../limits.mjs";
|
|
5
6
|
import { ResourceLimitError } from "../ports/extractor.mjs";
|
|
6
7
|
const V8_ALLOCATION_CODES = /* @__PURE__ */ new Set([
|
|
@@ -62,6 +63,10 @@ async function parse(request) {
|
|
|
62
63
|
send({ type: "failure", requestId: request.requestId, code: error.code }, true);
|
|
63
64
|
return;
|
|
64
65
|
}
|
|
66
|
+
if (error instanceof StructuredTextExtractionError) {
|
|
67
|
+
send({ type: "failure", requestId: request.requestId, code: error.code }, true);
|
|
68
|
+
return;
|
|
69
|
+
}
|
|
65
70
|
send({ type: "failure", requestId: request.requestId, code: classifyParseError(error) }, true);
|
|
66
71
|
}
|
|
67
72
|
}
|
|
@@ -16,6 +16,8 @@ const PARSE_ERROR_MESSAGES = {
|
|
|
16
16
|
parse_invalid_result: "Document parsing returned an invalid result.",
|
|
17
17
|
unsupported_media_type: "Document media type is not supported.",
|
|
18
18
|
invalid_media_hint: "Document media hint is invalid.",
|
|
19
|
+
extraction_invalid_json: "Structured JSON is invalid.",
|
|
20
|
+
extraction_invalid_yaml: "Structured YAML is invalid.",
|
|
19
21
|
source_too_large: "Source document exceeds the maximum supported size.",
|
|
20
22
|
too_many_pages: "Document contains more pages than allowed.",
|
|
21
23
|
archive_too_large: "Document archive exceeds the maximum supported size.",
|
|
@@ -43,6 +45,8 @@ const CHILD_FAILURE_CODES = /* @__PURE__ */ new Set([
|
|
|
43
45
|
"parse_resource_exhausted",
|
|
44
46
|
"unsupported_media_type",
|
|
45
47
|
"invalid_media_hint",
|
|
48
|
+
"extraction_invalid_json",
|
|
49
|
+
"extraction_invalid_yaml",
|
|
46
50
|
...RESOURCE_LIMIT_CODES
|
|
47
51
|
]);
|
|
48
52
|
const CHILD_DRAIN_GRACE_MS = 100;
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { createHash, randomUUID } from "node:crypto";
|
|
2
2
|
import * as path from "node:path";
|
|
3
3
|
import { UnsupportedMediaTypeError } from "../adapters/registry.mjs";
|
|
4
|
+
import { StructuredTextExtractionError } from "../adapters/structured-text.mjs";
|
|
4
5
|
import { ResourceLimitError } from "../ports/extractor.mjs";
|
|
5
6
|
import { clientDbPath, openDatabase } from "../store/db.mjs";
|
|
6
7
|
import { writeRootManifest } from "../store/backup.mjs";
|
|
@@ -75,9 +76,9 @@ async function ingestFile(input) {
|
|
|
75
76
|
}
|
|
76
77
|
} catch (error) {
|
|
77
78
|
const detail = errorMessage(error);
|
|
78
|
-
const resultCode = error instanceof UnsupportedMediaTypeError || error instanceof ResourceLimitError ? error.code : error instanceof ParseHostError ? error.code : "ingest_failed";
|
|
79
|
+
const resultCode = error instanceof UnsupportedMediaTypeError || error instanceof ResourceLimitError || error instanceof StructuredTextExtractionError ? error.code : error instanceof ParseHostError ? error.code : "ingest_failed";
|
|
79
80
|
recordFailedRun(db, projectId, input.sourcePath, sha256, runStartedAt, detail, resultCode);
|
|
80
|
-
if (error instanceof UnsupportedMediaTypeError || error instanceof ResourceLimitError || error instanceof ParseHostError) throw error;
|
|
81
|
+
if (error instanceof UnsupportedMediaTypeError || error instanceof ResourceLimitError || error instanceof StructuredTextExtractionError || error instanceof ParseHostError) throw error;
|
|
81
82
|
throw new IngestError(detail, error);
|
|
82
83
|
}
|
|
83
84
|
if (extracted.sections.length === 0) {
|
|
@@ -19,6 +19,20 @@ function retireDocument(db, projectId, documentId) {
|
|
|
19
19
|
}
|
|
20
20
|
return { documentId, status: updated.deleted_at === deletedAt ? "retired" : "already_retired", deletedAt: updated.deleted_at };
|
|
21
21
|
}
|
|
22
|
+
function relabelDocument(db, projectId, documentId, label) {
|
|
23
|
+
db.exec("BEGIN IMMEDIATE;");
|
|
24
|
+
try {
|
|
25
|
+
const update = db.prepare("UPDATE document SET label = ? WHERE document_id = ? AND project_id = ?").run(label, documentId, projectId);
|
|
26
|
+
if (Number(update.changes) !== 1) {
|
|
27
|
+
throw new LifecycleError("document_not_found", `Document not found in this project: ${documentId}.`);
|
|
28
|
+
}
|
|
29
|
+
db.exec("COMMIT;");
|
|
30
|
+
} catch (error) {
|
|
31
|
+
db.exec("ROLLBACK;");
|
|
32
|
+
throw error;
|
|
33
|
+
}
|
|
34
|
+
return { documentId, label, status: "relabeled" };
|
|
35
|
+
}
|
|
22
36
|
function renameEntity(db, projectId, previousEntityKey, entityKey) {
|
|
23
37
|
const entity = db.prepare("SELECT entity_id FROM entity WHERE project_id = ? AND entity_key = ?").get(projectId, previousEntityKey);
|
|
24
38
|
if (!entity) throw new LifecycleError("entity_not_found", `Entity key not found in this project: ${previousEntityKey}.`);
|
|
@@ -141,6 +155,7 @@ export {
|
|
|
141
155
|
LifecycleError,
|
|
142
156
|
linkReviewVersions,
|
|
143
157
|
listReviewCandidates,
|
|
158
|
+
relabelDocument,
|
|
144
159
|
renameEntity,
|
|
145
160
|
retireDocument
|
|
146
161
|
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "agent-rag-mcp",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.0",
|
|
4
4
|
"description": "Deterministic, namespace-isolated document retrieval with a CLI and MCP server.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -12,13 +12,25 @@
|
|
|
12
12
|
"url": "https://github.com/Lsarmientov/agent-rag/issues"
|
|
13
13
|
},
|
|
14
14
|
"author": "Luis Daniel Sarmiento Valdiviezo",
|
|
15
|
-
"keywords": [
|
|
15
|
+
"keywords": [
|
|
16
|
+
"rag",
|
|
17
|
+
"mcp",
|
|
18
|
+
"mcp-server",
|
|
19
|
+
"document-retrieval",
|
|
20
|
+
"knowledge-base"
|
|
21
|
+
],
|
|
16
22
|
"type": "module",
|
|
17
23
|
"bin": {
|
|
18
24
|
"rag": "bin/rag.mjs",
|
|
19
25
|
"agent-rag-mcp": "bin/agent-rag-mcp.mjs"
|
|
20
26
|
},
|
|
21
|
-
"files": [
|
|
27
|
+
"files": [
|
|
28
|
+
"bin",
|
|
29
|
+
"dist",
|
|
30
|
+
"extensions",
|
|
31
|
+
"README.md",
|
|
32
|
+
"LICENSE"
|
|
33
|
+
],
|
|
22
34
|
"pi": {
|
|
23
35
|
"extensions": [
|
|
24
36
|
"./extensions/agent-rag.ts"
|