agent-rag-mcp 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +92 -2
- package/bin/agent-rag-mcp.mjs +28 -0
- package/bin/rag.mjs +59 -0
- package/dist/src/adapters/docx.mjs +30 -0
- package/dist/src/adapters/image-ocr.mjs +102 -0
- package/dist/src/adapters/pdf.mjs +94 -0
- package/dist/src/adapters/plain-text.mjs +72 -0
- package/dist/src/adapters/pptx.mjs +214 -0
- package/dist/src/adapters/registry.mjs +45 -0
- package/dist/src/adapters/structured-text.mjs +150 -0
- package/dist/src/adapters/xlsx.mjs +506 -0
- package/dist/src/cli/index.mjs +677 -0
- package/dist/src/config/discovery.mjs +81 -0
- package/dist/src/ingest/import.mjs +71 -0
- package/dist/src/ingest/parse-child.mjs +112 -0
- package/dist/src/ingest/parse-host.mjs +270 -0
- package/dist/src/ingest/pipeline.mjs +223 -0
- package/dist/src/limits.mjs +52 -0
- package/dist/src/ports/extractor.mjs +11 -0
- package/dist/src/retrieve/find.mjs +101 -0
- package/dist/src/retrieve/read.mjs +92 -0
- package/dist/src/server.mjs +68 -0
- package/dist/src/store/backup.mjs +267 -0
- package/dist/src/store/db.mjs +54 -0
- package/dist/src/store/identity.mjs +90 -0
- package/dist/src/store/lifecycle.mjs +161 -0
- package/dist/src/store/manifest.mjs +121 -0
- package/dist/src/store/objects.mjs +361 -0
- package/dist/src/store/schema.mjs +178 -0
- package/dist/src/store/similarity.mjs +43 -0
- package/dist/src/tools/schema.mjs +150 -0
- package/extensions/agent-rag.ts +89 -0
- package/extensions/pi-host.d.ts +55 -0
- package/package.json +61 -4
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Luis Daniel Sarmiento Valdiviezo
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,3 +1,93 @@
|
|
|
1
|
-
#
|
|
1
|
+
# agent-rag
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
## What it is
|
|
4
|
+
|
|
5
|
+
agent-rag is a deterministic, per-client source of truth for documents such as user stories, contracts, specifications, and manuals, logically organized by project. Original bytes live in a content-addressed object store; SQLite is a rebuildable index, not the source of truth.
|
|
6
|
+
|
|
7
|
+
## What it is not
|
|
8
|
+
|
|
9
|
+
Engram is for memory points; agent-rag retrieves answers from documents. agent-rag is not a chatbot: it offers deterministic catalog and document-reading operations rather than autonomous conversation. In v1, it is not an LLM or an embedding index.
|
|
10
|
+
|
|
11
|
+
## Install
|
|
12
|
+
|
|
13
|
+
Install globally so an MCP adapter can resolve `agent-rag-mcp` from its `PATH`:
|
|
14
|
+
|
|
15
|
+
```sh
|
|
16
|
+
npm install --global agent-rag-mcp
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
The `rag` commands available today are:
|
|
20
|
+
|
|
21
|
+
- `rag init` — Initializes the namespace database and object store.
|
|
22
|
+
- `rag add <path>` — Ingests a file and copies its bytes to the object store at the same time, so OneDrive/Files-On-Demand eviction cannot lose the ingested document.
|
|
23
|
+
- `rag import <dir>` — Ingests files from a directory.
|
|
24
|
+
- `rag ls` — Lists active documents in the selected namespace.
|
|
25
|
+
- `rag rm <document-id>` — Logically retires a document and preserves its history.
|
|
26
|
+
- `rag relabel <document-id> --label <text>` — Replaces one document's stored label.
|
|
27
|
+
- `rag entity rename <from> <to>` — Corrects an entity key within the project.
|
|
28
|
+
- `rag review list` — Lists version pairs awaiting a human decision.
|
|
29
|
+
- `rag review link <from> <to> --type <conflicts_with|same_entity>` — Records a human link between versions with a note.
|
|
30
|
+
- `rag status` — Shows counts for the namespace.
|
|
31
|
+
- `rag doctor` — Verifies registered-object integrity and reports missing objects, orphan blobs, or temporary files.
|
|
32
|
+
- `rag backup --to <dir>` — Backs up the manifest and object store.
|
|
33
|
+
- `rag restore --from <dir>` — Restores a backup under this root.
|
|
34
|
+
- `rag reindex` — Rebuilds the database from the manifest and objects.
|
|
35
|
+
|
|
36
|
+
Structured files can use `.yaml`, `.yml`, `.json`, or `.jsonl`; each structured file yields one section per top-level key.
|
|
37
|
+
|
|
38
|
+
The default knowledge-base root is `%USERPROFILE%\.rag\`. Each client has its own `rag.db` under `.rag\clients\<client>\`; projects are logical within that client and are identified by `project_id`. The namespace is the pair `(client, project)`. By default, `failClosed` is `true`; without a namespace, reads are allowed, and when `failClosed` is set to `false`, writes go to the reserved `default/default` namespace. The Tier 3 human review queue is available through `rag review list`.
|
|
39
|
+
|
|
40
|
+
`rag add` is idempotent by `sha256`: adding the same bytes again returns the `{namespace, result}` envelope with `result.status` set to `"unchanged"` and creates no new version. A different hash creates a new `doc_version` without deleting the previous one. Deletion is a logical tombstone; version history is preserved. `label` is what re-ingestion matches on, so relabeling detaches the document from future imports of that same source path, which will then appear as a new document.
|
|
41
|
+
|
|
42
|
+
## MCP
|
|
43
|
+
|
|
44
|
+
Register two separate server processes in the adapter configuration (this example uses the common `mcpServers` format). Replace `/path/to/rag-data` with an absolute storage path:
|
|
45
|
+
|
|
46
|
+
```json
|
|
47
|
+
{
|
|
48
|
+
"mcpServers": {
|
|
49
|
+
"agent-rag-orchestrator": {
|
|
50
|
+
"command": "agent-rag-mcp",
|
|
51
|
+
"args": [],
|
|
52
|
+
"env": {
|
|
53
|
+
"RAG_ROOT": "/path/to/rag-data",
|
|
54
|
+
"RAG_CAPABILITIES": "find"
|
|
55
|
+
}
|
|
56
|
+
},
|
|
57
|
+
"agent-rag-explorer": {
|
|
58
|
+
"command": "agent-rag-mcp",
|
|
59
|
+
"args": [],
|
|
60
|
+
"env": {
|
|
61
|
+
"RAG_ROOT": "/path/to/rag-data",
|
|
62
|
+
"RAG_CAPABILITIES": "find,read"
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
The orchestrator receives only `rag_find`, which returns citations and bounded excerpts. The explorer also receives `rag_read`, which reads bounded text from a citation. Keeping them separate applies least privilege. The server reads `RAG_ROOT` to locate the knowledge base and `RAG_CAPABILITIES` to decide which tools to announce.
|
|
70
|
+
|
|
71
|
+
The namespace travels in every `rag_find` or `rag_read` call; it does not come from the environment. For example, include `"namespace": {"client":"acme","project":"research"}` in each tool call's arguments. Do not configure `RAG_CLIENT` or `RAG_PROJECT` as a substitute for the per-call namespace.
|
|
72
|
+
|
|
73
|
+
## Requirements
|
|
74
|
+
|
|
75
|
+
- Node.js >= 22.13.
|
|
76
|
+
- The `.mts` source files run with `--experimental-strip-types --experimental-sqlite`.
|
|
77
|
+
- `npm run build` builds each source file with esbuild (per-file).
|
|
78
|
+
- `npm test` runs the tests with Node's `node:test`.
|
|
79
|
+
- `npm run typecheck` runs the TypeScript typecheck without emitting files.
|
|
80
|
+
|
|
81
|
+
## Known limits in 0.1.0
|
|
82
|
+
|
|
83
|
+
Version 0.1.0 has no embeddings or Tier 2; scanned PDFs go through OCR. Document parsing runs in a child process with a 512 MB V8 heap cap.
|
|
84
|
+
|
|
85
|
+
## Design documents
|
|
86
|
+
|
|
87
|
+
- [Architecture](https://github.com/Lsarmientov/agent-rag/blob/main/docs/design/0001-arquitectura.md)
|
|
88
|
+
- [Data model](https://github.com/Lsarmientov/agent-rag/blob/main/docs/design/0002-modelo-de-datos.md)
|
|
89
|
+
- [Identity and duplicates](https://github.com/Lsarmientov/agent-rag/blob/main/docs/design/0003-identidad-duplicados.md)
|
|
90
|
+
|
|
91
|
+
## License
|
|
92
|
+
|
|
93
|
+
MIT.
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import * as fs from "node:fs";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { spawnSync } from "node:child_process";
|
|
5
|
+
import { fileURLToPath } from "node:url";
|
|
6
|
+
|
|
7
|
+
const packageRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
|
8
|
+
const builtEntry = path.join(packageRoot, "dist", "src", "server.mjs");
|
|
9
|
+
const sourceEntry = path.join(packageRoot, "src", "server.mts");
|
|
10
|
+
const entry = fs.existsSync(builtEntry) ? builtEntry : sourceEntry;
|
|
11
|
+
|
|
12
|
+
if (!fs.existsSync(entry)) {
|
|
13
|
+
console.error(`agent-rag-mcp: could not find ${path.relative(packageRoot, builtEntry)} or ${path.relative(packageRoot, sourceEntry)}`);
|
|
14
|
+
process.exitCode = 1;
|
|
15
|
+
} else {
|
|
16
|
+
const nodeArgs = entry === builtEntry
|
|
17
|
+
? ["--experimental-sqlite", entry, ...process.argv.slice(2)]
|
|
18
|
+
: ["--experimental-strip-types", "--experimental-sqlite", entry, ...process.argv.slice(2)];
|
|
19
|
+
const child = spawnSync(process.execPath, nodeArgs, { stdio: "inherit" });
|
|
20
|
+
if (child.error) {
|
|
21
|
+
console.error(`agent-rag-mcp: failed to start Node.js: ${child.error.message}`);
|
|
22
|
+
process.exitCode = 1;
|
|
23
|
+
} else if (child.signal) {
|
|
24
|
+
process.kill(process.pid, child.signal);
|
|
25
|
+
} else {
|
|
26
|
+
process.exitCode = child.status ?? 1;
|
|
27
|
+
}
|
|
28
|
+
}
|
package/bin/rag.mjs
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import * as fs from "node:fs";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { spawnSync } from "node:child_process";
|
|
5
|
+
import { fileURLToPath } from "node:url";
|
|
6
|
+
|
|
7
|
+
// Development/test seam only: RAG_ENTRY selects source or dist; unset/auto keeps the normal preference.
|
|
8
|
+
const projectRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
|
9
|
+
const builtEntry = path.join(projectRoot, "dist", "src", "cli", "index.mjs");
|
|
10
|
+
const sourceEntry = path.join(projectRoot, "src", "cli", "index.mts");
|
|
11
|
+
|
|
12
|
+
function resolveEntry(mode = process.env.RAG_ENTRY) {
|
|
13
|
+
const selected = mode ?? "auto";
|
|
14
|
+
if (!["auto", "source", "dist"].includes(selected)) {
|
|
15
|
+
return {
|
|
16
|
+
error: {
|
|
17
|
+
namespace: null,
|
|
18
|
+
error: { code: "invalid_entry", message: `Unknown RAG_ENTRY value: ${selected}` },
|
|
19
|
+
},
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
const candidates = selected === "source"
|
|
24
|
+
? [sourceEntry]
|
|
25
|
+
: selected === "dist"
|
|
26
|
+
? [builtEntry]
|
|
27
|
+
: [builtEntry, sourceEntry];
|
|
28
|
+
const entry = candidates.find((candidate) => fs.existsSync(candidate));
|
|
29
|
+
if (entry) {
|
|
30
|
+
const flags = entry === sourceEntry ? ["--experimental-strip-types", "--experimental-sqlite"] : [];
|
|
31
|
+
return { childArgs: [...flags, entry, ...process.argv.slice(2)] };
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
const missingMessage = selected === "auto"
|
|
35
|
+
? `Could not find ${path.relative(projectRoot, builtEntry)} or ${path.relative(projectRoot, sourceEntry)}`
|
|
36
|
+
: `Could not find ${path.relative(projectRoot, candidates[0]).split(path.sep).join("/")}`;
|
|
37
|
+
return {
|
|
38
|
+
error: {
|
|
39
|
+
namespace: null,
|
|
40
|
+
error: { code: selected === "auto" ? "internal_error" : "entry_not_found", message: missingMessage },
|
|
41
|
+
},
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
const resolution = resolveEntry();
|
|
46
|
+
if (resolution.error) {
|
|
47
|
+
process.stdout.write(`${JSON.stringify(resolution.error)}\n`);
|
|
48
|
+
process.exitCode = 1;
|
|
49
|
+
} else {
|
|
50
|
+
const child = spawnSync(process.execPath, resolution.childArgs, { stdio: "inherit" });
|
|
51
|
+
if (child.error) {
|
|
52
|
+
console.error(`rag: failed to start Node.js: ${child.error.message}`);
|
|
53
|
+
process.exitCode = 1;
|
|
54
|
+
} else if (child.signal) {
|
|
55
|
+
process.kill(process.pid, child.signal);
|
|
56
|
+
} else {
|
|
57
|
+
process.exitCode = child.status ?? 1;
|
|
58
|
+
}
|
|
59
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import { extractRawText } from "mammoth";
|
|
2
|
+
const DOCX_MEDIA_TYPE = "application/vnd.openxmlformats-officedocument.wordprocessingml.document";
|
|
3
|
+
function supports(ref) {
|
|
4
|
+
if (typeof ref === "string") return /\.docx$/i.test(ref);
|
|
5
|
+
if (typeof ref !== "object" || ref === null) return false;
|
|
6
|
+
const fields = ref;
|
|
7
|
+
const mediaType = fields.mediaType ?? fields.mimeType ?? fields.contentType;
|
|
8
|
+
if (typeof mediaType === "string") return mediaType.split(";")[0].trim().toLowerCase() === DOCX_MEDIA_TYPE;
|
|
9
|
+
const fileName = fields.filename ?? fields.name ?? fields.path ?? fields.extension;
|
|
10
|
+
return typeof fileName === "string" && /(?:^|[\\/])[^\\/]+\.docx$/i.test(fileName);
|
|
11
|
+
}
|
|
12
|
+
async function extract(bytes) {
|
|
13
|
+
const result = await extractRawText({ buffer: Buffer.from(bytes) });
|
|
14
|
+
const text = result.value.trim();
|
|
15
|
+
return {
|
|
16
|
+
sections: text.length > 0 ? [{ ordinal: 0, heading: null, level: null, text }] : [],
|
|
17
|
+
versionLabel: null,
|
|
18
|
+
mediaType: DOCX_MEDIA_TYPE
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
const docxExtractor = {
|
|
22
|
+
name: "docx",
|
|
23
|
+
version: "1.13.0",
|
|
24
|
+
mediaTypes: [DOCX_MEDIA_TYPE],
|
|
25
|
+
supports,
|
|
26
|
+
extract
|
|
27
|
+
};
|
|
28
|
+
export {
|
|
29
|
+
docxExtractor
|
|
30
|
+
};
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
import { createRequire } from "node:module";
|
|
2
|
+
import Tesseract from "tesseract.js";
|
|
3
|
+
import { ResourceLimitError } from "../ports/extractor.mjs";
|
|
4
|
+
const PNG_MEDIA_TYPE = "image/png";
|
|
5
|
+
const JPEG_MEDIA_TYPE = "image/jpeg";
|
|
6
|
+
const IMAGE_MEDIA_TYPES = /* @__PURE__ */ new Set([PNG_MEDIA_TYPE, JPEG_MEDIA_TYPE]);
|
|
7
|
+
const IMAGE_EXTENSION = /\.(?:png|jpe?g)$/i;
|
|
8
|
+
const MAX_IMAGE_PIXELS = 5e7;
|
|
9
|
+
const englishLanguageData = createRequire(import.meta.url)("@tesseract.js-data/eng");
|
|
10
|
+
const OCR_LANGUAGE_DATA_PATH = englishLanguageData.langPath;
|
|
11
|
+
const OCR_WORKER_OPTIONS = {
|
|
12
|
+
langPath: OCR_LANGUAGE_DATA_PATH,
|
|
13
|
+
cacheMethod: "none",
|
|
14
|
+
gzip: englishLanguageData.gzip
|
|
15
|
+
};
|
|
16
|
+
function supports(ref) {
|
|
17
|
+
if (typeof ref === "string") {
|
|
18
|
+
const mediaType2 = ref.split(";")[0]?.trim().toLowerCase();
|
|
19
|
+
return IMAGE_MEDIA_TYPES.has(mediaType2 ?? "") || IMAGE_EXTENSION.test(ref);
|
|
20
|
+
}
|
|
21
|
+
if (typeof ref !== "object" || ref === null) return false;
|
|
22
|
+
const fields = ref;
|
|
23
|
+
const mediaType = fields.mediaType ?? fields.mimeType ?? fields.contentType;
|
|
24
|
+
if (typeof mediaType === "string") {
|
|
25
|
+
return IMAGE_MEDIA_TYPES.has(mediaType.split(";")[0]?.trim().toLowerCase() ?? "");
|
|
26
|
+
}
|
|
27
|
+
const fileName = fields.filename ?? fields.name ?? fields.path ?? fields.extension;
|
|
28
|
+
return typeof fileName === "string" && IMAGE_EXTENSION.test(fileName);
|
|
29
|
+
}
|
|
30
|
+
function getImageMediaType(bytes) {
|
|
31
|
+
if (bytes.byteLength >= 8 && bytes[0] === 137 && bytes[1] === 80 && bytes[2] === 78 && bytes[3] === 71 && bytes[4] === 13 && bytes[5] === 10 && bytes[6] === 26 && bytes[7] === 10) {
|
|
32
|
+
return PNG_MEDIA_TYPE;
|
|
33
|
+
}
|
|
34
|
+
if (bytes.byteLength >= 3 && bytes[0] === 255 && bytes[1] === 216 && bytes[2] === 255) {
|
|
35
|
+
return JPEG_MEDIA_TYPE;
|
|
36
|
+
}
|
|
37
|
+
throw new Error("Unsupported image data; expected PNG or JPEG bytes.");
|
|
38
|
+
}
|
|
39
|
+
function imageDimensions(bytes, mediaType) {
|
|
40
|
+
const data = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
41
|
+
if (mediaType === PNG_MEDIA_TYPE) {
|
|
42
|
+
if (bytes.byteLength < 24 || String.fromCharCode(...bytes.subarray(12, 16)) !== "IHDR") {
|
|
43
|
+
throw new Error("Invalid PNG image header.");
|
|
44
|
+
}
|
|
45
|
+
return { width: data.getUint32(16), height: data.getUint32(20) };
|
|
46
|
+
}
|
|
47
|
+
let offset = 2;
|
|
48
|
+
while (offset + 3 < bytes.byteLength) {
|
|
49
|
+
if (bytes[offset] !== 255) {
|
|
50
|
+
offset += 1;
|
|
51
|
+
continue;
|
|
52
|
+
}
|
|
53
|
+
while (bytes[offset] === 255) offset += 1;
|
|
54
|
+
const marker = bytes[offset];
|
|
55
|
+
offset += 1;
|
|
56
|
+
if (marker === 217 || marker === 218) break;
|
|
57
|
+
if (marker === 216 || marker === 1 || marker !== void 0 && marker >= 208 && marker <= 215) continue;
|
|
58
|
+
if (offset + 2 > bytes.byteLength) break;
|
|
59
|
+
const segmentLength = data.getUint16(offset);
|
|
60
|
+
if (segmentLength < 2 || offset + segmentLength > bytes.byteLength) break;
|
|
61
|
+
const isStartOfFrame = marker !== void 0 && (marker >= 192 && marker <= 195 || marker >= 197 && marker <= 199 || marker >= 201 && marker <= 203 || marker >= 205 && marker <= 207);
|
|
62
|
+
if (isStartOfFrame) {
|
|
63
|
+
if (segmentLength < 7) break;
|
|
64
|
+
return { height: data.getUint16(offset + 3), width: data.getUint16(offset + 5) };
|
|
65
|
+
}
|
|
66
|
+
offset += segmentLength;
|
|
67
|
+
}
|
|
68
|
+
throw new Error("JPEG image dimensions could not be read.");
|
|
69
|
+
}
|
|
70
|
+
async function extract(bytes) {
|
|
71
|
+
const mediaType = getImageMediaType(bytes);
|
|
72
|
+
const { width, height } = imageDimensions(bytes, mediaType);
|
|
73
|
+
if (width === 0 || height === 0 || width > MAX_IMAGE_PIXELS / height) {
|
|
74
|
+
throw new ResourceLimitError("image_too_large", `Image exceeds MAX_IMAGE_PIXELS (${MAX_IMAGE_PIXELS} pixels).`);
|
|
75
|
+
}
|
|
76
|
+
const worker = await Tesseract.createWorker("eng", Tesseract.OEM.LSTM_ONLY, OCR_WORKER_OPTIONS);
|
|
77
|
+
try {
|
|
78
|
+
await worker.setParameters({ tessedit_pageseg_mode: Tesseract.PSM.SINGLE_BLOCK });
|
|
79
|
+
const { data } = await worker.recognize(Buffer.from(bytes));
|
|
80
|
+
const text = data.text.replace(/\s+/g, " ").trim();
|
|
81
|
+
return {
|
|
82
|
+
sections: text.length > 0 ? [{ ordinal: 0, heading: null, level: null, text }] : [],
|
|
83
|
+
versionLabel: null,
|
|
84
|
+
mediaType
|
|
85
|
+
};
|
|
86
|
+
} finally {
|
|
87
|
+
await worker.terminate();
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
const imageOcrExtractor = {
|
|
91
|
+
name: "image-ocr",
|
|
92
|
+
version: "7.0.0",
|
|
93
|
+
mediaTypes: [PNG_MEDIA_TYPE, JPEG_MEDIA_TYPE],
|
|
94
|
+
supports,
|
|
95
|
+
extract
|
|
96
|
+
};
|
|
97
|
+
export {
|
|
98
|
+
MAX_IMAGE_PIXELS,
|
|
99
|
+
OCR_LANGUAGE_DATA_PATH,
|
|
100
|
+
OCR_WORKER_OPTIONS,
|
|
101
|
+
imageOcrExtractor
|
|
102
|
+
};
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
import { createCanvas } from "@napi-rs/canvas";
|
|
2
|
+
import Tesseract from "tesseract.js";
|
|
3
|
+
import { getDocument } from "pdfjs-dist/legacy/build/pdf.mjs";
|
|
4
|
+
import { OCR_WORKER_OPTIONS } from "./image-ocr.mjs";
|
|
5
|
+
import { MAX_OCR_PDF_PAGES, MAX_PDF_OCR_RENDER_PIXELS, MAX_PDF_SOURCE_IMAGE_PIXELS } from "../limits.mjs";
|
|
6
|
+
import { ResourceLimitError } from "../ports/extractor.mjs";
|
|
7
|
+
const PDF_MEDIA_TYPE = "application/pdf";
|
|
8
|
+
const MAX_PDF_PAGES = 2e3;
|
|
9
|
+
const PDF_OCR_RENDER_SCALE = 2;
|
|
10
|
+
function supports(ref) {
|
|
11
|
+
if (typeof ref === "string") return /\.pdf$/i.test(ref);
|
|
12
|
+
if (typeof ref !== "object" || ref === null) return false;
|
|
13
|
+
const fields = ref;
|
|
14
|
+
const mediaType = fields.mediaType ?? fields.mimeType ?? fields.contentType;
|
|
15
|
+
if (typeof mediaType === "string") return mediaType.split(";")[0].trim().toLowerCase() === PDF_MEDIA_TYPE;
|
|
16
|
+
const fileName = fields.filename ?? fields.name ?? fields.path ?? fields.extension;
|
|
17
|
+
return typeof fileName === "string" && /(?:^|[\\/])[^\\/]+\.pdf$/i.test(fileName);
|
|
18
|
+
}
|
|
19
|
+
async function extract(bytes) {
|
|
20
|
+
const loadingTask = getDocument({
|
|
21
|
+
data: Uint8Array.from(bytes),
|
|
22
|
+
maxImageSize: MAX_PDF_SOURCE_IMAGE_PIXELS,
|
|
23
|
+
disableFontFace: true,
|
|
24
|
+
useWorkerFetch: false,
|
|
25
|
+
verbosity: 0
|
|
26
|
+
});
|
|
27
|
+
const document = await loadingTask.promise;
|
|
28
|
+
try {
|
|
29
|
+
if (document.numPages > MAX_PDF_PAGES) {
|
|
30
|
+
throw new ResourceLimitError("too_many_pages", `PDF exceeds MAX_PDF_PAGES (${MAX_PDF_PAGES} pages).`);
|
|
31
|
+
}
|
|
32
|
+
const sections = [];
|
|
33
|
+
for (let pageNumber = 1; pageNumber <= document.numPages; pageNumber += 1) {
|
|
34
|
+
const page = await document.getPage(pageNumber);
|
|
35
|
+
const textContent = await page.getTextContent();
|
|
36
|
+
const text = textContent.items.filter((item) => "str" in item).map((item) => "str" in item ? item.str : "").join(" ").replace(/\s+/g, " ").trim();
|
|
37
|
+
if (text.length > 0) {
|
|
38
|
+
sections.push({ ordinal: sections.length, heading: null, level: null, text });
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
if (sections.length > 0) return { sections, versionLabel: null, mediaType: PDF_MEDIA_TYPE };
|
|
42
|
+
if (document.numPages > MAX_OCR_PDF_PAGES) {
|
|
43
|
+
throw new ResourceLimitError(
|
|
44
|
+
"too_many_pages",
|
|
45
|
+
`PDF exceeds MAX_OCR_PDF_PAGES (${MAX_OCR_PDF_PAGES} pages) for OCR.`
|
|
46
|
+
);
|
|
47
|
+
}
|
|
48
|
+
const worker = await Tesseract.createWorker("eng", Tesseract.OEM.LSTM_ONLY, OCR_WORKER_OPTIONS);
|
|
49
|
+
try {
|
|
50
|
+
await worker.setParameters({ tessedit_pageseg_mode: Tesseract.PSM.SINGLE_BLOCK });
|
|
51
|
+
for (let pageNumber = 1; pageNumber <= document.numPages; pageNumber += 1) {
|
|
52
|
+
const page = await document.getPage(pageNumber);
|
|
53
|
+
const viewport = page.getViewport({ scale: PDF_OCR_RENDER_SCALE });
|
|
54
|
+
const width = Math.ceil(viewport.width);
|
|
55
|
+
const height = Math.ceil(viewport.height);
|
|
56
|
+
if (width === 0 || height === 0 || width > MAX_PDF_OCR_RENDER_PIXELS / height) {
|
|
57
|
+
throw new ResourceLimitError(
|
|
58
|
+
"image_too_large",
|
|
59
|
+
`PDF page exceeds MAX_PDF_OCR_RENDER_PIXELS (${MAX_PDF_OCR_RENDER_PIXELS} pixels).`
|
|
60
|
+
);
|
|
61
|
+
}
|
|
62
|
+
const canvas = createCanvas(width, height);
|
|
63
|
+
const canvasContext = canvas.getContext("2d");
|
|
64
|
+
await page.render({
|
|
65
|
+
canvas,
|
|
66
|
+
canvasContext,
|
|
67
|
+
viewport
|
|
68
|
+
}).promise;
|
|
69
|
+
const { data } = await worker.recognize(canvas.toBuffer("image/png"));
|
|
70
|
+
const text = data.text.replace(/\s+/g, " ").trim();
|
|
71
|
+
if (text.length > 0) {
|
|
72
|
+
sections.push({ ordinal: pageNumber - 1, heading: `Page ${pageNumber}`, level: null, text });
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
} finally {
|
|
76
|
+
await worker.terminate();
|
|
77
|
+
}
|
|
78
|
+
return { sections, versionLabel: null, mediaType: PDF_MEDIA_TYPE };
|
|
79
|
+
} finally {
|
|
80
|
+
await loadingTask.destroy();
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
const pdfExtractor = {
|
|
84
|
+
name: "pdf",
|
|
85
|
+
version: "6.4.299",
|
|
86
|
+
mediaTypes: [PDF_MEDIA_TYPE],
|
|
87
|
+
supports,
|
|
88
|
+
extract
|
|
89
|
+
};
|
|
90
|
+
export {
|
|
91
|
+
MAX_PDF_PAGES,
|
|
92
|
+
PDF_OCR_RENDER_SCALE,
|
|
93
|
+
pdfExtractor
|
|
94
|
+
};
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import { parseVersionLabel } from "../store/identity.mjs";
|
|
2
|
+
const TEXT_MEDIA_TYPES = /* @__PURE__ */ new Set(["text/plain", "text/markdown"]);
|
|
3
|
+
function getReferenceField(ref, names) {
|
|
4
|
+
if (typeof ref === "string") return ref;
|
|
5
|
+
if (typeof ref !== "object" || ref === null) return null;
|
|
6
|
+
const value = ref;
|
|
7
|
+
for (const name of names) {
|
|
8
|
+
if (typeof value[name] === "string") return value[name];
|
|
9
|
+
}
|
|
10
|
+
return null;
|
|
11
|
+
}
|
|
12
|
+
function supports(ref) {
|
|
13
|
+
const mediaType = getReferenceField(ref, ["mediaType", "mimeType", "contentType"]);
|
|
14
|
+
if (mediaType !== null) return TEXT_MEDIA_TYPES.has(mediaType.split(";")[0].trim().toLowerCase());
|
|
15
|
+
const fileName = getReferenceField(ref, ["filename", "name", "path", "extension"]);
|
|
16
|
+
if (fileName === null) return false;
|
|
17
|
+
return /(?:^|[\\/])[^\\/]+\.(?:txt|md)$/i.test(fileName) || /^\.(?:txt|md)$/i.test(fileName);
|
|
18
|
+
}
|
|
19
|
+
function splitMarkdownSections(content) {
|
|
20
|
+
const lines = content.split("\n");
|
|
21
|
+
const sections = [];
|
|
22
|
+
let current = null;
|
|
23
|
+
const headingPattern = /^ {0,3}(#{1,6})(?:[ \t]+|$)(.*)$/;
|
|
24
|
+
if (!lines.some((line) => headingPattern.test(line.replace(/\r$/, "")))) {
|
|
25
|
+
return content.length === 0 ? [] : [{ ordinal: 0, heading: null, level: null, text: content }];
|
|
26
|
+
}
|
|
27
|
+
const pushCurrent = () => {
|
|
28
|
+
if (!current) return;
|
|
29
|
+
sections.push({
|
|
30
|
+
ordinal: sections.length,
|
|
31
|
+
heading: current.heading,
|
|
32
|
+
level: current.level,
|
|
33
|
+
text: current.lines.join("\n").trim()
|
|
34
|
+
});
|
|
35
|
+
};
|
|
36
|
+
for (const line of lines) {
|
|
37
|
+
const match = headingPattern.exec(line.replace(/\r$/, ""));
|
|
38
|
+
if (!match) {
|
|
39
|
+
if (current) current.lines.push(line);
|
|
40
|
+
else if (line.length > 0) current = { heading: null, level: null, lines: [line] };
|
|
41
|
+
continue;
|
|
42
|
+
}
|
|
43
|
+
pushCurrent();
|
|
44
|
+
const heading = match[2].replace(/[ \t]+#+[ \t]*$/, "").trim();
|
|
45
|
+
current = { heading, level: match[1].length, lines: [] };
|
|
46
|
+
}
|
|
47
|
+
pushCurrent();
|
|
48
|
+
return sections;
|
|
49
|
+
}
|
|
50
|
+
async function extract(bytes) {
|
|
51
|
+
const decoded = new TextDecoder("utf-8", { fatal: false, ignoreBOM: true }).decode(bytes);
|
|
52
|
+
const content = decoded.startsWith("\uFEFF") ? decoded.slice(1) : decoded;
|
|
53
|
+
const sections = splitMarkdownSections(content);
|
|
54
|
+
const hasHeadings = sections.some((section) => section.heading !== null);
|
|
55
|
+
const versionLabel = sections.find(
|
|
56
|
+
(section) => section.heading !== null && parseVersionLabel(section.heading) !== null
|
|
57
|
+
)?.heading ?? null;
|
|
58
|
+
return {
|
|
59
|
+
sections,
|
|
60
|
+
versionLabel,
|
|
61
|
+
mediaType: hasHeadings ? "text/markdown" : content.length > 0 ? "text/plain" : null
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
const plainTextExtractor = {
|
|
65
|
+
name: "plain-text",
|
|
66
|
+
version: "1.0.0",
|
|
67
|
+
supports,
|
|
68
|
+
extract
|
|
69
|
+
};
|
|
70
|
+
export {
|
|
71
|
+
plainTextExtractor
|
|
72
|
+
};
|