abb-opencode-local-rag 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.de.md +416 -0
- package/README.es.md +416 -0
- package/README.fr.md +416 -0
- package/README.md +491 -0
- package/README.pt-BR.md +416 -0
- package/README.zh-CN.md +416 -0
- package/dist/bin/install-skills.d.ts +20 -0
- package/dist/bin/install-skills.d.ts.map +1 -0
- package/dist/bin/install-skills.js +195 -0
- package/dist/bin/install-skills.js.map +1 -0
- package/dist/chunker/index.d.ts +24 -0
- package/dist/chunker/index.d.ts.map +1 -0
- package/dist/chunker/index.js +2 -0
- package/dist/chunker/index.js.map +1 -0
- package/dist/chunker/semantic-chunker.d.ts +97 -0
- package/dist/chunker/semantic-chunker.d.ts.map +1 -0
- package/dist/chunker/semantic-chunker.js +294 -0
- package/dist/chunker/semantic-chunker.js.map +1 -0
- package/dist/chunker/sentence-splitter.d.ts +28 -0
- package/dist/chunker/sentence-splitter.d.ts.map +1 -0
- package/dist/chunker/sentence-splitter.js +219 -0
- package/dist/chunker/sentence-splitter.js.map +1 -0
- package/dist/cli/common.d.ts +65 -0
- package/dist/cli/common.d.ts.map +1 -0
- package/dist/cli/common.js +138 -0
- package/dist/cli/common.js.map +1 -0
- package/dist/cli/delete.d.ts +8 -0
- package/dist/cli/delete.d.ts.map +1 -0
- package/dist/cli/delete.js +173 -0
- package/dist/cli/delete.js.map +1 -0
- package/dist/cli/file-collection.d.ts +2 -0
- package/dist/cli/file-collection.d.ts.map +1 -0
- package/dist/cli/file-collection.js +53 -0
- package/dist/cli/file-collection.js.map +1 -0
- package/dist/cli/ingest.d.ts +100 -0
- package/dist/cli/ingest.d.ts.map +1 -0
- package/dist/cli/ingest.js +363 -0
- package/dist/cli/ingest.js.map +1 -0
- package/dist/cli/list.d.ts +35 -0
- package/dist/cli/list.d.ts.map +1 -0
- package/dist/cli/list.js +210 -0
- package/dist/cli/list.js.map +1 -0
- package/dist/cli/options.d.ts +100 -0
- package/dist/cli/options.d.ts.map +1 -0
- package/dist/cli/options.js +241 -0
- package/dist/cli/options.js.map +1 -0
- package/dist/cli/query.d.ts +24 -0
- package/dist/cli/query.d.ts.map +1 -0
- package/dist/cli/query.js +191 -0
- package/dist/cli/query.js.map +1 -0
- package/dist/cli/read-neighbors.d.ts +11 -0
- package/dist/cli/read-neighbors.d.ts.map +1 -0
- package/dist/cli/read-neighbors.js +224 -0
- package/dist/cli/read-neighbors.js.map +1 -0
- package/dist/cli/status.d.ts +8 -0
- package/dist/cli/status.d.ts.map +1 -0
- package/dist/cli/status.js +80 -0
- package/dist/cli/status.js.map +1 -0
- package/dist/cli/sync.d.ts +8 -0
- package/dist/cli/sync.d.ts.map +1 -0
- package/dist/cli/sync.js +244 -0
- package/dist/cli/sync.js.map +1 -0
- package/dist/cli-main.d.ts +12 -0
- package/dist/cli-main.d.ts.map +1 -0
- package/dist/cli-main.js +63 -0
- package/dist/cli-main.js.map +1 -0
- package/dist/embedder/index.d.ts +85 -0
- package/dist/embedder/index.d.ts.map +1 -0
- package/dist/embedder/index.js +284 -0
- package/dist/embedder/index.js.map +1 -0
- package/dist/features/list.d.ts +37 -0
- package/dist/features/list.d.ts.map +1 -0
- package/dist/features/list.js +40 -0
- package/dist/features/list.js.map +1 -0
- package/dist/features/sync.d.ts +207 -0
- package/dist/features/sync.d.ts.map +1 -0
- package/dist/features/sync.js +380 -0
- package/dist/features/sync.js.map +1 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +53 -0
- package/dist/index.js.map +1 -0
- package/dist/ingest/compute.d.ts +86 -0
- package/dist/ingest/compute.d.ts.map +1 -0
- package/dist/ingest/compute.js +177 -0
- package/dist/ingest/compute.js.map +1 -0
- package/dist/ingest/file.d.ts +27 -0
- package/dist/ingest/file.d.ts.map +1 -0
- package/dist/ingest/file.js +67 -0
- package/dist/ingest/file.js.map +1 -0
- package/dist/ingest/visual.d.ts +45 -0
- package/dist/ingest/visual.d.ts.map +1 -0
- package/dist/ingest/visual.js +234 -0
- package/dist/ingest/visual.js.map +1 -0
- package/dist/parser/docx-parser.d.ts +12 -0
- package/dist/parser/docx-parser.d.ts.map +1 -0
- package/dist/parser/docx-parser.js +328 -0
- package/dist/parser/docx-parser.js.map +1 -0
- package/dist/parser/html-parser.d.ts +18 -0
- package/dist/parser/html-parser.d.ts.map +1 -0
- package/dist/parser/html-parser.js +102 -0
- package/dist/parser/html-parser.js.map +1 -0
- package/dist/parser/index.d.ts +214 -0
- package/dist/parser/index.d.ts.map +1 -0
- package/dist/parser/index.js +454 -0
- package/dist/parser/index.js.map +1 -0
- package/dist/parser/pdf-extract.d.ts +81 -0
- package/dist/parser/pdf-extract.d.ts.map +1 -0
- package/dist/parser/pdf-extract.js +112 -0
- package/dist/parser/pdf-extract.js.map +1 -0
- package/dist/parser/pdf-filter.d.ts +117 -0
- package/dist/parser/pdf-filter.d.ts.map +1 -0
- package/dist/parser/pdf-filter.js +528 -0
- package/dist/parser/pdf-filter.js.map +1 -0
- package/dist/parser/title-extractor.d.ts +69 -0
- package/dist/parser/title-extractor.d.ts.map +1 -0
- package/dist/parser/title-extractor.js +145 -0
- package/dist/parser/title-extractor.js.map +1 -0
- package/dist/pdf-visual/captioner.d.ts +16 -0
- package/dist/pdf-visual/captioner.d.ts.map +1 -0
- package/dist/pdf-visual/captioner.js +63 -0
- package/dist/pdf-visual/captioner.js.map +1 -0
- package/dist/pdf-visual/captioners/fast.d.ts +7 -0
- package/dist/pdf-visual/captioners/fast.d.ts.map +1 -0
- package/dist/pdf-visual/captioners/fast.js +103 -0
- package/dist/pdf-visual/captioners/fast.js.map +1 -0
- package/dist/pdf-visual/captioners/quality.d.ts +7 -0
- package/dist/pdf-visual/captioners/quality.d.ts.map +1 -0
- package/dist/pdf-visual/captioners/quality.js +127 -0
- package/dist/pdf-visual/captioners/quality.js.map +1 -0
- package/dist/pdf-visual/captioners/shared.d.ts +44 -0
- package/dist/pdf-visual/captioners/shared.d.ts.map +1 -0
- package/dist/pdf-visual/captioners/shared.js +104 -0
- package/dist/pdf-visual/captioners/shared.js.map +1 -0
- package/dist/pdf-visual/detector.d.ts +9 -0
- package/dist/pdf-visual/detector.d.ts.map +1 -0
- package/dist/pdf-visual/detector.js +234 -0
- package/dist/pdf-visual/detector.js.map +1 -0
- package/dist/pdf-visual/index.d.ts +13 -0
- package/dist/pdf-visual/index.d.ts.map +1 -0
- package/dist/pdf-visual/index.js +45 -0
- package/dist/pdf-visual/index.js.map +1 -0
- package/dist/pdf-visual/renderer.d.ts +9 -0
- package/dist/pdf-visual/renderer.d.ts.map +1 -0
- package/dist/pdf-visual/renderer.js +177 -0
- package/dist/pdf-visual/renderer.js.map +1 -0
- package/dist/pdf-visual/types.d.ts +62 -0
- package/dist/pdf-visual/types.d.ts.map +1 -0
- package/dist/pdf-visual/types.js +32 -0
- package/dist/pdf-visual/types.js.map +1 -0
- package/dist/server/error-utils.d.ts +79 -0
- package/dist/server/error-utils.d.ts.map +1 -0
- package/dist/server/error-utils.js +148 -0
- package/dist/server/error-utils.js.map +1 -0
- package/dist/server/index.d.ts +258 -0
- package/dist/server/index.d.ts.map +1 -0
- package/dist/server/index.js +1104 -0
- package/dist/server/index.js.map +1 -0
- package/dist/server/list-scanner.d.ts +52 -0
- package/dist/server/list-scanner.d.ts.map +1 -0
- package/dist/server/list-scanner.js +72 -0
- package/dist/server/list-scanner.js.map +1 -0
- package/dist/server/tool-definitions.d.ts +8 -0
- package/dist/server/tool-definitions.d.ts.map +1 -0
- package/dist/server/tool-definitions.js +181 -0
- package/dist/server/tool-definitions.js.map +1 -0
- package/dist/server/tool-input.d.ts +37 -0
- package/dist/server/tool-input.d.ts.map +1 -0
- package/dist/server/tool-input.js +216 -0
- package/dist/server/tool-input.js.map +1 -0
- package/dist/server/types.d.ts +331 -0
- package/dist/server/types.d.ts.map +1 -0
- package/dist/server/types.js +3 -0
- package/dist/server/types.js.map +1 -0
- package/dist/server-main.d.ts +46 -0
- package/dist/server-main.d.ts.map +1 -0
- package/dist/server-main.js +242 -0
- package/dist/server-main.js.map +1 -0
- package/dist/utils/base-dirs.d.ts +212 -0
- package/dist/utils/base-dirs.d.ts.map +1 -0
- package/dist/utils/base-dirs.js +422 -0
- package/dist/utils/base-dirs.js.map +1 -0
- package/dist/utils/errors.d.ts +24 -0
- package/dist/utils/errors.d.ts.map +1 -0
- package/dist/utils/errors.js +53 -0
- package/dist/utils/errors.js.map +1 -0
- package/dist/utils/limits.d.ts +26 -0
- package/dist/utils/limits.d.ts.map +1 -0
- package/dist/utils/limits.js +28 -0
- package/dist/utils/limits.js.map +1 -0
- package/dist/utils/list-sources.d.ts +47 -0
- package/dist/utils/list-sources.d.ts.map +1 -0
- package/dist/utils/list-sources.js +50 -0
- package/dist/utils/list-sources.js.map +1 -0
- package/dist/utils/raw-data-utils.d.ts +131 -0
- package/dist/utils/raw-data-utils.d.ts.map +1 -0
- package/dist/utils/raw-data-utils.js +255 -0
- package/dist/utils/raw-data-utils.js.map +1 -0
- package/dist/utils/scan.d.ts +126 -0
- package/dist/utils/scan.d.ts.map +1 -0
- package/dist/utils/scan.js +221 -0
- package/dist/utils/scan.js.map +1 -0
- package/dist/utils/scope-match.d.ts +43 -0
- package/dist/utils/scope-match.d.ts.map +1 -0
- package/dist/utils/scope-match.js +87 -0
- package/dist/utils/scope-match.js.map +1 -0
- package/dist/utils/sensitive-path.d.ts +23 -0
- package/dist/utils/sensitive-path.d.ts.map +1 -0
- package/dist/utils/sensitive-path.js +91 -0
- package/dist/utils/sensitive-path.js.map +1 -0
- package/dist/utils/sync-path-key.d.ts +20 -0
- package/dist/utils/sync-path-key.d.ts.map +1 -0
- package/dist/utils/sync-path-key.js +33 -0
- package/dist/utils/sync-path-key.js.map +1 -0
- package/dist/vectordb/index.d.ts +168 -0
- package/dist/vectordb/index.d.ts.map +1 -0
- package/dist/vectordb/index.js +619 -0
- package/dist/vectordb/index.js.map +1 -0
- package/dist/vectordb/search-filters.d.ts +39 -0
- package/dist/vectordb/search-filters.d.ts.map +1 -0
- package/dist/vectordb/search-filters.js +136 -0
- package/dist/vectordb/search-filters.js.map +1 -0
- package/dist/vectordb/types.d.ts +196 -0
- package/dist/vectordb/types.d.ts.map +1 -0
- package/dist/vectordb/types.js +224 -0
- package/dist/vectordb/types.js.map +1 -0
- package/package.json +105 -0
- package/skills/mcp-local-rag/SKILL.md +308 -0
- package/skills/mcp-local-rag/references/cli-reference.md +175 -0
- package/skills/mcp-local-rag/references/html-ingestion.md +78 -0
- package/skills/mcp-local-rag/references/query-optimization.md +57 -0
- package/skills/mcp-local-rag/references/result-refinement.md +56 -0
package/dist/index.js
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Entry point for mcp-local-rag
|
|
3
|
+
// Routes to CLI subcommands or starts the MCP server
|
|
4
|
+
import { parseGlobalOptions } from './cli/options.js';
|
|
5
|
+
import { handleCli, SUBCOMMANDS } from './cli-main.js';
|
|
6
|
+
import { startServer } from './server-main.js';
|
|
7
|
+
// ============================================
|
|
8
|
+
// Routing helpers
|
|
9
|
+
// ============================================
|
|
10
|
+
const SUBCOMMAND_SET = new Set(SUBCOMMANDS);
|
|
11
|
+
function isSubcommand(value) {
|
|
12
|
+
return SUBCOMMAND_SET.has(value);
|
|
13
|
+
}
|
|
14
|
+
/** Replace control chars and truncate, so an unexpected argv value
|
|
15
|
+
* echoed to stderr cannot smuggle ANSI escapes or CR/LF into log lines. */
|
|
16
|
+
function sanitizeForEcho(s) {
|
|
17
|
+
return s.replace(/\p{Cc}/gu, '?').slice(0, 100);
|
|
18
|
+
}
|
|
19
|
+
// ============================================
|
|
20
|
+
// Routing
|
|
21
|
+
// ============================================
|
|
22
|
+
const { globalOptions, remainingArgs } = parseGlobalOptions(process.argv.slice(2));
|
|
23
|
+
const firstArg = remainingArgs[0];
|
|
24
|
+
if (firstArg !== undefined && isSubcommand(firstArg)) {
|
|
25
|
+
// CLI subcommand
|
|
26
|
+
handleCli(firstArg, remainingArgs.slice(1), globalOptions).catch((error) => {
|
|
27
|
+
console.error(error);
|
|
28
|
+
process.exit(1);
|
|
29
|
+
});
|
|
30
|
+
}
|
|
31
|
+
else if (remainingArgs.length === 0) {
|
|
32
|
+
if (Object.keys(globalOptions).length > 0) {
|
|
33
|
+
console.error('Global CLI options are not supported when launching the MCP server directly.');
|
|
34
|
+
console.error('Use environment variables like DB_PATH, CACHE_DIR, MODEL_NAME, BASE_DIR, BASE_DIRS, and MAX_FILE_SIZE instead.');
|
|
35
|
+
process.exit(1);
|
|
36
|
+
}
|
|
37
|
+
// Default: start MCP server (env-only, no CLI flags)
|
|
38
|
+
process.on('unhandledRejection', (reason, promise) => {
|
|
39
|
+
console.error('Unhandled Rejection at:', promise, 'reason:', reason);
|
|
40
|
+
process.exit(1);
|
|
41
|
+
});
|
|
42
|
+
process.on('uncaughtException', (error) => {
|
|
43
|
+
console.error('Uncaught Exception:', error);
|
|
44
|
+
process.exit(1);
|
|
45
|
+
});
|
|
46
|
+
startServer();
|
|
47
|
+
}
|
|
48
|
+
else {
|
|
49
|
+
console.error(`Unknown command: ${sanitizeForEcho(firstArg ?? '')}`);
|
|
50
|
+
console.error(`Available commands: ${SUBCOMMANDS.join(', ')}`);
|
|
51
|
+
process.exit(1);
|
|
52
|
+
}
|
|
53
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAEA,gCAAgC;AAChC,qDAAqD;AAErD,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AACrD,OAAO,EAAE,SAAS,EAAE,WAAW,EAAmB,MAAM,eAAe,CAAA;AACvE,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAA;AAE9C,+CAA+C;AAC/C,kBAAkB;AAClB,+CAA+C;AAE/C,MAAM,cAAc,GAAwB,IAAI,GAAG,CAAC,WAAW,CAAC,CAAA;AAEhE,SAAS,YAAY,CAAC,KAAa;IACjC,OAAO,cAAc,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;AAClC,CAAC;AAED;4EAC4E;AAC5E,SAAS,eAAe,CAAC,CAAS;IAChC,OAAO,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAA;AACjD,CAAC;AAED,+CAA+C;AAC/C,UAAU;AACV,+CAA+C;AAE/C,MAAM,EAAE,aAAa,EAAE,aAAa,EAAE,GAAG,kBAAkB,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA;AAClF,MAAM,QAAQ,GAAG,aAAa,CAAC,CAAC,CAAC,CAAA;AAEjC,IAAI,QAAQ,KAAK,SAAS,IAAI,YAAY,CAAC,QAAQ,CAAC,EAAE,CAAC;IACrD,iBAAiB;IACjB,SAAS,CAAC,QAAQ,EAAE,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,aAAa,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;QACzE,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;QACpB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACjB,CAAC,CAAC,CAAA;AACJ,CAAC;KAAM,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;IACtC,IAAI,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1C,OAAO,CAAC,KAAK,CAAC,8EAA8E,CAAC,CAAA;QAC7F,OAAO,CAAC,KAAK,CACX,gHAAgH,CACjH,CAAA;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACjB,CAAC;IAED,qDAAqD;IACrD,OAAO,CAAC,EAAE,CAAC,oBAAoB,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE;QACnD,OAAO,CAAC,KAAK,CAAC,yBAAyB,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,CAAC,CAAA;QACpE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACjB,CAAC,CAAC,CAAA;IAEF,OAAO,CAAC,EAAE,CAAC,mBAAmB,EAAE,CAAC,KAAK,EAAE,EAAE;QACxC,OAAO,CAAC,KAAK,CAAC,qBAAqB,EAAE,KAAK,CAAC,CAAA;QAC3C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACjB,CAAC,CAAC,CAAA;IAEF,WAAW,EAAE,CAAA;AACf,CAAC;KAAM,CAAC;IACN,OAAO,CAAC,KAAK,CAAC,oBAAoB,eAAe,CAAC,QAAQ,IAAI,EAAE,CAAC,EAAE,CAAC,CAAA;IACpE,OAAO,CAAC,KAAK,CAAC,uBAAuB,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAC9D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;AACjB,CAAC"}
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import type { AtomicTextRange, SemanticChunker, TextChunk } from '../chunker/index.js';
|
|
2
|
+
import type { EmbedderInterface } from '../chunker/semantic-chunker.js';
|
|
3
|
+
import type { ParseResult } from '../parser/index.js';
|
|
4
|
+
import type { ImageRendition } from '../pdf-visual/types.js';
|
|
5
|
+
import type { VectorChunk, VisualAttachment } from '../vectordb/index.js';
|
|
6
|
+
/**
|
|
7
|
+
* Result of the shared chunk + embed computation.
|
|
8
|
+
*
|
|
9
|
+
* - `chunks` is the result of a single `chunker.chunkText` call.
|
|
10
|
+
* - `embeddings` is the result of `embedder.embedBatch(chunks.map(c => c.text))`
|
|
11
|
+
* and has the same length as `chunks`.
|
|
12
|
+
*/
|
|
13
|
+
export interface BuildChunksAndEmbeddingsResult {
|
|
14
|
+
chunks: TextChunk[];
|
|
15
|
+
embeddings: number[][];
|
|
16
|
+
}
|
|
17
|
+
export interface BuildChunksFromParseResultResult extends BuildChunksAndEmbeddingsResult {
|
|
18
|
+
visualAttachments: Map<number, VisualAttachment[]>;
|
|
19
|
+
omittedImageCount: number;
|
|
20
|
+
}
|
|
21
|
+
export declare function findNearestChunk(chunks: readonly TextChunk[], sourceOffset: number): TextChunk | undefined;
|
|
22
|
+
export declare function createVisualAttachment(imageIndex: number, rendition: ImageRendition): VisualAttachment;
|
|
23
|
+
/**
|
|
24
|
+
* Compute semantic chunks and their embeddings for already-extracted text.
|
|
25
|
+
*
|
|
26
|
+
* Calls `chunker.chunkText` exactly once and then
|
|
27
|
+
* `embedder.embedBatch` on the resulting chunk texts. Does NOT touch
|
|
28
|
+
* `vectorStore`. Does NOT fail-fast on zero chunks — callers decide
|
|
29
|
+
* how to handle an empty result (the MCP handler throws `McpError`;
|
|
30
|
+
* the CLI logs a warning and returns 0).
|
|
31
|
+
*
|
|
32
|
+
* Errors from the chunker or embedder propagate verbatim.
|
|
33
|
+
*
|
|
34
|
+
* @param text Already-extracted document text (parser output, raw-data
|
|
35
|
+
* payload, or joined visual-enriched per-page text).
|
|
36
|
+
* @param chunker Semantic chunker instance (owned by the caller).
|
|
37
|
+
* @param embedder Embedder implementing the structural `EmbedderInterface`
|
|
38
|
+
* (only `embedBatch` is required).
|
|
39
|
+
*/
|
|
40
|
+
export declare function buildChunksAndEmbeddings(text: string, chunker: SemanticChunker, embedder: EmbedderInterface, atomicRanges?: readonly AtomicTextRange[]): Promise<BuildChunksAndEmbeddingsResult>;
|
|
41
|
+
/**
|
|
42
|
+
* Preserve the parser content/range mapping at one shared boundary. Display
|
|
43
|
+
* title handling stays in each dispatch root because it does not affect chunks.
|
|
44
|
+
*/
|
|
45
|
+
export declare function buildChunksFromParseResult(result: ParseResult, chunker: SemanticChunker, embedder: EmbedderInterface): Promise<BuildChunksFromParseResultResult>;
|
|
46
|
+
/**
|
|
47
|
+
* Content identity of a source file: the lowercase SHA-256 hex digest of its
|
|
48
|
+
* raw bytes.
|
|
49
|
+
*
|
|
50
|
+
* Hashing the bytes (not the parsed or normalized text) keeps the value
|
|
51
|
+
* reproducible by any caller that can read the file, which is what lets a later
|
|
52
|
+
* sync pass decide "unchanged" without re-parsing. Pure: the caller reads the
|
|
53
|
+
* file and passes the bytes in.
|
|
54
|
+
*/
|
|
55
|
+
export declare function computeContentHash(bytes: Uint8Array): string;
|
|
56
|
+
/**
|
|
57
|
+
* Build persistable `VectorChunk`s from computed chunks + embeddings.
|
|
58
|
+
*
|
|
59
|
+
* Single source of truth for the chunk→VectorChunk mapping shared by the MCP
|
|
60
|
+
* ingest handler (`handleIngestFile`) and both CLI ingest paths (default +
|
|
61
|
+
* visual). Assigns one shared `timestamp` to every chunk, a fresh `id`, and
|
|
62
|
+
* derives `fileName`/`fileType` from `filePath` via `node:path` (cross-platform).
|
|
63
|
+
* Does NOT touch `vectorStore` — persistence stays in each caller.
|
|
64
|
+
*
|
|
65
|
+
* Throws when a chunk has no corresponding embedding (index mismatch);
|
|
66
|
+
* `embeddings` must align 1:1 with `chunks`.
|
|
67
|
+
*
|
|
68
|
+
* @param fileSize Length value recorded in `metadata.fileSize`. The caller
|
|
69
|
+
* chooses the source: the default path passes parsed text length; the visual
|
|
70
|
+
* path passes the joined enriched-page text length (pre-chunking).
|
|
71
|
+
* @param contentHash {@link computeContentHash} of the source file bytes,
|
|
72
|
+
* shared by every chunk of that file exactly like `timestamp`. `null` for a
|
|
73
|
+
* chunk set with no source file; the key is then omitted rather than stored
|
|
74
|
+
* empty, so a hashless row is never mistaken for a hash of nothing. Required
|
|
75
|
+
* (not optional) so a new call site cannot silently write hashless rows.
|
|
76
|
+
*/
|
|
77
|
+
export declare function buildVectorChunks(params: {
|
|
78
|
+
filePath: string;
|
|
79
|
+
chunks: TextChunk[];
|
|
80
|
+
embeddings: number[][];
|
|
81
|
+
fileSize: number;
|
|
82
|
+
fileTitle: string | null;
|
|
83
|
+
contentHash: string | null;
|
|
84
|
+
visualAttachments?: ReadonlyMap<number, readonly VisualAttachment[]>;
|
|
85
|
+
}): VectorChunk[];
|
|
86
|
+
//# sourceMappingURL=compute.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"compute.d.ts","sourceRoot":"","sources":["../../src/ingest/compute.ts"],"names":[],"mappings":"AAcA,OAAO,KAAK,EAAE,eAAe,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAA;AACtF,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,gCAAgC,CAAA;AACvE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAA;AACrD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAA;AAE5D,OAAO,KAAK,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAA;AAEzE;;;;;;GAMG;AACH,MAAM,WAAW,8BAA8B;IAC7C,MAAM,EAAE,SAAS,EAAE,CAAA;IACnB,UAAU,EAAE,MAAM,EAAE,EAAE,CAAA;CACvB;AAED,MAAM,WAAW,gCAAiC,SAAQ,8BAA8B;IACtF,iBAAiB,EAAE,GAAG,CAAC,MAAM,EAAE,gBAAgB,EAAE,CAAC,CAAA;IAClD,iBAAiB,EAAE,MAAM,CAAA;CAC1B;AAED,wBAAgB,gBAAgB,CAC9B,MAAM,EAAE,SAAS,SAAS,EAAE,EAC5B,YAAY,EAAE,MAAM,GACnB,SAAS,GAAG,SAAS,CA2BvB;AAED,wBAAgB,sBAAsB,CACpC,UAAU,EAAE,MAAM,EAClB,SAAS,EAAE,cAAc,GACxB,gBAAgB,CAclB;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAsB,wBAAwB,CAC5C,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,eAAe,EACxB,QAAQ,EAAE,iBAAiB,EAC3B,YAAY,CAAC,EAAE,SAAS,eAAe,EAAE,GACxC,OAAO,CAAC,8BAA8B,CAAC,CAWzC;AAED;;;GAGG;AACH,wBAAsB,0BAA0B,CAC9C,MAAM,EAAE,WAAW,EACnB,OAAO,EAAE,eAAe,EACxB,QAAQ,EAAE,iBAAiB,GAC1B,OAAO,CAAC,gCAAgC,CAAC,CA+B3C;AAED;;;;;;;;GAQG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,UAAU,GAAG,MAAM,CAE5D;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE;IACxC,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,SAAS,EAAE,CAAA;IACnB,UAAU,EAAE,MAAM,EAAE,EAAE,CAAA;IACtB,QAAQ,EAAE,MAAM,CAAA;IAChB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAA;IACxB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAA;IAC1B,iBAAiB,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE,SAAS,gBAAgB,EAAE,CAAC,CAAA;CACrE,GAAG,WAAW,EAAE,CAwChB"}
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
// Shared chunk + embed computation for the ingest pipeline.
|
|
2
|
+
//
|
|
3
|
+
// Lifts the duplicated `chunker.chunkText -> embedder.embedBatch` sequence
|
|
4
|
+
// out of `handleIngestFile` and `ingestSingleFile` into this single shared
|
|
5
|
+
// function. Persistence (delete + insert + rollback + optimize) stays in each
|
|
6
|
+
// caller because the rollback semantics differ between the MCP path and the
|
|
7
|
+
// CLI path.
|
|
8
|
+
//
|
|
9
|
+
// The function is dispatch-agnostic: it takes already-extracted `text`
|
|
10
|
+
// and `title` and does not touch `vectorStore`. It is the single
|
|
11
|
+
// chunker call site for any ingest path.
|
|
12
|
+
import { createHash, randomUUID } from 'node:crypto';
|
|
13
|
+
import { basename, extname } from 'node:path';
|
|
14
|
+
import { MAX_VISUAL_RENDITION_BYTES } from '../utils/limits.js';
|
|
15
|
+
export function findNearestChunk(chunks, sourceOffset) {
|
|
16
|
+
let owner;
|
|
17
|
+
let bestDistance = Number.POSITIVE_INFINITY;
|
|
18
|
+
for (const chunk of chunks) {
|
|
19
|
+
const distance = sourceOffset < chunk.sourceStart
|
|
20
|
+
? chunk.sourceStart - sourceOffset
|
|
21
|
+
: sourceOffset > chunk.sourceEnd
|
|
22
|
+
? sourceOffset - chunk.sourceEnd
|
|
23
|
+
: 0;
|
|
24
|
+
if (!owner) {
|
|
25
|
+
owner = chunk;
|
|
26
|
+
bestDistance = distance;
|
|
27
|
+
continue;
|
|
28
|
+
}
|
|
29
|
+
const isPreceding = chunk.sourceEnd <= sourceOffset;
|
|
30
|
+
const ownerIsPreceding = owner.sourceEnd <= sourceOffset;
|
|
31
|
+
if (distance < bestDistance ||
|
|
32
|
+
(distance === bestDistance && isPreceding && !ownerIsPreceding) ||
|
|
33
|
+
(distance === bestDistance && isPreceding === ownerIsPreceding && chunk.index < owner.index)) {
|
|
34
|
+
owner = chunk;
|
|
35
|
+
bestDistance = distance;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
return owner;
|
|
39
|
+
}
|
|
40
|
+
export function createVisualAttachment(imageIndex, rendition) {
|
|
41
|
+
if (!Number.isInteger(imageIndex) ||
|
|
42
|
+
imageIndex < 0 ||
|
|
43
|
+
rendition.bytes.byteLength === 0 ||
|
|
44
|
+
rendition.bytes.byteLength > MAX_VISUAL_RENDITION_BYTES) {
|
|
45
|
+
throw new Error('Invalid bounded image rendition');
|
|
46
|
+
}
|
|
47
|
+
return {
|
|
48
|
+
imageIndex,
|
|
49
|
+
mimeType: rendition.mimeType,
|
|
50
|
+
data: Buffer.from(rendition.bytes).toString('base64'),
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Compute semantic chunks and their embeddings for already-extracted text.
|
|
55
|
+
*
|
|
56
|
+
* Calls `chunker.chunkText` exactly once and then
|
|
57
|
+
* `embedder.embedBatch` on the resulting chunk texts. Does NOT touch
|
|
58
|
+
* `vectorStore`. Does NOT fail-fast on zero chunks — callers decide
|
|
59
|
+
* how to handle an empty result (the MCP handler throws `McpError`;
|
|
60
|
+
* the CLI logs a warning and returns 0).
|
|
61
|
+
*
|
|
62
|
+
* Errors from the chunker or embedder propagate verbatim.
|
|
63
|
+
*
|
|
64
|
+
* @param text Already-extracted document text (parser output, raw-data
|
|
65
|
+
* payload, or joined visual-enriched per-page text).
|
|
66
|
+
* @param chunker Semantic chunker instance (owned by the caller).
|
|
67
|
+
* @param embedder Embedder implementing the structural `EmbedderInterface`
|
|
68
|
+
* (only `embedBatch` is required).
|
|
69
|
+
*/
|
|
70
|
+
export async function buildChunksAndEmbeddings(text, chunker, embedder, atomicRanges) {
|
|
71
|
+
const chunks = await chunker.chunkText(text, embedder, atomicRanges);
|
|
72
|
+
// F5: Skip `embedBatch` entirely on zero chunks. `embedBatch` runs
|
|
73
|
+
// `ensureInitialized()` (which triggers the ~90MB MiniLM download on a
|
|
74
|
+
// cold cache) BEFORE checking for the empty-array short-circuit, so an
|
|
75
|
+
// empty file would otherwise pay the model-load cost for no work.
|
|
76
|
+
if (chunks.length === 0) {
|
|
77
|
+
return { chunks: [], embeddings: [] };
|
|
78
|
+
}
|
|
79
|
+
const embeddings = await embedder.embedBatch(chunks.map((chunk) => chunk.text));
|
|
80
|
+
return { chunks, embeddings };
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Preserve the parser content/range mapping at one shared boundary. Display
|
|
84
|
+
* title handling stays in each dispatch root because it does not affect chunks.
|
|
85
|
+
*/
|
|
86
|
+
export async function buildChunksFromParseResult(result, chunker, embedder) {
|
|
87
|
+
const computed = await buildChunksAndEmbeddings(result.content, chunker, embedder, result.atomicRanges);
|
|
88
|
+
const visualAttachments = new Map();
|
|
89
|
+
if (!result.imageAnchors?.length || computed.chunks.length === 0) {
|
|
90
|
+
return { ...computed, visualAttachments, omittedImageCount: 0 };
|
|
91
|
+
}
|
|
92
|
+
const { renderImageRendition } = await import('../pdf-visual/renderer.js');
|
|
93
|
+
let omittedCount = 0;
|
|
94
|
+
for (const anchor of [...result.imageAnchors].sort((left, right) => left.imageIndex - right.imageIndex)) {
|
|
95
|
+
const owner = findNearestChunk(computed.chunks, anchor.offset);
|
|
96
|
+
if (!owner)
|
|
97
|
+
throw new Error(`Image ${anchor.imageIndex} has no owning chunk`);
|
|
98
|
+
try {
|
|
99
|
+
const rendition = renderImageRendition(anchor.bytes, anchor.mimeType);
|
|
100
|
+
const attachment = createVisualAttachment(anchor.imageIndex, rendition);
|
|
101
|
+
const owned = visualAttachments.get(owner.index) ?? [];
|
|
102
|
+
owned.push(attachment);
|
|
103
|
+
visualAttachments.set(owner.index, owned);
|
|
104
|
+
}
|
|
105
|
+
catch {
|
|
106
|
+
omittedCount += 1;
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
return { ...computed, visualAttachments, omittedImageCount: omittedCount };
|
|
110
|
+
}
|
|
111
|
+
/**
|
|
112
|
+
* Content identity of a source file: the lowercase SHA-256 hex digest of its
|
|
113
|
+
* raw bytes.
|
|
114
|
+
*
|
|
115
|
+
* Hashing the bytes (not the parsed or normalized text) keeps the value
|
|
116
|
+
* reproducible by any caller that can read the file, which is what lets a later
|
|
117
|
+
* sync pass decide "unchanged" without re-parsing. Pure: the caller reads the
|
|
118
|
+
* file and passes the bytes in.
|
|
119
|
+
*/
|
|
120
|
+
export function computeContentHash(bytes) {
|
|
121
|
+
return createHash('sha256').update(bytes).digest('hex');
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* Build persistable `VectorChunk`s from computed chunks + embeddings.
|
|
125
|
+
*
|
|
126
|
+
* Single source of truth for the chunk→VectorChunk mapping shared by the MCP
|
|
127
|
+
* ingest handler (`handleIngestFile`) and both CLI ingest paths (default +
|
|
128
|
+
* visual). Assigns one shared `timestamp` to every chunk, a fresh `id`, and
|
|
129
|
+
* derives `fileName`/`fileType` from `filePath` via `node:path` (cross-platform).
|
|
130
|
+
* Does NOT touch `vectorStore` — persistence stays in each caller.
|
|
131
|
+
*
|
|
132
|
+
* Throws when a chunk has no corresponding embedding (index mismatch);
|
|
133
|
+
* `embeddings` must align 1:1 with `chunks`.
|
|
134
|
+
*
|
|
135
|
+
* @param fileSize Length value recorded in `metadata.fileSize`. The caller
|
|
136
|
+
* chooses the source: the default path passes parsed text length; the visual
|
|
137
|
+
* path passes the joined enriched-page text length (pre-chunking).
|
|
138
|
+
* @param contentHash {@link computeContentHash} of the source file bytes,
|
|
139
|
+
* shared by every chunk of that file exactly like `timestamp`. `null` for a
|
|
140
|
+
* chunk set with no source file; the key is then omitted rather than stored
|
|
141
|
+
* empty, so a hashless row is never mistaken for a hash of nothing. Required
|
|
142
|
+
* (not optional) so a new call site cannot silently write hashless rows.
|
|
143
|
+
*/
|
|
144
|
+
export function buildVectorChunks(params) {
|
|
145
|
+
const { filePath, chunks, embeddings, fileSize, fileTitle, contentHash, visualAttachments = new Map(), } = params;
|
|
146
|
+
const timestamp = new Date().toISOString();
|
|
147
|
+
return chunks.map((chunk, index) => {
|
|
148
|
+
const embedding = embeddings[index];
|
|
149
|
+
if (!embedding) {
|
|
150
|
+
throw new Error(`Missing embedding for chunk ${index}`);
|
|
151
|
+
}
|
|
152
|
+
const attachments = [...(visualAttachments.get(chunk.index) ?? [])]
|
|
153
|
+
.sort((left, right) => left.imageIndex - right.imageIndex)
|
|
154
|
+
.map(({ imageIndex, mimeType, data }) => ({
|
|
155
|
+
imageIndex,
|
|
156
|
+
mimeType,
|
|
157
|
+
data,
|
|
158
|
+
}));
|
|
159
|
+
return {
|
|
160
|
+
id: randomUUID(),
|
|
161
|
+
filePath,
|
|
162
|
+
chunkIndex: chunk.index,
|
|
163
|
+
text: chunk.text,
|
|
164
|
+
vector: embedding,
|
|
165
|
+
metadata: {
|
|
166
|
+
fileName: basename(filePath),
|
|
167
|
+
fileSize,
|
|
168
|
+
fileType: extname(filePath).slice(1),
|
|
169
|
+
},
|
|
170
|
+
fileTitle,
|
|
171
|
+
...(contentHash === null ? {} : { contentHash }),
|
|
172
|
+
visualAttachments: JSON.stringify(attachments),
|
|
173
|
+
timestamp,
|
|
174
|
+
};
|
|
175
|
+
});
|
|
176
|
+
}
|
|
177
|
+
//# sourceMappingURL=compute.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"compute.js","sourceRoot":"","sources":["../../src/ingest/compute.ts"],"names":[],"mappings":"AAAA,4DAA4D;AAC5D,EAAE;AACF,2EAA2E;AAC3E,2EAA2E;AAC3E,8EAA8E;AAC9E,4EAA4E;AAC5E,YAAY;AACZ,EAAE;AACF,uEAAuE;AACvE,iEAAiE;AACjE,yCAAyC;AAEzC,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACpD,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AAK7C,OAAO,EAAE,0BAA0B,EAAE,MAAM,oBAAoB,CAAA;AAoB/D,MAAM,UAAU,gBAAgB,CAC9B,MAA4B,EAC5B,YAAoB;IAEpB,IAAI,KAA4B,CAAA;IAChC,IAAI,YAAY,GAAG,MAAM,CAAC,iBAAiB,CAAA;IAC3C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,QAAQ,GACZ,YAAY,GAAG,KAAK,CAAC,WAAW;YAC9B,CAAC,CAAC,KAAK,CAAC,WAAW,GAAG,YAAY;YAClC,CAAC,CAAC,YAAY,GAAG,KAAK,CAAC,SAAS;gBAC9B,CAAC,CAAC,YAAY,GAAG,KAAK,CAAC,SAAS;gBAChC,CAAC,CAAC,CAAC,CAAA;QACT,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,KAAK,GAAG,KAAK,CAAA;YACb,YAAY,GAAG,QAAQ,CAAA;YACvB,SAAQ;QACV,CAAC;QACD,MAAM,WAAW,GAAG,KAAK,CAAC,SAAS,IAAI,YAAY,CAAA;QACnD,MAAM,gBAAgB,GAAG,KAAK,CAAC,SAAS,IAAI,YAAY,CAAA;QACxD,IACE,QAAQ,GAAG,YAAY;YACvB,CAAC,QAAQ,KAAK,YAAY,IAAI,WAAW,IAAI,CAAC,gBAAgB,CAAC;YAC/D,CAAC,QAAQ,KAAK,YAAY,IAAI,WAAW,KAAK,gBAAgB,IAAI,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,EAC5F,CAAC;YACD,KAAK,GAAG,KAAK,CAAA;YACb,YAAY,GAAG,QAAQ,CAAA;QACzB,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,MAAM,UAAU,sBAAsB,CACpC,UAAkB,EAClB,SAAyB;IAEzB,IACE,CAAC,MAAM,CAAC,SAAS,CAAC,UAAU,CAAC;QAC7B,UAAU,GAAG,CAAC;QACd,SAAS,CAAC,KAAK,CAAC,UAAU,KAAK,CAAC;QAChC,SAAS,CAAC,KAAK,CAAC,UAAU,GAAG,0BAA0B,EACvD,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAA;IACpD,CAAC;IACD,OAAO;QACL,UAAU;QACV,QAAQ,EAAE,SAAS,CAAC,QAAQ;QAC5B,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC;KACtD,CAAA;AACH,CAAC;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC5C,IAAY,EACZ,OAAwB,EACxB,QAA2B,EAC3B,YAAyC;IAEzC,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,IAAI,EAAE,QAAQ,EAAE,YAAY,CAAC,CAAA;IACpE,mEAAmE;IACnE,uEAAuE;IACvE,uEAAuE;IACvE,kEAAkE;IAClE,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE,CAAA;IACvC,CAAC;IACD,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAA;IAC/E,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,CAAA;AAC/B,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAC9C,MAAmB,EACnB,OAAwB,EACxB,QAA2B;IAE3B,MAAM,QAAQ,GAAG,MAAM,wBAAwB,CAC7C,MAAM,CAAC,OAAO,EACd,OAAO,EACP,QAAQ,EACR,MAAM,CAAC,YAAY,CACpB,CAAA;IACD,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAA8B,CAAA;IAC/D,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,MAAM,IAAI,QAAQ,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACjE,OAAO,EAAE,GAAG,QAAQ,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,CAAC,EAAE,CAAA;IACjE,CAAC;IAED,MAAM,EAAE,oBAAoB,EAAE,GAAG,MAAM,MAAM,CAAC,2BAA2B,CAAC,CAAA;IAC1E,IAAI,YAAY,GAAG,CAAC,CAAA;IACpB,KAAK,MAAM,MAAM,IAAI,CAAC,GAAG,MAAM,CAAC,YAAY,CAAC,CAAC,IAAI,CAChD,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC,UAAU,CACpD,EAAE,CAAC;QACF,MAAM,KAAK,GAAG,gBAAgB,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAA;QAC9D,IAAI,CAAC,KAAK;YAAE,MAAM,IAAI,KAAK,CAAC,SAAS,MAAM,CAAC,UAAU,sBAAsB,CAAC,CAAA;QAE7E,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,oBAAoB,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAA;YACrE,MAAM,UAAU,GAAG,sBAAsB,CAAC,MAAM,CAAC,UAAU,EAAE,SAAS,CAAC,CAAA;YACvE,MAAM,KAAK,GAAG,iBAAiB,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,CAAA;YACtD,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;YACtB,iBAAiB,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,CAAA;QAC3C,CAAC;QAAC,MAAM,CAAC;YACP,YAAY,IAAI,CAAC,CAAA;QACnB,CAAC;IACH,CAAC;IACD,OAAO,EAAE,GAAG,QAAQ,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,YAAY,EAAE,CAAA;AAC5E,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,kBAAkB,CAAC,KAAiB;IAClD,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;AACzD,CAAC;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,UAAU,iBAAiB,CAAC,MAQjC;IACC,MAAM,EACJ,QAAQ,EACR,MAAM,EACN,UAAU,EACV,QAAQ,EACR,SAAS,EACT,WAAW,EACX,iBAAiB,GAAG,IAAI,GAAG,EAAE,GAC9B,GAAG,MAAM,CAAA;IACV,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;IAC1C,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;QACjC,MAAM,SAAS,GAAG,UAAU,CAAC,KAAK,CAAC,CAAA;QACnC,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,+BAA+B,KAAK,EAAE,CAAC,CAAA;QACzD,CAAC;QACD,MAAM,WAAW,GAAG,CAAC,GAAG,CAAC,iBAAiB,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;aAChE,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC,UAAU,CAAC;aACzD,GAAG,CAAC,CAAC,EAAE,UAAU,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;YACxC,UAAU;YACV,QAAQ;YACR,IAAI;SACL,CAAC,CAAC,CAAA;QACL,OAAO;YACL,EAAE,EAAE,UAAU,EAAE;YAChB,QAAQ;YACR,UAAU,EAAE,KAAK,CAAC,KAAK;YACvB,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,MAAM,EAAE,SAAS;YACjB,QAAQ,EAAE;gBACR,QAAQ,EAAE,QAAQ,CAAC,QAAQ,CAAC;gBAC5B,QAAQ;gBACR,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;aACrC;YACD,SAAS;YACT,GAAG,CAAC,WAAW,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC;YAChD,iBAAiB,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC;YAC9C,SAAS;SACV,CAAA;IACH,CAAC,CAAC,CAAA;AACJ,CAAC"}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import type { SemanticChunker, TextChunk } from '../chunker/index.js';
|
|
2
|
+
import type { EmbedderInterface } from '../chunker/semantic-chunker.js';
|
|
3
|
+
import type { DocumentParser } from '../parser/index.js';
|
|
4
|
+
import type { VectorChunk, VisualAttachment } from '../vectordb/index.js';
|
|
5
|
+
import { type CaptionerConfig } from './visual.js';
|
|
6
|
+
export interface PrepareFileForIngestOptions {
|
|
7
|
+
images: boolean;
|
|
8
|
+
captioner?: CaptionerConfig;
|
|
9
|
+
}
|
|
10
|
+
export interface PreparedFileIngest {
|
|
11
|
+
filePath: string;
|
|
12
|
+
chunks: TextChunk[];
|
|
13
|
+
embeddings: number[][];
|
|
14
|
+
textLength: number;
|
|
15
|
+
title: string | null;
|
|
16
|
+
contentHash: string;
|
|
17
|
+
visualAttachments: Map<number, VisualAttachment[]>;
|
|
18
|
+
omittedImageCount: number;
|
|
19
|
+
}
|
|
20
|
+
export declare function buildPreparedFileVectorChunks(prepared: PreparedFileIngest): VectorChunk[];
|
|
21
|
+
/**
|
|
22
|
+
* Prepare one validated source file for persistence without touching storage or
|
|
23
|
+
* process output. The source hash is read before parsing so a concurrent rewrite
|
|
24
|
+
* remains visible to the next sync instead of being paired with stale chunks.
|
|
25
|
+
*/
|
|
26
|
+
export declare function prepareFileForIngest(filePath: string, parser: DocumentParser, chunker: SemanticChunker, embedder: EmbedderInterface, options: PrepareFileForIngestOptions): Promise<PreparedFileIngest>;
|
|
27
|
+
//# sourceMappingURL=file.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"file.d.ts","sourceRoot":"","sources":["../../src/ingest/file.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAA;AACrE,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,gCAAgC,CAAA;AACvE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAA;AACxD,OAAO,KAAK,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAA;AAEzE,OAAO,EACL,KAAK,eAAe,EAGrB,MAAM,aAAa,CAAA;AAEpB,MAAM,WAAW,2BAA2B;IAC1C,MAAM,EAAE,OAAO,CAAA;IACf,SAAS,CAAC,EAAE,eAAe,CAAA;CAC5B;AAED,MAAM,WAAW,kBAAkB;IACjC,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,SAAS,EAAE,CAAA;IACnB,UAAU,EAAE,MAAM,EAAE,EAAE,CAAA;IACtB,UAAU,EAAE,MAAM,CAAA;IAClB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;IACpB,WAAW,EAAE,MAAM,CAAA;IACnB,iBAAiB,EAAE,GAAG,CAAC,MAAM,EAAE,gBAAgB,EAAE,CAAC,CAAA;IAClD,iBAAiB,EAAE,MAAM,CAAA;CAC1B;AAED,wBAAgB,6BAA6B,CAAC,QAAQ,EAAE,kBAAkB,GAAG,WAAW,EAAE,CAUzF;AAQD;;;;GAIG;AACH,wBAAsB,oBAAoB,CACxC,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,cAAc,EACtB,OAAO,EAAE,eAAe,EACxB,QAAQ,EAAE,iBAAiB,EAC3B,OAAO,EAAE,2BAA2B,GACnC,OAAO,CAAC,kBAAkB,CAAC,CA2C7B"}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import { readFile } from 'node:fs/promises';
|
|
2
|
+
import { buildChunksFromParseResult, buildVectorChunks, computeContentHash } from './compute.js';
|
|
3
|
+
import { prepareVisualPdfChunks, } from './visual.js';
|
|
4
|
+
export function buildPreparedFileVectorChunks(prepared) {
|
|
5
|
+
return buildVectorChunks({
|
|
6
|
+
filePath: prepared.filePath,
|
|
7
|
+
chunks: prepared.chunks,
|
|
8
|
+
embeddings: prepared.embeddings,
|
|
9
|
+
fileSize: prepared.textLength,
|
|
10
|
+
fileTitle: prepared.title,
|
|
11
|
+
contentHash: prepared.contentHash,
|
|
12
|
+
visualAttachments: prepared.visualAttachments,
|
|
13
|
+
});
|
|
14
|
+
}
|
|
15
|
+
async function readPreParseContentHash(filePath, parser) {
|
|
16
|
+
await parser.validateFilePath(filePath);
|
|
17
|
+
parser.validateFileSize(filePath);
|
|
18
|
+
return computeContentHash(await readFile(filePath));
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Prepare one validated source file for persistence without touching storage or
|
|
22
|
+
* process output. The source hash is read before parsing so a concurrent rewrite
|
|
23
|
+
* remains visible to the next sync instead of being paired with stale chunks.
|
|
24
|
+
*/
|
|
25
|
+
export async function prepareFileForIngest(filePath, parser, chunker, embedder, options) {
|
|
26
|
+
const contentHash = await readPreParseContentHash(filePath, parser);
|
|
27
|
+
const isPdf = filePath.toLowerCase().endsWith('.pdf');
|
|
28
|
+
let text;
|
|
29
|
+
let title;
|
|
30
|
+
let chunks;
|
|
31
|
+
let embeddings;
|
|
32
|
+
let visualAttachments;
|
|
33
|
+
let omittedImageCount;
|
|
34
|
+
if (isPdf && (options.captioner !== undefined || options.images)) {
|
|
35
|
+
const visualOptions = {
|
|
36
|
+
images: options.images,
|
|
37
|
+
...(options.captioner === undefined ? {} : { captioner: options.captioner }),
|
|
38
|
+
};
|
|
39
|
+
const result = await prepareVisualPdfChunks(filePath, parser, chunker, embedder, visualOptions);
|
|
40
|
+
text = result.text;
|
|
41
|
+
title = result.title;
|
|
42
|
+
chunks = result.chunks;
|
|
43
|
+
embeddings = result.embeddings;
|
|
44
|
+
visualAttachments = result.visualAttachments;
|
|
45
|
+
omittedImageCount = result.omittedImageCount;
|
|
46
|
+
}
|
|
47
|
+
else {
|
|
48
|
+
const result = isPdf
|
|
49
|
+
? await parser.parsePdf(filePath, embedder)
|
|
50
|
+
: await parser.parseFile(filePath, { images: options.images });
|
|
51
|
+
text = result.content;
|
|
52
|
+
title = result.title || null;
|
|
53
|
+
({ chunks, embeddings, visualAttachments, omittedImageCount } =
|
|
54
|
+
await buildChunksFromParseResult(result, chunker, embedder));
|
|
55
|
+
}
|
|
56
|
+
return {
|
|
57
|
+
filePath,
|
|
58
|
+
chunks,
|
|
59
|
+
embeddings,
|
|
60
|
+
textLength: text.length,
|
|
61
|
+
title,
|
|
62
|
+
contentHash,
|
|
63
|
+
visualAttachments,
|
|
64
|
+
omittedImageCount,
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
//# sourceMappingURL=file.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"file.js","sourceRoot":"","sources":["../../src/ingest/file.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAA;AAM3C,OAAO,EAAE,0BAA0B,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAA;AAChG,OAAO,EAGL,sBAAsB,GACvB,MAAM,aAAa,CAAA;AAkBpB,MAAM,UAAU,6BAA6B,CAAC,QAA4B;IACxE,OAAO,iBAAiB,CAAC;QACvB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;QAC3B,MAAM,EAAE,QAAQ,CAAC,MAAM;QACvB,UAAU,EAAE,QAAQ,CAAC,UAAU;QAC/B,QAAQ,EAAE,QAAQ,CAAC,UAAU;QAC7B,SAAS,EAAE,QAAQ,CAAC,KAAK;QACzB,WAAW,EAAE,QAAQ,CAAC,WAAW;QACjC,iBAAiB,EAAE,QAAQ,CAAC,iBAAiB;KAC9C,CAAC,CAAA;AACJ,CAAC;AAED,KAAK,UAAU,uBAAuB,CAAC,QAAgB,EAAE,MAAsB;IAC7E,MAAM,MAAM,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAA;IACvC,MAAM,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAA;IACjC,OAAO,kBAAkB,CAAC,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAA;AACrD,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,QAAgB,EAChB,MAAsB,EACtB,OAAwB,EACxB,QAA2B,EAC3B,OAAoC;IAEpC,MAAM,WAAW,GAAG,MAAM,uBAAuB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;IACnE,MAAM,KAAK,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAA;IAErD,IAAI,IAAY,CAAA;IAChB,IAAI,KAAoB,CAAA;IACxB,IAAI,MAAwE,CAAA;IAC5E,IAAI,UAAgF,CAAA;IACpF,IAAI,iBAA8F,CAAA;IAClG,IAAI,iBAAyB,CAAA;IAE7B,IAAI,KAAK,IAAI,CAAC,OAAO,CAAC,SAAS,KAAK,SAAS,IAAI,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QACjE,MAAM,aAAa,GAAkC;YACnD,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,GAAG,CAAC,OAAO,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC;SAC7E,CAAA;QACD,MAAM,MAAM,GAAG,MAAM,sBAAsB,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,aAAa,CAAC,CAAA;QAC/F,IAAI,GAAG,MAAM,CAAC,IAAI,CAAA;QAClB,KAAK,GAAG,MAAM,CAAC,KAAK,CAAA;QACpB,MAAM,GAAG,MAAM,CAAC,MAAM,CAAA;QACtB,UAAU,GAAG,MAAM,CAAC,UAAU,CAAA;QAC9B,iBAAiB,GAAG,MAAM,CAAC,iBAAiB,CAAA;QAC5C,iBAAiB,GAAG,MAAM,CAAC,iBAAiB,CAAA;IAC9C,CAAC;SAAM,CAAC;QACN,MAAM,MAAM,GAAG,KAAK;YAClB,CAAC,CAAC,MAAM,MAAM,CAAC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC;YAC3C,CAAC,CAAC,MAAM,MAAM,CAAC,SAAS,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;QAChE,IAAI,GAAG,MAAM,CAAC,OAAO,CAAA;QACrB,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,IAAI,CAC3B;QAAA,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,iBAAiB,EAAE,iBAAiB,EAAE;YAC5D,MAAM,0BAA0B,CAAC,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAA;IAChE,CAAC;IAED,OAAO;QACL,QAAQ;QACR,MAAM;QACN,UAAU;QACV,UAAU,EAAE,IAAI,CAAC,MAAM;QACvB,KAAK;QACL,WAAW;QACX,iBAAiB;QACjB,iBAAiB;KAClB,CAAA;AACH,CAAC"}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import type { AtomicTextRange, SemanticChunker, TextChunk } from '../chunker/index.js';
|
|
2
|
+
import type { EmbedderInterface } from '../chunker/semantic-chunker.js';
|
|
3
|
+
import type { DocumentParser } from '../parser/index.js';
|
|
4
|
+
import type { FilteredTextFragment } from '../parser/pdf-filter.js';
|
|
5
|
+
import type { ProcessedVisualRegion, QualityProfile } from '../pdf-visual/types.js';
|
|
6
|
+
import type { VisualAttachment } from '../vectordb/types.js';
|
|
7
|
+
export interface VisualPdfParser {
|
|
8
|
+
parsePdfPages: DocumentParser['parsePdfPages'];
|
|
9
|
+
}
|
|
10
|
+
export interface CaptionerConfig {
|
|
11
|
+
profile: QualityProfile;
|
|
12
|
+
cacheDir: string;
|
|
13
|
+
device?: string | undefined;
|
|
14
|
+
}
|
|
15
|
+
export interface PrepareVisualPdfChunksOptions {
|
|
16
|
+
images: boolean;
|
|
17
|
+
captioner?: CaptionerConfig;
|
|
18
|
+
}
|
|
19
|
+
export interface PrepareVisualPdfChunksResult {
|
|
20
|
+
chunks: TextChunk[];
|
|
21
|
+
embeddings: number[][];
|
|
22
|
+
title: string | null;
|
|
23
|
+
text: string;
|
|
24
|
+
atomicRanges: AtomicTextRange[];
|
|
25
|
+
visualAttachments: Map<number, VisualAttachment[]>;
|
|
26
|
+
omittedImageCount: number;
|
|
27
|
+
}
|
|
28
|
+
export interface OrderedVisualPage {
|
|
29
|
+
pageNum: number;
|
|
30
|
+
text: string;
|
|
31
|
+
textFragments: readonly FilteredTextFragment[];
|
|
32
|
+
}
|
|
33
|
+
export interface OrderedVisualRegion extends ProcessedVisualRegion {
|
|
34
|
+
visualIndex: number;
|
|
35
|
+
anchorOffset: number;
|
|
36
|
+
captionRange?: AtomicTextRange;
|
|
37
|
+
}
|
|
38
|
+
export interface OrderedVisualDocument {
|
|
39
|
+
text: string;
|
|
40
|
+
atomicRanges: AtomicTextRange[];
|
|
41
|
+
regions: OrderedVisualRegion[];
|
|
42
|
+
}
|
|
43
|
+
export declare function buildOrderedVisualDocument(pages: readonly OrderedVisualPage[], processedRegions: readonly ProcessedVisualRegion[]): OrderedVisualDocument;
|
|
44
|
+
export declare function prepareVisualPdfChunks(filePath: string, parser: VisualPdfParser, chunker: SemanticChunker, embedder: EmbedderInterface, options: PrepareVisualPdfChunksOptions): Promise<PrepareVisualPdfChunksResult>;
|
|
45
|
+
//# sourceMappingURL=visual.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"visual.d.ts","sourceRoot":"","sources":["../../src/ingest/visual.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAA;AACtF,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,gCAAgC,CAAA;AACvE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAA;AACxD,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAA;AACnE,OAAO,KAAK,EAEV,qBAAqB,EACrB,cAAc,EACf,MAAM,wBAAwB,CAAA;AAC/B,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAA;AAG5D,MAAM,WAAW,eAAe;IAC9B,aAAa,EAAE,cAAc,CAAC,eAAe,CAAC,CAAA;CAC/C;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,cAAc,CAAA;IACvB,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;CAC5B;AAED,MAAM,WAAW,6BAA6B;IAC5C,MAAM,EAAE,OAAO,CAAA;IACf,SAAS,CAAC,EAAE,eAAe,CAAA;CAC5B;AAED,MAAM,WAAW,4BAA4B;IAC3C,MAAM,EAAE,SAAS,EAAE,CAAA;IACnB,UAAU,EAAE,MAAM,EAAE,EAAE,CAAA;IACtB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;IACpB,IAAI,EAAE,MAAM,CAAA;IACZ,YAAY,EAAE,eAAe,EAAE,CAAA;IAC/B,iBAAiB,EAAE,GAAG,CAAC,MAAM,EAAE,gBAAgB,EAAE,CAAC,CAAA;IAClD,iBAAiB,EAAE,MAAM,CAAA;CAC1B;AAED,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,MAAM,CAAA;IACf,IAAI,EAAE,MAAM,CAAA;IACZ,aAAa,EAAE,SAAS,oBAAoB,EAAE,CAAA;CAC/C;AAED,MAAM,WAAW,mBAAoB,SAAQ,qBAAqB;IAChE,WAAW,EAAE,MAAM,CAAA;IACnB,YAAY,EAAE,MAAM,CAAA;IACpB,YAAY,CAAC,EAAE,eAAe,CAAA;CAC/B;AAED,MAAM,WAAW,qBAAqB;IACpC,IAAI,EAAE,MAAM,CAAA;IACZ,YAAY,EAAE,eAAe,EAAE,CAAA;IAC/B,OAAO,EAAE,mBAAmB,EAAE,CAAA;CAC/B;AAyDD,wBAAgB,0BAA0B,CACxC,KAAK,EAAE,SAAS,iBAAiB,EAAE,EACnC,gBAAgB,EAAE,SAAS,qBAAqB,EAAE,GACjD,qBAAqB,CA4GvB;AA6DD,wBAAsB,sBAAsB,CAC1C,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,eAAe,EACvB,OAAO,EAAE,eAAe,EACxB,QAAQ,EAAE,iBAAiB,EAC3B,OAAO,EAAE,6BAA6B,GACrC,OAAO,CAAC,4BAA4B,CAAC,CAoEvC"}
|