abb-opencode-local-rag 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.de.md +416 -0
- package/README.es.md +416 -0
- package/README.fr.md +416 -0
- package/README.md +491 -0
- package/README.pt-BR.md +416 -0
- package/README.zh-CN.md +416 -0
- package/dist/bin/install-skills.d.ts +20 -0
- package/dist/bin/install-skills.d.ts.map +1 -0
- package/dist/bin/install-skills.js +195 -0
- package/dist/bin/install-skills.js.map +1 -0
- package/dist/chunker/index.d.ts +24 -0
- package/dist/chunker/index.d.ts.map +1 -0
- package/dist/chunker/index.js +2 -0
- package/dist/chunker/index.js.map +1 -0
- package/dist/chunker/semantic-chunker.d.ts +97 -0
- package/dist/chunker/semantic-chunker.d.ts.map +1 -0
- package/dist/chunker/semantic-chunker.js +294 -0
- package/dist/chunker/semantic-chunker.js.map +1 -0
- package/dist/chunker/sentence-splitter.d.ts +28 -0
- package/dist/chunker/sentence-splitter.d.ts.map +1 -0
- package/dist/chunker/sentence-splitter.js +219 -0
- package/dist/chunker/sentence-splitter.js.map +1 -0
- package/dist/cli/common.d.ts +65 -0
- package/dist/cli/common.d.ts.map +1 -0
- package/dist/cli/common.js +138 -0
- package/dist/cli/common.js.map +1 -0
- package/dist/cli/delete.d.ts +8 -0
- package/dist/cli/delete.d.ts.map +1 -0
- package/dist/cli/delete.js +173 -0
- package/dist/cli/delete.js.map +1 -0
- package/dist/cli/file-collection.d.ts +2 -0
- package/dist/cli/file-collection.d.ts.map +1 -0
- package/dist/cli/file-collection.js +53 -0
- package/dist/cli/file-collection.js.map +1 -0
- package/dist/cli/ingest.d.ts +100 -0
- package/dist/cli/ingest.d.ts.map +1 -0
- package/dist/cli/ingest.js +363 -0
- package/dist/cli/ingest.js.map +1 -0
- package/dist/cli/list.d.ts +35 -0
- package/dist/cli/list.d.ts.map +1 -0
- package/dist/cli/list.js +210 -0
- package/dist/cli/list.js.map +1 -0
- package/dist/cli/options.d.ts +100 -0
- package/dist/cli/options.d.ts.map +1 -0
- package/dist/cli/options.js +241 -0
- package/dist/cli/options.js.map +1 -0
- package/dist/cli/query.d.ts +24 -0
- package/dist/cli/query.d.ts.map +1 -0
- package/dist/cli/query.js +191 -0
- package/dist/cli/query.js.map +1 -0
- package/dist/cli/read-neighbors.d.ts +11 -0
- package/dist/cli/read-neighbors.d.ts.map +1 -0
- package/dist/cli/read-neighbors.js +224 -0
- package/dist/cli/read-neighbors.js.map +1 -0
- package/dist/cli/status.d.ts +8 -0
- package/dist/cli/status.d.ts.map +1 -0
- package/dist/cli/status.js +80 -0
- package/dist/cli/status.js.map +1 -0
- package/dist/cli/sync.d.ts +8 -0
- package/dist/cli/sync.d.ts.map +1 -0
- package/dist/cli/sync.js +244 -0
- package/dist/cli/sync.js.map +1 -0
- package/dist/cli-main.d.ts +12 -0
- package/dist/cli-main.d.ts.map +1 -0
- package/dist/cli-main.js +63 -0
- package/dist/cli-main.js.map +1 -0
- package/dist/embedder/index.d.ts +85 -0
- package/dist/embedder/index.d.ts.map +1 -0
- package/dist/embedder/index.js +284 -0
- package/dist/embedder/index.js.map +1 -0
- package/dist/features/list.d.ts +37 -0
- package/dist/features/list.d.ts.map +1 -0
- package/dist/features/list.js +40 -0
- package/dist/features/list.js.map +1 -0
- package/dist/features/sync.d.ts +207 -0
- package/dist/features/sync.d.ts.map +1 -0
- package/dist/features/sync.js +380 -0
- package/dist/features/sync.js.map +1 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +53 -0
- package/dist/index.js.map +1 -0
- package/dist/ingest/compute.d.ts +86 -0
- package/dist/ingest/compute.d.ts.map +1 -0
- package/dist/ingest/compute.js +177 -0
- package/dist/ingest/compute.js.map +1 -0
- package/dist/ingest/file.d.ts +27 -0
- package/dist/ingest/file.d.ts.map +1 -0
- package/dist/ingest/file.js +67 -0
- package/dist/ingest/file.js.map +1 -0
- package/dist/ingest/visual.d.ts +45 -0
- package/dist/ingest/visual.d.ts.map +1 -0
- package/dist/ingest/visual.js +234 -0
- package/dist/ingest/visual.js.map +1 -0
- package/dist/parser/docx-parser.d.ts +12 -0
- package/dist/parser/docx-parser.d.ts.map +1 -0
- package/dist/parser/docx-parser.js +328 -0
- package/dist/parser/docx-parser.js.map +1 -0
- package/dist/parser/html-parser.d.ts +18 -0
- package/dist/parser/html-parser.d.ts.map +1 -0
- package/dist/parser/html-parser.js +102 -0
- package/dist/parser/html-parser.js.map +1 -0
- package/dist/parser/index.d.ts +214 -0
- package/dist/parser/index.d.ts.map +1 -0
- package/dist/parser/index.js +454 -0
- package/dist/parser/index.js.map +1 -0
- package/dist/parser/pdf-extract.d.ts +81 -0
- package/dist/parser/pdf-extract.d.ts.map +1 -0
- package/dist/parser/pdf-extract.js +112 -0
- package/dist/parser/pdf-extract.js.map +1 -0
- package/dist/parser/pdf-filter.d.ts +117 -0
- package/dist/parser/pdf-filter.d.ts.map +1 -0
- package/dist/parser/pdf-filter.js +528 -0
- package/dist/parser/pdf-filter.js.map +1 -0
- package/dist/parser/title-extractor.d.ts +69 -0
- package/dist/parser/title-extractor.d.ts.map +1 -0
- package/dist/parser/title-extractor.js +145 -0
- package/dist/parser/title-extractor.js.map +1 -0
- package/dist/pdf-visual/captioner.d.ts +16 -0
- package/dist/pdf-visual/captioner.d.ts.map +1 -0
- package/dist/pdf-visual/captioner.js +63 -0
- package/dist/pdf-visual/captioner.js.map +1 -0
- package/dist/pdf-visual/captioners/fast.d.ts +7 -0
- package/dist/pdf-visual/captioners/fast.d.ts.map +1 -0
- package/dist/pdf-visual/captioners/fast.js +103 -0
- package/dist/pdf-visual/captioners/fast.js.map +1 -0
- package/dist/pdf-visual/captioners/quality.d.ts +7 -0
- package/dist/pdf-visual/captioners/quality.d.ts.map +1 -0
- package/dist/pdf-visual/captioners/quality.js +127 -0
- package/dist/pdf-visual/captioners/quality.js.map +1 -0
- package/dist/pdf-visual/captioners/shared.d.ts +44 -0
- package/dist/pdf-visual/captioners/shared.d.ts.map +1 -0
- package/dist/pdf-visual/captioners/shared.js +104 -0
- package/dist/pdf-visual/captioners/shared.js.map +1 -0
- package/dist/pdf-visual/detector.d.ts +9 -0
- package/dist/pdf-visual/detector.d.ts.map +1 -0
- package/dist/pdf-visual/detector.js +234 -0
- package/dist/pdf-visual/detector.js.map +1 -0
- package/dist/pdf-visual/index.d.ts +13 -0
- package/dist/pdf-visual/index.d.ts.map +1 -0
- package/dist/pdf-visual/index.js +45 -0
- package/dist/pdf-visual/index.js.map +1 -0
- package/dist/pdf-visual/renderer.d.ts +9 -0
- package/dist/pdf-visual/renderer.d.ts.map +1 -0
- package/dist/pdf-visual/renderer.js +177 -0
- package/dist/pdf-visual/renderer.js.map +1 -0
- package/dist/pdf-visual/types.d.ts +62 -0
- package/dist/pdf-visual/types.d.ts.map +1 -0
- package/dist/pdf-visual/types.js +32 -0
- package/dist/pdf-visual/types.js.map +1 -0
- package/dist/server/error-utils.d.ts +79 -0
- package/dist/server/error-utils.d.ts.map +1 -0
- package/dist/server/error-utils.js +148 -0
- package/dist/server/error-utils.js.map +1 -0
- package/dist/server/index.d.ts +258 -0
- package/dist/server/index.d.ts.map +1 -0
- package/dist/server/index.js +1104 -0
- package/dist/server/index.js.map +1 -0
- package/dist/server/list-scanner.d.ts +52 -0
- package/dist/server/list-scanner.d.ts.map +1 -0
- package/dist/server/list-scanner.js +72 -0
- package/dist/server/list-scanner.js.map +1 -0
- package/dist/server/tool-definitions.d.ts +8 -0
- package/dist/server/tool-definitions.d.ts.map +1 -0
- package/dist/server/tool-definitions.js +181 -0
- package/dist/server/tool-definitions.js.map +1 -0
- package/dist/server/tool-input.d.ts +37 -0
- package/dist/server/tool-input.d.ts.map +1 -0
- package/dist/server/tool-input.js +216 -0
- package/dist/server/tool-input.js.map +1 -0
- package/dist/server/types.d.ts +331 -0
- package/dist/server/types.d.ts.map +1 -0
- package/dist/server/types.js +3 -0
- package/dist/server/types.js.map +1 -0
- package/dist/server-main.d.ts +46 -0
- package/dist/server-main.d.ts.map +1 -0
- package/dist/server-main.js +242 -0
- package/dist/server-main.js.map +1 -0
- package/dist/utils/base-dirs.d.ts +212 -0
- package/dist/utils/base-dirs.d.ts.map +1 -0
- package/dist/utils/base-dirs.js +422 -0
- package/dist/utils/base-dirs.js.map +1 -0
- package/dist/utils/errors.d.ts +24 -0
- package/dist/utils/errors.d.ts.map +1 -0
- package/dist/utils/errors.js +53 -0
- package/dist/utils/errors.js.map +1 -0
- package/dist/utils/limits.d.ts +26 -0
- package/dist/utils/limits.d.ts.map +1 -0
- package/dist/utils/limits.js +28 -0
- package/dist/utils/limits.js.map +1 -0
- package/dist/utils/list-sources.d.ts +47 -0
- package/dist/utils/list-sources.d.ts.map +1 -0
- package/dist/utils/list-sources.js +50 -0
- package/dist/utils/list-sources.js.map +1 -0
- package/dist/utils/raw-data-utils.d.ts +131 -0
- package/dist/utils/raw-data-utils.d.ts.map +1 -0
- package/dist/utils/raw-data-utils.js +255 -0
- package/dist/utils/raw-data-utils.js.map +1 -0
- package/dist/utils/scan.d.ts +126 -0
- package/dist/utils/scan.d.ts.map +1 -0
- package/dist/utils/scan.js +221 -0
- package/dist/utils/scan.js.map +1 -0
- package/dist/utils/scope-match.d.ts +43 -0
- package/dist/utils/scope-match.d.ts.map +1 -0
- package/dist/utils/scope-match.js +87 -0
- package/dist/utils/scope-match.js.map +1 -0
- package/dist/utils/sensitive-path.d.ts +23 -0
- package/dist/utils/sensitive-path.d.ts.map +1 -0
- package/dist/utils/sensitive-path.js +91 -0
- package/dist/utils/sensitive-path.js.map +1 -0
- package/dist/utils/sync-path-key.d.ts +20 -0
- package/dist/utils/sync-path-key.d.ts.map +1 -0
- package/dist/utils/sync-path-key.js +33 -0
- package/dist/utils/sync-path-key.js.map +1 -0
- package/dist/vectordb/index.d.ts +168 -0
- package/dist/vectordb/index.d.ts.map +1 -0
- package/dist/vectordb/index.js +619 -0
- package/dist/vectordb/index.js.map +1 -0
- package/dist/vectordb/search-filters.d.ts +39 -0
- package/dist/vectordb/search-filters.d.ts.map +1 -0
- package/dist/vectordb/search-filters.js +136 -0
- package/dist/vectordb/search-filters.js.map +1 -0
- package/dist/vectordb/types.d.ts +196 -0
- package/dist/vectordb/types.d.ts.map +1 -0
- package/dist/vectordb/types.js +224 -0
- package/dist/vectordb/types.js.map +1 -0
- package/package.json +105 -0
- package/skills/mcp-local-rag/SKILL.md +308 -0
- package/skills/mcp-local-rag/references/cli-reference.md +175 -0
- package/skills/mcp-local-rag/references/html-ingestion.md +78 -0
- package/skills/mcp-local-rag/references/query-optimization.md +57 -0
- package/skills/mcp-local-rag/references/result-refinement.md +56 -0
package/dist/cli-main.js
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
// CLI entry point for subcommands (skills install, etc.)
|
|
2
|
+
import { run as runSkillsInstall } from './bin/install-skills.js';
|
|
3
|
+
import { runDelete } from './cli/delete.js';
|
|
4
|
+
import { runIngest } from './cli/ingest.js';
|
|
5
|
+
import { runList } from './cli/list.js';
|
|
6
|
+
import { runQuery } from './cli/query.js';
|
|
7
|
+
import { runReadNeighbors } from './cli/read-neighbors.js';
|
|
8
|
+
import { runStatus } from './cli/status.js';
|
|
9
|
+
import { runSync } from './cli/sync.js';
|
|
10
|
+
export const SUBCOMMANDS = [
|
|
11
|
+
'skills',
|
|
12
|
+
'ingest',
|
|
13
|
+
'list',
|
|
14
|
+
'query',
|
|
15
|
+
'status',
|
|
16
|
+
'delete',
|
|
17
|
+
'read-neighbors',
|
|
18
|
+
'sync',
|
|
19
|
+
];
|
|
20
|
+
/**
|
|
21
|
+
* Handle CLI subcommands. The caller is expected to have already validated
|
|
22
|
+
* `subcommand` against `SUBCOMMANDS`; the union type makes the switch exhaustive.
|
|
23
|
+
* @param subcommand - The validated subcommand name
|
|
24
|
+
* @param args - Arguments following the subcommand (subcommand itself excluded)
|
|
25
|
+
* @param globalOptions - Global options parsed before the subcommand
|
|
26
|
+
*/
|
|
27
|
+
export async function handleCli(subcommand, args, globalOptions = {}) {
|
|
28
|
+
switch (subcommand) {
|
|
29
|
+
case 'skills':
|
|
30
|
+
if (args[0] === 'install') {
|
|
31
|
+
runSkillsInstall(args.slice(1));
|
|
32
|
+
process.exit(0);
|
|
33
|
+
}
|
|
34
|
+
else {
|
|
35
|
+
console.error('Unknown skills subcommand. Usage: npx mcp-local-rag skills install [options]');
|
|
36
|
+
console.error('Run "npx mcp-local-rag skills install --help" for more information.');
|
|
37
|
+
process.exit(1);
|
|
38
|
+
}
|
|
39
|
+
break;
|
|
40
|
+
case 'ingest':
|
|
41
|
+
await runIngest(args, globalOptions);
|
|
42
|
+
break;
|
|
43
|
+
case 'list':
|
|
44
|
+
await runList(args, globalOptions);
|
|
45
|
+
break;
|
|
46
|
+
case 'query':
|
|
47
|
+
await runQuery(args, globalOptions);
|
|
48
|
+
break;
|
|
49
|
+
case 'status':
|
|
50
|
+
await runStatus(args, globalOptions);
|
|
51
|
+
break;
|
|
52
|
+
case 'delete':
|
|
53
|
+
await runDelete(args, globalOptions);
|
|
54
|
+
break;
|
|
55
|
+
case 'read-neighbors':
|
|
56
|
+
await runReadNeighbors(args, globalOptions);
|
|
57
|
+
break;
|
|
58
|
+
case 'sync':
|
|
59
|
+
await runSync(args, globalOptions);
|
|
60
|
+
break;
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
//# sourceMappingURL=cli-main.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cli-main.js","sourceRoot":"","sources":["../src/cli-main.ts"],"names":[],"mappings":"AAAA,yDAAyD;AACzD,OAAO,EAAE,GAAG,IAAI,gBAAgB,EAAE,MAAM,yBAAyB,CAAA;AACjE,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAA;AAC3C,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAA;AAC3C,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAA;AAEvC,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAA;AACzC,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAA;AAC1D,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAA;AAC3C,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAA;AAEvC,MAAM,CAAC,MAAM,WAAW,GAAG;IACzB,QAAQ;IACR,QAAQ;IACR,MAAM;IACN,OAAO;IACP,QAAQ;IACR,QAAQ;IACR,gBAAgB;IAChB,MAAM;CACE,CAAA;AAIV;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,UAAsB,EACtB,IAAc,EACd,aAAa,GAAkB,EAAE;IAEjC,QAAQ,UAAU,EAAE,CAAC;QACnB,KAAK,QAAQ;YACX,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,SAAS,EAAE,CAAC;gBAC1B,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA;gBAC/B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACjB,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,KAAK,CACX,8EAA8E,CAC/E,CAAA;gBACD,OAAO,CAAC,KAAK,CAAC,qEAAqE,CAAC,CAAA;gBACpF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACjB,CAAC;YACD,MAAK;QAEP,KAAK,QAAQ;YACX,MAAM,SAAS,CAAC,IAAI,EAAE,aAAa,CAAC,CAAA;YACpC,MAAK;QAEP,KAAK,MAAM;YACT,MAAM,OAAO,CAAC,IAAI,EAAE,aAAa,CAAC,CAAA;YAClC,MAAK;QAEP,KAAK,OAAO;YACV,MAAM,QAAQ,CAAC,IAAI,EAAE,aAAa,CAAC,CAAA;YACnC,MAAK;QAEP,KAAK,QAAQ;YACX,MAAM,SAAS,CAAC,IAAI,EAAE,aAAa,CAAC,CAAA;YACpC,MAAK;QAEP,KAAK,QAAQ;YACX,MAAM,SAAS,CAAC,IAAI,EAAE,aAAa,CAAC,CAAA;YACpC,MAAK;QAEP,KAAK,gBAAgB;YACnB,MAAM,gBAAgB,CAAC,IAAI,EAAE,aAAa,CAAC,CAAA;YAC3C,MAAK;QAEP,KAAK,MAAM;YACT,MAAM,OAAO,CAAC,IAAI,EAAE,aAAa,CAAC,CAAA;YAClC,MAAK;IACT,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
import { AppError } from '../utils/errors.js';
|
|
2
|
+
/**
|
|
3
|
+
* Embedder configuration
|
|
4
|
+
*/
|
|
5
|
+
export interface EmbedderConfig {
|
|
6
|
+
/** HuggingFace model path */
|
|
7
|
+
modelPath: string;
|
|
8
|
+
/** Batch size */
|
|
9
|
+
batchSize: number;
|
|
10
|
+
/** Model cache directory */
|
|
11
|
+
cacheDir: string;
|
|
12
|
+
/** Device type */
|
|
13
|
+
device?: string;
|
|
14
|
+
/**
|
|
15
|
+
* Embedding quantization dtype (fp32, fp16, q8, int8, ...). Passed through to
|
|
16
|
+
* transformers.js — no allowlist. Undefined means "unset": initialize() then
|
|
17
|
+
* applies the fp32 default. The unset-vs-explicit-fp32 distinction is
|
|
18
|
+
* preserved on purpose (it gates failure-path error enrichment).
|
|
19
|
+
*/
|
|
20
|
+
dtype?: string;
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Embedding generation error
|
|
24
|
+
*/
|
|
25
|
+
export declare class EmbeddingError extends AppError {
|
|
26
|
+
constructor(message: string, cause?: Error);
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Embedding generation class using Transformers.js
|
|
30
|
+
*
|
|
31
|
+
* Responsibilities:
|
|
32
|
+
* - Generate embedding vectors (dimension depends on model)
|
|
33
|
+
* - Transformers.js wrapper
|
|
34
|
+
* - Batch processing (size 8)
|
|
35
|
+
*/
|
|
36
|
+
export declare class Embedder {
|
|
37
|
+
private model;
|
|
38
|
+
private initPromise;
|
|
39
|
+
private readonly config;
|
|
40
|
+
constructor(config: EmbedderConfig);
|
|
41
|
+
/**
|
|
42
|
+
* Release resources held by the Embedder pipeline
|
|
43
|
+
*/
|
|
44
|
+
dispose(): Promise<void>;
|
|
45
|
+
/**
|
|
46
|
+
* Initialize Transformers.js model
|
|
47
|
+
*/
|
|
48
|
+
initialize(): Promise<void>;
|
|
49
|
+
/**
|
|
50
|
+
* Best-effort failure-path enrichment for an explicit `RAG_DTYPE`.
|
|
51
|
+
*
|
|
52
|
+
* When the load failed and a dtype was explicitly requested, consult the
|
|
53
|
+
* model's available dtypes and, if the requested one is absent, return a
|
|
54
|
+
* message that names what the model provides. The enumeration is a Hub
|
|
55
|
+
* network call wrapped in its own try/catch: if it fails (e.g. air-gapped
|
|
56
|
+
* after caching) it degrades to a generic clear, dtype-aware message rather
|
|
57
|
+
* than surfacing a confusing secondary error (TD-3). This method never throws
|
|
58
|
+
* and never converts the load failure into a fallback — the caller always
|
|
59
|
+
* re-throws.
|
|
60
|
+
*
|
|
61
|
+
* @param nativeMessage - The underlying load-failure message.
|
|
62
|
+
* @returns The message to wrap in the thrown `EmbeddingError`.
|
|
63
|
+
*/
|
|
64
|
+
private enrichDtypeFailureMessage;
|
|
65
|
+
/**
|
|
66
|
+
* Ensure model is initialized (lazy initialization)
|
|
67
|
+
* This method is called automatically by embed() and embedBatch()
|
|
68
|
+
*/
|
|
69
|
+
private ensureInitialized;
|
|
70
|
+
/**
|
|
71
|
+
* Convert single text to embedding vector
|
|
72
|
+
*
|
|
73
|
+
* @param text - Text
|
|
74
|
+
* @returns Embedding vector (dimension depends on model)
|
|
75
|
+
*/
|
|
76
|
+
embed(text: string): Promise<number[]>;
|
|
77
|
+
/**
|
|
78
|
+
* Convert multiple texts to embedding vectors with batch processing
|
|
79
|
+
*
|
|
80
|
+
* @param texts - Array of texts
|
|
81
|
+
* @returns Array of embedding vectors (dimension depends on model)
|
|
82
|
+
*/
|
|
83
|
+
embedBatch(texts: string[]): Promise<number[][]>;
|
|
84
|
+
}
|
|
85
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/embedder/index.ts"],"names":[],"mappings":"AASA,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAA;AAM7C;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,6BAA6B;IAC7B,SAAS,EAAE,MAAM,CAAA;IACjB,iBAAiB;IACjB,SAAS,EAAE,MAAM,CAAA;IACjB,4BAA4B;IAC5B,QAAQ,EAAE,MAAM,CAAA;IAChB,kBAAkB;IAClB,MAAM,CAAC,EAAE,MAAM,CAAA;IACf;;;;;OAKG;IACH,KAAK,CAAC,EAAE,MAAM,CAAA;CACf;AAgED;;GAEG;AACH,qBAAa,cAAe,SAAQ,QAAQ;IAC1C,YAAY,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,KAAK,EAGzC;CACF;AAMD;;;;;;;GAOG;AACH,qBAAa,QAAQ;IAEnB,OAAO,CAAC,KAAK,CAAgB;IAC7B,OAAO,CAAC,WAAW,CAA6B;IAChD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgB;IAEvC,YAAY,MAAM,EAAE,cAAc,EAEjC;IAED;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAW7B;IAED;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC,CAkChC;IAED;;;;;;;;;;;;;;OAcG;YACW,yBAAyB;IAqBvC;;;OAGG;YACW,iBAAiB;IAyB/B;;;;;OAKG;IACG,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CA6B3C;IAED;;;;;OAKG;IACG,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAkGrD;CACF"}
|
|
@@ -0,0 +1,284 @@
|
|
|
1
|
+
// Embedder implementation with Transformers.js
|
|
2
|
+
import { env, ModelRegistry, pipeline, } from '@huggingface/transformers';
|
|
3
|
+
import { AppError } from '../utils/errors.js';
|
|
4
|
+
// Keep estimated padding waste below one-third of dense self-attention work.
|
|
5
|
+
const MAX_PADDING_AMPLIFICATION = 1.5;
|
|
6
|
+
function estimatePaddingAmplification(inputs) {
|
|
7
|
+
let maxTokenLength = 0;
|
|
8
|
+
let individualWork = 0;
|
|
9
|
+
for (const input of inputs) {
|
|
10
|
+
maxTokenLength = Math.max(maxTokenLength, input.tokenLength);
|
|
11
|
+
individualWork += input.tokenLength ** 2;
|
|
12
|
+
}
|
|
13
|
+
return (inputs.length * maxTokenLength ** 2) / individualWork;
|
|
14
|
+
}
|
|
15
|
+
function deferBatchOutliers(inputs) {
|
|
16
|
+
const batch = [...inputs];
|
|
17
|
+
const deferred = [];
|
|
18
|
+
while (batch.length > 1) {
|
|
19
|
+
if (estimatePaddingAmplification(batch) <= MAX_PADDING_AMPLIFICATION) {
|
|
20
|
+
break;
|
|
21
|
+
}
|
|
22
|
+
let longestIndex = 0;
|
|
23
|
+
for (let index = 1; index < batch.length; index++) {
|
|
24
|
+
if (batch[index].tokenLength > batch[longestIndex].tokenLength) {
|
|
25
|
+
longestIndex = index;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
const longest = batch[longestIndex];
|
|
29
|
+
deferred.push(longest);
|
|
30
|
+
batch.splice(longestIndex, 1);
|
|
31
|
+
}
|
|
32
|
+
return { batch, deferred };
|
|
33
|
+
}
|
|
34
|
+
// ============================================
|
|
35
|
+
// Error Classes
|
|
36
|
+
// ============================================
|
|
37
|
+
/**
|
|
38
|
+
* Embedding generation error
|
|
39
|
+
*/
|
|
40
|
+
export class EmbeddingError extends AppError {
|
|
41
|
+
constructor(message, cause) {
|
|
42
|
+
super(message, 'embedder', 'internal', cause);
|
|
43
|
+
this.name = 'EmbeddingError';
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
// ============================================
|
|
47
|
+
// Embedder Class
|
|
48
|
+
// ============================================
|
|
49
|
+
/**
|
|
50
|
+
* Embedding generation class using Transformers.js
|
|
51
|
+
*
|
|
52
|
+
* Responsibilities:
|
|
53
|
+
* - Generate embedding vectors (dimension depends on model)
|
|
54
|
+
* - Transformers.js wrapper
|
|
55
|
+
* - Batch processing (size 8)
|
|
56
|
+
*/
|
|
57
|
+
export class Embedder {
|
|
58
|
+
// Using unknown to avoid TS2590 (union type too complex with @types/jsdom)
|
|
59
|
+
model = null;
|
|
60
|
+
initPromise = null;
|
|
61
|
+
config;
|
|
62
|
+
constructor(config) {
|
|
63
|
+
this.config = config;
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Release resources held by the Embedder pipeline
|
|
67
|
+
*/
|
|
68
|
+
async dispose() {
|
|
69
|
+
const model = this.model;
|
|
70
|
+
if (model && typeof model.dispose === 'function') {
|
|
71
|
+
try {
|
|
72
|
+
await model.dispose();
|
|
73
|
+
}
|
|
74
|
+
catch (error) {
|
|
75
|
+
console.error('Error disposing embedder model:', error);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
this.model = null;
|
|
79
|
+
this.initPromise = null;
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Initialize Transformers.js model
|
|
83
|
+
*/
|
|
84
|
+
async initialize() {
|
|
85
|
+
// Skip if already initialized
|
|
86
|
+
if (this.model) {
|
|
87
|
+
return;
|
|
88
|
+
}
|
|
89
|
+
// Set cache directory BEFORE creating pipeline
|
|
90
|
+
env.cacheDir = this.config.cacheDir;
|
|
91
|
+
// No fallback — if the requested device fails, init throws.
|
|
92
|
+
const device = this.config.device || 'cpu';
|
|
93
|
+
console.error(`Embedder: Setting cache directory to "${this.config.cacheDir}"`);
|
|
94
|
+
console.error(`Embedder: Loading model "${this.config.modelPath}" on device "${device}"...`);
|
|
95
|
+
try {
|
|
96
|
+
this.model = await pipeline('feature-extraction', this.config.modelPath, {
|
|
97
|
+
// The sole fp32 default literal: unset dtype loads fp32, unchanged from
|
|
98
|
+
// before this knob existed. `as DataType` mirrors the `as DeviceType`
|
|
99
|
+
// cast below — a single typed pipeline boundary, no allowlist.
|
|
100
|
+
dtype: (this.config.dtype ?? 'fp32'),
|
|
101
|
+
device: device,
|
|
102
|
+
});
|
|
103
|
+
console.error(`Embedder: Model loaded successfully (device=${device})`);
|
|
104
|
+
}
|
|
105
|
+
catch (error) {
|
|
106
|
+
const nativeError = error;
|
|
107
|
+
// Only enrich when RAG_DTYPE was explicitly set (unset is `undefined` per
|
|
108
|
+
// TD-5). Enrichment never runs on the happy path and never on the unset
|
|
109
|
+
// path, so normal operation adds zero network. Always re-throw — an
|
|
110
|
+
// unavailable dtype fails loud, never silently downgrades (TD-2).
|
|
111
|
+
const message = await this.enrichDtypeFailureMessage(nativeError.message);
|
|
112
|
+
throw new EmbeddingError(message, nativeError);
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* Best-effort failure-path enrichment for an explicit `RAG_DTYPE`.
|
|
117
|
+
*
|
|
118
|
+
* When the load failed and a dtype was explicitly requested, consult the
|
|
119
|
+
* model's available dtypes and, if the requested one is absent, return a
|
|
120
|
+
* message that names what the model provides. The enumeration is a Hub
|
|
121
|
+
* network call wrapped in its own try/catch: if it fails (e.g. air-gapped
|
|
122
|
+
* after caching) it degrades to a generic clear, dtype-aware message rather
|
|
123
|
+
* than surfacing a confusing secondary error (TD-3). This method never throws
|
|
124
|
+
* and never converts the load failure into a fallback — the caller always
|
|
125
|
+
* re-throws.
|
|
126
|
+
*
|
|
127
|
+
* @param nativeMessage - The underlying load-failure message.
|
|
128
|
+
* @returns The message to wrap in the thrown `EmbeddingError`.
|
|
129
|
+
*/
|
|
130
|
+
async enrichDtypeFailureMessage(nativeMessage) {
|
|
131
|
+
const requestedDtype = this.config.dtype;
|
|
132
|
+
if (requestedDtype === undefined) {
|
|
133
|
+
return nativeMessage;
|
|
134
|
+
}
|
|
135
|
+
try {
|
|
136
|
+
const availableDtypes = await ModelRegistry.get_available_dtypes(this.config.modelPath);
|
|
137
|
+
if (availableDtypes.includes(requestedDtype)) {
|
|
138
|
+
// The requested dtype exists for this model, so the load failed for some
|
|
139
|
+
// other reason — keep the native message, don't misattribute it to dtype.
|
|
140
|
+
return nativeMessage;
|
|
141
|
+
}
|
|
142
|
+
return `Model "${this.config.modelPath}" provides dtypes [${availableDtypes.join(', ')}]; requested dtype "${requestedDtype}" is unavailable. Set RAG_DTYPE to one of the available dtypes, or leave it unset for the fp32 default.`;
|
|
143
|
+
}
|
|
144
|
+
catch {
|
|
145
|
+
// Enumeration unavailable (e.g. offline). Degrade to a generic clear,
|
|
146
|
+
// dtype-aware message — no secondary error, still re-thrown by the caller.
|
|
147
|
+
return `Failed to load model "${this.config.modelPath}" with requested dtype "${requestedDtype}". The model may not provide this dtype, and the available-dtype list could not be retrieved. Set RAG_DTYPE to a dtype the model provides, or leave it unset for the fp32 default. (${nativeMessage})`;
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
/**
|
|
151
|
+
* Ensure model is initialized (lazy initialization)
|
|
152
|
+
* This method is called automatically by embed() and embedBatch()
|
|
153
|
+
*/
|
|
154
|
+
async ensureInitialized() {
|
|
155
|
+
// Already initialized
|
|
156
|
+
if (this.model) {
|
|
157
|
+
return;
|
|
158
|
+
}
|
|
159
|
+
// Initialization already in progress, wait for it
|
|
160
|
+
if (this.initPromise) {
|
|
161
|
+
await this.initPromise;
|
|
162
|
+
return;
|
|
163
|
+
}
|
|
164
|
+
console.error('Embedder: First use detected. Initializing model (downloading ~90MB, may take 1-2 minutes)...');
|
|
165
|
+
this.initPromise = this.initialize().catch((error) => {
|
|
166
|
+
// Clear initPromise on failure to allow retry on the next call.
|
|
167
|
+
this.initPromise = null;
|
|
168
|
+
throw error;
|
|
169
|
+
});
|
|
170
|
+
await this.initPromise;
|
|
171
|
+
}
|
|
172
|
+
/**
|
|
173
|
+
* Convert single text to embedding vector
|
|
174
|
+
*
|
|
175
|
+
* @param text - Text
|
|
176
|
+
* @returns Embedding vector (dimension depends on model)
|
|
177
|
+
*/
|
|
178
|
+
async embed(text) {
|
|
179
|
+
// Reject empty input before paying for model init.
|
|
180
|
+
if (text.length === 0) {
|
|
181
|
+
throw new EmbeddingError('Cannot generate embedding for empty text');
|
|
182
|
+
}
|
|
183
|
+
// Lazy initialization: initialize on first use if not already initialized
|
|
184
|
+
await this.ensureInitialized();
|
|
185
|
+
try {
|
|
186
|
+
const options = { pooling: 'mean', normalize: true };
|
|
187
|
+
const modelCall = this.model;
|
|
188
|
+
const output = await modelCall(text, options);
|
|
189
|
+
// Access raw data via .data property
|
|
190
|
+
const embedding = Array.from(output.data);
|
|
191
|
+
return embedding;
|
|
192
|
+
}
|
|
193
|
+
catch (error) {
|
|
194
|
+
if (error instanceof EmbeddingError) {
|
|
195
|
+
throw error;
|
|
196
|
+
}
|
|
197
|
+
throw new EmbeddingError(`Failed to generate embedding: ${error.message}`, error);
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
/**
|
|
201
|
+
* Convert multiple texts to embedding vectors with batch processing
|
|
202
|
+
*
|
|
203
|
+
* @param texts - Array of texts
|
|
204
|
+
* @returns Array of embedding vectors (dimension depends on model)
|
|
205
|
+
*/
|
|
206
|
+
async embedBatch(texts) {
|
|
207
|
+
// Nothing to embed → skip model init entirely.
|
|
208
|
+
if (texts.length === 0) {
|
|
209
|
+
return [];
|
|
210
|
+
}
|
|
211
|
+
// Preserve embed()'s empty-text contract for batch elements (the previous
|
|
212
|
+
// per-text implementation rejected empty strings via embed()).
|
|
213
|
+
if (texts.some((text) => text.length === 0)) {
|
|
214
|
+
throw new EmbeddingError('Cannot generate embedding for empty text');
|
|
215
|
+
}
|
|
216
|
+
// Lazy initialization: initialize on first use if not already initialized
|
|
217
|
+
await this.ensureInitialized();
|
|
218
|
+
try {
|
|
219
|
+
const options = { pooling: 'mean', normalize: true };
|
|
220
|
+
// True batched inference: the feature-extraction pipeline accepts an
|
|
221
|
+
// array of texts and returns a single [batchLen, dim] tensor in one
|
|
222
|
+
// forward pass. The previous implementation called the model once per
|
|
223
|
+
// text via Promise.all, so `batchSize` had no real effect (onnxruntime
|
|
224
|
+
// inference is not parallelized by Promise.all). Passing the whole batch
|
|
225
|
+
// lets the runtime batch the matmuls. Mean-pooling honors the attention
|
|
226
|
+
// mask, so per-row vectors match the single-text result.
|
|
227
|
+
const modelCall = this.model;
|
|
228
|
+
const embeddings = Array.from({ length: texts.length });
|
|
229
|
+
const deferred = [];
|
|
230
|
+
const embedInputs = async (inputs) => {
|
|
231
|
+
const output = await modelCall(inputs.map((input) => input.text), options);
|
|
232
|
+
// Validate the output shape before slicing so a runtime/model contract
|
|
233
|
+
// change surfaces as a clear error rather than silently wrong vectors.
|
|
234
|
+
const dim = output?.dims?.[output.dims.length - 1];
|
|
235
|
+
if (!output ||
|
|
236
|
+
!(output.data instanceof Float32Array) ||
|
|
237
|
+
typeof dim !== 'number' ||
|
|
238
|
+
dim <= 0 ||
|
|
239
|
+
output.data.length !== inputs.length * dim) {
|
|
240
|
+
throw new EmbeddingError('Unexpected embedder batch output shape');
|
|
241
|
+
}
|
|
242
|
+
for (let row = 0; row < inputs.length; row++) {
|
|
243
|
+
const input = inputs[row];
|
|
244
|
+
embeddings[input.originalIndex] = Array.from(output.data.subarray(row * dim, (row + 1) * dim));
|
|
245
|
+
}
|
|
246
|
+
};
|
|
247
|
+
for (let i = 0; i < texts.length; i += this.config.batchSize) {
|
|
248
|
+
const batchTexts = texts.slice(i, i + this.config.batchSize);
|
|
249
|
+
const tokenized = modelCall.tokenizer(batchTexts, {
|
|
250
|
+
padding: false,
|
|
251
|
+
truncation: true,
|
|
252
|
+
return_tensor: false,
|
|
253
|
+
});
|
|
254
|
+
if (!tokenized ||
|
|
255
|
+
!Array.isArray(tokenized.input_ids) ||
|
|
256
|
+
tokenized.input_ids.length !== batchTexts.length) {
|
|
257
|
+
throw new EmbeddingError('Unexpected embedder tokenizer output shape');
|
|
258
|
+
}
|
|
259
|
+
const indexedInputs = batchTexts.map((text, batchIndex) => ({
|
|
260
|
+
text,
|
|
261
|
+
originalIndex: i + batchIndex,
|
|
262
|
+
tokenLength: tokenized.input_ids[batchIndex].length,
|
|
263
|
+
}));
|
|
264
|
+
const selected = deferBatchOutliers(indexedInputs);
|
|
265
|
+
deferred.push(...selected.deferred);
|
|
266
|
+
await embedInputs(selected.batch);
|
|
267
|
+
}
|
|
268
|
+
for (const input of deferred) {
|
|
269
|
+
await embedInputs([input]);
|
|
270
|
+
}
|
|
271
|
+
if (embeddings.some((embedding) => embedding === undefined)) {
|
|
272
|
+
throw new EmbeddingError('Missing embedder batch output row');
|
|
273
|
+
}
|
|
274
|
+
return embeddings;
|
|
275
|
+
}
|
|
276
|
+
catch (error) {
|
|
277
|
+
if (error instanceof EmbeddingError) {
|
|
278
|
+
throw error;
|
|
279
|
+
}
|
|
280
|
+
throw new EmbeddingError(`Failed to generate batch embeddings: ${error.message}`, error);
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/embedder/index.ts"],"names":[],"mappings":"AAAA,+CAA+C;AAE/C,OAAO,EAGL,GAAG,EACH,aAAa,EACb,QAAQ,GACT,MAAM,2BAA2B,CAAA;AAClC,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAA;AA6C7C,6EAA6E;AAC7E,MAAM,yBAAyB,GAAG,GAAG,CAAA;AAErC,SAAS,4BAA4B,CAAC,MAA+B;IACnE,IAAI,cAAc,GAAG,CAAC,CAAA;IACtB,IAAI,cAAc,GAAG,CAAC,CAAA;IACtB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,cAAc,EAAE,KAAK,CAAC,WAAW,CAAC,CAAA;QAC5D,cAAc,IAAI,KAAK,CAAC,WAAW,IAAI,CAAC,CAAA;IAC1C,CAAC;IACD,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,cAAc,IAAI,CAAC,CAAC,GAAG,cAAc,CAAA;AAC/D,CAAC;AAED,SAAS,kBAAkB,CAAC,MAA+B;IAIzD,MAAM,KAAK,GAAG,CAAC,GAAG,MAAM,CAAC,CAAA;IACzB,MAAM,QAAQ,GAA4B,EAAE,CAAA;IAE5C,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,IAAI,4BAA4B,CAAC,KAAK,CAAC,IAAI,yBAAyB,EAAE,CAAC;YACrE,MAAK;QACP,CAAC;QAED,IAAI,YAAY,GAAG,CAAC,CAAA;QACpB,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;YAClD,IAAI,KAAK,CAAC,KAAK,CAAE,CAAC,WAAW,GAAG,KAAK,CAAC,YAAY,CAAE,CAAC,WAAW,EAAE,CAAC;gBACjE,YAAY,GAAG,KAAK,CAAA;YACtB,CAAC;QACH,CAAC;QAED,MAAM,OAAO,GAAG,KAAK,CAAC,YAAY,CAAE,CAAA;QACpC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;QACtB,KAAK,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC,CAAC,CAAA;IAC/B,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAA;AAC5B,CAAC;AAED,+CAA+C;AAC/C,gBAAgB;AAChB,+CAA+C;AAE/C;;GAEG;AACH,MAAM,OAAO,cAAe,SAAQ,QAAQ;IAC1C,YAAY,OAAe,EAAE,KAAa;QACxC,KAAK,CAAC,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,KAAK,CAAC,CAAA;QAC7C,IAAI,CAAC,IAAI,GAAG,gBAAgB,CAAA;IAC9B,CAAC;CACF;AAED,+CAA+C;AAC/C,iBAAiB;AACjB,+CAA+C;AAE/C;;;;;;;GAOG;AACH,MAAM,OAAO,QAAQ;IACnB,2EAA2E;IACnE,KAAK,GAAY,IAAI,CAAA;IACrB,WAAW,GAAyB,IAAI,CAAA;IAC/B,MAAM,CAAgB;IAEvC,YAAY,MAAsB;QAChC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAA;IACtB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO;QACX,MAAM,KAAK,GAAG,IAAI,CAAC,KAAiD,CAAA;QACpE,IAAI,KAAK,IAAI,OAAO,KAAK,CAAC,OAAO,KAAK,UAAU,EAAE,CAAC;YACjD,IAAI,CAAC;gBACH,MAAM,KAAK,CAAC,OAAO,EAAE,CAAA;YACvB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,KAAK,CAAC,iCAAiC,EAAE,KAAK,CAAC,CAAA;YACzD,CAAC;QACH,CAAC;QACD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAA;QACjB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAA;IACzB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU;QACd,8BAA8B;QAC9B,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,OAAM;QACR,CAAC;QAED,+CAA+C;QAC/C,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAA;QAEnC,4DAA4D;QAC5D,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,KAAK,CAAA;QAE1C,OAAO,CAAC,KAAK,CAAC,yCAAyC,IAAI,CAAC,MAAM,CAAC,QAAQ,GAAG,CAAC,CAAA;QAC/E,OAAO,CAAC,KAAK,CAAC,4BAA4B,IAAI,CAAC,MAAM,CAAC,SAAS,gBAAgB,MAAM,MAAM,CAAC,CAAA;QAE5F,IAAI,CAAC;YACH,IAAI,CAAC,KAAK,GAAG,MAAM,QAAQ,CAAC,oBAAoB,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE;gBACvE,wEAAwE;gBACxE,sEAAsE;gBACtE,+DAA+D;gBAC/D,KAAK,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,MAAM,CAAa;gBAChD,MAAM,EAAE,MAAoB;aAC7B,CAAC,CAAA;YACF,OAAO,CAAC,KAAK,CAAC,+CAA+C,MAAM,GAAG,CAAC,CAAA;QACzE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,WAAW,GAAG,KAAc,CAAA;YAElC,0EAA0E;YAC1E,wEAAwE;YACxE,oEAAoE;YACpE,kEAAkE;YAClE,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,yBAAyB,CAAC,WAAW,CAAC,OAAO,CAAC,CAAA;YACzE,MAAM,IAAI,cAAc,CAAC,OAAO,EAAE,WAAW,CAAC,CAAA;QAChD,CAAC;IACH,CAAC;IAED;;;;;;;;;;;;;;OAcG;IACK,KAAK,CAAC,yBAAyB,CAAC,aAAqB;QAC3D,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAA;QACxC,IAAI,cAAc,KAAK,SAAS,EAAE,CAAC;YACjC,OAAO,aAAa,CAAA;QACtB,CAAC;QAED,IAAI,CAAC;YACH,MAAM,eAAe,GAAG,MAAM,aAAa,CAAC,oBAAoB,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;YACvF,IAAI,eAAe,CAAC,QAAQ,CAAC,cAAc,CAAC,EAAE,CAAC;gBAC7C,yEAAyE;gBACzE,0EAA0E;gBAC1E,OAAO,aAAa,CAAA;YACtB,CAAC;YACD,OAAO,UAAU,IAAI,CAAC,MAAM,CAAC,SAAS,sBAAsB,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,uBAAuB,cAAc,yGAAyG,CAAA;QACtO,CAAC;QAAC,MAAM,CAAC;YACP,sEAAsE;YACtE,2EAA2E;YAC3E,OAAO,yBAAyB,IAAI,CAAC,MAAM,CAAC,SAAS,2BAA2B,cAAc,uLAAuL,aAAa,GAAG,CAAA;QACvS,CAAC;IACH,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,iBAAiB;QAC7B,sBAAsB;QACtB,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,OAAM;QACR,CAAC;QAED,kDAAkD;QAClD,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,MAAM,IAAI,CAAC,WAAW,CAAA;YACtB,OAAM;QACR,CAAC;QAED,OAAO,CAAC,KAAK,CACX,+FAA+F,CAChG,CAAA;QAED,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;YACnD,gEAAgE;YAChE,IAAI,CAAC,WAAW,GAAG,IAAI,CAAA;YACvB,MAAM,KAAK,CAAA;QACb,CAAC,CAAC,CAAA;QAEF,MAAM,IAAI,CAAC,WAAW,CAAA;IACxB,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,KAAK,CAAC,IAAY;QACtB,mDAAmD;QACnD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,MAAM,IAAI,cAAc,CAAC,0CAA0C,CAAC,CAAA;QACtE,CAAC;QAED,0EAA0E;QAC1E,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAA;QAE9B,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAA;YACpD,MAAM,SAAS,GAAG,IAAI,CAAC,KAGa,CAAA;YACpC,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;YAE7C,qCAAqC;YACrC,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YACzC,OAAO,SAAS,CAAA;QAClB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,cAAc,EAAE,CAAC;gBACpC,MAAM,KAAK,CAAA;YACb,CAAC;YACD,MAAM,IAAI,cAAc,CACtB,iCAAkC,KAAe,CAAC,OAAO,EAAE,EAC3D,KAAc,CACf,CAAA;QACH,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,UAAU,CAAC,KAAe;QAC9B,+CAA+C;QAC/C,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,EAAE,CAAA;QACX,CAAC;QAED,0EAA0E;QAC1E,+DAA+D;QAC/D,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC,EAAE,CAAC;YAC5C,MAAM,IAAI,cAAc,CAAC,0CAA0C,CAAC,CAAA;QACtE,CAAC;QAED,0EAA0E;QAC1E,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAA;QAE9B,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAA;YACpD,qEAAqE;YACrE,oEAAoE;YACpE,sEAAsE;YACtE,uEAAuE;YACvE,yEAAyE;YACzE,wEAAwE;YACxE,yDAAyD;YACzD,MAAM,SAAS,GAAG,IAAI,CAAC,KAA+B,CAAA;YACtD,MAAM,UAAU,GAA6B,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAA;YACjF,MAAM,QAAQ,GAA4B,EAAE,CAAA;YAE5C,MAAM,WAAW,GAAG,KAAK,EAAE,MAA+B,EAAiB,EAAE;gBAC3E,MAAM,MAAM,GAAG,MAAM,SAAS,CAC5B,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,EACjC,OAAO,CACR,CAAA;gBAED,uEAAuE;gBACvE,uEAAuE;gBACvE,MAAM,GAAG,GAAG,MAAM,EAAE,IAAI,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;gBAClD,IACE,CAAC,MAAM;oBACP,CAAC,CAAC,MAAM,CAAC,IAAI,YAAY,YAAY,CAAC;oBACtC,OAAO,GAAG,KAAK,QAAQ;oBACvB,GAAG,IAAI,CAAC;oBACR,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,GAAG,GAAG,EAC1C,CAAC;oBACD,MAAM,IAAI,cAAc,CAAC,wCAAwC,CAAC,CAAA;gBACpE,CAAC;gBAED,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE,CAAC;oBAC7C,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAE,CAAA;oBAC1B,UAAU,CAAC,KAAK,CAAC,aAAa,CAAC,GAAG,KAAK,CAAC,IAAI,CAC1C,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,GAAG,GAAG,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CACjD,CAAA;gBACH,CAAC;YACH,CAAC,CAAA;YAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;gBAC7D,MAAM,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;gBAC5D,MAAM,SAAS,GAAG,SAAS,CAAC,SAAS,CAAC,UAAU,EAAE;oBAChD,OAAO,EAAE,KAAK;oBACd,UAAU,EAAE,IAAI;oBAChB,aAAa,EAAE,KAAK;iBACrB,CAAC,CAAA;gBACF,IACE,CAAC,SAAS;oBACV,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC;oBACnC,SAAS,CAAC,SAAS,CAAC,MAAM,KAAK,UAAU,CAAC,MAAM,EAChD,CAAC;oBACD,MAAM,IAAI,cAAc,CAAC,4CAA4C,CAAC,CAAA;gBACxE,CAAC;gBAED,MAAM,aAAa,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,UAAU,EAAE,EAAE,CAAC,CAAC;oBAC1D,IAAI;oBACJ,aAAa,EAAE,CAAC,GAAG,UAAU;oBAC7B,WAAW,EAAE,SAAS,CAAC,SAAS,CAAC,UAAU,CAAE,CAAC,MAAM;iBACrD,CAAC,CAAC,CAAA;gBACH,MAAM,QAAQ,GAAG,kBAAkB,CAAC,aAAa,CAAC,CAAA;gBAClD,QAAQ,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAA;gBACnC,MAAM,WAAW,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAA;YACnC,CAAC;YAED,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;gBAC7B,MAAM,WAAW,CAAC,CAAC,KAAK,CAAC,CAAC,CAAA;YAC5B,CAAC;YAED,IAAI,UAAU,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,KAAK,SAAS,CAAC,EAAE,CAAC;gBAC5D,MAAM,IAAI,cAAc,CAAC,mCAAmC,CAAC,CAAA;YAC/D,CAAC;YAED,OAAO,UAAwB,CAAA;QACjC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,cAAc,EAAE,CAAC;gBACpC,MAAM,KAAK,CAAA;YACb,CAAC;YACD,MAAM,IAAI,cAAc,CACtB,wCAAyC,KAAe,CAAC,OAAO,EAAE,EAClE,KAAc,CACf,CAAA;QACH,CAAC;IACH,CAAC;CACF"}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import { type ClassifiedSource } from '../utils/list-sources.js';
|
|
2
|
+
export interface IngestedFileSummary {
|
|
3
|
+
filePath: string;
|
|
4
|
+
chunkCount: number;
|
|
5
|
+
timestamp: string;
|
|
6
|
+
}
|
|
7
|
+
export type ListedFile = {
|
|
8
|
+
filePath: string;
|
|
9
|
+
baseDir: string;
|
|
10
|
+
ingested: true;
|
|
11
|
+
chunkCount: number;
|
|
12
|
+
timestamp: string;
|
|
13
|
+
} | {
|
|
14
|
+
filePath: string;
|
|
15
|
+
baseDir: string;
|
|
16
|
+
ingested: false;
|
|
17
|
+
};
|
|
18
|
+
export interface RootScanResult {
|
|
19
|
+
files: string[];
|
|
20
|
+
warnings: string[];
|
|
21
|
+
}
|
|
22
|
+
export interface ListDocumentsResult {
|
|
23
|
+
files: ListedFile[];
|
|
24
|
+
sources: ClassifiedSource[];
|
|
25
|
+
warnings: Array<{
|
|
26
|
+
baseDir: string;
|
|
27
|
+
message: string;
|
|
28
|
+
}>;
|
|
29
|
+
}
|
|
30
|
+
export declare function listDocuments(input: {
|
|
31
|
+
roots: readonly string[];
|
|
32
|
+
dbPath: string;
|
|
33
|
+
ingested: IngestedFileSummary[];
|
|
34
|
+
scope?: string[] | undefined;
|
|
35
|
+
scan: (root: string, scope?: string[] | undefined) => Promise<RootScanResult>;
|
|
36
|
+
}): Promise<ListDocumentsResult>;
|
|
37
|
+
//# sourceMappingURL=list.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"list.d.ts","sourceRoot":"","sources":["../../src/features/list.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,gBAAgB,EAA2B,MAAM,0BAA0B,CAAA;AAGzF,MAAM,WAAW,mBAAmB;IAClC,QAAQ,EAAE,MAAM,CAAA;IAChB,UAAU,EAAE,MAAM,CAAA;IAClB,SAAS,EAAE,MAAM,CAAA;CAClB;AAED,MAAM,MAAM,UAAU,GAClB;IACE,QAAQ,EAAE,MAAM,CAAA;IAChB,OAAO,EAAE,MAAM,CAAA;IACf,QAAQ,EAAE,IAAI,CAAA;IACd,UAAU,EAAE,MAAM,CAAA;IAClB,SAAS,EAAE,MAAM,CAAA;CAClB,GACD;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,KAAK,CAAA;CAAE,CAAA;AAE1D,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,MAAM,EAAE,CAAA;IACf,QAAQ,EAAE,MAAM,EAAE,CAAA;CACnB;AAED,MAAM,WAAW,mBAAmB;IAClC,KAAK,EAAE,UAAU,EAAE,CAAA;IACnB,OAAO,EAAE,gBAAgB,EAAE,CAAA;IAC3B,QAAQ,EAAE,KAAK,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;CACtD;AAED,wBAAsB,aAAa,CAAC,KAAK,EAAE;IACzC,KAAK,EAAE,SAAS,MAAM,EAAE,CAAA;IACxB,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,mBAAmB,EAAE,CAAA;IAC/B,KAAK,CAAC,EAAE,MAAM,EAAE,GAAG,SAAS,CAAA;IAC5B,IAAI,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,EAAE,GAAG,SAAS,KAAK,OAAO,CAAC,cAAc,CAAC,CAAA;CAC9E,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAwC/B"}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { classifyIngestedSources } from '../utils/list-sources.js';
|
|
2
|
+
import { realpathForMatch } from '../utils/scan.js';
|
|
3
|
+
export async function listDocuments(input) {
|
|
4
|
+
const ingestedKeyed = await Promise.all(input.ingested.map(async (entry) => ({ entry, key: await realpathForMatch(entry.filePath) })));
|
|
5
|
+
const ingestedByKey = new Map(ingestedKeyed.map(({ entry, key }) => [key, entry]));
|
|
6
|
+
const seenKeys = new Set();
|
|
7
|
+
const matchedKeys = new Set();
|
|
8
|
+
const files = [];
|
|
9
|
+
const warnings = [];
|
|
10
|
+
for (const baseDir of input.roots) {
|
|
11
|
+
const scanned = await input.scan(baseDir, input.scope);
|
|
12
|
+
warnings.push(...scanned.warnings.map((message) => ({ baseDir, message })));
|
|
13
|
+
for (const scannedPath of scanned.files) {
|
|
14
|
+
const key = await realpathForMatch(scannedPath);
|
|
15
|
+
if (seenKeys.has(key))
|
|
16
|
+
continue;
|
|
17
|
+
seenKeys.add(key);
|
|
18
|
+
const entry = ingestedByKey.get(key);
|
|
19
|
+
if (entry) {
|
|
20
|
+
matchedKeys.add(key);
|
|
21
|
+
files.push({
|
|
22
|
+
filePath: entry.filePath,
|
|
23
|
+
baseDir,
|
|
24
|
+
ingested: true,
|
|
25
|
+
chunkCount: entry.chunkCount,
|
|
26
|
+
timestamp: entry.timestamp,
|
|
27
|
+
});
|
|
28
|
+
}
|
|
29
|
+
else {
|
|
30
|
+
files.push({ filePath: scannedPath, baseDir, ingested: false });
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
return {
|
|
35
|
+
files,
|
|
36
|
+
sources: classifyIngestedSources(ingestedKeyed, matchedKeys, input.dbPath, input.scope),
|
|
37
|
+
warnings,
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
//# sourceMappingURL=list.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"list.js","sourceRoot":"","sources":["../../src/features/list.ts"],"names":[],"mappings":"AAAA,OAAO,EAAyB,uBAAuB,EAAE,MAAM,0BAA0B,CAAA;AACzF,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AA6BnD,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,KAMnC;IACC,MAAM,aAAa,GAAG,MAAM,OAAO,CAAC,GAAG,CACrC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,gBAAgB,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAC9F,CAAA;IACD,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,CAAA;IAClF,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAA;IAClC,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAA;IACrC,MAAM,KAAK,GAAiB,EAAE,CAAA;IAC9B,MAAM,QAAQ,GAAgD,EAAE,CAAA;IAEhE,KAAK,MAAM,OAAO,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;QAClC,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,KAAK,CAAC,CAAA;QACtD,QAAQ,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,CAAA;QAE3E,KAAK,MAAM,WAAW,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;YACxC,MAAM,GAAG,GAAG,MAAM,gBAAgB,CAAC,WAAW,CAAC,CAAA;YAC/C,IAAI,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,SAAQ;YAC/B,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YAEjB,MAAM,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YACpC,IAAI,KAAK,EAAE,CAAC;gBACV,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;gBACpB,KAAK,CAAC,IAAI,CAAC;oBACT,QAAQ,EAAE,KAAK,CAAC,QAAQ;oBACxB,OAAO;oBACP,QAAQ,EAAE,IAAI;oBACd,UAAU,EAAE,KAAK,CAAC,UAAU;oBAC5B,SAAS,EAAE,KAAK,CAAC,SAAS;iBAC3B,CAAC,CAAA;YACJ,CAAC;iBAAM,CAAC;gBACN,KAAK,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,WAAW,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAA;YACjE,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO;QACL,KAAK;QACL,OAAO,EAAE,uBAAuB,CAAC,aAAa,EAAE,WAAW,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC;QACvF,QAAQ;KACT,CAAA;AACH,CAAC"}
|