abb-opencode-local-rag 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (233) hide show
  1. package/LICENSE +21 -0
  2. package/README.de.md +416 -0
  3. package/README.es.md +416 -0
  4. package/README.fr.md +416 -0
  5. package/README.md +491 -0
  6. package/README.pt-BR.md +416 -0
  7. package/README.zh-CN.md +416 -0
  8. package/dist/bin/install-skills.d.ts +20 -0
  9. package/dist/bin/install-skills.d.ts.map +1 -0
  10. package/dist/bin/install-skills.js +195 -0
  11. package/dist/bin/install-skills.js.map +1 -0
  12. package/dist/chunker/index.d.ts +24 -0
  13. package/dist/chunker/index.d.ts.map +1 -0
  14. package/dist/chunker/index.js +2 -0
  15. package/dist/chunker/index.js.map +1 -0
  16. package/dist/chunker/semantic-chunker.d.ts +97 -0
  17. package/dist/chunker/semantic-chunker.d.ts.map +1 -0
  18. package/dist/chunker/semantic-chunker.js +294 -0
  19. package/dist/chunker/semantic-chunker.js.map +1 -0
  20. package/dist/chunker/sentence-splitter.d.ts +28 -0
  21. package/dist/chunker/sentence-splitter.d.ts.map +1 -0
  22. package/dist/chunker/sentence-splitter.js +219 -0
  23. package/dist/chunker/sentence-splitter.js.map +1 -0
  24. package/dist/cli/common.d.ts +65 -0
  25. package/dist/cli/common.d.ts.map +1 -0
  26. package/dist/cli/common.js +138 -0
  27. package/dist/cli/common.js.map +1 -0
  28. package/dist/cli/delete.d.ts +8 -0
  29. package/dist/cli/delete.d.ts.map +1 -0
  30. package/dist/cli/delete.js +173 -0
  31. package/dist/cli/delete.js.map +1 -0
  32. package/dist/cli/file-collection.d.ts +2 -0
  33. package/dist/cli/file-collection.d.ts.map +1 -0
  34. package/dist/cli/file-collection.js +53 -0
  35. package/dist/cli/file-collection.js.map +1 -0
  36. package/dist/cli/ingest.d.ts +100 -0
  37. package/dist/cli/ingest.d.ts.map +1 -0
  38. package/dist/cli/ingest.js +363 -0
  39. package/dist/cli/ingest.js.map +1 -0
  40. package/dist/cli/list.d.ts +35 -0
  41. package/dist/cli/list.d.ts.map +1 -0
  42. package/dist/cli/list.js +210 -0
  43. package/dist/cli/list.js.map +1 -0
  44. package/dist/cli/options.d.ts +100 -0
  45. package/dist/cli/options.d.ts.map +1 -0
  46. package/dist/cli/options.js +241 -0
  47. package/dist/cli/options.js.map +1 -0
  48. package/dist/cli/query.d.ts +24 -0
  49. package/dist/cli/query.d.ts.map +1 -0
  50. package/dist/cli/query.js +191 -0
  51. package/dist/cli/query.js.map +1 -0
  52. package/dist/cli/read-neighbors.d.ts +11 -0
  53. package/dist/cli/read-neighbors.d.ts.map +1 -0
  54. package/dist/cli/read-neighbors.js +224 -0
  55. package/dist/cli/read-neighbors.js.map +1 -0
  56. package/dist/cli/status.d.ts +8 -0
  57. package/dist/cli/status.d.ts.map +1 -0
  58. package/dist/cli/status.js +80 -0
  59. package/dist/cli/status.js.map +1 -0
  60. package/dist/cli/sync.d.ts +8 -0
  61. package/dist/cli/sync.d.ts.map +1 -0
  62. package/dist/cli/sync.js +244 -0
  63. package/dist/cli/sync.js.map +1 -0
  64. package/dist/cli-main.d.ts +12 -0
  65. package/dist/cli-main.d.ts.map +1 -0
  66. package/dist/cli-main.js +63 -0
  67. package/dist/cli-main.js.map +1 -0
  68. package/dist/embedder/index.d.ts +85 -0
  69. package/dist/embedder/index.d.ts.map +1 -0
  70. package/dist/embedder/index.js +284 -0
  71. package/dist/embedder/index.js.map +1 -0
  72. package/dist/features/list.d.ts +37 -0
  73. package/dist/features/list.d.ts.map +1 -0
  74. package/dist/features/list.js +40 -0
  75. package/dist/features/list.js.map +1 -0
  76. package/dist/features/sync.d.ts +207 -0
  77. package/dist/features/sync.d.ts.map +1 -0
  78. package/dist/features/sync.js +380 -0
  79. package/dist/features/sync.js.map +1 -0
  80. package/dist/index.d.ts +3 -0
  81. package/dist/index.d.ts.map +1 -0
  82. package/dist/index.js +53 -0
  83. package/dist/index.js.map +1 -0
  84. package/dist/ingest/compute.d.ts +86 -0
  85. package/dist/ingest/compute.d.ts.map +1 -0
  86. package/dist/ingest/compute.js +177 -0
  87. package/dist/ingest/compute.js.map +1 -0
  88. package/dist/ingest/file.d.ts +27 -0
  89. package/dist/ingest/file.d.ts.map +1 -0
  90. package/dist/ingest/file.js +67 -0
  91. package/dist/ingest/file.js.map +1 -0
  92. package/dist/ingest/visual.d.ts +45 -0
  93. package/dist/ingest/visual.d.ts.map +1 -0
  94. package/dist/ingest/visual.js +234 -0
  95. package/dist/ingest/visual.js.map +1 -0
  96. package/dist/parser/docx-parser.d.ts +12 -0
  97. package/dist/parser/docx-parser.d.ts.map +1 -0
  98. package/dist/parser/docx-parser.js +328 -0
  99. package/dist/parser/docx-parser.js.map +1 -0
  100. package/dist/parser/html-parser.d.ts +18 -0
  101. package/dist/parser/html-parser.d.ts.map +1 -0
  102. package/dist/parser/html-parser.js +102 -0
  103. package/dist/parser/html-parser.js.map +1 -0
  104. package/dist/parser/index.d.ts +214 -0
  105. package/dist/parser/index.d.ts.map +1 -0
  106. package/dist/parser/index.js +454 -0
  107. package/dist/parser/index.js.map +1 -0
  108. package/dist/parser/pdf-extract.d.ts +81 -0
  109. package/dist/parser/pdf-extract.d.ts.map +1 -0
  110. package/dist/parser/pdf-extract.js +112 -0
  111. package/dist/parser/pdf-extract.js.map +1 -0
  112. package/dist/parser/pdf-filter.d.ts +117 -0
  113. package/dist/parser/pdf-filter.d.ts.map +1 -0
  114. package/dist/parser/pdf-filter.js +528 -0
  115. package/dist/parser/pdf-filter.js.map +1 -0
  116. package/dist/parser/title-extractor.d.ts +69 -0
  117. package/dist/parser/title-extractor.d.ts.map +1 -0
  118. package/dist/parser/title-extractor.js +145 -0
  119. package/dist/parser/title-extractor.js.map +1 -0
  120. package/dist/pdf-visual/captioner.d.ts +16 -0
  121. package/dist/pdf-visual/captioner.d.ts.map +1 -0
  122. package/dist/pdf-visual/captioner.js +63 -0
  123. package/dist/pdf-visual/captioner.js.map +1 -0
  124. package/dist/pdf-visual/captioners/fast.d.ts +7 -0
  125. package/dist/pdf-visual/captioners/fast.d.ts.map +1 -0
  126. package/dist/pdf-visual/captioners/fast.js +103 -0
  127. package/dist/pdf-visual/captioners/fast.js.map +1 -0
  128. package/dist/pdf-visual/captioners/quality.d.ts +7 -0
  129. package/dist/pdf-visual/captioners/quality.d.ts.map +1 -0
  130. package/dist/pdf-visual/captioners/quality.js +127 -0
  131. package/dist/pdf-visual/captioners/quality.js.map +1 -0
  132. package/dist/pdf-visual/captioners/shared.d.ts +44 -0
  133. package/dist/pdf-visual/captioners/shared.d.ts.map +1 -0
  134. package/dist/pdf-visual/captioners/shared.js +104 -0
  135. package/dist/pdf-visual/captioners/shared.js.map +1 -0
  136. package/dist/pdf-visual/detector.d.ts +9 -0
  137. package/dist/pdf-visual/detector.d.ts.map +1 -0
  138. package/dist/pdf-visual/detector.js +234 -0
  139. package/dist/pdf-visual/detector.js.map +1 -0
  140. package/dist/pdf-visual/index.d.ts +13 -0
  141. package/dist/pdf-visual/index.d.ts.map +1 -0
  142. package/dist/pdf-visual/index.js +45 -0
  143. package/dist/pdf-visual/index.js.map +1 -0
  144. package/dist/pdf-visual/renderer.d.ts +9 -0
  145. package/dist/pdf-visual/renderer.d.ts.map +1 -0
  146. package/dist/pdf-visual/renderer.js +177 -0
  147. package/dist/pdf-visual/renderer.js.map +1 -0
  148. package/dist/pdf-visual/types.d.ts +62 -0
  149. package/dist/pdf-visual/types.d.ts.map +1 -0
  150. package/dist/pdf-visual/types.js +32 -0
  151. package/dist/pdf-visual/types.js.map +1 -0
  152. package/dist/server/error-utils.d.ts +79 -0
  153. package/dist/server/error-utils.d.ts.map +1 -0
  154. package/dist/server/error-utils.js +148 -0
  155. package/dist/server/error-utils.js.map +1 -0
  156. package/dist/server/index.d.ts +258 -0
  157. package/dist/server/index.d.ts.map +1 -0
  158. package/dist/server/index.js +1104 -0
  159. package/dist/server/index.js.map +1 -0
  160. package/dist/server/list-scanner.d.ts +52 -0
  161. package/dist/server/list-scanner.d.ts.map +1 -0
  162. package/dist/server/list-scanner.js +72 -0
  163. package/dist/server/list-scanner.js.map +1 -0
  164. package/dist/server/tool-definitions.d.ts +8 -0
  165. package/dist/server/tool-definitions.d.ts.map +1 -0
  166. package/dist/server/tool-definitions.js +181 -0
  167. package/dist/server/tool-definitions.js.map +1 -0
  168. package/dist/server/tool-input.d.ts +37 -0
  169. package/dist/server/tool-input.d.ts.map +1 -0
  170. package/dist/server/tool-input.js +216 -0
  171. package/dist/server/tool-input.js.map +1 -0
  172. package/dist/server/types.d.ts +331 -0
  173. package/dist/server/types.d.ts.map +1 -0
  174. package/dist/server/types.js +3 -0
  175. package/dist/server/types.js.map +1 -0
  176. package/dist/server-main.d.ts +46 -0
  177. package/dist/server-main.d.ts.map +1 -0
  178. package/dist/server-main.js +242 -0
  179. package/dist/server-main.js.map +1 -0
  180. package/dist/utils/base-dirs.d.ts +212 -0
  181. package/dist/utils/base-dirs.d.ts.map +1 -0
  182. package/dist/utils/base-dirs.js +422 -0
  183. package/dist/utils/base-dirs.js.map +1 -0
  184. package/dist/utils/errors.d.ts +24 -0
  185. package/dist/utils/errors.d.ts.map +1 -0
  186. package/dist/utils/errors.js +53 -0
  187. package/dist/utils/errors.js.map +1 -0
  188. package/dist/utils/limits.d.ts +26 -0
  189. package/dist/utils/limits.d.ts.map +1 -0
  190. package/dist/utils/limits.js +28 -0
  191. package/dist/utils/limits.js.map +1 -0
  192. package/dist/utils/list-sources.d.ts +47 -0
  193. package/dist/utils/list-sources.d.ts.map +1 -0
  194. package/dist/utils/list-sources.js +50 -0
  195. package/dist/utils/list-sources.js.map +1 -0
  196. package/dist/utils/raw-data-utils.d.ts +131 -0
  197. package/dist/utils/raw-data-utils.d.ts.map +1 -0
  198. package/dist/utils/raw-data-utils.js +255 -0
  199. package/dist/utils/raw-data-utils.js.map +1 -0
  200. package/dist/utils/scan.d.ts +126 -0
  201. package/dist/utils/scan.d.ts.map +1 -0
  202. package/dist/utils/scan.js +221 -0
  203. package/dist/utils/scan.js.map +1 -0
  204. package/dist/utils/scope-match.d.ts +43 -0
  205. package/dist/utils/scope-match.d.ts.map +1 -0
  206. package/dist/utils/scope-match.js +87 -0
  207. package/dist/utils/scope-match.js.map +1 -0
  208. package/dist/utils/sensitive-path.d.ts +23 -0
  209. package/dist/utils/sensitive-path.d.ts.map +1 -0
  210. package/dist/utils/sensitive-path.js +91 -0
  211. package/dist/utils/sensitive-path.js.map +1 -0
  212. package/dist/utils/sync-path-key.d.ts +20 -0
  213. package/dist/utils/sync-path-key.d.ts.map +1 -0
  214. package/dist/utils/sync-path-key.js +33 -0
  215. package/dist/utils/sync-path-key.js.map +1 -0
  216. package/dist/vectordb/index.d.ts +168 -0
  217. package/dist/vectordb/index.d.ts.map +1 -0
  218. package/dist/vectordb/index.js +619 -0
  219. package/dist/vectordb/index.js.map +1 -0
  220. package/dist/vectordb/search-filters.d.ts +39 -0
  221. package/dist/vectordb/search-filters.d.ts.map +1 -0
  222. package/dist/vectordb/search-filters.js +136 -0
  223. package/dist/vectordb/search-filters.js.map +1 -0
  224. package/dist/vectordb/types.d.ts +196 -0
  225. package/dist/vectordb/types.d.ts.map +1 -0
  226. package/dist/vectordb/types.js +224 -0
  227. package/dist/vectordb/types.js.map +1 -0
  228. package/package.json +105 -0
  229. package/skills/mcp-local-rag/SKILL.md +308 -0
  230. package/skills/mcp-local-rag/references/cli-reference.md +175 -0
  231. package/skills/mcp-local-rag/references/html-ingestion.md +78 -0
  232. package/skills/mcp-local-rag/references/query-optimization.md +57 -0
  233. package/skills/mcp-local-rag/references/result-refinement.md +56 -0
@@ -0,0 +1,63 @@
1
+ // CLI entry point for subcommands (skills install, etc.)
2
+ import { run as runSkillsInstall } from './bin/install-skills.js';
3
+ import { runDelete } from './cli/delete.js';
4
+ import { runIngest } from './cli/ingest.js';
5
+ import { runList } from './cli/list.js';
6
+ import { runQuery } from './cli/query.js';
7
+ import { runReadNeighbors } from './cli/read-neighbors.js';
8
+ import { runStatus } from './cli/status.js';
9
+ import { runSync } from './cli/sync.js';
10
+ export const SUBCOMMANDS = [
11
+ 'skills',
12
+ 'ingest',
13
+ 'list',
14
+ 'query',
15
+ 'status',
16
+ 'delete',
17
+ 'read-neighbors',
18
+ 'sync',
19
+ ];
20
+ /**
21
+ * Handle CLI subcommands. The caller is expected to have already validated
22
+ * `subcommand` against `SUBCOMMANDS`; the union type makes the switch exhaustive.
23
+ * @param subcommand - The validated subcommand name
24
+ * @param args - Arguments following the subcommand (subcommand itself excluded)
25
+ * @param globalOptions - Global options parsed before the subcommand
26
+ */
27
+ export async function handleCli(subcommand, args, globalOptions = {}) {
28
+ switch (subcommand) {
29
+ case 'skills':
30
+ if (args[0] === 'install') {
31
+ runSkillsInstall(args.slice(1));
32
+ process.exit(0);
33
+ }
34
+ else {
35
+ console.error('Unknown skills subcommand. Usage: npx mcp-local-rag skills install [options]');
36
+ console.error('Run "npx mcp-local-rag skills install --help" for more information.');
37
+ process.exit(1);
38
+ }
39
+ break;
40
+ case 'ingest':
41
+ await runIngest(args, globalOptions);
42
+ break;
43
+ case 'list':
44
+ await runList(args, globalOptions);
45
+ break;
46
+ case 'query':
47
+ await runQuery(args, globalOptions);
48
+ break;
49
+ case 'status':
50
+ await runStatus(args, globalOptions);
51
+ break;
52
+ case 'delete':
53
+ await runDelete(args, globalOptions);
54
+ break;
55
+ case 'read-neighbors':
56
+ await runReadNeighbors(args, globalOptions);
57
+ break;
58
+ case 'sync':
59
+ await runSync(args, globalOptions);
60
+ break;
61
+ }
62
+ }
63
+ //# sourceMappingURL=cli-main.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"cli-main.js","sourceRoot":"","sources":["../src/cli-main.ts"],"names":[],"mappings":"AAAA,yDAAyD;AACzD,OAAO,EAAE,GAAG,IAAI,gBAAgB,EAAE,MAAM,yBAAyB,CAAA;AACjE,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAA;AAC3C,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAA;AAC3C,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAA;AAEvC,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAA;AACzC,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAA;AAC1D,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAA;AAC3C,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAA;AAEvC,MAAM,CAAC,MAAM,WAAW,GAAG;IACzB,QAAQ;IACR,QAAQ;IACR,MAAM;IACN,OAAO;IACP,QAAQ;IACR,QAAQ;IACR,gBAAgB;IAChB,MAAM;CACE,CAAA;AAIV;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,UAAsB,EACtB,IAAc,EACd,aAAa,GAAkB,EAAE;IAEjC,QAAQ,UAAU,EAAE,CAAC;QACnB,KAAK,QAAQ;YACX,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,SAAS,EAAE,CAAC;gBAC1B,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA;gBAC/B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACjB,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,KAAK,CACX,8EAA8E,CAC/E,CAAA;gBACD,OAAO,CAAC,KAAK,CAAC,qEAAqE,CAAC,CAAA;gBACpF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACjB,CAAC;YACD,MAAK;QAEP,KAAK,QAAQ;YACX,MAAM,SAAS,CAAC,IAAI,EAAE,aAAa,CAAC,CAAA;YACpC,MAAK;QAEP,KAAK,MAAM;YACT,MAAM,OAAO,CAAC,IAAI,EAAE,aAAa,CAAC,CAAA;YAClC,MAAK;QAEP,KAAK,OAAO;YACV,MAAM,QAAQ,CAAC,IAAI,EAAE,aAAa,CAAC,CAAA;YACnC,MAAK;QAEP,KAAK,QAAQ;YACX,MAAM,SAAS,CAAC,IAAI,EAAE,aAAa,CAAC,CAAA;YACpC,MAAK;QAEP,KAAK,QAAQ;YACX,MAAM,SAAS,CAAC,IAAI,EAAE,aAAa,CAAC,CAAA;YACpC,MAAK;QAEP,KAAK,gBAAgB;YACnB,MAAM,gBAAgB,CAAC,IAAI,EAAE,aAAa,CAAC,CAAA;YAC3C,MAAK;QAEP,KAAK,MAAM;YACT,MAAM,OAAO,CAAC,IAAI,EAAE,aAAa,CAAC,CAAA;YAClC,MAAK;IACT,CAAC;AACH,CAAC"}
@@ -0,0 +1,85 @@
1
+ import { AppError } from '../utils/errors.js';
2
+ /**
3
+ * Embedder configuration
4
+ */
5
+ export interface EmbedderConfig {
6
+ /** HuggingFace model path */
7
+ modelPath: string;
8
+ /** Batch size */
9
+ batchSize: number;
10
+ /** Model cache directory */
11
+ cacheDir: string;
12
+ /** Device type */
13
+ device?: string;
14
+ /**
15
+ * Embedding quantization dtype (fp32, fp16, q8, int8, ...). Passed through to
16
+ * transformers.js — no allowlist. Undefined means "unset": initialize() then
17
+ * applies the fp32 default. The unset-vs-explicit-fp32 distinction is
18
+ * preserved on purpose (it gates failure-path error enrichment).
19
+ */
20
+ dtype?: string;
21
+ }
22
+ /**
23
+ * Embedding generation error
24
+ */
25
+ export declare class EmbeddingError extends AppError {
26
+ constructor(message: string, cause?: Error);
27
+ }
28
+ /**
29
+ * Embedding generation class using Transformers.js
30
+ *
31
+ * Responsibilities:
32
+ * - Generate embedding vectors (dimension depends on model)
33
+ * - Transformers.js wrapper
34
+ * - Batch processing (size 8)
35
+ */
36
+ export declare class Embedder {
37
+ private model;
38
+ private initPromise;
39
+ private readonly config;
40
+ constructor(config: EmbedderConfig);
41
+ /**
42
+ * Release resources held by the Embedder pipeline
43
+ */
44
+ dispose(): Promise<void>;
45
+ /**
46
+ * Initialize Transformers.js model
47
+ */
48
+ initialize(): Promise<void>;
49
+ /**
50
+ * Best-effort failure-path enrichment for an explicit `RAG_DTYPE`.
51
+ *
52
+ * When the load failed and a dtype was explicitly requested, consult the
53
+ * model's available dtypes and, if the requested one is absent, return a
54
+ * message that names what the model provides. The enumeration is a Hub
55
+ * network call wrapped in its own try/catch: if it fails (e.g. air-gapped
56
+ * after caching) it degrades to a generic clear, dtype-aware message rather
57
+ * than surfacing a confusing secondary error (TD-3). This method never throws
58
+ * and never converts the load failure into a fallback — the caller always
59
+ * re-throws.
60
+ *
61
+ * @param nativeMessage - The underlying load-failure message.
62
+ * @returns The message to wrap in the thrown `EmbeddingError`.
63
+ */
64
+ private enrichDtypeFailureMessage;
65
+ /**
66
+ * Ensure model is initialized (lazy initialization)
67
+ * This method is called automatically by embed() and embedBatch()
68
+ */
69
+ private ensureInitialized;
70
+ /**
71
+ * Convert single text to embedding vector
72
+ *
73
+ * @param text - Text
74
+ * @returns Embedding vector (dimension depends on model)
75
+ */
76
+ embed(text: string): Promise<number[]>;
77
+ /**
78
+ * Convert multiple texts to embedding vectors with batch processing
79
+ *
80
+ * @param texts - Array of texts
81
+ * @returns Array of embedding vectors (dimension depends on model)
82
+ */
83
+ embedBatch(texts: string[]): Promise<number[][]>;
84
+ }
85
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/embedder/index.ts"],"names":[],"mappings":"AASA,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAA;AAM7C;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,6BAA6B;IAC7B,SAAS,EAAE,MAAM,CAAA;IACjB,iBAAiB;IACjB,SAAS,EAAE,MAAM,CAAA;IACjB,4BAA4B;IAC5B,QAAQ,EAAE,MAAM,CAAA;IAChB,kBAAkB;IAClB,MAAM,CAAC,EAAE,MAAM,CAAA;IACf;;;;;OAKG;IACH,KAAK,CAAC,EAAE,MAAM,CAAA;CACf;AAgED;;GAEG;AACH,qBAAa,cAAe,SAAQ,QAAQ;IAC1C,YAAY,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,KAAK,EAGzC;CACF;AAMD;;;;;;;GAOG;AACH,qBAAa,QAAQ;IAEnB,OAAO,CAAC,KAAK,CAAgB;IAC7B,OAAO,CAAC,WAAW,CAA6B;IAChD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgB;IAEvC,YAAY,MAAM,EAAE,cAAc,EAEjC;IAED;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAW7B;IAED;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC,CAkChC;IAED;;;;;;;;;;;;;;OAcG;YACW,yBAAyB;IAqBvC;;;OAGG;YACW,iBAAiB;IAyB/B;;;;;OAKG;IACG,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CA6B3C;IAED;;;;;OAKG;IACG,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAkGrD;CACF"}
@@ -0,0 +1,284 @@
1
+ // Embedder implementation with Transformers.js
2
+ import { env, ModelRegistry, pipeline, } from '@huggingface/transformers';
3
+ import { AppError } from '../utils/errors.js';
4
+ // Keep estimated padding waste below one-third of dense self-attention work.
5
+ const MAX_PADDING_AMPLIFICATION = 1.5;
6
+ function estimatePaddingAmplification(inputs) {
7
+ let maxTokenLength = 0;
8
+ let individualWork = 0;
9
+ for (const input of inputs) {
10
+ maxTokenLength = Math.max(maxTokenLength, input.tokenLength);
11
+ individualWork += input.tokenLength ** 2;
12
+ }
13
+ return (inputs.length * maxTokenLength ** 2) / individualWork;
14
+ }
15
+ function deferBatchOutliers(inputs) {
16
+ const batch = [...inputs];
17
+ const deferred = [];
18
+ while (batch.length > 1) {
19
+ if (estimatePaddingAmplification(batch) <= MAX_PADDING_AMPLIFICATION) {
20
+ break;
21
+ }
22
+ let longestIndex = 0;
23
+ for (let index = 1; index < batch.length; index++) {
24
+ if (batch[index].tokenLength > batch[longestIndex].tokenLength) {
25
+ longestIndex = index;
26
+ }
27
+ }
28
+ const longest = batch[longestIndex];
29
+ deferred.push(longest);
30
+ batch.splice(longestIndex, 1);
31
+ }
32
+ return { batch, deferred };
33
+ }
34
+ // ============================================
35
+ // Error Classes
36
+ // ============================================
37
+ /**
38
+ * Embedding generation error
39
+ */
40
+ export class EmbeddingError extends AppError {
41
+ constructor(message, cause) {
42
+ super(message, 'embedder', 'internal', cause);
43
+ this.name = 'EmbeddingError';
44
+ }
45
+ }
46
+ // ============================================
47
+ // Embedder Class
48
+ // ============================================
49
+ /**
50
+ * Embedding generation class using Transformers.js
51
+ *
52
+ * Responsibilities:
53
+ * - Generate embedding vectors (dimension depends on model)
54
+ * - Transformers.js wrapper
55
+ * - Batch processing (size 8)
56
+ */
57
+ export class Embedder {
58
+ // Using unknown to avoid TS2590 (union type too complex with @types/jsdom)
59
+ model = null;
60
+ initPromise = null;
61
+ config;
62
+ constructor(config) {
63
+ this.config = config;
64
+ }
65
+ /**
66
+ * Release resources held by the Embedder pipeline
67
+ */
68
+ async dispose() {
69
+ const model = this.model;
70
+ if (model && typeof model.dispose === 'function') {
71
+ try {
72
+ await model.dispose();
73
+ }
74
+ catch (error) {
75
+ console.error('Error disposing embedder model:', error);
76
+ }
77
+ }
78
+ this.model = null;
79
+ this.initPromise = null;
80
+ }
81
+ /**
82
+ * Initialize Transformers.js model
83
+ */
84
+ async initialize() {
85
+ // Skip if already initialized
86
+ if (this.model) {
87
+ return;
88
+ }
89
+ // Set cache directory BEFORE creating pipeline
90
+ env.cacheDir = this.config.cacheDir;
91
+ // No fallback — if the requested device fails, init throws.
92
+ const device = this.config.device || 'cpu';
93
+ console.error(`Embedder: Setting cache directory to "${this.config.cacheDir}"`);
94
+ console.error(`Embedder: Loading model "${this.config.modelPath}" on device "${device}"...`);
95
+ try {
96
+ this.model = await pipeline('feature-extraction', this.config.modelPath, {
97
+ // The sole fp32 default literal: unset dtype loads fp32, unchanged from
98
+ // before this knob existed. `as DataType` mirrors the `as DeviceType`
99
+ // cast below — a single typed pipeline boundary, no allowlist.
100
+ dtype: (this.config.dtype ?? 'fp32'),
101
+ device: device,
102
+ });
103
+ console.error(`Embedder: Model loaded successfully (device=${device})`);
104
+ }
105
+ catch (error) {
106
+ const nativeError = error;
107
+ // Only enrich when RAG_DTYPE was explicitly set (unset is `undefined` per
108
+ // TD-5). Enrichment never runs on the happy path and never on the unset
109
+ // path, so normal operation adds zero network. Always re-throw — an
110
+ // unavailable dtype fails loud, never silently downgrades (TD-2).
111
+ const message = await this.enrichDtypeFailureMessage(nativeError.message);
112
+ throw new EmbeddingError(message, nativeError);
113
+ }
114
+ }
115
+ /**
116
+ * Best-effort failure-path enrichment for an explicit `RAG_DTYPE`.
117
+ *
118
+ * When the load failed and a dtype was explicitly requested, consult the
119
+ * model's available dtypes and, if the requested one is absent, return a
120
+ * message that names what the model provides. The enumeration is a Hub
121
+ * network call wrapped in its own try/catch: if it fails (e.g. air-gapped
122
+ * after caching) it degrades to a generic clear, dtype-aware message rather
123
+ * than surfacing a confusing secondary error (TD-3). This method never throws
124
+ * and never converts the load failure into a fallback — the caller always
125
+ * re-throws.
126
+ *
127
+ * @param nativeMessage - The underlying load-failure message.
128
+ * @returns The message to wrap in the thrown `EmbeddingError`.
129
+ */
130
+ async enrichDtypeFailureMessage(nativeMessage) {
131
+ const requestedDtype = this.config.dtype;
132
+ if (requestedDtype === undefined) {
133
+ return nativeMessage;
134
+ }
135
+ try {
136
+ const availableDtypes = await ModelRegistry.get_available_dtypes(this.config.modelPath);
137
+ if (availableDtypes.includes(requestedDtype)) {
138
+ // The requested dtype exists for this model, so the load failed for some
139
+ // other reason — keep the native message, don't misattribute it to dtype.
140
+ return nativeMessage;
141
+ }
142
+ return `Model "${this.config.modelPath}" provides dtypes [${availableDtypes.join(', ')}]; requested dtype "${requestedDtype}" is unavailable. Set RAG_DTYPE to one of the available dtypes, or leave it unset for the fp32 default.`;
143
+ }
144
+ catch {
145
+ // Enumeration unavailable (e.g. offline). Degrade to a generic clear,
146
+ // dtype-aware message — no secondary error, still re-thrown by the caller.
147
+ return `Failed to load model "${this.config.modelPath}" with requested dtype "${requestedDtype}". The model may not provide this dtype, and the available-dtype list could not be retrieved. Set RAG_DTYPE to a dtype the model provides, or leave it unset for the fp32 default. (${nativeMessage})`;
148
+ }
149
+ }
150
+ /**
151
+ * Ensure model is initialized (lazy initialization)
152
+ * This method is called automatically by embed() and embedBatch()
153
+ */
154
+ async ensureInitialized() {
155
+ // Already initialized
156
+ if (this.model) {
157
+ return;
158
+ }
159
+ // Initialization already in progress, wait for it
160
+ if (this.initPromise) {
161
+ await this.initPromise;
162
+ return;
163
+ }
164
+ console.error('Embedder: First use detected. Initializing model (downloading ~90MB, may take 1-2 minutes)...');
165
+ this.initPromise = this.initialize().catch((error) => {
166
+ // Clear initPromise on failure to allow retry on the next call.
167
+ this.initPromise = null;
168
+ throw error;
169
+ });
170
+ await this.initPromise;
171
+ }
172
+ /**
173
+ * Convert single text to embedding vector
174
+ *
175
+ * @param text - Text
176
+ * @returns Embedding vector (dimension depends on model)
177
+ */
178
+ async embed(text) {
179
+ // Reject empty input before paying for model init.
180
+ if (text.length === 0) {
181
+ throw new EmbeddingError('Cannot generate embedding for empty text');
182
+ }
183
+ // Lazy initialization: initialize on first use if not already initialized
184
+ await this.ensureInitialized();
185
+ try {
186
+ const options = { pooling: 'mean', normalize: true };
187
+ const modelCall = this.model;
188
+ const output = await modelCall(text, options);
189
+ // Access raw data via .data property
190
+ const embedding = Array.from(output.data);
191
+ return embedding;
192
+ }
193
+ catch (error) {
194
+ if (error instanceof EmbeddingError) {
195
+ throw error;
196
+ }
197
+ throw new EmbeddingError(`Failed to generate embedding: ${error.message}`, error);
198
+ }
199
+ }
200
+ /**
201
+ * Convert multiple texts to embedding vectors with batch processing
202
+ *
203
+ * @param texts - Array of texts
204
+ * @returns Array of embedding vectors (dimension depends on model)
205
+ */
206
+ async embedBatch(texts) {
207
+ // Nothing to embed → skip model init entirely.
208
+ if (texts.length === 0) {
209
+ return [];
210
+ }
211
+ // Preserve embed()'s empty-text contract for batch elements (the previous
212
+ // per-text implementation rejected empty strings via embed()).
213
+ if (texts.some((text) => text.length === 0)) {
214
+ throw new EmbeddingError('Cannot generate embedding for empty text');
215
+ }
216
+ // Lazy initialization: initialize on first use if not already initialized
217
+ await this.ensureInitialized();
218
+ try {
219
+ const options = { pooling: 'mean', normalize: true };
220
+ // True batched inference: the feature-extraction pipeline accepts an
221
+ // array of texts and returns a single [batchLen, dim] tensor in one
222
+ // forward pass. The previous implementation called the model once per
223
+ // text via Promise.all, so `batchSize` had no real effect (onnxruntime
224
+ // inference is not parallelized by Promise.all). Passing the whole batch
225
+ // lets the runtime batch the matmuls. Mean-pooling honors the attention
226
+ // mask, so per-row vectors match the single-text result.
227
+ const modelCall = this.model;
228
+ const embeddings = Array.from({ length: texts.length });
229
+ const deferred = [];
230
+ const embedInputs = async (inputs) => {
231
+ const output = await modelCall(inputs.map((input) => input.text), options);
232
+ // Validate the output shape before slicing so a runtime/model contract
233
+ // change surfaces as a clear error rather than silently wrong vectors.
234
+ const dim = output?.dims?.[output.dims.length - 1];
235
+ if (!output ||
236
+ !(output.data instanceof Float32Array) ||
237
+ typeof dim !== 'number' ||
238
+ dim <= 0 ||
239
+ output.data.length !== inputs.length * dim) {
240
+ throw new EmbeddingError('Unexpected embedder batch output shape');
241
+ }
242
+ for (let row = 0; row < inputs.length; row++) {
243
+ const input = inputs[row];
244
+ embeddings[input.originalIndex] = Array.from(output.data.subarray(row * dim, (row + 1) * dim));
245
+ }
246
+ };
247
+ for (let i = 0; i < texts.length; i += this.config.batchSize) {
248
+ const batchTexts = texts.slice(i, i + this.config.batchSize);
249
+ const tokenized = modelCall.tokenizer(batchTexts, {
250
+ padding: false,
251
+ truncation: true,
252
+ return_tensor: false,
253
+ });
254
+ if (!tokenized ||
255
+ !Array.isArray(tokenized.input_ids) ||
256
+ tokenized.input_ids.length !== batchTexts.length) {
257
+ throw new EmbeddingError('Unexpected embedder tokenizer output shape');
258
+ }
259
+ const indexedInputs = batchTexts.map((text, batchIndex) => ({
260
+ text,
261
+ originalIndex: i + batchIndex,
262
+ tokenLength: tokenized.input_ids[batchIndex].length,
263
+ }));
264
+ const selected = deferBatchOutliers(indexedInputs);
265
+ deferred.push(...selected.deferred);
266
+ await embedInputs(selected.batch);
267
+ }
268
+ for (const input of deferred) {
269
+ await embedInputs([input]);
270
+ }
271
+ if (embeddings.some((embedding) => embedding === undefined)) {
272
+ throw new EmbeddingError('Missing embedder batch output row');
273
+ }
274
+ return embeddings;
275
+ }
276
+ catch (error) {
277
+ if (error instanceof EmbeddingError) {
278
+ throw error;
279
+ }
280
+ throw new EmbeddingError(`Failed to generate batch embeddings: ${error.message}`, error);
281
+ }
282
+ }
283
+ }
284
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/embedder/index.ts"],"names":[],"mappings":"AAAA,+CAA+C;AAE/C,OAAO,EAGL,GAAG,EACH,aAAa,EACb,QAAQ,GACT,MAAM,2BAA2B,CAAA;AAClC,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAA;AA6C7C,6EAA6E;AAC7E,MAAM,yBAAyB,GAAG,GAAG,CAAA;AAErC,SAAS,4BAA4B,CAAC,MAA+B;IACnE,IAAI,cAAc,GAAG,CAAC,CAAA;IACtB,IAAI,cAAc,GAAG,CAAC,CAAA;IACtB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,cAAc,EAAE,KAAK,CAAC,WAAW,CAAC,CAAA;QAC5D,cAAc,IAAI,KAAK,CAAC,WAAW,IAAI,CAAC,CAAA;IAC1C,CAAC;IACD,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,cAAc,IAAI,CAAC,CAAC,GAAG,cAAc,CAAA;AAC/D,CAAC;AAED,SAAS,kBAAkB,CAAC,MAA+B;IAIzD,MAAM,KAAK,GAAG,CAAC,GAAG,MAAM,CAAC,CAAA;IACzB,MAAM,QAAQ,GAA4B,EAAE,CAAA;IAE5C,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,IAAI,4BAA4B,CAAC,KAAK,CAAC,IAAI,yBAAyB,EAAE,CAAC;YACrE,MAAK;QACP,CAAC;QAED,IAAI,YAAY,GAAG,CAAC,CAAA;QACpB,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;YAClD,IAAI,KAAK,CAAC,KAAK,CAAE,CAAC,WAAW,GAAG,KAAK,CAAC,YAAY,CAAE,CAAC,WAAW,EAAE,CAAC;gBACjE,YAAY,GAAG,KAAK,CAAA;YACtB,CAAC;QACH,CAAC;QAED,MAAM,OAAO,GAAG,KAAK,CAAC,YAAY,CAAE,CAAA;QACpC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;QACtB,KAAK,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC,CAAC,CAAA;IAC/B,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAA;AAC5B,CAAC;AAED,+CAA+C;AAC/C,gBAAgB;AAChB,+CAA+C;AAE/C;;GAEG;AACH,MAAM,OAAO,cAAe,SAAQ,QAAQ;IAC1C,YAAY,OAAe,EAAE,KAAa;QACxC,KAAK,CAAC,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,KAAK,CAAC,CAAA;QAC7C,IAAI,CAAC,IAAI,GAAG,gBAAgB,CAAA;IAC9B,CAAC;CACF;AAED,+CAA+C;AAC/C,iBAAiB;AACjB,+CAA+C;AAE/C;;;;;;;GAOG;AACH,MAAM,OAAO,QAAQ;IACnB,2EAA2E;IACnE,KAAK,GAAY,IAAI,CAAA;IACrB,WAAW,GAAyB,IAAI,CAAA;IAC/B,MAAM,CAAgB;IAEvC,YAAY,MAAsB;QAChC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAA;IACtB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,OAAO;QACX,MAAM,KAAK,GAAG,IAAI,CAAC,KAAiD,CAAA;QACpE,IAAI,KAAK,IAAI,OAAO,KAAK,CAAC,OAAO,KAAK,UAAU,EAAE,CAAC;YACjD,IAAI,CAAC;gBACH,MAAM,KAAK,CAAC,OAAO,EAAE,CAAA;YACvB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,KAAK,CAAC,iCAAiC,EAAE,KAAK,CAAC,CAAA;YACzD,CAAC;QACH,CAAC;QACD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAA;QACjB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAA;IACzB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU;QACd,8BAA8B;QAC9B,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,OAAM;QACR,CAAC;QAED,+CAA+C;QAC/C,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAA;QAEnC,4DAA4D;QAC5D,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,KAAK,CAAA;QAE1C,OAAO,CAAC,KAAK,CAAC,yCAAyC,IAAI,CAAC,MAAM,CAAC,QAAQ,GAAG,CAAC,CAAA;QAC/E,OAAO,CAAC,KAAK,CAAC,4BAA4B,IAAI,CAAC,MAAM,CAAC,SAAS,gBAAgB,MAAM,MAAM,CAAC,CAAA;QAE5F,IAAI,CAAC;YACH,IAAI,CAAC,KAAK,GAAG,MAAM,QAAQ,CAAC,oBAAoB,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE;gBACvE,wEAAwE;gBACxE,sEAAsE;gBACtE,+DAA+D;gBAC/D,KAAK,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,MAAM,CAAa;gBAChD,MAAM,EAAE,MAAoB;aAC7B,CAAC,CAAA;YACF,OAAO,CAAC,KAAK,CAAC,+CAA+C,MAAM,GAAG,CAAC,CAAA;QACzE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,WAAW,GAAG,KAAc,CAAA;YAElC,0EAA0E;YAC1E,wEAAwE;YACxE,oEAAoE;YACpE,kEAAkE;YAClE,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,yBAAyB,CAAC,WAAW,CAAC,OAAO,CAAC,CAAA;YACzE,MAAM,IAAI,cAAc,CAAC,OAAO,EAAE,WAAW,CAAC,CAAA;QAChD,CAAC;IACH,CAAC;IAED;;;;;;;;;;;;;;OAcG;IACK,KAAK,CAAC,yBAAyB,CAAC,aAAqB;QAC3D,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAA;QACxC,IAAI,cAAc,KAAK,SAAS,EAAE,CAAC;YACjC,OAAO,aAAa,CAAA;QACtB,CAAC;QAED,IAAI,CAAC;YACH,MAAM,eAAe,GAAG,MAAM,aAAa,CAAC,oBAAoB,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;YACvF,IAAI,eAAe,CAAC,QAAQ,CAAC,cAAc,CAAC,EAAE,CAAC;gBAC7C,yEAAyE;gBACzE,0EAA0E;gBAC1E,OAAO,aAAa,CAAA;YACtB,CAAC;YACD,OAAO,UAAU,IAAI,CAAC,MAAM,CAAC,SAAS,sBAAsB,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,uBAAuB,cAAc,yGAAyG,CAAA;QACtO,CAAC;QAAC,MAAM,CAAC;YACP,sEAAsE;YACtE,2EAA2E;YAC3E,OAAO,yBAAyB,IAAI,CAAC,MAAM,CAAC,SAAS,2BAA2B,cAAc,uLAAuL,aAAa,GAAG,CAAA;QACvS,CAAC;IACH,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,iBAAiB;QAC7B,sBAAsB;QACtB,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,OAAM;QACR,CAAC;QAED,kDAAkD;QAClD,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,MAAM,IAAI,CAAC,WAAW,CAAA;YACtB,OAAM;QACR,CAAC;QAED,OAAO,CAAC,KAAK,CACX,+FAA+F,CAChG,CAAA;QAED,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;YACnD,gEAAgE;YAChE,IAAI,CAAC,WAAW,GAAG,IAAI,CAAA;YACvB,MAAM,KAAK,CAAA;QACb,CAAC,CAAC,CAAA;QAEF,MAAM,IAAI,CAAC,WAAW,CAAA;IACxB,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,KAAK,CAAC,IAAY;QACtB,mDAAmD;QACnD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,MAAM,IAAI,cAAc,CAAC,0CAA0C,CAAC,CAAA;QACtE,CAAC;QAED,0EAA0E;QAC1E,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAA;QAE9B,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAA;YACpD,MAAM,SAAS,GAAG,IAAI,CAAC,KAGa,CAAA;YACpC,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;YAE7C,qCAAqC;YACrC,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YACzC,OAAO,SAAS,CAAA;QAClB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,cAAc,EAAE,CAAC;gBACpC,MAAM,KAAK,CAAA;YACb,CAAC;YACD,MAAM,IAAI,cAAc,CACtB,iCAAkC,KAAe,CAAC,OAAO,EAAE,EAC3D,KAAc,CACf,CAAA;QACH,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,UAAU,CAAC,KAAe;QAC9B,+CAA+C;QAC/C,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,EAAE,CAAA;QACX,CAAC;QAED,0EAA0E;QAC1E,+DAA+D;QAC/D,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC,EAAE,CAAC;YAC5C,MAAM,IAAI,cAAc,CAAC,0CAA0C,CAAC,CAAA;QACtE,CAAC;QAED,0EAA0E;QAC1E,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAA;QAE9B,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAA;YACpD,qEAAqE;YACrE,oEAAoE;YACpE,sEAAsE;YACtE,uEAAuE;YACvE,yEAAyE;YACzE,wEAAwE;YACxE,yDAAyD;YACzD,MAAM,SAAS,GAAG,IAAI,CAAC,KAA+B,CAAA;YACtD,MAAM,UAAU,GAA6B,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAA;YACjF,MAAM,QAAQ,GAA4B,EAAE,CAAA;YAE5C,MAAM,WAAW,GAAG,KAAK,EAAE,MAA+B,EAAiB,EAAE;gBAC3E,MAAM,MAAM,GAAG,MAAM,SAAS,CAC5B,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,EACjC,OAAO,CACR,CAAA;gBAED,uEAAuE;gBACvE,uEAAuE;gBACvE,MAAM,GAAG,GAAG,MAAM,EAAE,IAAI,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;gBAClD,IACE,CAAC,MAAM;oBACP,CAAC,CAAC,MAAM,CAAC,IAAI,YAAY,YAAY,CAAC;oBACtC,OAAO,GAAG,KAAK,QAAQ;oBACvB,GAAG,IAAI,CAAC;oBACR,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,GAAG,GAAG,EAC1C,CAAC;oBACD,MAAM,IAAI,cAAc,CAAC,wCAAwC,CAAC,CAAA;gBACpE,CAAC;gBAED,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE,CAAC;oBAC7C,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAE,CAAA;oBAC1B,UAAU,CAAC,KAAK,CAAC,aAAa,CAAC,GAAG,KAAK,CAAC,IAAI,CAC1C,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,GAAG,GAAG,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CACjD,CAAA;gBACH,CAAC;YACH,CAAC,CAAA;YAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;gBAC7D,MAAM,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;gBAC5D,MAAM,SAAS,GAAG,SAAS,CAAC,SAAS,CAAC,UAAU,EAAE;oBAChD,OAAO,EAAE,KAAK;oBACd,UAAU,EAAE,IAAI;oBAChB,aAAa,EAAE,KAAK;iBACrB,CAAC,CAAA;gBACF,IACE,CAAC,SAAS;oBACV,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC;oBACnC,SAAS,CAAC,SAAS,CAAC,MAAM,KAAK,UAAU,CAAC,MAAM,EAChD,CAAC;oBACD,MAAM,IAAI,cAAc,CAAC,4CAA4C,CAAC,CAAA;gBACxE,CAAC;gBAED,MAAM,aAAa,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,UAAU,EAAE,EAAE,CAAC,CAAC;oBAC1D,IAAI;oBACJ,aAAa,EAAE,CAAC,GAAG,UAAU;oBAC7B,WAAW,EAAE,SAAS,CAAC,SAAS,CAAC,UAAU,CAAE,CAAC,MAAM;iBACrD,CAAC,CAAC,CAAA;gBACH,MAAM,QAAQ,GAAG,kBAAkB,CAAC,aAAa,CAAC,CAAA;gBAClD,QAAQ,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAA;gBACnC,MAAM,WAAW,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAA;YACnC,CAAC;YAED,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;gBAC7B,MAAM,WAAW,CAAC,CAAC,KAAK,CAAC,CAAC,CAAA;YAC5B,CAAC;YAED,IAAI,UAAU,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,KAAK,SAAS,CAAC,EAAE,CAAC;gBAC5D,MAAM,IAAI,cAAc,CAAC,mCAAmC,CAAC,CAAA;YAC/D,CAAC;YAED,OAAO,UAAwB,CAAA;QACjC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,cAAc,EAAE,CAAC;gBACpC,MAAM,KAAK,CAAA;YACb,CAAC;YACD,MAAM,IAAI,cAAc,CACtB,wCAAyC,KAAe,CAAC,OAAO,EAAE,EAClE,KAAc,CACf,CAAA;QACH,CAAC;IACH,CAAC;CACF"}
@@ -0,0 +1,37 @@
1
+ import { type ClassifiedSource } from '../utils/list-sources.js';
2
+ export interface IngestedFileSummary {
3
+ filePath: string;
4
+ chunkCount: number;
5
+ timestamp: string;
6
+ }
7
+ export type ListedFile = {
8
+ filePath: string;
9
+ baseDir: string;
10
+ ingested: true;
11
+ chunkCount: number;
12
+ timestamp: string;
13
+ } | {
14
+ filePath: string;
15
+ baseDir: string;
16
+ ingested: false;
17
+ };
18
+ export interface RootScanResult {
19
+ files: string[];
20
+ warnings: string[];
21
+ }
22
+ export interface ListDocumentsResult {
23
+ files: ListedFile[];
24
+ sources: ClassifiedSource[];
25
+ warnings: Array<{
26
+ baseDir: string;
27
+ message: string;
28
+ }>;
29
+ }
30
+ export declare function listDocuments(input: {
31
+ roots: readonly string[];
32
+ dbPath: string;
33
+ ingested: IngestedFileSummary[];
34
+ scope?: string[] | undefined;
35
+ scan: (root: string, scope?: string[] | undefined) => Promise<RootScanResult>;
36
+ }): Promise<ListDocumentsResult>;
37
+ //# sourceMappingURL=list.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"list.d.ts","sourceRoot":"","sources":["../../src/features/list.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,gBAAgB,EAA2B,MAAM,0BAA0B,CAAA;AAGzF,MAAM,WAAW,mBAAmB;IAClC,QAAQ,EAAE,MAAM,CAAA;IAChB,UAAU,EAAE,MAAM,CAAA;IAClB,SAAS,EAAE,MAAM,CAAA;CAClB;AAED,MAAM,MAAM,UAAU,GAClB;IACE,QAAQ,EAAE,MAAM,CAAA;IAChB,OAAO,EAAE,MAAM,CAAA;IACf,QAAQ,EAAE,IAAI,CAAA;IACd,UAAU,EAAE,MAAM,CAAA;IAClB,SAAS,EAAE,MAAM,CAAA;CAClB,GACD;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,KAAK,CAAA;CAAE,CAAA;AAE1D,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,MAAM,EAAE,CAAA;IACf,QAAQ,EAAE,MAAM,EAAE,CAAA;CACnB;AAED,MAAM,WAAW,mBAAmB;IAClC,KAAK,EAAE,UAAU,EAAE,CAAA;IACnB,OAAO,EAAE,gBAAgB,EAAE,CAAA;IAC3B,QAAQ,EAAE,KAAK,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;CACtD;AAED,wBAAsB,aAAa,CAAC,KAAK,EAAE;IACzC,KAAK,EAAE,SAAS,MAAM,EAAE,CAAA;IACxB,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,mBAAmB,EAAE,CAAA;IAC/B,KAAK,CAAC,EAAE,MAAM,EAAE,GAAG,SAAS,CAAA;IAC5B,IAAI,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,EAAE,GAAG,SAAS,KAAK,OAAO,CAAC,cAAc,CAAC,CAAA;CAC9E,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAwC/B"}
@@ -0,0 +1,40 @@
1
+ import { classifyIngestedSources } from '../utils/list-sources.js';
2
+ import { realpathForMatch } from '../utils/scan.js';
3
+ export async function listDocuments(input) {
4
+ const ingestedKeyed = await Promise.all(input.ingested.map(async (entry) => ({ entry, key: await realpathForMatch(entry.filePath) })));
5
+ const ingestedByKey = new Map(ingestedKeyed.map(({ entry, key }) => [key, entry]));
6
+ const seenKeys = new Set();
7
+ const matchedKeys = new Set();
8
+ const files = [];
9
+ const warnings = [];
10
+ for (const baseDir of input.roots) {
11
+ const scanned = await input.scan(baseDir, input.scope);
12
+ warnings.push(...scanned.warnings.map((message) => ({ baseDir, message })));
13
+ for (const scannedPath of scanned.files) {
14
+ const key = await realpathForMatch(scannedPath);
15
+ if (seenKeys.has(key))
16
+ continue;
17
+ seenKeys.add(key);
18
+ const entry = ingestedByKey.get(key);
19
+ if (entry) {
20
+ matchedKeys.add(key);
21
+ files.push({
22
+ filePath: entry.filePath,
23
+ baseDir,
24
+ ingested: true,
25
+ chunkCount: entry.chunkCount,
26
+ timestamp: entry.timestamp,
27
+ });
28
+ }
29
+ else {
30
+ files.push({ filePath: scannedPath, baseDir, ingested: false });
31
+ }
32
+ }
33
+ }
34
+ return {
35
+ files,
36
+ sources: classifyIngestedSources(ingestedKeyed, matchedKeys, input.dbPath, input.scope),
37
+ warnings,
38
+ };
39
+ }
40
+ //# sourceMappingURL=list.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"list.js","sourceRoot":"","sources":["../../src/features/list.ts"],"names":[],"mappings":"AAAA,OAAO,EAAyB,uBAAuB,EAAE,MAAM,0BAA0B,CAAA;AACzF,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AA6BnD,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,KAMnC;IACC,MAAM,aAAa,GAAG,MAAM,OAAO,CAAC,GAAG,CACrC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,gBAAgB,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAC9F,CAAA;IACD,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,CAAA;IAClF,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAA;IAClC,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAA;IACrC,MAAM,KAAK,GAAiB,EAAE,CAAA;IAC9B,MAAM,QAAQ,GAAgD,EAAE,CAAA;IAEhE,KAAK,MAAM,OAAO,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;QAClC,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,KAAK,CAAC,CAAA;QACtD,QAAQ,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,CAAA;QAE3E,KAAK,MAAM,WAAW,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;YACxC,MAAM,GAAG,GAAG,MAAM,gBAAgB,CAAC,WAAW,CAAC,CAAA;YAC/C,IAAI,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,SAAQ;YAC/B,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YAEjB,MAAM,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YACpC,IAAI,KAAK,EAAE,CAAC;gBACV,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;gBACpB,KAAK,CAAC,IAAI,CAAC;oBACT,QAAQ,EAAE,KAAK,CAAC,QAAQ;oBACxB,OAAO;oBACP,QAAQ,EAAE,IAAI;oBACd,UAAU,EAAE,KAAK,CAAC,UAAU;oBAC5B,SAAS,EAAE,KAAK,CAAC,SAAS;iBAC3B,CAAC,CAAA;YACJ,CAAC;iBAAM,CAAC;gBACN,KAAK,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,WAAW,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAA;YACjE,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO;QACL,KAAK;QACL,OAAO,EAAE,uBAAuB,CAAC,aAAa,EAAE,WAAW,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC;QACvF,QAAQ;KACT,CAAA;AACH,CAAC"}