mcp-local-rag 0.14.2 → 0.15.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -0
- package/dist/cli/common.d.ts +14 -0
- package/dist/cli/common.d.ts.map +1 -1
- package/dist/cli/common.js +34 -3
- package/dist/cli/common.js.map +1 -1
- package/dist/cli/delete.d.ts.map +1 -1
- package/dist/cli/delete.js +4 -3
- package/dist/cli/delete.js.map +1 -1
- package/dist/cli/file-collection.d.ts +2 -0
- package/dist/cli/file-collection.d.ts.map +1 -0
- package/dist/cli/file-collection.js +53 -0
- package/dist/cli/file-collection.js.map +1 -0
- package/dist/cli/ingest.d.ts +3 -5
- package/dist/cli/ingest.d.ts.map +1 -1
- package/dist/cli/ingest.js +36 -164
- package/dist/cli/ingest.js.map +1 -1
- package/dist/cli/list.d.ts.map +1 -1
- package/dist/cli/list.js +63 -112
- package/dist/cli/list.js.map +1 -1
- package/dist/cli/options.d.ts +21 -2
- package/dist/cli/options.d.ts.map +1 -1
- package/dist/cli/options.js +41 -25
- package/dist/cli/options.js.map +1 -1
- package/dist/cli/query.d.ts.map +1 -1
- package/dist/cli/query.js +5 -9
- package/dist/cli/query.js.map +1 -1
- package/dist/cli/read-neighbors.d.ts.map +1 -1
- package/dist/cli/read-neighbors.js +6 -5
- package/dist/cli/read-neighbors.js.map +1 -1
- package/dist/cli/status.js +2 -2
- package/dist/cli/status.js.map +1 -1
- package/dist/embedder/index.d.ts +26 -3
- package/dist/embedder/index.d.ts.map +1 -1
- package/dist/embedder/index.js +78 -13
- package/dist/embedder/index.js.map +1 -1
- package/dist/ingest/compute.d.ts +24 -0
- package/dist/ingest/compute.d.ts.map +1 -1
- package/dist/ingest/compute.js +48 -9
- package/dist/ingest/compute.js.map +1 -1
- package/dist/ingest/visual.d.ts +1 -1
- package/dist/ingest/visual.d.ts.map +1 -1
- package/dist/ingest/visual.js +9 -15
- package/dist/ingest/visual.js.map +1 -1
- package/dist/parser/index.d.ts +12 -10
- package/dist/parser/index.d.ts.map +1 -1
- package/dist/parser/index.js +43 -94
- package/dist/parser/index.js.map +1 -1
- package/dist/parser/pdf-extract.d.ts +74 -0
- package/dist/parser/pdf-extract.d.ts.map +1 -0
- package/dist/parser/pdf-extract.js +87 -0
- package/dist/parser/pdf-extract.js.map +1 -0
- package/dist/parser/pdf-filter.d.ts +0 -21
- package/dist/parser/pdf-filter.d.ts.map +1 -1
- package/dist/parser/pdf-filter.js +63 -36
- package/dist/parser/pdf-filter.js.map +1 -1
- package/dist/parser/title-extractor.d.ts.map +1 -1
- package/dist/parser/title-extractor.js +7 -2
- package/dist/parser/title-extractor.js.map +1 -1
- package/dist/pdf-visual/captioner.d.ts +0 -5
- package/dist/pdf-visual/captioner.d.ts.map +1 -1
- package/dist/pdf-visual/captioner.js +0 -8
- package/dist/pdf-visual/captioner.js.map +1 -1
- package/dist/pdf-visual/captioners/fast.d.ts.map +1 -1
- package/dist/pdf-visual/captioners/fast.js +20 -46
- package/dist/pdf-visual/captioners/fast.js.map +1 -1
- package/dist/pdf-visual/captioners/quality.d.ts.map +1 -1
- package/dist/pdf-visual/captioners/quality.js +22 -45
- package/dist/pdf-visual/captioners/quality.js.map +1 -1
- package/dist/pdf-visual/captioners/shared.d.ts +36 -3
- package/dist/pdf-visual/captioners/shared.d.ts.map +1 -1
- package/dist/pdf-visual/captioners/shared.js +67 -3
- package/dist/pdf-visual/captioners/shared.js.map +1 -1
- package/dist/pdf-visual/detector.d.ts +2 -4
- package/dist/pdf-visual/detector.d.ts.map +1 -1
- package/dist/pdf-visual/detector.js.map +1 -1
- package/dist/pdf-visual/index.d.ts.map +1 -1
- package/dist/pdf-visual/index.js +13 -15
- package/dist/pdf-visual/index.js.map +1 -1
- package/dist/pdf-visual/renderer.d.ts.map +1 -1
- package/dist/pdf-visual/renderer.js +2 -7
- package/dist/pdf-visual/renderer.js.map +1 -1
- package/dist/pdf-visual/types.d.ts +2 -2
- package/dist/pdf-visual/types.d.ts.map +1 -1
- package/dist/pdf-visual/types.js +9 -13
- package/dist/pdf-visual/types.js.map +1 -1
- package/dist/server/error-utils.d.ts +44 -18
- package/dist/server/error-utils.d.ts.map +1 -1
- package/dist/server/error-utils.js +74 -15
- package/dist/server/error-utils.js.map +1 -1
- package/dist/server/index.d.ts +27 -51
- package/dist/server/index.d.ts.map +1 -1
- package/dist/server/index.js +524 -627
- package/dist/server/index.js.map +1 -1
- package/dist/server/list-scanner.d.ts +44 -0
- package/dist/server/list-scanner.d.ts.map +1 -0
- package/dist/server/list-scanner.js +100 -0
- package/dist/server/list-scanner.js.map +1 -0
- package/dist/server/tool-definitions.d.ts.map +1 -1
- package/dist/server/tool-definitions.js +4 -2
- package/dist/server/tool-definitions.js.map +1 -1
- package/dist/server/tool-input.d.ts +14 -0
- package/dist/server/tool-input.d.ts.map +1 -0
- package/dist/server/tool-input.js +62 -0
- package/dist/server/tool-input.js.map +1 -0
- package/dist/server/types.d.ts +27 -17
- package/dist/server/types.d.ts.map +1 -1
- package/dist/server-main.d.ts +11 -0
- package/dist/server-main.d.ts.map +1 -1
- package/dist/server-main.js +125 -133
- package/dist/server-main.js.map +1 -1
- package/dist/utils/base-dirs.d.ts +19 -10
- package/dist/utils/base-dirs.d.ts.map +1 -1
- package/dist/utils/base-dirs.js +34 -19
- package/dist/utils/base-dirs.js.map +1 -1
- package/dist/utils/errors.d.ts +24 -0
- package/dist/utils/errors.d.ts.map +1 -0
- package/dist/utils/errors.js +53 -0
- package/dist/utils/errors.js.map +1 -0
- package/dist/utils/limits.d.ts +17 -0
- package/dist/utils/limits.d.ts.map +1 -0
- package/dist/utils/limits.js +19 -0
- package/dist/utils/limits.js.map +1 -0
- package/dist/utils/scan.d.ts +35 -0
- package/dist/utils/scan.d.ts.map +1 -0
- package/dist/utils/scan.js +84 -0
- package/dist/utils/scan.js.map +1 -0
- package/dist/vectordb/index.d.ts +14 -8
- package/dist/vectordb/index.d.ts.map +1 -1
- package/dist/vectordb/index.js +68 -34
- package/dist/vectordb/index.js.map +1 -1
- package/dist/vectordb/search-filters.d.ts +0 -6
- package/dist/vectordb/search-filters.d.ts.map +1 -1
- package/dist/vectordb/search-filters.js +1 -1
- package/dist/vectordb/search-filters.js.map +1 -1
- package/dist/vectordb/types.d.ts +18 -14
- package/dist/vectordb/types.d.ts.map +1 -1
- package/dist/vectordb/types.js +52 -8
- package/dist/vectordb/types.js.map +1 -1
- package/package.json +10 -9
- package/dist/cli/sync.d.ts +0 -3
- package/dist/cli/sync.d.ts.map +0 -1
- package/dist/cli/sync.js +0 -81
- package/dist/cli/sync.js.map +0 -1
- package/dist/utils/sync-runner.d.ts +0 -68
- package/dist/utils/sync-runner.d.ts.map +0 -1
- package/dist/utils/sync-runner.js +0 -73
- package/dist/utils/sync-runner.js.map +0 -1
- package/dist/utils/sync-utils.d.ts +0 -26
- package/dist/utils/sync-utils.d.ts.map +0 -1
- package/dist/utils/sync-utils.js +0 -39
- package/dist/utils/sync-utils.js.map +0 -1
package/README.md
CHANGED
|
@@ -383,6 +383,7 @@ The MCP server is configured by environment variables only — pass them through
|
|
|
383
383
|
| `MAX_FILE_SIZE` | `--max-file-size` | `104857600` (100MB) | Maximum file size in bytes |
|
|
384
384
|
| `CHUNK_MIN_LENGTH` | `--chunk-min-length` | `50` | Minimum chunk length in characters (1–10000) |
|
|
385
385
|
| `RAG_DEVICE` | — | `cpu` | Execution device. Passed straight to ONNX Runtime. See the [Transformers.js device source code](https://github.com/huggingface/transformers.js/blob/main/packages/transformers/src/utils/devices.js) for the live list of supported backend names. If initialization fails, the server throws an error. |
|
|
386
|
+
| `RAG_DTYPE` | — | `fp32` | Embedding quantization dtype. Opt-in and passed straight through; accepts any dtype the chosen model provides (`fp32`, `fp16`, `q8`, `int8`, …). If the model lacks the requested variant, the server throws an error naming the dtypes it does provide. Changing `RAG_DEVICE`/`RAG_DTYPE` changes the embedding space — re-ingest existing data. |
|
|
386
387
|
|
|
387
388
|
**Model choice tips:**
|
|
388
389
|
- Multilingual docs → e.g., `onnx-community/embeddinggemma-300m-ONNX` (100+ languages)
|
|
@@ -607,6 +608,9 @@ Yes, but you must delete your database and re-ingest all documents. Different mo
|
|
|
607
608
|
**GPU acceleration?**
|
|
608
609
|
Opt-in via `RAG_DEVICE`. Devices are passed straight to ONNX Runtime. GPU support is highly dependent on your system, Node.js version, and the underlying ONNX backend. See the [Transformers.js device source code](https://github.com/huggingface/transformers.js/blob/main/packages/transformers/src/utils/devices.js) for the live list of supported backend names. If the requested device fails to initialize, the server throws an error — set `RAG_DEVICE=cpu` to revert.
|
|
609
610
|
|
|
611
|
+
**Can I change the embedding precision (dtype)?**
|
|
612
|
+
Opt-in via `RAG_DTYPE` (default `fp32`); accepted values are in the env-var table above. A recognized dtype the model lacks errors and lists the available ones; an unrecognized value (a typo) silently falls back to `fp32`. Changing `RAG_DEVICE`/`RAG_DTYPE` changes the embedding space — delete `DB_PATH` and re-ingest.
|
|
613
|
+
|
|
610
614
|
**Multi-user support?**
|
|
611
615
|
No. Designed for single-user, local access. Multi-user would require authentication/access control.
|
|
612
616
|
|
package/dist/cli/common.d.ts
CHANGED
|
@@ -2,6 +2,20 @@ import { Embedder } from '../embedder/index.js';
|
|
|
2
2
|
import { type BaseDirsConfig, type BaseDirsConfigWarning } from '../utils/base-dirs.js';
|
|
3
3
|
import { VectorStore } from '../vectordb/index.js';
|
|
4
4
|
import { type ResolvedGlobalConfig } from './options.js';
|
|
5
|
+
/**
|
|
6
|
+
* Render an unknown caught value for a CLI failure site: every link of the
|
|
7
|
+
* `.cause` chain (via {@link getCauseChain}) followed by its stack, joined so
|
|
8
|
+
* the operator sees the full diagnostic. Deeper links are prefixed
|
|
9
|
+
* `Caused by: `; the outermost link is not.
|
|
10
|
+
*
|
|
11
|
+
* The CLI is operator-facing, so — unlike the MCP client boundary, which never
|
|
12
|
+
* leaks the cause chain — the full chain IS printed (Contract-Delta CLI row:
|
|
13
|
+
* `message` → `message` + cause chain on stderr). Callers keep their own
|
|
14
|
+
* operation prefix and exit-code policy; this function only produces the
|
|
15
|
+
* rendered reason. Single source for the CLI catch-block error-rendering
|
|
16
|
+
* pattern, replacing the former message-only renderer.
|
|
17
|
+
*/
|
|
18
|
+
export declare function formatCliError(error: unknown): string;
|
|
5
19
|
/**
|
|
6
20
|
* Create an uninitialized VectorStore from resolved global config.
|
|
7
21
|
* Callers are responsible for calling initialize() before use.
|
package/dist/cli/common.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"common.d.ts","sourceRoot":"","sources":["../../src/cli/common.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAA;AAC/C,OAAO,EACL,KAAK,cAAc,EACnB,KAAK,qBAAqB,EAG3B,MAAM,uBAAuB,CAAA;
|
|
1
|
+
{"version":3,"file":"common.d.ts","sourceRoot":"","sources":["../../src/cli/common.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAA;AAC/C,OAAO,EACL,KAAK,cAAc,EACnB,KAAK,qBAAqB,EAG3B,MAAM,uBAAuB,CAAA;AAG9B,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAA;AAClD,OAAO,EAAE,KAAK,oBAAoB,EAA6C,MAAM,cAAc,CAAA;AAEnG;;;;;;;;;;;;GAYG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAQrD;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,oBAAoB,GAAG,WAAW,CAK3E;AAED;;;GAGG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,oBAAoB,GAAG,QAAQ,CAerE;AAED;;;;GAIG;AACH,MAAM,WAAW,qBAAqB;IACpC,MAAM,EAAE,cAAc,CAAA;IACtB,QAAQ,EAAE,qBAAqB,EAAE,CAAA;CAClC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,wBAAsB,wBAAwB,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAqDjG"}
|
package/dist/cli/common.js
CHANGED
|
@@ -2,9 +2,32 @@
|
|
|
2
2
|
// plus base-directory resolution shared by every subcommand that scans files.
|
|
3
3
|
import { Embedder } from '../embedder/index.js';
|
|
4
4
|
import { parseBaseDirsEnv, resolveBaseDirs, } from '../utils/base-dirs.js';
|
|
5
|
+
import { getCauseChain } from '../utils/errors.js';
|
|
5
6
|
import { checkSensitivePath } from '../utils/sensitive-path.js';
|
|
6
7
|
import { VectorStore } from '../vectordb/index.js';
|
|
7
|
-
import { resolveDevice, validatePath } from './options.js';
|
|
8
|
+
import { resolveDevice, resolveDtype, validatePath } from './options.js';
|
|
9
|
+
/**
|
|
10
|
+
* Render an unknown caught value for a CLI failure site: every link of the
|
|
11
|
+
* `.cause` chain (via {@link getCauseChain}) followed by its stack, joined so
|
|
12
|
+
* the operator sees the full diagnostic. Deeper links are prefixed
|
|
13
|
+
* `Caused by: `; the outermost link is not.
|
|
14
|
+
*
|
|
15
|
+
* The CLI is operator-facing, so — unlike the MCP client boundary, which never
|
|
16
|
+
* leaks the cause chain — the full chain IS printed (Contract-Delta CLI row:
|
|
17
|
+
* `message` → `message` + cause chain on stderr). Callers keep their own
|
|
18
|
+
* operation prefix and exit-code policy; this function only produces the
|
|
19
|
+
* rendered reason. Single source for the CLI catch-block error-rendering
|
|
20
|
+
* pattern, replacing the former message-only renderer.
|
|
21
|
+
*/
|
|
22
|
+
export function formatCliError(error) {
|
|
23
|
+
const err = error instanceof Error ? error : new Error(String(error));
|
|
24
|
+
return getCauseChain(err)
|
|
25
|
+
.map((link, index) => {
|
|
26
|
+
const header = index === 0 ? '' : 'Caused by: ';
|
|
27
|
+
return `${header}${link.stack || `${link.name}: ${link.message}`}`;
|
|
28
|
+
})
|
|
29
|
+
.join('\n');
|
|
30
|
+
}
|
|
8
31
|
/**
|
|
9
32
|
* Create an uninitialized VectorStore from resolved global config.
|
|
10
33
|
* Callers are responsible for calling initialize() before use.
|
|
@@ -20,12 +43,20 @@ export function createVectorStore(config) {
|
|
|
20
43
|
* Callers are responsible for managing the Embedder lifecycle.
|
|
21
44
|
*/
|
|
22
45
|
export function createEmbedder(config) {
|
|
23
|
-
|
|
46
|
+
const embedderConfig = {
|
|
24
47
|
modelPath: config.modelName,
|
|
25
48
|
batchSize: 16,
|
|
26
49
|
cacheDir: config.cacheDir,
|
|
27
50
|
device: resolveDevice(process.env['RAG_DEVICE']),
|
|
28
|
-
}
|
|
51
|
+
};
|
|
52
|
+
// Set dtype only when RAG_DTYPE resolves to a defined value, mirroring the
|
|
53
|
+
// server path — an unset RAG_DTYPE leaves config.dtype undefined so the
|
|
54
|
+
// embedder applies its fp32 default.
|
|
55
|
+
const dtype = resolveDtype(process.env['RAG_DTYPE']);
|
|
56
|
+
if (dtype !== undefined) {
|
|
57
|
+
embedderConfig.dtype = dtype;
|
|
58
|
+
}
|
|
59
|
+
return new Embedder(embedderConfig);
|
|
29
60
|
}
|
|
30
61
|
/**
|
|
31
62
|
* Resolve effective base directories for a CLI subcommand using the shared
|
package/dist/cli/common.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"common.js","sourceRoot":"","sources":["../../src/cli/common.ts"],"names":[],"mappings":"AAAA,gFAAgF;AAChF,8EAA8E;AAE9E,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAA;AAC/C,OAAO,EAGL,gBAAgB,EAChB,eAAe,GAChB,MAAM,uBAAuB,CAAA;AAC9B,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAA;AAC/D,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAA;AAClD,OAAO,EAA6B,aAAa,EAAE,YAAY,EAAE,MAAM,cAAc,CAAA;
|
|
1
|
+
{"version":3,"file":"common.js","sourceRoot":"","sources":["../../src/cli/common.ts"],"names":[],"mappings":"AAAA,gFAAgF;AAChF,8EAA8E;AAE9E,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAA;AAC/C,OAAO,EAGL,gBAAgB,EAChB,eAAe,GAChB,MAAM,uBAAuB,CAAA;AAC9B,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAA;AAClD,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAA;AAC/D,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAA;AAClD,OAAO,EAA6B,aAAa,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,cAAc,CAAA;AAEnG;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,cAAc,CAAC,KAAc;IAC3C,MAAM,GAAG,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAA;IACrE,OAAO,aAAa,CAAC,GAAG,CAAC;SACtB,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QACnB,MAAM,MAAM,GAAG,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,aAAa,CAAA;QAC/C,OAAO,GAAG,MAAM,GAAG,IAAI,CAAC,KAAK,IAAI,GAAG,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,OAAO,EAAE,EAAE,CAAA;IACpE,CAAC,CAAC;SACD,IAAI,CAAC,IAAI,CAAC,CAAA;AACf,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,MAA4B;IAC5D,OAAO,IAAI,WAAW,CAAC;QACrB,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,SAAS,EAAE,QAAQ;KACpB,CAAC,CAAA;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,MAA4B;IACzD,MAAM,cAAc,GAA8C;QAChE,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,SAAS,EAAE,EAAE;QACb,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,MAAM,EAAE,aAAa,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;KACjD,CAAA;IACD,2EAA2E;IAC3E,wEAAwE;IACxE,qCAAqC;IACrC,MAAM,KAAK,GAAG,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAA;IACpD,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,cAAc,CAAC,KAAK,GAAG,KAAK,CAAA;IAC9B,CAAC;IACD,OAAO,IAAI,QAAQ,CAAC,cAAc,CAAC,CAAA;AACrC,CAAC;AAYD;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAAC,QAAkB;IAC/D,kEAAkE;IAClE,qEAAqE;IACrE,sCAAsC;IACtC,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,IAAI,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,KAAK,SAAS,IAAI,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClF,MAAM,MAAM,GAAG,gBAAgB,CAAC,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAA;YACzD,IAAI,MAAM,CAAC,EAAE,EAAE,CAAC;gBACd,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;oBAC/B,MAAM,SAAS,GAAG,kBAAkB,CAAC,GAAG,EAAE,WAAW,CAAC,CAAA;oBACtD,IAAI,SAAS,EAAE,CAAC;wBACd,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;wBACxB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;oBACjB,CAAC;gBACH,CAAC;YACH,CAAC;YACD,0DAA0D;QAC5D,CAAC;aAAM,IAAI,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,KAAK,SAAS,IAAI,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC9F,MAAM,SAAS,GAAG,kBAAkB,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC,CAAA;YACzE,IAAI,SAAS,EAAE,CAAC;gBACd,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;gBACxB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACjB,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC;QACnC,QAAQ;QACR,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC;QACrC,UAAU,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC;QACnC,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE;KACnB,CAAC,CAAA;IAEF,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAA;QACnC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACjB,CAAC;IAED,oEAAoE;IACpE,yEAAyE;IACzE,uEAAuE;IACvE,oEAAoE;IACpE,sEAAsE;IACtE,4DAA4D;IAC5D,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;QAC1C,MAAM,SAAS,GAAG,YAAY,CAAC,IAAI,EAAE,YAAY,CAAC,CAAA;QAClD,IAAI,SAAS,EAAE,CAAC;YACd,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;YACxB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACjB,CAAC;IACH,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAA;AAC7D,CAAC"}
|
package/dist/cli/delete.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"delete.d.ts","sourceRoot":"","sources":["../../src/cli/delete.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,cAAc,CAAA;AAkFjD;;;;GAIG;AACH,wBAAsB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,aAAa,GAAE,aAAkB,GAAG,OAAO,CAAC,IAAI,CAAC,
|
|
1
|
+
{"version":3,"file":"delete.d.ts","sourceRoot":"","sources":["../../src/cli/delete.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,cAAc,CAAA;AAkFjD;;;;GAIG;AACH,wBAAsB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,aAAa,GAAE,aAAkB,GAAG,OAAO,CAAC,IAAI,CAAC,CAkGhG"}
|
package/dist/cli/delete.js
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
import { unlink } from 'node:fs/promises';
|
|
3
3
|
import { resolve } from 'node:path';
|
|
4
4
|
import { generateMetaJsonPath, generateRawDataPath, isPathInRawDataDirLexical, } from '../utils/raw-data-utils.js';
|
|
5
|
-
import { createVectorStore } from './common.js';
|
|
5
|
+
import { createVectorStore, formatCliError } from './common.js';
|
|
6
6
|
import { resolveGlobalConfig, validatePath } from './options.js';
|
|
7
7
|
// ============================================
|
|
8
8
|
// Help
|
|
@@ -108,7 +108,8 @@ export async function runDelete(args, globalOptions = {}) {
|
|
|
108
108
|
targetPath = generateRawDataPath(globalConfig.dbPath, parsed.source, 'markdown');
|
|
109
109
|
}
|
|
110
110
|
else {
|
|
111
|
-
//
|
|
111
|
+
// DB key is the resolve()'d ingest path, so look up by resolve() (never
|
|
112
|
+
// realpath) — realpath stays in validatePath/validateFilePath.
|
|
112
113
|
targetPath = resolve(parsed.filePath);
|
|
113
114
|
// Validate path (reject sensitive system directories)
|
|
114
115
|
const pathError = validatePath(targetPath, '<file-path>');
|
|
@@ -156,7 +157,7 @@ export async function runDelete(args, globalOptions = {}) {
|
|
|
156
157
|
process.stdout.write(JSON.stringify(result));
|
|
157
158
|
}
|
|
158
159
|
catch (error) {
|
|
159
|
-
const reason =
|
|
160
|
+
const reason = formatCliError(error);
|
|
160
161
|
console.error(`Error: ${reason}`);
|
|
161
162
|
process.exit(1);
|
|
162
163
|
}
|
package/dist/cli/delete.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"delete.js","sourceRoot":"","sources":["../../src/cli/delete.ts"],"names":[],"mappings":"AAAA,6EAA6E;AAE7E,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAA;AACzC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AACnC,OAAO,EACL,oBAAoB,EACpB,mBAAmB,EACnB,yBAAyB,GAC1B,MAAM,4BAA4B,CAAA;AACnC,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAA;AAE/
|
|
1
|
+
{"version":3,"file":"delete.js","sourceRoot":"","sources":["../../src/cli/delete.ts"],"names":[],"mappings":"AAAA,6EAA6E;AAE7E,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAA;AACzC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AACnC,OAAO,EACL,oBAAoB,EACpB,mBAAmB,EACnB,yBAAyB,GAC1B,MAAM,4BAA4B,CAAA;AACnC,OAAO,EAAE,iBAAiB,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAE/D,OAAO,EAAE,mBAAmB,EAAE,YAAY,EAAE,MAAM,cAAc,CAAA;AAEhE,+CAA+C;AAC/C,OAAO;AACP,+CAA+C;AAE/C,MAAM,SAAS,GAAG;;;;;;;;;;;;;;;;yCAgBuB,CAAA;AAYzC;;;;GAIG;AACH,SAAS,SAAS,CAAC,IAAc;IAC/B,IAAI,IAAI,GAAG,KAAK,CAAA;IAChB,IAAI,MAA0B,CAAA;IAC9B,IAAI,QAA4B,CAAA;IAEhC,IAAI,CAAC,GAAG,CAAC,CAAA;IACT,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QACvB,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAE,CAAA;QAEpB,IAAI,GAAG,KAAK,IAAI,IAAI,GAAG,KAAK,QAAQ,EAAE,CAAC;YACrC,IAAI,GAAG,IAAI,CAAA;YACX,CAAC,EAAE,CAAA;QACL,CAAC;aAAM,IAAI,GAAG,KAAK,UAAU,EAAE,CAAC;YAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC,CAAA;YACvB,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBACjD,OAAO,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAA;gBAC3C,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;gBACxB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACjB,CAAC;YACD,MAAM,GAAG,KAAK,CAAA;YACd,CAAC,EAAE,CAAA;QACL,CAAC;aAAM,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YAC/B,OAAO,CAAC,KAAK,CAAC,mBAAmB,GAAG,EAAE,CAAC,CAAA;YACvC,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;YACxB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACjB,CAAC;aAAM,CAAC;YACN,iCAAiC;YACjC,QAAQ,GAAG,GAAG,CAAA;YACd,CAAC,EAAE,CAAA;QACL,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAe,EAAE,IAAI,EAAE,CAAA;IACnC,IAAI,MAAM,KAAK,SAAS;QAAE,MAAM,CAAC,MAAM,GAAG,MAAM,CAAA;IAChD,IAAI,QAAQ,KAAK,SAAS;QAAE,MAAM,CAAC,QAAQ,GAAG,QAAQ,CAAA;IACtD,OAAO,MAAM,CAAA;AACf,CAAC;AAED,+CAA+C;AAC/C,mBAAmB;AACnB,+CAA+C;AAE/C;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAc,EAAE,gBAA+B,EAAE;IAC/E,oBAAoB;IACpB,MAAM,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,CAAA;IAE9B,gBAAgB;IAChB,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;QAChB,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;QACxB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACjB,CAAC;IAED,4DAA4D;IAC5D,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QACvC,OAAO,CAAC,KAAK,CAAC,4CAA4C,CAAC,CAAA;QAC3D,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;QACxB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACjB,CAAC;IAED,IAAI,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QACrC,OAAO,CAAC,KAAK,CAAC,8CAA8C,CAAC,CAAA;QAC7D,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;QACxB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACjB,CAAC;IAED,wBAAwB;IACxB,MAAM,YAAY,GAAG,mBAAmB,CAAC,aAAa,CAAC,CAAA;IAEvD,IAAI,CAAC;QACH,oEAAoE;QACpE,MAAM,WAAW,GAAG,iBAAiB,CAAC,YAAY,CAAC,CAAA;QACnD,MAAM,WAAW,CAAC,UAAU,EAAE,CAAA;QAE9B,6BAA6B;QAC7B,IAAI,UAAkB,CAAA;QAEtB,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;YAClB,yCAAyC;YACzC,UAAU,GAAG,mBAAmB,CAAC,YAAY,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAA;QAClF,CAAC;aAAM,CAAC;YACN,wEAAwE;YACxE,+DAA+D;YAC/D,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,QAAS,CAAC,CAAA;YAEtC,sDAAsD;YACtD,MAAM,SAAS,GAAG,YAAY,CAAC,UAAU,EAAE,aAAa,CAAC,CAAA;YACzD,IAAI,SAAS,EAAE,CAAC;gBACd,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;gBACxB,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAA;gBACpB,OAAM;YACR,CAAC;QACH,CAAC;QAED,iCAAiC;QACjC,MAAM,WAAW,CAAC,YAAY,CAAC,UAAU,CAAC,CAAA;QAE1C,kDAAkD;QAClD,IAAI,yBAAyB,CAAC,UAAU,EAAE,YAAY,CAAC,MAAM,CAAC,EAAE,CAAC;YAC/D,IAAI,CAAC;gBACH,MAAM,MAAM,CAAC,UAAU,CAAC,CAAA;YAC1B,CAAC;YAAC,OAAO,KAAc,EAAE,CAAC;gBACxB,uDAAuD;gBACvD,IACE,CAAC,CAAC,KAAK,YAAY,KAAK,CAAC;oBACzB,CAAC,CAAC,MAAM,IAAI,KAAK,CAAC;oBACjB,KAA+B,CAAC,IAAI,KAAK,QAAQ,EAClD,CAAC;oBACD,MAAM,KAAK,CAAA;gBACb,CAAC;YACH,CAAC;YAED,IAAI,CAAC;gBACH,MAAM,MAAM,CAAC,oBAAoB,CAAC,UAAU,CAAC,CAAC,CAAA;YAChD,CAAC;YAAC,OAAO,KAAc,EAAE,CAAC;gBACxB,gBAAgB;gBAChB,IACE,CAAC,CAAC,KAAK,YAAY,KAAK,CAAC;oBACzB,CAAC,CAAC,MAAM,IAAI,KAAK,CAAC;oBACjB,KAA+B,CAAC,IAAI,KAAK,QAAQ,EAClD,CAAC;oBACD,MAAM,KAAK,CAAA;gBACb,CAAC;YACH,CAAC;QACH,CAAC;QAED,sCAAsC;QACtC,MAAM,WAAW,CAAC,QAAQ,EAAE,CAAA;QAE5B,+BAA+B;QAC/B,MAAM,MAAM,GAAG;YACb,QAAQ,EAAE,UAAU;YACpB,OAAO,EAAE,IAAI;YACb,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACpC,CAAA;QACD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAA;IAC9C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,MAAM,GAAG,cAAc,CAAC,KAAK,CAAC,CAAA;QACpC,OAAO,CAAC,KAAK,CAAC,UAAU,MAAM,EAAE,CAAC,CAAA;QACjC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACjB,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"file-collection.d.ts","sourceRoot":"","sources":["../../src/cli/file-collection.ts"],"names":[],"mappings":"AAeA,wBAAsB,YAAY,CAChC,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,SAAS,MAAM,EAAE,EAC3B,YAAY,EAAE,MAAM,EAAE,GACrB,OAAO,CAAC,MAAM,EAAE,CAAC,CA0DnB"}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
// CLI ingest file collection.
|
|
2
|
+
//
|
|
3
|
+
// Resolves the positional ingest path into the concrete list of supported
|
|
4
|
+
// files to ingest. Single-file mode validates the extension; directory mode
|
|
5
|
+
// confirms the path is inside a configured root and delegates the bounded
|
|
6
|
+
// BFS walk to `bfsCollectSupportedFiles`. Out-of-root directory targets exit
|
|
7
|
+
// the process (preserved from the original inline implementation).
|
|
8
|
+
import { realpath, stat } from 'node:fs/promises';
|
|
9
|
+
import { extname, resolve, sep } from 'node:path';
|
|
10
|
+
import { SUPPORTED_EXTENSIONS } from '../parser/index.js';
|
|
11
|
+
import { MAX_SCAN_DEPTH } from '../utils/limits.js';
|
|
12
|
+
import { bfsCollectSupportedFiles } from '../utils/scan.js';
|
|
13
|
+
export async function collectFiles(targetPath, baseDirs, excludePaths) {
|
|
14
|
+
const resolved = resolve(targetPath);
|
|
15
|
+
const info = await stat(resolved);
|
|
16
|
+
if (info.isFile()) {
|
|
17
|
+
const ext = extname(resolved).toLowerCase();
|
|
18
|
+
if (!SUPPORTED_EXTENSIONS.has(ext)) {
|
|
19
|
+
console.error(`Unsupported file extension: ${ext} (supported: ${[...SUPPORTED_EXTENSIONS].join(', ')})`);
|
|
20
|
+
return [];
|
|
21
|
+
}
|
|
22
|
+
// Store the resolve()'d path (the DB key), not realpath — path policy:
|
|
23
|
+
// realpath only at the security boundary, resolve() everywhere user-facing.
|
|
24
|
+
return [resolved];
|
|
25
|
+
}
|
|
26
|
+
if (info.isDirectory()) {
|
|
27
|
+
// realpath both sides so a symlinked positional path still matches a
|
|
28
|
+
// root whose realpath agrees. baseDirs from resolveCliBaseDirsOrExit
|
|
29
|
+
// are already realpath-normalized with a trailing sep.
|
|
30
|
+
const realResolved = await realpath(resolved);
|
|
31
|
+
const realResolvedWithSep = realResolved.endsWith(sep) ? realResolved : realResolved + sep;
|
|
32
|
+
const insideAnyRoot = baseDirs.some((root) => realResolvedWithSep === root || realResolvedWithSep.startsWith(root));
|
|
33
|
+
if (!insideAnyRoot) {
|
|
34
|
+
console.error(`Error: ${targetPath} is not under any configured base directory. ` +
|
|
35
|
+
`Allowed roots: ${baseDirs.join(', ')}. ` +
|
|
36
|
+
`Provide a path inside one of the configured roots, or set BASE_DIRS / --base-dir to include the desired tree.`);
|
|
37
|
+
process.exit(1);
|
|
38
|
+
}
|
|
39
|
+
// `realResolved` is used ONLY for the in-root containment check (security).
|
|
40
|
+
// The walk uses the resolve()'d `resolved` so the stored DB keys match what
|
|
41
|
+
// `list`/`delete`/`read_chunk_neighbors` use (resolve(), not realpath).
|
|
42
|
+
const { files: collected, unreadableDirs, depthLimited, } = await bfsCollectSupportedFiles(resolved, excludePaths);
|
|
43
|
+
for (const { dirPath } of unreadableDirs) {
|
|
44
|
+
console.error(`Warning: cannot read directory: ${dirPath}`);
|
|
45
|
+
}
|
|
46
|
+
if (depthLimited) {
|
|
47
|
+
console.error(`Warning: some directories were skipped because they exceed the maximum depth (${MAX_SCAN_DEPTH})`);
|
|
48
|
+
}
|
|
49
|
+
return [...new Set(collected)].sort();
|
|
50
|
+
}
|
|
51
|
+
return [];
|
|
52
|
+
}
|
|
53
|
+
//# sourceMappingURL=file-collection.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"file-collection.js","sourceRoot":"","sources":["../../src/cli/file-collection.ts"],"names":[],"mappings":"AAAA,8BAA8B;AAC9B,EAAE;AACF,0EAA0E;AAC1E,4EAA4E;AAC5E,0EAA0E;AAC1E,6EAA6E;AAC7E,mEAAmE;AAEnE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAA;AACjD,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,WAAW,CAAA;AAEjD,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAA;AACzD,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAA;AACnD,OAAO,EAAE,wBAAwB,EAAE,MAAM,kBAAkB,CAAA;AAE3D,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,UAAkB,EAClB,QAA2B,EAC3B,YAAsB;IAEtB,MAAM,QAAQ,GAAG,OAAO,CAAC,UAAU,CAAC,CAAA;IACpC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAA;IAEjC,IAAI,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;QAClB,MAAM,GAAG,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAA;QAC3C,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACnC,OAAO,CAAC,KAAK,CACX,+BAA+B,GAAG,gBAAgB,CAAC,GAAG,oBAAoB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAC1F,CAAA;YACD,OAAO,EAAE,CAAA;QACX,CAAC;QACD,uEAAuE;QACvE,4EAA4E;QAC5E,OAAO,CAAC,QAAQ,CAAC,CAAA;IACnB,CAAC;IAED,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;QACvB,qEAAqE;QACrE,qEAAqE;QACrE,uDAAuD;QACvD,MAAM,YAAY,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAA;QAC7C,MAAM,mBAAmB,GAAG,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,YAAY,GAAG,GAAG,CAAA;QAC1F,MAAM,aAAa,GAAG,QAAQ,CAAC,IAAI,CACjC,CAAC,IAAI,EAAE,EAAE,CAAC,mBAAmB,KAAK,IAAI,IAAI,mBAAmB,CAAC,UAAU,CAAC,IAAI,CAAC,CAC/E,CAAA;QACD,IAAI,CAAC,aAAa,EAAE,CAAC;YACnB,OAAO,CAAC,KAAK,CACX,UAAU,UAAU,+CAA+C;gBACjE,kBAAkB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI;gBACzC,+GAA+G,CAClH,CAAA;YACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACjB,CAAC;QAED,4EAA4E;QAC5E,4EAA4E;QAC5E,wEAAwE;QACxE,MAAM,EACJ,KAAK,EAAE,SAAS,EAChB,cAAc,EACd,YAAY,GACb,GAAG,MAAM,wBAAwB,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAA;QAE1D,KAAK,MAAM,EAAE,OAAO,EAAE,IAAI,cAAc,EAAE,CAAC;YACzC,OAAO,CAAC,KAAK,CAAC,mCAAmC,OAAO,EAAE,CAAC,CAAA;QAC7D,CAAC;QAED,IAAI,YAAY,EAAE,CAAC;YACjB,OAAO,CAAC,KAAK,CACX,iFAAiF,cAAc,GAAG,CACnG,CAAA;QACH,CAAC;QAED,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;IACvC,CAAC;IAED,OAAO,EAAE,CAAA;AACX,CAAC"}
|
package/dist/cli/ingest.d.ts
CHANGED
|
@@ -86,12 +86,10 @@ export type IngestSingleFileOptions = {
|
|
|
86
86
|
* visual-enrichment path: `parsePdfPages` + VLM captioning (`pdf-visual`
|
|
87
87
|
* orchestrator) + joined-text chunking. `pdf-visual` is loaded via dynamic
|
|
88
88
|
* `await import('../pdf-visual/index.js')` so the default (non-visual) path
|
|
89
|
-
* never pulls the VLM module into the bundle
|
|
90
|
-
* verified by AC-001 Proxy sentinel in T4.6).
|
|
89
|
+
* never pulls the VLM module into the bundle.
|
|
91
90
|
*
|
|
92
|
-
* Non-visual, non-PDF, and `visual: true` + non-PDF
|
|
93
|
-
*
|
|
94
|
-
* state and never load `pdf-visual`.
|
|
91
|
+
* Non-visual, non-PDF, and `visual: true` + non-PDF paths all use the default
|
|
92
|
+
* text-only branch and never load `pdf-visual`.
|
|
95
93
|
*/
|
|
96
94
|
export declare function ingestSingleFile(filePath: string, parser: DocumentParser, chunker: SemanticChunker, embedder: Embedder, vectorStore: VectorStore, options?: IngestSingleFileOptions): Promise<number>;
|
|
97
95
|
/**
|
package/dist/cli/ingest.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ingest.d.ts","sourceRoot":"","sources":["../../src/cli/ingest.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"ingest.d.ts","sourceRoot":"","sources":["../../src/cli/ingest.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAA;AACrD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAA;AAGpD,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAA;AACnD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAA;AAC5D,OAAO,KAAK,EAAE,cAAc,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAA;AAElF,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAA;AAQvD,OAAO,KAAK,EAAE,aAAa,EAAE,oBAAoB,EAAE,MAAM,cAAc,CAAA;AAevE,UAAU,YAAY;IACpB,QAAQ,EAAE,cAAc,CAAA;IACxB,gBAAgB,EAAE,qBAAqB,EAAE,CAAA;IACzC,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB,SAAS,EAAE,MAAM,CAAA;IACjB,WAAW,EAAE,MAAM,CAAA;IACnB,cAAc,CAAC,EAAE,MAAM,CAAA;CACxB;AAQD,UAAU,gBAAgB;IACxB;;;;OAIG;IACH,QAAQ,CAAC,EAAE,MAAM,EAAE,GAAG,SAAS,CAAA;IAC/B,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IAChC,cAAc,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IACnC,MAAM,CAAC,EAAE,OAAO,GAAG,SAAS,CAAA;IAC5B;;;;OAIG;IACH,aAAa,CAAC,EAAE,cAAc,GAAG,SAAS,CAAA;CAC3C;AAED,UAAU,UAAU;IAClB,UAAU,EAAE,MAAM,GAAG,SAAS,CAAA;IAC9B,OAAO,EAAE,gBAAgB,CAAA;IACzB,IAAI,EAAE,OAAO,CAAA;CACd;AAmCD;;;;GAIG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,UAAU,CAmFpD;AAMD;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,aAAa,CACjC,YAAY,EAAE,oBAAoB,EAClC,aAAa,GAAE,gBAAqB,GACnC,OAAO,CAAC,YAAY,CAAC,CAyDvB;AAMD;;;;;;;;;;;GAWG;AACH,MAAM,MAAM,uBAAuB,GAC/B;IAAE,MAAM,CAAC,EAAE,KAAK,GAAG,SAAS,CAAA;CAAE,GAC9B;IACE,MAAM,EAAE,IAAI,CAAA;IACZ,OAAO,EAAE,cAAc,CAAA;IACvB,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;CAC5B,CAAA;AAEL;;;;;;;;;;;;GAYG;AACH,wBAAsB,gBAAgB,CACpC,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,cAAc,EACtB,OAAO,EAAE,eAAe,EACxB,QAAQ,EAAE,QAAQ,EAClB,WAAW,EAAE,WAAW,EACxB,OAAO,CAAC,EAAE,uBAAuB,GAChC,OAAO,CAAC,MAAM,CAAC,CA2EjB;AAMD;;;;GAIG;AACH,wBAAsB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,aAAa,GAAE,aAAkB,GAAG,OAAO,CAAC,IAAI,CAAC,CAuIhG"}
|
package/dist/cli/ingest.js
CHANGED
|
@@ -1,22 +1,19 @@
|
|
|
1
1
|
// CLI ingest subcommand — bulk file ingestion with single optimize() at end
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import { basename, extname, join, resolve, sep } from 'node:path';
|
|
2
|
+
import { stat } from 'node:fs/promises';
|
|
3
|
+
import { resolve, sep } from 'node:path';
|
|
5
4
|
import { SemanticChunker } from '../chunker/index.js';
|
|
6
|
-
import { buildChunksAndEmbeddings } from '../ingest/compute.js';
|
|
5
|
+
import { buildChunksAndEmbeddings, buildVectorChunks } from '../ingest/compute.js';
|
|
7
6
|
import { prepareVisualPdfChunks } from '../ingest/visual.js';
|
|
8
|
-
import { DocumentParser
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
// ============================================
|
|
14
|
-
const MAX_DEPTH = 10;
|
|
7
|
+
import { DocumentParser } from '../parser/index.js';
|
|
8
|
+
import { DEFAULT_MAX_FILE_SIZE } from '../utils/limits.js';
|
|
9
|
+
import { createEmbedder, createVectorStore, formatCliError, resolveCliBaseDirsOrExit, } from './common.js';
|
|
10
|
+
import { collectFiles } from './file-collection.js';
|
|
11
|
+
import { consumeBaseDirArg, requireFlagValue, resolveDevice, resolveGlobalConfig, validateChunkMinLength, validateMaxFileSize, validatePath, } from './options.js';
|
|
15
12
|
// ============================================
|
|
16
13
|
// Defaults
|
|
17
14
|
// ============================================
|
|
18
15
|
const INGEST_DEFAULTS = {
|
|
19
|
-
maxFileSize:
|
|
16
|
+
maxFileSize: DEFAULT_MAX_FILE_SIZE,
|
|
20
17
|
};
|
|
21
18
|
// ============================================
|
|
22
19
|
// Help
|
|
@@ -71,31 +68,23 @@ export function parseArgs(args) {
|
|
|
71
68
|
break;
|
|
72
69
|
}
|
|
73
70
|
case '--max-file-size': {
|
|
74
|
-
const raw = args
|
|
75
|
-
if (raw === undefined || raw.startsWith('-')) {
|
|
76
|
-
console.error('Missing value for --max-file-size');
|
|
77
|
-
process.exit(1);
|
|
78
|
-
}
|
|
71
|
+
const raw = requireFlagValue(args, i, '--max-file-size');
|
|
79
72
|
if (!/^\d+$/.test(raw)) {
|
|
80
73
|
console.error(`Invalid value for --max-file-size: "${raw.slice(0, 100)}"`);
|
|
81
74
|
process.exit(1);
|
|
82
75
|
}
|
|
83
76
|
options.maxFileSize = Number.parseInt(raw, 10);
|
|
84
|
-
i
|
|
77
|
+
i += 2;
|
|
85
78
|
break;
|
|
86
79
|
}
|
|
87
80
|
case '--chunk-min-length': {
|
|
88
|
-
const raw = args
|
|
89
|
-
if (raw === undefined || raw.startsWith('-')) {
|
|
90
|
-
console.error('Missing value for --chunk-min-length');
|
|
91
|
-
process.exit(1);
|
|
92
|
-
}
|
|
81
|
+
const raw = requireFlagValue(args, i, '--chunk-min-length');
|
|
93
82
|
if (!/^\d+$/.test(raw)) {
|
|
94
83
|
console.error(`Invalid value for --chunk-min-length: "${raw.slice(0, 100)}"`);
|
|
95
84
|
process.exit(1);
|
|
96
85
|
}
|
|
97
86
|
options.chunkMinLength = Number.parseInt(raw, 10);
|
|
98
|
-
i
|
|
87
|
+
i += 2;
|
|
99
88
|
break;
|
|
100
89
|
}
|
|
101
90
|
case '--visual':
|
|
@@ -104,17 +93,13 @@ export function parseArgs(args) {
|
|
|
104
93
|
i++;
|
|
105
94
|
break;
|
|
106
95
|
case '--visual-quality': {
|
|
107
|
-
const value = args
|
|
108
|
-
if (value === undefined || value.startsWith('-')) {
|
|
109
|
-
console.error('Missing value for --visual-quality');
|
|
110
|
-
process.exit(1);
|
|
111
|
-
}
|
|
96
|
+
const value = requireFlagValue(args, i, '--visual-quality');
|
|
112
97
|
if (value !== 'fast' && value !== 'quality') {
|
|
113
98
|
console.error(`Invalid value for --visual-quality: "${value.slice(0, 100)}". Expected "fast" or "quality".`);
|
|
114
99
|
process.exit(1);
|
|
115
100
|
}
|
|
116
101
|
options.visualQuality = value;
|
|
117
|
-
i
|
|
102
|
+
i += 2;
|
|
118
103
|
break;
|
|
119
104
|
}
|
|
120
105
|
default:
|
|
@@ -202,86 +187,6 @@ export async function resolveConfig(globalConfig, ingestOptions = {}) {
|
|
|
202
187
|
}
|
|
203
188
|
return resolved;
|
|
204
189
|
}
|
|
205
|
-
// ============================================
|
|
206
|
-
// File Collection
|
|
207
|
-
// ============================================
|
|
208
|
-
/**
|
|
209
|
-
* BFS-walk a single directory up to {@link MAX_DEPTH} levels, returning every
|
|
210
|
-
* supported file path under it. Skips symlinks, permission errors, and excluded
|
|
211
|
-
* directories. Reports a single shared `depthLimited` flag via an out-param so
|
|
212
|
-
* the caller can emit one combined warning across multiple roots instead of
|
|
213
|
-
* one per root.
|
|
214
|
-
*/
|
|
215
|
-
async function walkDirectory(rootPath, excludePaths, state) {
|
|
216
|
-
const files = [];
|
|
217
|
-
const queue = [{ dirPath: rootPath, depth: 0 }];
|
|
218
|
-
while (queue.length > 0) {
|
|
219
|
-
const { dirPath, depth } = queue.shift();
|
|
220
|
-
if (depth >= MAX_DEPTH) {
|
|
221
|
-
state.depthLimited = true;
|
|
222
|
-
continue;
|
|
223
|
-
}
|
|
224
|
-
let dir;
|
|
225
|
-
try {
|
|
226
|
-
dir = await opendir(dirPath);
|
|
227
|
-
}
|
|
228
|
-
catch {
|
|
229
|
-
console.error(`Warning: cannot read directory: ${dirPath}`);
|
|
230
|
-
continue;
|
|
231
|
-
}
|
|
232
|
-
for await (const entry of dir) {
|
|
233
|
-
const fullPath = join(dirPath, entry.name);
|
|
234
|
-
// `join(dirPath, entry.name)` always produces a path under `dirPath`,
|
|
235
|
-
// which itself stays under `rootPath` because the BFS only enqueues
|
|
236
|
-
// descendants of `rootPath`. We therefore do not need a per-entry
|
|
237
|
-
// `startsWith(rootPath)` re-check here.
|
|
238
|
-
if (entry.isSymbolicLink())
|
|
239
|
-
continue;
|
|
240
|
-
if (excludePaths.some((ep) => fullPath.startsWith(ep)))
|
|
241
|
-
continue;
|
|
242
|
-
if (entry.isDirectory()) {
|
|
243
|
-
queue.push({ dirPath: fullPath, depth: depth + 1 });
|
|
244
|
-
}
|
|
245
|
-
else if (entry.isFile() && SUPPORTED_EXTENSIONS.has(extname(entry.name).toLowerCase())) {
|
|
246
|
-
files.push(fullPath);
|
|
247
|
-
}
|
|
248
|
-
}
|
|
249
|
-
}
|
|
250
|
-
return files;
|
|
251
|
-
}
|
|
252
|
-
async function collectFiles(targetPath, baseDirs, excludePaths) {
|
|
253
|
-
const resolved = resolve(targetPath);
|
|
254
|
-
const info = await stat(resolved);
|
|
255
|
-
if (info.isFile()) {
|
|
256
|
-
const ext = extname(resolved).toLowerCase();
|
|
257
|
-
if (!SUPPORTED_EXTENSIONS.has(ext)) {
|
|
258
|
-
console.error(`Unsupported file extension: ${ext} (supported: ${[...SUPPORTED_EXTENSIONS].join(', ')})`);
|
|
259
|
-
return [];
|
|
260
|
-
}
|
|
261
|
-
return [resolved];
|
|
262
|
-
}
|
|
263
|
-
if (info.isDirectory()) {
|
|
264
|
-
// realpath both sides so a symlinked positional path still matches a
|
|
265
|
-
// root whose realpath agrees. baseDirs from resolveCliBaseDirsOrExit
|
|
266
|
-
// are already realpath-normalized with a trailing sep.
|
|
267
|
-
const realResolved = await realpath(resolved);
|
|
268
|
-
const realResolvedWithSep = realResolved.endsWith(sep) ? realResolved : realResolved + sep;
|
|
269
|
-
const insideAnyRoot = baseDirs.some((root) => realResolvedWithSep === root || realResolvedWithSep.startsWith(root));
|
|
270
|
-
if (!insideAnyRoot) {
|
|
271
|
-
console.error(`Error: ${targetPath} is not under any configured base directory. ` +
|
|
272
|
-
`Allowed roots: ${baseDirs.join(', ')}. ` +
|
|
273
|
-
`Provide a path inside one of the configured roots, or set BASE_DIRS / --base-dir to include the desired tree.`);
|
|
274
|
-
process.exit(1);
|
|
275
|
-
}
|
|
276
|
-
const state = { depthLimited: false };
|
|
277
|
-
const collected = await walkDirectory(resolved, excludePaths, state);
|
|
278
|
-
if (state.depthLimited) {
|
|
279
|
-
console.error(`Warning: some directories were skipped because they exceed the maximum depth (${MAX_DEPTH})`);
|
|
280
|
-
}
|
|
281
|
-
return [...new Set(collected)].sort();
|
|
282
|
-
}
|
|
283
|
-
return [];
|
|
284
|
-
}
|
|
285
190
|
/**
|
|
286
191
|
* Ingest a single file: parse, chunk, embed, delete old chunks, insert new chunks.
|
|
287
192
|
* Returns the number of chunks inserted.
|
|
@@ -290,12 +195,10 @@ async function collectFiles(targetPath, baseDirs, excludePaths) {
|
|
|
290
195
|
* visual-enrichment path: `parsePdfPages` + VLM captioning (`pdf-visual`
|
|
291
196
|
* orchestrator) + joined-text chunking. `pdf-visual` is loaded via dynamic
|
|
292
197
|
* `await import('../pdf-visual/index.js')` so the default (non-visual) path
|
|
293
|
-
* never pulls the VLM module into the bundle
|
|
294
|
-
* verified by AC-001 Proxy sentinel in T4.6).
|
|
198
|
+
* never pulls the VLM module into the bundle.
|
|
295
199
|
*
|
|
296
|
-
* Non-visual, non-PDF, and `visual: true` + non-PDF
|
|
297
|
-
*
|
|
298
|
-
* state and never load `pdf-visual`.
|
|
200
|
+
* Non-visual, non-PDF, and `visual: true` + non-PDF paths all use the default
|
|
201
|
+
* text-only branch and never load `pdf-visual`.
|
|
299
202
|
*/
|
|
300
203
|
export async function ingestSingleFile(filePath, parser, chunker, embedder, vectorStore, options) {
|
|
301
204
|
// Parse file
|
|
@@ -329,26 +232,12 @@ export async function ingestSingleFile(filePath, parser, chunker, embedder, vect
|
|
|
329
232
|
// pre-existing `metadata.fileSize` semantics (post-enrichment,
|
|
330
233
|
// pre-chunking text length).
|
|
331
234
|
await vectorStore.deleteChunks(filePath);
|
|
332
|
-
const
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
return {
|
|
339
|
-
id: randomUUID(),
|
|
340
|
-
filePath,
|
|
341
|
-
chunkIndex: chunk.index,
|
|
342
|
-
text: chunk.text,
|
|
343
|
-
vector: embedding,
|
|
344
|
-
metadata: {
|
|
345
|
-
fileName: basename(filePath),
|
|
346
|
-
fileSize: visualResult.text.length,
|
|
347
|
-
fileType: extname(filePath).slice(1),
|
|
348
|
-
},
|
|
349
|
-
fileTitle: title,
|
|
350
|
-
timestamp,
|
|
351
|
-
};
|
|
235
|
+
const vectorChunks = buildVectorChunks({
|
|
236
|
+
filePath,
|
|
237
|
+
chunks,
|
|
238
|
+
embeddings,
|
|
239
|
+
fileSize: visualResult.text.length,
|
|
240
|
+
fileTitle: title,
|
|
352
241
|
});
|
|
353
242
|
await vectorStore.insertChunks(vectorChunks);
|
|
354
243
|
return vectorChunks.length;
|
|
@@ -363,7 +252,7 @@ export async function ingestSingleFile(filePath, parser, chunker, embedder, vect
|
|
|
363
252
|
text = result.content;
|
|
364
253
|
title = result.title || null;
|
|
365
254
|
}
|
|
366
|
-
// Chunk text + generate embeddings via the shared computation layer
|
|
255
|
+
// Chunk text + generate embeddings via the shared computation layer.
|
|
367
256
|
const { chunks, embeddings } = await buildChunksAndEmbeddings(text, title, chunker, embedder);
|
|
368
257
|
if (chunks.length === 0) {
|
|
369
258
|
console.error(` Warning: 0 chunks generated (file may be empty or too short)`);
|
|
@@ -372,26 +261,12 @@ export async function ingestSingleFile(filePath, parser, chunker, embedder, vect
|
|
|
372
261
|
// Delete existing chunks for this file
|
|
373
262
|
await vectorStore.deleteChunks(filePath);
|
|
374
263
|
// Build vector chunks
|
|
375
|
-
const
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
return {
|
|
382
|
-
id: randomUUID(),
|
|
383
|
-
filePath,
|
|
384
|
-
chunkIndex: chunk.index,
|
|
385
|
-
text: chunk.text,
|
|
386
|
-
vector: embedding,
|
|
387
|
-
metadata: {
|
|
388
|
-
fileName: basename(filePath),
|
|
389
|
-
fileSize: text.length,
|
|
390
|
-
fileType: extname(filePath).slice(1),
|
|
391
|
-
},
|
|
392
|
-
fileTitle: title,
|
|
393
|
-
timestamp,
|
|
394
|
-
};
|
|
264
|
+
const vectorChunks = buildVectorChunks({
|
|
265
|
+
filePath,
|
|
266
|
+
chunks,
|
|
267
|
+
embeddings,
|
|
268
|
+
fileSize: text.length,
|
|
269
|
+
fileTitle: title,
|
|
395
270
|
});
|
|
396
271
|
// Insert chunks
|
|
397
272
|
await vectorStore.insertChunks(vectorChunks);
|
|
@@ -434,14 +309,13 @@ export async function runIngest(args, globalOptions = {}) {
|
|
|
434
309
|
const config = await resolveConfig(globalConfig, options);
|
|
435
310
|
const excludePaths = [`${resolve(config.dbPath)}${sep}`, `${resolve(config.cacheDir)}${sep}`];
|
|
436
311
|
// Surface resolver warnings (precedence, nested-root pruning) on stderr
|
|
437
|
-
// before scan output starts.
|
|
438
|
-
// task only wires the resolver so the warnings are visible today.
|
|
312
|
+
// before scan output starts.
|
|
439
313
|
for (const warning of config.baseDirsWarnings) {
|
|
440
314
|
console.error(warning.message);
|
|
441
315
|
}
|
|
442
316
|
// Collect files: when `targetPath` is a directory, the scan iterates every
|
|
443
|
-
// effective root in `config.baseDirs.baseDirs
|
|
444
|
-
//
|
|
317
|
+
// effective root in `config.baseDirs.baseDirs`; the positional directory
|
|
318
|
+
// only triggers directory mode and is no longer the scan target.
|
|
445
319
|
// Single-file mode is unchanged. See `collectFiles` for the full rationale.
|
|
446
320
|
const files = await collectFiles(targetPath, config.baseDirs.baseDirs, excludePaths);
|
|
447
321
|
if (files.length === 0) {
|
|
@@ -452,9 +326,7 @@ export async function runIngest(args, globalOptions = {}) {
|
|
|
452
326
|
// Initialize components (single instances reused across all files).
|
|
453
327
|
// The parser receives the full multi-root config. The directory-scan loop
|
|
454
328
|
// (`collectFiles`) iterates every effective root in `config.baseDirs.baseDirs`
|
|
455
|
-
//
|
|
456
|
-
// pre-P2-T2 single-root scan; under multiple roots it walks each one and
|
|
457
|
-
// dedupes overlap.
|
|
329
|
+
// and dedupes overlap.
|
|
458
330
|
const parser = new DocumentParser({
|
|
459
331
|
baseDirs: config.baseDirs.baseDirs,
|
|
460
332
|
maxFileSize: config.maxFileSize,
|
|
@@ -502,7 +374,7 @@ export async function runIngest(args, globalOptions = {}) {
|
|
|
502
374
|
}
|
|
503
375
|
}
|
|
504
376
|
catch (error) {
|
|
505
|
-
const reason =
|
|
377
|
+
const reason = formatCliError(error);
|
|
506
378
|
console.error(`${label} ${filePath} ... FAILED: ${reason}`);
|
|
507
379
|
summary.failed++;
|
|
508
380
|
}
|