@equinor/fusion-framework-cli-plugin-ai-index 3.0.6 → 3.0.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +32 -0
- package/dist/esm/bin/apply-metadata.js +13 -5
- package/dist/esm/bin/apply-metadata.js.map +1 -1
- package/dist/esm/bin/apply-schema.js +7 -1
- package/dist/esm/bin/apply-schema.js.map +1 -1
- package/dist/esm/bin/apply-schema.test.js +2 -1
- package/dist/esm/bin/apply-schema.test.js.map +1 -1
- package/dist/esm/bin/{delete-removed-files.js → create-delete-removed-files-stream.js} +12 -1
- package/dist/esm/bin/create-delete-removed-files-stream.js.map +1 -0
- package/dist/esm/bin/embed.js +260 -97
- package/dist/esm/bin/embed.js.map +1 -1
- package/dist/esm/bin/execute-pipeline.js +5 -0
- package/dist/esm/bin/execute-pipeline.js.map +1 -1
- package/dist/esm/bin/get-diff.js +6 -0
- package/dist/esm/bin/get-diff.js.map +1 -1
- package/dist/esm/{embeddings-command.options.js → command-options-schema.js} +1 -1
- package/dist/esm/command-options-schema.js.map +1 -0
- package/dist/esm/{embeddings-command.js → command.js} +4 -2
- package/dist/esm/command.js.map +1 -0
- package/dist/esm/{create-command.js → create-index-command.js} +13 -5
- package/dist/esm/create-index-command.js.map +1 -0
- package/dist/esm/{schema.js → define-index-schema.js} +1 -1
- package/dist/esm/define-index-schema.js.map +1 -0
- package/dist/esm/delete-command.js +11 -2
- package/dist/esm/delete-command.js.map +1 -1
- package/dist/esm/delete-index-command.js +4 -0
- package/dist/esm/delete-index-command.js.map +1 -1
- package/dist/esm/{delete-command.options.js → delete-options-schema.js} +1 -1
- package/dist/esm/delete-options-schema.js.map +1 -0
- package/dist/esm/embed-command.js +3 -4
- package/dist/esm/embed-command.js.map +1 -1
- package/dist/esm/index.js +3 -3
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/search-command.js +20 -0
- package/dist/esm/search-command.js.map +1 -1
- package/dist/esm/utils/git/{metadata.js → extract-git-metadata.js} +4 -2
- package/dist/esm/utils/git/extract-git-metadata.js.map +1 -0
- package/dist/esm/utils/git/get-changed-files.js +87 -0
- package/dist/esm/utils/git/get-changed-files.js.map +1 -0
- package/dist/esm/utils/git/{file-changes.js → get-file-status.js} +9 -95
- package/dist/esm/utils/git/get-file-status.js.map +1 -0
- package/dist/esm/utils/git/{status.js → get-git-status.js} +5 -2
- package/dist/esm/utils/git/get-git-status.js.map +1 -0
- package/dist/esm/utils/git/get-git.js +30 -0
- package/dist/esm/utils/git/get-git.js.map +1 -0
- package/dist/esm/utils/git/index.js +7 -4
- package/dist/esm/utils/git/index.js.map +1 -1
- package/dist/esm/utils/git/is-file-changed.js +19 -0
- package/dist/esm/utils/git/is-file-changed.js.map +1 -0
- package/dist/esm/utils/git/resolve-project-root.js +22 -0
- package/dist/esm/utils/git/resolve-project-root.js.map +1 -0
- package/dist/esm/utils/markdown/index.js +3 -1
- package/dist/esm/utils/markdown/index.js.map +1 -1
- package/dist/esm/utils/markdown/is-markdown-file.js +10 -0
- package/dist/esm/utils/markdown/is-markdown-file.js.map +1 -0
- package/dist/esm/utils/markdown/parse-markdown-file.js +29 -0
- package/dist/esm/utils/markdown/parse-markdown-file.js.map +1 -0
- package/dist/esm/utils/markdown/{parser.js → parse-markdown.js} +8 -37
- package/dist/esm/utils/markdown/parse-markdown.js.map +1 -0
- package/dist/esm/utils/{embedding-dimensions.js → resolve-embedding-dimensions.js} +3 -1
- package/dist/esm/utils/resolve-embedding-dimensions.js.map +1 -0
- package/dist/esm/utils/{package-resolver.js → resolve-package.js} +4 -1
- package/dist/esm/utils/resolve-package.js.map +1 -0
- package/dist/esm/utils/ts-doc/create-typescript-document.js +27 -0
- package/dist/esm/utils/ts-doc/create-typescript-document.js.map +1 -0
- package/dist/esm/utils/ts-doc/extract-document-from-class-node.js +86 -0
- package/dist/esm/utils/ts-doc/extract-document-from-class-node.js.map +1 -0
- package/dist/esm/utils/ts-doc/extract-document-from-node.js +82 -0
- package/dist/esm/utils/ts-doc/extract-document-from-node.js.map +1 -0
- package/dist/esm/utils/ts-doc/index.js +3 -1
- package/dist/esm/utils/ts-doc/index.js.map +1 -1
- package/dist/esm/utils/ts-doc/is-typescript-file.js +10 -0
- package/dist/esm/utils/ts-doc/is-typescript-file.js.map +1 -0
- package/dist/esm/utils/ts-doc/{constants.js → node-kinds.js} +1 -1
- package/dist/esm/utils/ts-doc/node-kinds.js.map +1 -0
- package/dist/esm/utils/ts-doc/parse-ts-doc-from-file-sync.js +23 -0
- package/dist/esm/utils/ts-doc/parse-ts-doc-from-file-sync.js.map +1 -0
- package/dist/esm/utils/ts-doc/parse-ts-doc-sync.js +18 -0
- package/dist/esm/utils/ts-doc/parse-ts-doc-sync.js.map +1 -0
- package/dist/esm/utils/ts-doc/process-source-file.js +22 -0
- package/dist/esm/utils/ts-doc/process-source-file.js.map +1 -0
- package/dist/esm/utils/zod-to-azure-fields.js +7 -0
- package/dist/esm/utils/zod-to-azure-fields.js.map +1 -1
- package/dist/esm/utils/zod-to-azure-fields.test.js +8 -7
- package/dist/esm/utils/zod-to-azure-fields.test.js.map +1 -1
- package/dist/esm/version.js +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/dist/types/bin/apply-schema.d.ts +1 -1
- package/dist/types/config.d.ts +1 -1
- package/dist/types/index.d.ts +1 -1
- package/dist/types/utils/git/get-changed-files.d.ts +13 -0
- package/dist/types/utils/git/get-file-status.d.ts +13 -0
- package/dist/types/utils/git/{git-client.d.ts → get-git.d.ts} +0 -10
- package/dist/types/utils/git/index.d.ts +7 -4
- package/dist/types/utils/git/is-file-changed.d.ts +12 -0
- package/dist/types/utils/git/resolve-project-root.d.ts +10 -0
- package/dist/types/utils/markdown/index.d.ts +3 -1
- package/dist/types/utils/markdown/is-markdown-file.d.ts +7 -0
- package/dist/types/utils/markdown/parse-markdown-file.d.ts +14 -0
- package/dist/types/utils/markdown/parse-markdown.d.ts +14 -0
- package/dist/types/utils/ts-doc/create-typescript-document.d.ts +12 -0
- package/dist/types/utils/ts-doc/extract-document-from-class-node.d.ts +15 -0
- package/dist/types/utils/ts-doc/extract-document-from-node.d.ts +20 -0
- package/dist/types/utils/ts-doc/index.d.ts +3 -1
- package/dist/types/utils/ts-doc/is-typescript-file.d.ts +7 -0
- package/dist/types/utils/ts-doc/parse-ts-doc-from-file-sync.d.ts +14 -0
- package/dist/types/utils/ts-doc/parse-ts-doc-sync.d.ts +12 -0
- package/dist/types/utils/ts-doc/process-source-file.d.ts +11 -0
- package/dist/types/version.d.ts +1 -1
- package/package.json +8 -8
- package/src/bin/apply-metadata.ts +69 -57
- package/src/bin/apply-schema.test.ts +2 -1
- package/src/bin/apply-schema.ts +6 -1
- package/src/bin/{delete-removed-files.ts → create-delete-removed-files-stream.ts} +12 -1
- package/src/bin/embed.ts +389 -145
- package/src/bin/execute-pipeline.ts +5 -0
- package/src/bin/get-diff.ts +7 -1
- package/src/bin/types.ts +1 -1
- package/src/{embeddings-command.ts → command.ts} +3 -1
- package/src/config.ts +1 -1
- package/src/{create-command.ts → create-index-command.ts} +15 -5
- package/src/delete-command.ts +11 -2
- package/src/delete-index-command.ts +4 -0
- package/src/embed-command.ts +3 -6
- package/src/index.ts +3 -3
- package/src/search-command.ts +20 -0
- package/src/utils/git/{metadata.ts → extract-git-metadata.ts} +3 -1
- package/src/utils/git/get-changed-files.ts +93 -0
- package/src/utils/git/{file-changes.ts → get-file-status.ts} +9 -103
- package/src/utils/git/{status.ts → get-git-status.ts} +4 -1
- package/src/utils/git/{git-client.ts → get-git.ts} +4 -22
- package/src/utils/git/index.ts +7 -4
- package/src/utils/git/is-file-changed.ts +21 -0
- package/src/utils/git/resolve-project-root.ts +22 -0
- package/src/utils/markdown/index.ts +3 -1
- package/src/utils/markdown/is-markdown-file.ts +9 -0
- package/src/utils/markdown/parse-markdown-file.ts +36 -0
- package/src/utils/markdown/{parser.ts → parse-markdown.ts} +16 -51
- package/src/utils/{embedding-dimensions.ts → resolve-embedding-dimensions.ts} +2 -0
- package/src/utils/{package-resolver.ts → resolve-package.ts} +3 -0
- package/src/utils/ts-doc/create-typescript-document.ts +36 -0
- package/src/utils/ts-doc/extract-document-from-class-node.ts +106 -0
- package/src/utils/ts-doc/extract-document-from-node.ts +120 -0
- package/src/utils/ts-doc/index.ts +3 -1
- package/src/utils/ts-doc/is-typescript-file.ts +9 -0
- package/src/utils/ts-doc/parse-ts-doc-from-file-sync.ts +32 -0
- package/src/utils/ts-doc/parse-ts-doc-sync.ts +23 -0
- package/src/utils/ts-doc/process-source-file.ts +30 -0
- package/src/utils/zod-to-azure-fields.test.ts +10 -7
- package/src/utils/zod-to-azure-fields.ts +7 -0
- package/src/version.ts +1 -1
- package/dist/esm/bin/delete-removed-files.js.map +0 -1
- package/dist/esm/bin/file-stream.js +0 -22
- package/dist/esm/bin/file-stream.js.map +0 -1
- package/dist/esm/create-command.js.map +0 -1
- package/dist/esm/delete-command.options.js.map +0 -1
- package/dist/esm/embeddings-command.js.map +0 -1
- package/dist/esm/embeddings-command.options.js.map +0 -1
- package/dist/esm/schema.js.map +0 -1
- package/dist/esm/utils/embedding-dimensions.js.map +0 -1
- package/dist/esm/utils/git/file-changes.js.map +0 -1
- package/dist/esm/utils/git/git-client.js +0 -48
- package/dist/esm/utils/git/git-client.js.map +0 -1
- package/dist/esm/utils/git/metadata.js.map +0 -1
- package/dist/esm/utils/git/status.js.map +0 -1
- package/dist/esm/utils/markdown/parser.js.map +0 -1
- package/dist/esm/utils/package-resolver.js.map +0 -1
- package/dist/esm/utils/ts-doc/constants.js.map +0 -1
- package/dist/esm/utils/ts-doc/extractors.js +0 -189
- package/dist/esm/utils/ts-doc/extractors.js.map +0 -1
- package/dist/esm/utils/ts-doc/parser.js +0 -46
- package/dist/esm/utils/ts-doc/parser.js.map +0 -1
- package/dist/types/bin/file-stream.d.ts +0 -1
- package/dist/types/utils/git/file-changes.d.ts +0 -36
- package/dist/types/utils/markdown/parser.d.ts +0 -34
- package/dist/types/utils/ts-doc/extractors.d.ts +0 -42
- package/dist/types/utils/ts-doc/parser.d.ts +0 -32
- package/src/bin/file-stream.ts +0 -34
- package/src/utils/ts-doc/extractors.ts +0 -260
- package/src/utils/ts-doc/parser.ts +0 -60
- /package/dist/types/bin/{delete-removed-files.d.ts → create-delete-removed-files-stream.d.ts} +0 -0
- /package/dist/types/{embeddings-command.options.d.ts → command-options-schema.d.ts} +0 -0
- /package/dist/types/{embeddings-command.d.ts → command.d.ts} +0 -0
- /package/dist/types/{create-command.d.ts → create-index-command.d.ts} +0 -0
- /package/dist/types/{schema.d.ts → define-index-schema.d.ts} +0 -0
- /package/dist/types/{delete-command.options.d.ts → delete-options-schema.d.ts} +0 -0
- /package/dist/types/utils/git/{metadata.d.ts → extract-git-metadata.d.ts} +0 -0
- /package/dist/types/utils/git/{status.d.ts → get-git-status.d.ts} +0 -0
- /package/dist/types/utils/{embedding-dimensions.d.ts → resolve-embedding-dimensions.d.ts} +0 -0
- /package/dist/types/utils/{package-resolver.d.ts → resolve-package.d.ts} +0 -0
- /package/dist/types/utils/ts-doc/{constants.d.ts → node-kinds.d.ts} +0 -0
- /package/src/{embeddings-command.options.ts → command-options-schema.ts} +0 -0
- /package/src/{schema.ts → define-index-schema.ts} +0 -0
- /package/src/{delete-command.options.ts → delete-options-schema.ts} +0 -0
- /package/src/utils/ts-doc/{constants.ts → node-kinds.ts} +0 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,37 @@
|
|
|
1
1
|
# @equinor/fusion-framework-cli-plugin-ai-index
|
|
2
2
|
|
|
3
|
+
## 3.0.7
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- 80c3e4a: Internal: added missing intent comments for object-merge logic; no public API changes.
|
|
8
|
+
- 80c3e4a: Fix `getChangedFiles` to correctly handle renamed files in `--diff` mode. Previously, renamed files (`git diff --name-status` status `R###`) were silently skipped, so renamed files were never picked up for re-indexing. Renames are now treated as a removal of the old path plus an addition of the new path.
|
|
9
|
+
- 80c3e4a: Internal: add clarifying intent comments to control-flow and array-transform logic in the `create`, `delete`, `delete-index`, `embed`, `embeddings`, and `search` CLI commands; no public API or behavior changes.
|
|
10
|
+
- 80c3e4a: Internal: add clarifying intent comments in `utils/git/metadata.ts` and `utils/git/status.ts`, and split `utils/git/git-client.ts` so `resolveProjectRoot` moves into its own `resolve-project-root.ts` module alongside the single-export `getGit` in `git-client.ts`, re-exported from the existing barrel; no public API or behavior changes.
|
|
11
|
+
- 80c3e4a: Internal: add clarifying intent comments and split `utils/git/file-changes.ts` into single-export `get-changed-files.ts`, `get-file-status.ts`, and `is-file-changed.ts` modules, re-exported from the existing barrel; no public API or behavior changes.
|
|
12
|
+
- 80c3e4a: Internal: add clarifying intent comments in `zod-to-azure-fields`, and split `utils/markdown/parser.ts` into single-export `parse-markdown.ts` and `parse-markdown-file.ts` modules re-exported from the existing barrel; no public API or behavior changes.
|
|
13
|
+
- 80c3e4a: Internal: add clarifying intent comments to control-flow, iterator, and RxJS pipeline logic in `apply-metadata`, `file-stream`, `execute-pipeline`, `embedding-dimensions`, `package-resolver`, and `get-diff`; no public API or behavior changes.
|
|
14
|
+
- 80c3e4a: Internal: add clarifying intent comments to control-flow logic in `apply-schema` and `delete-removed-files`; no public API or behavior changes.
|
|
15
|
+
- 80c3e4a: Internal: add clarifying intent comments and split `utils/ts-doc/extractors.ts` into single-export `create-typescript-document.ts`, `extract-document-from-class-node.ts`, `extract-document-from-node.ts`, and `process-source-file.ts` modules; rename `utils/ts-doc/parser.ts` to `is-typescript-file.ts` to match its single export, re-exported from the existing barrel; no public API or behavior changes.
|
|
16
|
+
- 80c3e4a: Internal: split the monolithic `embed()` bin function into focused helper functions (file streaming, parsing, metadata, embedding, and upserting stages) for readability and maintainability; no public API or behavior changes.
|
|
17
|
+
- 80c3e4a: Internal: Remove dead `src/bin/create-file-stream.ts` left over from the fusion-lint file-splitting cleanup. Its `createFileStream` export had zero importers and was superseded by the actively-used, diverged local `createFileStream$` in `embed.ts`. No change to CLI behavior.
|
|
18
|
+
- 80c3e4a: Internal: renamed 53 source files across `ai-base`, `ai-chat`, `ai-index`, and `copilot` to comply with the `filename-convention` lint rule (e.g. `config.ts` → `load-fusion-ai-config.ts`, `tools/write-file.ts` → `tools/create-write-file-tool.ts`, `prompts/plan.prompt.ts` → `prompts/create-plan-prompt.ts`). No public API changes.
|
|
19
|
+
- Updated dependencies [80c3e4a]
|
|
20
|
+
- Updated dependencies [80c3e4a]
|
|
21
|
+
- Updated dependencies [80c3e4a]
|
|
22
|
+
- Updated dependencies [80c3e4a]
|
|
23
|
+
- Updated dependencies [80c3e4a]
|
|
24
|
+
- Updated dependencies [80c3e4a]
|
|
25
|
+
- Updated dependencies [80c3e4a]
|
|
26
|
+
- Updated dependencies [80c3e4a]
|
|
27
|
+
- Updated dependencies [80c3e4a]
|
|
28
|
+
- Updated dependencies [80c3e4a]
|
|
29
|
+
- Updated dependencies [80c3e4a]
|
|
30
|
+
- @equinor/fusion-framework-cli-plugin-ai-base@4.0.6
|
|
31
|
+
- @equinor/fusion-framework-module-ai@6.0.2
|
|
32
|
+
- @equinor/fusion-framework-module@6.1.1
|
|
33
|
+
- @equinor/fusion-imports@2.0.1
|
|
34
|
+
|
|
3
35
|
## 3.0.6
|
|
4
36
|
|
|
5
37
|
### Patch Changes
|
|
@@ -1,9 +1,14 @@
|
|
|
1
1
|
import path from 'node:path';
|
|
2
2
|
import { from, mergeMap, map, tap, toArray } from 'rxjs';
|
|
3
3
|
import { extractGitMetadata } from '../utils/git/index.js';
|
|
4
|
-
import { resolvePackage } from '../utils/package
|
|
4
|
+
import { resolvePackage } from '../utils/resolve-package.js';
|
|
5
5
|
/**
|
|
6
6
|
* Creates a stream that applies metadata to documents.
|
|
7
|
+
*
|
|
8
|
+
* @param document$ - Stream of document entries to enrich with metadata.
|
|
9
|
+
* @param indexConfig - Index configuration controlling git/package metadata resolution.
|
|
10
|
+
* @param onProgress - Optional callback invoked after each document is enriched.
|
|
11
|
+
* @returns A stream emitting enriched documents grouped by source entry.
|
|
7
12
|
* @internal
|
|
8
13
|
*/
|
|
9
14
|
export function applyMetadata(document$, indexConfig, onProgress) {
|
|
@@ -17,10 +22,11 @@ export function applyMetadata(document$, indexConfig, onProgress) {
|
|
|
17
22
|
* total concurrent git processes ≤ ENTRY_CONCURRENCY × GIT_CONCURRENCY.
|
|
18
23
|
*/
|
|
19
24
|
const ENTRY_CONCURRENCY = 20;
|
|
20
|
-
return document
|
|
21
|
-
|
|
25
|
+
return (document$
|
|
26
|
+
// Enrich each document entry with git/package metadata and re-batch the results
|
|
27
|
+
.pipe(mergeMap((entry) => {
|
|
22
28
|
// Extract git metadata concurrently (capped to limit parallel git processes)
|
|
23
|
-
mergeMap(async (document) => {
|
|
29
|
+
return from(entry.documents).pipe(mergeMap(async (document) => {
|
|
24
30
|
const rootPath = document.metadata.rootPath ?? process.cwd();
|
|
25
31
|
const sourcePath = path.join(rootPath, document.metadata.source);
|
|
26
32
|
const gitMetadata = document.metadata.source && indexConfig?.metadata?.resolveGit !== false
|
|
@@ -28,6 +34,7 @@ export function applyMetadata(document$, indexConfig, onProgress) {
|
|
|
28
34
|
: {};
|
|
29
35
|
// Resolve package information if enabled
|
|
30
36
|
let packageMetadata = {};
|
|
37
|
+
// Only attempt package resolution when explicitly enabled and a source path is present.
|
|
31
38
|
if (shouldResolvePackage && document.metadata.source) {
|
|
32
39
|
packageMetadata = await resolvePackage(sourcePath)
|
|
33
40
|
.then((pkg) => {
|
|
@@ -39,6 +46,7 @@ export function applyMetadata(document$, indexConfig, onProgress) {
|
|
|
39
46
|
})
|
|
40
47
|
.catch(() => ({}));
|
|
41
48
|
}
|
|
49
|
+
// Merge git and package metadata into the document's existing attributes
|
|
42
50
|
return {
|
|
43
51
|
...document,
|
|
44
52
|
metadata: {
|
|
@@ -68,6 +76,6 @@ export function applyMetadata(document$, indexConfig, onProgress) {
|
|
|
68
76
|
}),
|
|
69
77
|
// Group back by file for batch deletion in next step
|
|
70
78
|
toArray());
|
|
71
|
-
}, ENTRY_CONCURRENCY));
|
|
79
|
+
}, ENTRY_CONCURRENCY)));
|
|
72
80
|
}
|
|
73
81
|
//# sourceMappingURL=apply-metadata.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"apply-metadata.js","sourceRoot":"","sources":["../../../src/bin/apply-metadata.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AAGzD,OAAO,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AAC3D,OAAO,EAAE,cAAc,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"apply-metadata.js","sourceRoot":"","sources":["../../../src/bin/apply-metadata.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AAGzD,OAAO,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AAC3D,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAO7D;;;;;;;;GAQG;AACH,MAAM,UAAU,aAAa,CAC3B,SAAoC,EACpC,WAA6C,EAC7C,UAAqC;IAErC,8BAA8B;IAC9B,MAAM,oBAAoB,GAAG,WAAW,EAAE,QAAQ,EAAE,cAAc,IAAI,KAAK,CAAC;IAE5E,sFAAsF;IACtF,MAAM,eAAe,GAAG,EAAE,CAAC;IAE3B;;;;OAIG;IACH,MAAM,iBAAiB,GAAG,EAAE,CAAC;IAE7B,OAAO,CACL,SAAS;QACP,gFAAgF;SAC/E,IAAI,CACH,QAAQ,CAAC,CAAC,KAAK,EAAE,EAAE;QACjB,6EAA6E;QAC7E,OAAO,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,IAAI,CAC/B,QAAQ,CAAC,KAAK,EAAE,QAAQ,EAAgC,EAAE;YACxD,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,QAAQ,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;YAC7D,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YACjE,MAAM,WAAW,GACf,QAAQ,CAAC,QAAQ,CAAC,MAAM,IAAI,WAAW,EAAE,QAAQ,EAAE,UAAU,KAAK,KAAK;gBACrE,CAAC,CAAC,MAAM,kBAAkB,CAAC,UAAU,CAAC;gBACtC,CAAC,CAAC,EAAE,CAAC;YAET,yCAAyC;YACzC,IAAI,eAAe,GAAG,EAAE,CAAC;YACzB,wFAAwF;YACxF,IAAI,oBAAoB,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;gBACrD,eAAe,GAAG,MAAM,cAAc,CAAC,UAAU,CAAC;qBAC/C,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE;oBACZ,OAAO;wBACL,QAAQ,EAAE,GAAG,EAAE,IAAI;wBACnB,WAAW,EAAE,GAAG,EAAE,OAAO;wBACzB,YAAY,EAAE,GAAG,EAAE,QAAQ;qBAC5B,CAAC;gBACJ,CAAC,CAAC;qBACD,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YACvB,CAAC;YACD,yEAAyE;YACzE,OAAO;gBACL,GAAG,QAAQ;gBACX,QAAQ,EAAE;oBACR,GAAG,QAAQ,CAAC,QAAQ;oBACpB,UAAU,EAAE;wBACV,GAAG,QAAQ,CAAC,QAAQ,CAAC,UAAU;wBAC/B,GAAG,WAAW;wBACd,GAAG,eAAe;qBACnB;iBACF;aACF,CAAC;QACJ,CAAC,EAAE,eAAe,CAAC;QACnB,gDAAgD;QAChD,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,UAAU,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACzD,+CAA+C;QAC/C,GAAG,CAAC,CAAC,QAA6B,EAAE,EAAE;YACpC,MAAM,kBAAkB,GACtB,WAAW,EAAE,QAAQ,EAAE,kBAAkB;gBACzC,CAAC,CAAC,UAAmC,EAAE,SAA8B,EAAE,EAAE,CACvE,UAAU,CAAC,CAAC;YAChB,MAAM,UAAU,GAAG,kBAAkB,CAAC,QAAQ,CAAC,QAAQ,CAAC,UAAU,IAAI,EAAE,EAAE,QAAQ,CAAC,CAAC;YACpF,OAAO;gBACL,GAAG,QAAQ;gBACX,QAAQ,EAAE;oBACR,GAAG,QAAQ,CAAC,QAAQ;oBACpB,UAAU;iBACX;aACF,CAAC;QACJ,CAAC,CAAC;QACF,qDAAqD;QACrD,OAAO,EAAE,CACV,CAAC;IACJ,CAAC,EAAE,iBAAiB,CAAC,CACtB,CACJ,CAAC;AACJ,CAAC"}
|
|
@@ -23,10 +23,14 @@ export function applySchema(document$, schema) {
|
|
|
23
23
|
return document$;
|
|
24
24
|
}
|
|
25
25
|
const promotedKeys = new Set(Object.keys(schema.shape.shape));
|
|
26
|
-
|
|
26
|
+
// Resolve and validate promoted schema fields for each document batch
|
|
27
|
+
return document$.pipe(map((documents) =>
|
|
28
|
+
// Process each document in the batch individually
|
|
29
|
+
documents.map((document) => {
|
|
27
30
|
// Run typed attribute processor before schema resolution so the
|
|
28
31
|
// resolver receives fully enriched attributes
|
|
29
32
|
let enrichedDocument = document;
|
|
33
|
+
// Only run the attribute processor when the schema declares one
|
|
30
34
|
if (schema.prepareAttributes) {
|
|
31
35
|
const enrichedAttributes = schema.prepareAttributes((document.metadata.attributes ?? {}), document);
|
|
32
36
|
enrichedDocument = {
|
|
@@ -46,7 +50,9 @@ export function applySchema(document$, schema) {
|
|
|
46
50
|
// in both top-level fields and the generic attributes array
|
|
47
51
|
const currentAttributes = (enrichedDocument.metadata.attributes ?? {});
|
|
48
52
|
const remainingAttributes = {};
|
|
53
|
+
// Copy over only attributes that were not promoted to schema fields
|
|
49
54
|
for (const [key, value] of Object.entries(currentAttributes)) {
|
|
55
|
+
// Skip keys already promoted to schemaFields to avoid duplication
|
|
50
56
|
if (!promotedKeys.has(key)) {
|
|
51
57
|
remainingAttributes[key] = value;
|
|
52
58
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"apply-schema.js","sourceRoot":"","sources":["../../../src/bin/apply-schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,MAAM,CAAC;AAK3B;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,UAAU,WAAW,CACzB,SAA4C,EAC5C,MAAqC;IAErC,sEAAsE;IACtE,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAgC,CAAC,CAAC,CAAC;IAEzF,OAAO,SAAS,CAAC,IAAI,CACnB,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,
|
|
1
|
+
{"version":3,"file":"apply-schema.js","sourceRoot":"","sources":["../../../src/bin/apply-schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,MAAM,CAAC;AAK3B;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,UAAU,WAAW,CACzB,SAA4C,EAC5C,MAAqC;IAErC,sEAAsE;IACtE,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAgC,CAAC,CAAC,CAAC;IAEzF,sEAAsE;IACtE,OAAO,SAAS,CAAC,IAAI,CACnB,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE;IAChB,kDAAkD;IAClD,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE;QACzB,gEAAgE;QAChE,8CAA8C;QAC9C,IAAI,gBAAgB,GAAG,QAAQ,CAAC;QAChC,gEAAgE;QAChE,IAAI,MAAM,CAAC,iBAAiB,EAAE,CAAC;YAC7B,MAAM,kBAAkB,GAAG,MAAM,CAAC,iBAAiB,CACjD,CAAC,QAAQ,CAAC,QAAQ,CAAC,UAAU,IAAI,EAAE,CAA4B,EAC/D,QAAQ,CACT,CAAC;YACF,gBAAgB,GAAG;gBACjB,GAAG,QAAQ;gBACX,QAAQ,EAAE;oBACR,GAAG,QAAQ,CAAC,QAAQ;oBACpB,UAAU,EAAE,kBAA6C;iBAC1D;aACF,CAAC;QACJ,CAAC;QAED,iEAAiE;QACjE,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC;QAElD,+DAA+D;QAC/D,wDAAwD;QACxD,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,CAA4B,CAAC;QAE1E,6DAA6D;QAC7D,4DAA4D;QAC5D,MAAM,iBAAiB,GAAG,CAAC,gBAAgB,CAAC,QAAQ,CAAC,UAAU,IAAI,EAAE,CAGpE,CAAC;QACF,MAAM,mBAAmB,GAA4B,EAAE,CAAC;QACxD,oEAAoE;QACpE,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,iBAAiB,CAAC,EAAE,CAAC;YAC7D,kEAAkE;YAClE,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC3B,mBAAmB,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;YACnC,CAAC;QACH,CAAC;QAED,OAAO;YACL,GAAG,gBAAgB;YACnB,QAAQ,EAAE;gBACR,GAAG,gBAAgB,CAAC,QAAQ;gBAC5B,UAAU,EAAE,mBAAmB;gBAC/B,YAAY,EAAE,SAAS;aACxB;SACF,CAAC;IACJ,CAAC,CAAC,CACH,CACF,CAAC;AACJ,CAAC"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { describe, it, expect } from 'vitest';
|
|
2
2
|
import { z } from 'zod';
|
|
3
3
|
import { of, lastValueFrom } from 'rxjs';
|
|
4
|
-
import { defineIndexSchema } from '../schema.js';
|
|
4
|
+
import { defineIndexSchema } from '../define-index-schema.js';
|
|
5
5
|
import { applySchema } from './apply-schema.js';
|
|
6
6
|
/** Helper to create a minimal VectorStoreDocument for testing. */
|
|
7
7
|
function makeDocument(overrides) {
|
|
@@ -115,6 +115,7 @@ describe('applySchema', () => {
|
|
|
115
115
|
prepareAttributes: (attrs, doc) => {
|
|
116
116
|
// Type-safe: attrs.tags is string[] | undefined
|
|
117
117
|
attrs.tags ??= [];
|
|
118
|
+
// Only tag documents that live under a package directory
|
|
118
119
|
if (doc.metadata.source.includes('packages/')) {
|
|
119
120
|
attrs.tags.push('package');
|
|
120
121
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"apply-schema.test.js","sourceRoot":"","sources":["../../../src/bin/apply-schema.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,EAAE,EAAE,aAAa,EAAE,MAAM,MAAM,CAAC;AAGzC,OAAO,EAAE,iBAAiB,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"apply-schema.test.js","sourceRoot":"","sources":["../../../src/bin/apply-schema.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,EAAE,EAAE,aAAa,EAAE,MAAM,MAAM,CAAC;AAGzC,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,kEAAkE;AAClE,SAAS,YAAY,CACnB,SAAuF;IAEvF,OAAO;QACL,EAAE,EAAE,SAAS;QACb,WAAW,EAAE,cAAc;QAC3B,GAAG,SAAS;KACb,CAAC;AACJ,CAAC;AAED,QAAQ,CAAC,mBAAmB,EAAE,GAAG,EAAE;IACjC,EAAE,CAAC,sDAAsD,EAAE,GAAG,EAAE;QAC9D,MAAM,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAC7C,MAAM,OAAO,GAAG,GAAG,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;QAE1C,MAAM,MAAM,GAAG,iBAAiB,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;QAErD,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACjC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACvC,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;IAC3B,MAAM,MAAM,GAAG,iBAAiB,CAAC;QAC/B,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC;YACd,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;YAC/B,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;YAChB,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;YACrC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE;SACvB,CAAC;QACF,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACjB,QAAQ,EAAE,GAAG,CAAC,QAAQ,CAAC,UAAU,EAAE,QAA8B;YACjE,IAAI,EAAG,GAAG,CAAC,QAAQ,CAAC,UAAU,EAAE,IAAe,IAAI,SAAS;YAC5D,IAAI,EAAG,GAAG,CAAC,QAAQ,CAAC,UAAU,EAAE,IAAiB,IAAI,EAAE;YACvD,UAAU,EAAE,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;SAC9C,CAAC;KACH,CAAC,CAAC;IAEH,EAAE,CAAC,mDAAmD,EAAE,KAAK,IAAI,EAAE;QACjE,MAAM,GAAG,GAAG,YAAY,CAAC;YACvB,QAAQ,EAAE,EAAE,MAAM,EAAE,2BAA2B,EAAE,UAAU,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE;SACjF,CAAC,CAAC;QACH,MAAM,KAAK,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAExB,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,WAAW,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC;QAElE,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAChC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mEAAmE,EAAE,KAAK,IAAI,EAAE;QACjF,MAAM,GAAG,GAAG,YAAY,CAAC;YACvB,QAAQ,EAAE;gBACR,MAAM,EAAE,2BAA2B;gBACnC,UAAU,EAAE;oBACV,IAAI,EAAE,OAAO;oBACb,QAAQ,EAAE,2BAA2B;oBACrC,IAAI,EAAE,CAAC,SAAS,EAAE,OAAO,CAAC;oBAC1B,UAAU,EAAE,SAAS;iBACtB;aACF;SACF,CAAC,CAAC;QACH,MAAM,KAAK,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAExB,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,WAAW,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;QAE/D,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,OAAO,CAAC;YAC9C,QAAQ,EAAE,2BAA2B;YACrC,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,CAAC,SAAS,EAAE,OAAO,CAAC;YAC1B,UAAU,EAAE,UAAU;SACvB,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4DAA4D,EAAE,KAAK,IAAI,EAAE;QAC1E,MAAM,GAAG,GAAG,YAAY,CAAC;YACvB,QAAQ,EAAE;gBACR,MAAM,EAAE,2BAA2B;gBACnC,UAAU,EAAE;oBACV,IAAI,EAAE,OAAO;oBACb,QAAQ,EAAE,2BAA2B;oBACrC,IAAI,EAAE,CAAC,SAAS,CAAC;oBACjB,eAAe,EAAE,QAAQ;iBAC1B;aACF;SACF,CAAC,CAAC;QACH,MAAM,KAAK,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAExB,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,WAAW,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;QAE/D,qDAAqD;QACrD,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,OAAO,CAAC,EAAE,eAAe,EAAE,QAAQ,EAAE,CAAC,CAAC;IAC/E,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iDAAiD,EAAE,KAAK,IAAI,EAAE;QAC/D,MAAM,GAAG,GAAG,YAAY,CAAC;YACvB,QAAQ,EAAE,EAAE,MAAM,EAAE,iCAAiC,EAAE;SACxD,CAAC,CAAC;QACH,MAAM,KAAK,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAExB,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,WAAW,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;QAE/D,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,OAAO,CAAC;YAC9C,QAAQ,EAAE,SAAS;YACnB,IAAI,EAAE,SAAS;YACf,IAAI,EAAE,EAAE;YACR,UAAU,EAAE,WAAW;SACxB,CAAC,CAAC;QACH,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IACpD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iDAAiD,EAAE,KAAK,IAAI,EAAE;QAC/D,MAAM,SAAS,GAAG,iBAAiB,CAAC;YAClC,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;YAC5C,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,EAAE,4BAA4B;SAC5D,CAAC,CAAC;QAEH,MAAM,GAAG,GAAG,YAAY,CAAC;YACvB,QAAQ,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,EAAE,EAAE;SAChD,CAAC,CAAC;QACH,MAAM,KAAK,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAExB,MAAM,MAAM,CAAC,aAAa,CAAC,WAAW,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;IAC/E,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4DAA4D,EAAE,KAAK,IAAI,EAAE;QAC1E,MAAM,iBAAiB,GAAG,iBAAiB,CAAC;YAC1C,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC;gBACd,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;gBACrC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;aACjB,CAAC;YACF,iBAAiB,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;gBAChC,gDAAgD;gBAChD,KAAK,CAAC,IAAI,KAAK,EAAE,CAAC;gBAClB,yDAAyD;gBACzD,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;oBAC9C,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBAC7B,CAAC;gBACD,OAAO,KAAK,CAAC;YACf,CAAC;YACD,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;gBACjB,IAAI,EAAG,GAAG,CAAC,QAAQ,CAAC,UAAU,EAAE,IAAiB,IAAI,EAAE;gBACvD,IAAI,EAAG,GAAG,CAAC,QAAQ,CAAC,UAAU,EAAE,IAAe,IAAI,SAAS;aAC7D,CAAC;SACH,CAAC,CAAC;QAEH,MAAM,GAAG,GAAG,YAAY,CAAC;YACvB,QAAQ,EAAE;gBACR,MAAM,EAAE,iCAAiC;gBACzC,UAAU,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE;aAC9B;SACF,CAAC,CAAC;QACH,MAAM,KAAK,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAExB,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,WAAW,CAAC,KAAK,EAAE,iBAAiB,CAAC,CAAC,CAAC;QAE1E,mEAAmE;QACnE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,OAAO,CAAC;YAC9C,IAAI,EAAE,CAAC,SAAS,CAAC;YACjB,IAAI,EAAE,OAAO;SACd,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -1,25 +1,36 @@
|
|
|
1
1
|
import { map, mergeMap, toArray, filter } from 'rxjs';
|
|
2
2
|
/**
|
|
3
3
|
* Creates a stream that deletes removed files from the vector store.
|
|
4
|
+
*
|
|
5
|
+
* @param removedFiles$ - Stream of files detected as removed from the source tree.
|
|
6
|
+
* @param framework - Framework instance used to access the configured vector store.
|
|
7
|
+
* @param options - Command options controlling target index and dry-run behavior.
|
|
8
|
+
* @returns A stream emitting the result of each batch deletion.
|
|
4
9
|
* @internal
|
|
5
10
|
*/
|
|
6
11
|
export function createDeleteRemovedFilesStream(removedFiles$, framework, options) {
|
|
12
|
+
// Batch removed files, then build a single OData filter for deletion
|
|
7
13
|
return removedFiles$.pipe(toArray(), map((files) => {
|
|
14
|
+
// Nothing to delete when there are no removed files
|
|
8
15
|
if (files.length === 0) {
|
|
9
16
|
return { files: [], filterExpression: null };
|
|
10
17
|
}
|
|
11
18
|
// Build OData filter: "metadata/source eq 'path1' or metadata/source eq 'path2'"
|
|
19
|
+
// Escape each removed file into an OData equality clause
|
|
12
20
|
const filterExpression = files
|
|
13
21
|
.map((file) => `metadata/source eq '${file.relativePath}'`)
|
|
14
22
|
.join(' or ');
|
|
15
23
|
return { files, filterExpression };
|
|
16
24
|
}), mergeMap(async ({ files, filterExpression }) => {
|
|
25
|
+
// Skip the deletion call entirely when there is nothing to remove
|
|
17
26
|
if (files.length === 0) {
|
|
18
27
|
return undefined;
|
|
19
28
|
}
|
|
29
|
+
// Log each removed file for visibility before deleting
|
|
20
30
|
for (const file of files) {
|
|
21
31
|
console.log('Removing entry from vector store', file.relativePath);
|
|
22
32
|
}
|
|
33
|
+
// Skip the actual vector store mutation when running in dry-run mode
|
|
23
34
|
if (!options.dryRun) {
|
|
24
35
|
const vectorStoreService = framework.ai.useIndex(options.indexName);
|
|
25
36
|
// Single batch deletion - one file can produce multiple document chunks
|
|
@@ -33,4 +44,4 @@ export function createDeleteRemovedFilesStream(removedFiles$, framework, options
|
|
|
33
44
|
};
|
|
34
45
|
}), filter((result) => Boolean(result)));
|
|
35
46
|
}
|
|
36
|
-
//# sourceMappingURL=delete-removed-files.js.map
|
|
47
|
+
//# sourceMappingURL=create-delete-removed-files-stream.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"create-delete-removed-files-stream.js","sourceRoot":"","sources":["../../../src/bin/create-delete-removed-files-stream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AAMtD;;;;;;;;GAQG;AACH,MAAM,UAAU,8BAA8B,CAC5C,aAAwC,EACxC,SAA4B,EAC5B,OAAuB;IAEvB,qEAAqE;IACrE,OAAO,aAAa,CAAC,IAAI,CACvB,OAAO,EAAE,EACT,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QACZ,oDAAoD;QACpD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAAC;QAC/C,CAAC;QACD,iFAAiF;QACjF,yDAAyD;QACzD,MAAM,gBAAgB,GAAG,KAAK;aAC3B,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,uBAAuB,IAAI,CAAC,YAAY,GAAG,CAAC;aAC1D,IAAI,CAAC,MAAM,CAAC,CAAC;QAChB,OAAO,EAAE,KAAK,EAAE,gBAAgB,EAAE,CAAC;IACrC,CAAC,CAAC,EACF,QAAQ,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,gBAAgB,EAAE,EAAE,EAAE;QAC7C,kEAAkE;QAClE,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,uDAAuD;QACvD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,OAAO,CAAC,GAAG,CAAC,kCAAkC,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;QACrE,CAAC;QACD,qEAAqE;QACrE,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;YACpB,MAAM,kBAAkB,GAAG,SAAS,CAAC,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;YACpE,wEAAwE;YACxE,MAAM,kBAAkB,CAAC,eAAe,CAAC;gBACvC,MAAM,EAAE,EAAE,gBAAgB,EAAE,gBAAgB,IAAI,SAAS,EAAE;aAC5D,CAAC,CAAC;QACL,CAAC;QACD,OAAO;YACL,MAAM,EAAE,SAAS;YACjB,KAAK,EAAE,KAAmC;SAC3C,CAAC;IACJ,CAAC,CAAC,EACF,MAAM,CAAC,CAAC,MAAM,EAAsC,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CACxE,CAAC;AACJ,CAAC"}
|