@equinor/fusion-framework-cli-plugin-ai-index 3.0.6 → 3.0.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +32 -0
- package/dist/esm/bin/apply-metadata.js +13 -5
- package/dist/esm/bin/apply-metadata.js.map +1 -1
- package/dist/esm/bin/apply-schema.js +7 -1
- package/dist/esm/bin/apply-schema.js.map +1 -1
- package/dist/esm/bin/apply-schema.test.js +2 -1
- package/dist/esm/bin/apply-schema.test.js.map +1 -1
- package/dist/esm/bin/{delete-removed-files.js → create-delete-removed-files-stream.js} +12 -1
- package/dist/esm/bin/create-delete-removed-files-stream.js.map +1 -0
- package/dist/esm/bin/embed.js +260 -97
- package/dist/esm/bin/embed.js.map +1 -1
- package/dist/esm/bin/execute-pipeline.js +5 -0
- package/dist/esm/bin/execute-pipeline.js.map +1 -1
- package/dist/esm/bin/get-diff.js +6 -0
- package/dist/esm/bin/get-diff.js.map +1 -1
- package/dist/esm/{embeddings-command.options.js → command-options-schema.js} +1 -1
- package/dist/esm/command-options-schema.js.map +1 -0
- package/dist/esm/{embeddings-command.js → command.js} +4 -2
- package/dist/esm/command.js.map +1 -0
- package/dist/esm/{create-command.js → create-index-command.js} +13 -5
- package/dist/esm/create-index-command.js.map +1 -0
- package/dist/esm/{schema.js → define-index-schema.js} +1 -1
- package/dist/esm/define-index-schema.js.map +1 -0
- package/dist/esm/delete-command.js +11 -2
- package/dist/esm/delete-command.js.map +1 -1
- package/dist/esm/delete-index-command.js +4 -0
- package/dist/esm/delete-index-command.js.map +1 -1
- package/dist/esm/{delete-command.options.js → delete-options-schema.js} +1 -1
- package/dist/esm/delete-options-schema.js.map +1 -0
- package/dist/esm/embed-command.js +3 -4
- package/dist/esm/embed-command.js.map +1 -1
- package/dist/esm/index.js +3 -3
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/search-command.js +20 -0
- package/dist/esm/search-command.js.map +1 -1
- package/dist/esm/utils/git/{metadata.js → extract-git-metadata.js} +4 -2
- package/dist/esm/utils/git/extract-git-metadata.js.map +1 -0
- package/dist/esm/utils/git/get-changed-files.js +87 -0
- package/dist/esm/utils/git/get-changed-files.js.map +1 -0
- package/dist/esm/utils/git/{file-changes.js → get-file-status.js} +9 -95
- package/dist/esm/utils/git/get-file-status.js.map +1 -0
- package/dist/esm/utils/git/{status.js → get-git-status.js} +5 -2
- package/dist/esm/utils/git/get-git-status.js.map +1 -0
- package/dist/esm/utils/git/get-git.js +30 -0
- package/dist/esm/utils/git/get-git.js.map +1 -0
- package/dist/esm/utils/git/index.js +7 -4
- package/dist/esm/utils/git/index.js.map +1 -1
- package/dist/esm/utils/git/is-file-changed.js +19 -0
- package/dist/esm/utils/git/is-file-changed.js.map +1 -0
- package/dist/esm/utils/git/resolve-project-root.js +22 -0
- package/dist/esm/utils/git/resolve-project-root.js.map +1 -0
- package/dist/esm/utils/markdown/index.js +3 -1
- package/dist/esm/utils/markdown/index.js.map +1 -1
- package/dist/esm/utils/markdown/is-markdown-file.js +10 -0
- package/dist/esm/utils/markdown/is-markdown-file.js.map +1 -0
- package/dist/esm/utils/markdown/parse-markdown-file.js +29 -0
- package/dist/esm/utils/markdown/parse-markdown-file.js.map +1 -0
- package/dist/esm/utils/markdown/{parser.js → parse-markdown.js} +8 -37
- package/dist/esm/utils/markdown/parse-markdown.js.map +1 -0
- package/dist/esm/utils/{embedding-dimensions.js → resolve-embedding-dimensions.js} +3 -1
- package/dist/esm/utils/resolve-embedding-dimensions.js.map +1 -0
- package/dist/esm/utils/{package-resolver.js → resolve-package.js} +4 -1
- package/dist/esm/utils/resolve-package.js.map +1 -0
- package/dist/esm/utils/ts-doc/create-typescript-document.js +27 -0
- package/dist/esm/utils/ts-doc/create-typescript-document.js.map +1 -0
- package/dist/esm/utils/ts-doc/extract-document-from-class-node.js +86 -0
- package/dist/esm/utils/ts-doc/extract-document-from-class-node.js.map +1 -0
- package/dist/esm/utils/ts-doc/extract-document-from-node.js +82 -0
- package/dist/esm/utils/ts-doc/extract-document-from-node.js.map +1 -0
- package/dist/esm/utils/ts-doc/index.js +3 -1
- package/dist/esm/utils/ts-doc/index.js.map +1 -1
- package/dist/esm/utils/ts-doc/is-typescript-file.js +10 -0
- package/dist/esm/utils/ts-doc/is-typescript-file.js.map +1 -0
- package/dist/esm/utils/ts-doc/{constants.js → node-kinds.js} +1 -1
- package/dist/esm/utils/ts-doc/node-kinds.js.map +1 -0
- package/dist/esm/utils/ts-doc/parse-ts-doc-from-file-sync.js +23 -0
- package/dist/esm/utils/ts-doc/parse-ts-doc-from-file-sync.js.map +1 -0
- package/dist/esm/utils/ts-doc/parse-ts-doc-sync.js +18 -0
- package/dist/esm/utils/ts-doc/parse-ts-doc-sync.js.map +1 -0
- package/dist/esm/utils/ts-doc/process-source-file.js +22 -0
- package/dist/esm/utils/ts-doc/process-source-file.js.map +1 -0
- package/dist/esm/utils/zod-to-azure-fields.js +7 -0
- package/dist/esm/utils/zod-to-azure-fields.js.map +1 -1
- package/dist/esm/utils/zod-to-azure-fields.test.js +8 -7
- package/dist/esm/utils/zod-to-azure-fields.test.js.map +1 -1
- package/dist/esm/version.js +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/dist/types/bin/apply-schema.d.ts +1 -1
- package/dist/types/config.d.ts +1 -1
- package/dist/types/index.d.ts +1 -1
- package/dist/types/utils/git/get-changed-files.d.ts +13 -0
- package/dist/types/utils/git/get-file-status.d.ts +13 -0
- package/dist/types/utils/git/{git-client.d.ts → get-git.d.ts} +0 -10
- package/dist/types/utils/git/index.d.ts +7 -4
- package/dist/types/utils/git/is-file-changed.d.ts +12 -0
- package/dist/types/utils/git/resolve-project-root.d.ts +10 -0
- package/dist/types/utils/markdown/index.d.ts +3 -1
- package/dist/types/utils/markdown/is-markdown-file.d.ts +7 -0
- package/dist/types/utils/markdown/parse-markdown-file.d.ts +14 -0
- package/dist/types/utils/markdown/parse-markdown.d.ts +14 -0
- package/dist/types/utils/ts-doc/create-typescript-document.d.ts +12 -0
- package/dist/types/utils/ts-doc/extract-document-from-class-node.d.ts +15 -0
- package/dist/types/utils/ts-doc/extract-document-from-node.d.ts +20 -0
- package/dist/types/utils/ts-doc/index.d.ts +3 -1
- package/dist/types/utils/ts-doc/is-typescript-file.d.ts +7 -0
- package/dist/types/utils/ts-doc/parse-ts-doc-from-file-sync.d.ts +14 -0
- package/dist/types/utils/ts-doc/parse-ts-doc-sync.d.ts +12 -0
- package/dist/types/utils/ts-doc/process-source-file.d.ts +11 -0
- package/dist/types/version.d.ts +1 -1
- package/package.json +8 -8
- package/src/bin/apply-metadata.ts +69 -57
- package/src/bin/apply-schema.test.ts +2 -1
- package/src/bin/apply-schema.ts +6 -1
- package/src/bin/{delete-removed-files.ts → create-delete-removed-files-stream.ts} +12 -1
- package/src/bin/embed.ts +389 -145
- package/src/bin/execute-pipeline.ts +5 -0
- package/src/bin/get-diff.ts +7 -1
- package/src/bin/types.ts +1 -1
- package/src/{embeddings-command.ts → command.ts} +3 -1
- package/src/config.ts +1 -1
- package/src/{create-command.ts → create-index-command.ts} +15 -5
- package/src/delete-command.ts +11 -2
- package/src/delete-index-command.ts +4 -0
- package/src/embed-command.ts +3 -6
- package/src/index.ts +3 -3
- package/src/search-command.ts +20 -0
- package/src/utils/git/{metadata.ts → extract-git-metadata.ts} +3 -1
- package/src/utils/git/get-changed-files.ts +93 -0
- package/src/utils/git/{file-changes.ts → get-file-status.ts} +9 -103
- package/src/utils/git/{status.ts → get-git-status.ts} +4 -1
- package/src/utils/git/{git-client.ts → get-git.ts} +4 -22
- package/src/utils/git/index.ts +7 -4
- package/src/utils/git/is-file-changed.ts +21 -0
- package/src/utils/git/resolve-project-root.ts +22 -0
- package/src/utils/markdown/index.ts +3 -1
- package/src/utils/markdown/is-markdown-file.ts +9 -0
- package/src/utils/markdown/parse-markdown-file.ts +36 -0
- package/src/utils/markdown/{parser.ts → parse-markdown.ts} +16 -51
- package/src/utils/{embedding-dimensions.ts → resolve-embedding-dimensions.ts} +2 -0
- package/src/utils/{package-resolver.ts → resolve-package.ts} +3 -0
- package/src/utils/ts-doc/create-typescript-document.ts +36 -0
- package/src/utils/ts-doc/extract-document-from-class-node.ts +106 -0
- package/src/utils/ts-doc/extract-document-from-node.ts +120 -0
- package/src/utils/ts-doc/index.ts +3 -1
- package/src/utils/ts-doc/is-typescript-file.ts +9 -0
- package/src/utils/ts-doc/parse-ts-doc-from-file-sync.ts +32 -0
- package/src/utils/ts-doc/parse-ts-doc-sync.ts +23 -0
- package/src/utils/ts-doc/process-source-file.ts +30 -0
- package/src/utils/zod-to-azure-fields.test.ts +10 -7
- package/src/utils/zod-to-azure-fields.ts +7 -0
- package/src/version.ts +1 -1
- package/dist/esm/bin/delete-removed-files.js.map +0 -1
- package/dist/esm/bin/file-stream.js +0 -22
- package/dist/esm/bin/file-stream.js.map +0 -1
- package/dist/esm/create-command.js.map +0 -1
- package/dist/esm/delete-command.options.js.map +0 -1
- package/dist/esm/embeddings-command.js.map +0 -1
- package/dist/esm/embeddings-command.options.js.map +0 -1
- package/dist/esm/schema.js.map +0 -1
- package/dist/esm/utils/embedding-dimensions.js.map +0 -1
- package/dist/esm/utils/git/file-changes.js.map +0 -1
- package/dist/esm/utils/git/git-client.js +0 -48
- package/dist/esm/utils/git/git-client.js.map +0 -1
- package/dist/esm/utils/git/metadata.js.map +0 -1
- package/dist/esm/utils/git/status.js.map +0 -1
- package/dist/esm/utils/markdown/parser.js.map +0 -1
- package/dist/esm/utils/package-resolver.js.map +0 -1
- package/dist/esm/utils/ts-doc/constants.js.map +0 -1
- package/dist/esm/utils/ts-doc/extractors.js +0 -189
- package/dist/esm/utils/ts-doc/extractors.js.map +0 -1
- package/dist/esm/utils/ts-doc/parser.js +0 -46
- package/dist/esm/utils/ts-doc/parser.js.map +0 -1
- package/dist/types/bin/file-stream.d.ts +0 -1
- package/dist/types/utils/git/file-changes.d.ts +0 -36
- package/dist/types/utils/markdown/parser.d.ts +0 -34
- package/dist/types/utils/ts-doc/extractors.d.ts +0 -42
- package/dist/types/utils/ts-doc/parser.d.ts +0 -32
- package/src/bin/file-stream.ts +0 -34
- package/src/utils/ts-doc/extractors.ts +0 -260
- package/src/utils/ts-doc/parser.ts +0 -60
- /package/dist/types/bin/{delete-removed-files.d.ts → create-delete-removed-files-stream.d.ts} +0 -0
- /package/dist/types/{embeddings-command.options.d.ts → command-options-schema.d.ts} +0 -0
- /package/dist/types/{embeddings-command.d.ts → command.d.ts} +0 -0
- /package/dist/types/{create-command.d.ts → create-index-command.d.ts} +0 -0
- /package/dist/types/{schema.d.ts → define-index-schema.d.ts} +0 -0
- /package/dist/types/{delete-command.options.d.ts → delete-options-schema.d.ts} +0 -0
- /package/dist/types/utils/git/{metadata.d.ts → extract-git-metadata.d.ts} +0 -0
- /package/dist/types/utils/git/{status.d.ts → get-git-status.d.ts} +0 -0
- /package/dist/types/utils/{embedding-dimensions.d.ts → resolve-embedding-dimensions.d.ts} +0 -0
- /package/dist/types/utils/{package-resolver.d.ts → resolve-package.d.ts} +0 -0
- /package/dist/types/utils/ts-doc/{constants.d.ts → node-kinds.d.ts} +0 -0
- /package/src/{embeddings-command.options.ts → command-options-schema.ts} +0 -0
- /package/src/{schema.ts → define-index-schema.ts} +0 -0
- /package/src/{delete-command.options.ts → delete-options-schema.ts} +0 -0
- /package/src/utils/ts-doc/{constants.ts → node-kinds.ts} +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { Observable } from 'rxjs';
|
|
2
2
|
import type { VectorStoreDocument } from '@equinor/fusion-framework-module-ai/lib';
|
|
3
|
-
import type { IndexSchemaConfig } from '../schema.js';
|
|
3
|
+
import type { IndexSchemaConfig } from '../define-index-schema.js';
|
|
4
4
|
/**
|
|
5
5
|
* Creates an RxJS operator that resolves promoted schema fields for each
|
|
6
6
|
* document and separates them from the generic `attributes` bag.
|
package/dist/types/config.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { VectorStoreDocument } from '@equinor/fusion-framework-module-ai/lib';
|
|
2
2
|
import type { FusionAIConfig } from '@equinor/fusion-framework-cli-plugin-ai-base';
|
|
3
|
-
import type { IndexSchemaConfig } from './schema.js';
|
|
3
|
+
import type { IndexSchemaConfig } from './define-index-schema.js';
|
|
4
4
|
/**
|
|
5
5
|
* Index-specific configuration for Fusion AI document indexing operations.
|
|
6
6
|
*
|
package/dist/types/index.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { Command } from 'commander';
|
|
2
2
|
export { FusionAIConfigWithIndex, IndexConfig } from './config.js';
|
|
3
|
-
export { defineIndexSchema, IndexSchemaConfig } from './schema.js';
|
|
3
|
+
export { defineIndexSchema, IndexSchemaConfig } from './define-index-schema.js';
|
|
4
4
|
/**
|
|
5
5
|
* Registers the `ai index` command with the Fusion Framework CLI.
|
|
6
6
|
*
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { ChangedFile, GitDiffOptions } from './types.js';
|
|
2
|
+
/**
|
|
3
|
+
* Returns a list of files changed between `baseRef` and HEAD.
|
|
4
|
+
*
|
|
5
|
+
* Parses the output of `git diff --name-status` to classify each file as
|
|
6
|
+
* `'new'`, `'modified'`, or `'removed'`. Renames are expanded into a
|
|
7
|
+
* `'removed'` entry for the old path and a `'new'` entry for the new path.
|
|
8
|
+
*
|
|
9
|
+
* @param options - Configuration controlling the diff reference and working directory.
|
|
10
|
+
* @returns Array of changed files with their status.
|
|
11
|
+
* @throws {Error} If the working directory is not inside a git repository.
|
|
12
|
+
*/
|
|
13
|
+
export declare const getChangedFiles: (options: GitDiffOptions) => Promise<ChangedFile[]>;
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { ChangedFile } from './types.js';
|
|
2
|
+
/**
|
|
3
|
+
* Determines the git change status of a single file.
|
|
4
|
+
*
|
|
5
|
+
* Checks tracked status, porcelain output, and rename/copy detection to
|
|
6
|
+
* produce one or two {@link ChangedFile} entries (two when a rename is
|
|
7
|
+
* detected — one `'removed'` for the old path and one `'new'` for the
|
|
8
|
+
* current path).
|
|
9
|
+
*
|
|
10
|
+
* @param filePath - Absolute path to the file to inspect.
|
|
11
|
+
* @returns Array with one or two changed-file entries.
|
|
12
|
+
*/
|
|
13
|
+
export declare const getFileStatus: (filePath: string) => Promise<ChangedFile[]>;
|
|
@@ -1,14 +1,4 @@
|
|
|
1
1
|
import { type SimpleGit } from 'simple-git';
|
|
2
|
-
/**
|
|
3
|
-
* Resolves the git repository root for a given file path.
|
|
4
|
-
*
|
|
5
|
-
* Walks up the directory tree looking for a `.git` directory or file
|
|
6
|
-
* (to support worktrees) and returns the enclosing directory.
|
|
7
|
-
*
|
|
8
|
-
* @param filePath - Absolute file or directory path to resolve from.
|
|
9
|
-
* @returns Absolute path to the repository root, or `undefined` if not inside a git repo.
|
|
10
|
-
*/
|
|
11
|
-
export declare const resolveProjectRoot: (filePath: string) => string | undefined;
|
|
12
2
|
/**
|
|
13
3
|
* Returns a cached `SimpleGit` instance scoped to the repository that
|
|
14
4
|
* contains `filePath`.
|
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
export type { GitMetadata, GitDiffOptions, FileChangeStatus, ChangedFile, } from './types.js';
|
|
2
|
-
export { resolveProjectRoot
|
|
3
|
-
export {
|
|
4
|
-
export {
|
|
5
|
-
export {
|
|
2
|
+
export { resolveProjectRoot } from './resolve-project-root.js';
|
|
3
|
+
export { getGit } from './get-git.js';
|
|
4
|
+
export { extractGitMetadata } from './extract-git-metadata.js';
|
|
5
|
+
export { getChangedFiles } from './get-changed-files.js';
|
|
6
|
+
export { getFileStatus } from './get-file-status.js';
|
|
7
|
+
export { isFileChanged } from './is-file-changed.js';
|
|
8
|
+
export { getGitStatus } from './get-git-status.js';
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { ChangedFile } from './types.js';
|
|
2
|
+
/**
|
|
3
|
+
* Checks whether a file path appears in a list of changed files.
|
|
4
|
+
*
|
|
5
|
+
* When the changed-files list is empty (no diff filtering active), every
|
|
6
|
+
* file is considered changed so that all files are processed.
|
|
7
|
+
*
|
|
8
|
+
* @param filePath - Absolute file path to look up.
|
|
9
|
+
* @param changedFiles - Array of {@link ChangedFile} entries to search.
|
|
10
|
+
* @returns `true` if the file has changed or if diff filtering is disabled.
|
|
11
|
+
*/
|
|
12
|
+
export declare const isFileChanged: (filePath: string, changedFiles: ChangedFile[]) => boolean;
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Resolves the git repository root for a given file path.
|
|
3
|
+
*
|
|
4
|
+
* Walks up the directory tree looking for a `.git` directory or file
|
|
5
|
+
* (to support worktrees) and returns the enclosing directory.
|
|
6
|
+
*
|
|
7
|
+
* @param filePath - Absolute file or directory path to resolve from.
|
|
8
|
+
* @returns Absolute path to the repository root, or `undefined` if not inside a git repo.
|
|
9
|
+
*/
|
|
10
|
+
export declare const resolveProjectRoot: (filePath: string) => string | undefined;
|
|
@@ -1,2 +1,4 @@
|
|
|
1
1
|
export type { MarkdownMetadata, MarkdownDocument } from './types.js';
|
|
2
|
-
export { isMarkdownFile
|
|
2
|
+
export { isMarkdownFile } from './is-markdown-file.js';
|
|
3
|
+
export { parseMarkdown } from './parse-markdown.js';
|
|
4
|
+
export { parseMarkdownFile } from './parse-markdown-file.js';
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Checks whether a file path has a Markdown (`.md`) or MDX (`.mdx`) extension.
|
|
3
|
+
*
|
|
4
|
+
* @param filePath - Absolute or relative file path.
|
|
5
|
+
* @returns `true` when the extension is `.md` or `.mdx`.
|
|
6
|
+
*/
|
|
7
|
+
export declare const isMarkdownFile: (filePath: string) => boolean;
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { SourceFile } from '../types.js';
|
|
2
|
+
import type { MarkdownDocument } from './types.js';
|
|
3
|
+
/**
|
|
4
|
+
* Reads a Markdown or MDX file from disk and parses it into chunked documents.
|
|
5
|
+
*
|
|
6
|
+
* Delegates to {@link parseMarkdown} after reading the file content, then
|
|
7
|
+
* enriches each resulting document with the `rootPath` from the source file.
|
|
8
|
+
*
|
|
9
|
+
* @template T - Additional frontmatter attributes.
|
|
10
|
+
* @param file - Source file descriptor with path and optional project root.
|
|
11
|
+
* @returns Array of Markdown documents with root-path metadata.
|
|
12
|
+
* @throws {AssertionError} If the file does not have a `.md` or `.mdx` extension.
|
|
13
|
+
*/
|
|
14
|
+
export declare const parseMarkdownFile: <T extends Record<string, unknown> = Record<string, unknown>>(file: SourceFile) => Promise<MarkdownDocument<T>[]>;
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { MarkdownDocument } from './types.js';
|
|
2
|
+
/**
|
|
3
|
+
* Parses Markdown or MDX content into chunked vector-store documents.
|
|
4
|
+
*
|
|
5
|
+
* Extracts YAML frontmatter via `gray-matter`, splits the body using
|
|
6
|
+
* {@link RecursiveCharacterTextSplitter}, and returns one
|
|
7
|
+
* {@link MarkdownDocument} per valid chunk.
|
|
8
|
+
*
|
|
9
|
+
* @template T - Additional frontmatter attributes.
|
|
10
|
+
* @param content - Raw Markdown / MDX string.
|
|
11
|
+
* @param source - Relative source file path used as the document key.
|
|
12
|
+
* @returns Array of chunked Markdown documents.
|
|
13
|
+
*/
|
|
14
|
+
export declare const parseMarkdown: <T extends Record<string, unknown> = Record<string, unknown>>(content: string, source: string) => Promise<MarkdownDocument<T>[]>;
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { SourceFile as ProjectSourceFile } from 'ts-morph';
|
|
2
|
+
import type { TypescriptDocument, ParseTsDocOptions } from './types.js';
|
|
3
|
+
/**
|
|
4
|
+
* Creates a TypescriptDocument from the given parameters.
|
|
5
|
+
* @param sourceFile - The source file containing the node.
|
|
6
|
+
* @param pageContent - The content of the document.
|
|
7
|
+
* @param tsKind - The TypeScript node kind name.
|
|
8
|
+
* @param tsName - The name of the TypeScript node.
|
|
9
|
+
* @param options - Optional parsing configuration.
|
|
10
|
+
* @returns A TypeScript document with metadata.
|
|
11
|
+
*/
|
|
12
|
+
export declare const createTypescriptDocument: (sourceFile: ProjectSourceFile, pageContent: string, tsKind: string, tsName: string, options?: ParseTsDocOptions) => TypescriptDocument;
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { type SourceFile as ProjectSourceFile, type ClassDeclaration } from 'ts-morph';
|
|
2
|
+
import type { TypescriptDocument, ParseTsDocOptions } from './types.js';
|
|
3
|
+
/**
|
|
4
|
+
* Extracts a vector-store document from a TypeScript class declaration.
|
|
5
|
+
*
|
|
6
|
+
* Collects the class-level TSDoc comment, constructor signature (if documented),
|
|
7
|
+
* and all public members with TSDoc into a single document whose `pageContent`
|
|
8
|
+
* mirrors a minimal class interface.
|
|
9
|
+
*
|
|
10
|
+
* @param classNode - The `ts-morph` {@link ClassDeclaration} node to inspect.
|
|
11
|
+
* @param sourceFile - The project source file that contains the class.
|
|
12
|
+
* @param _options - Optional parsing configuration.
|
|
13
|
+
* @returns A {@link TypescriptDocument}, or `null` when the class has no TSDoc.
|
|
14
|
+
*/
|
|
15
|
+
export declare const extractDocumentFromClassNode: (classNode: ClassDeclaration, sourceFile: ProjectSourceFile, _options?: ParseTsDocOptions) => TypescriptDocument | null;
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { type SourceFile as ProjectSourceFile, Node } from 'ts-morph';
|
|
2
|
+
import type { TypescriptDocument, ParseTsDocOptions } from './types.js';
|
|
3
|
+
/**
|
|
4
|
+
* Extracts a vector-store document from a single TypeScript AST node.
|
|
5
|
+
*
|
|
6
|
+
* Handles function declarations, variable statements (arrow / function
|
|
7
|
+
* expressions), interfaces, type aliases, enums, and classes. Delegates
|
|
8
|
+
* to {@link extractDocumentFromClassNode} for class declarations.
|
|
9
|
+
*
|
|
10
|
+
* @param node - The `ts-morph` AST node to inspect.
|
|
11
|
+
* @param sourceFile - The project source file that contains the node.
|
|
12
|
+
* @param options - Optional parsing configuration.
|
|
13
|
+
* @param nodeOptions - Optional flags (e.g. `skipKindCheck`) to override default
|
|
14
|
+
* kind filtering.
|
|
15
|
+
* @returns A {@link TypescriptDocument}, or `null` when the node has no TSDoc or
|
|
16
|
+
* is not a supported kind.
|
|
17
|
+
*/
|
|
18
|
+
export declare const extractDocumentFromNode: (node: Node, sourceFile: ProjectSourceFile, options?: ParseTsDocOptions, nodeOptions?: {
|
|
19
|
+
skipKindCheck?: boolean;
|
|
20
|
+
}) => TypescriptDocument | null;
|
|
@@ -1,2 +1,4 @@
|
|
|
1
1
|
export type { TypescriptMetadata, TypescriptDocument, ParseTsDocOptions } from './types.js';
|
|
2
|
-
export { isTypescriptFile
|
|
2
|
+
export { isTypescriptFile } from './is-typescript-file.js';
|
|
3
|
+
export { parseTsDocSync } from './parse-ts-doc-sync.js';
|
|
4
|
+
export { parseTsDocFromFileSync } from './parse-ts-doc-from-file-sync.js';
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Checks whether a file path has a TypeScript (`.ts`) or TSX (`.tsx`) extension.
|
|
3
|
+
*
|
|
4
|
+
* @param filePath - Absolute or relative file path.
|
|
5
|
+
* @returns `true` if the file extension is `.ts` or `.tsx`.
|
|
6
|
+
*/
|
|
7
|
+
export declare const isTypescriptFile: (filePath: string) => boolean;
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { SourceFile } from '../types.js';
|
|
2
|
+
import type { ParseTsDocOptions, TypescriptDocument } from './types.js';
|
|
3
|
+
/**
|
|
4
|
+
* Parses TSDoc comments from a TypeScript file on disk.
|
|
5
|
+
*
|
|
6
|
+
* Reads the file synchronously, creates a `ts-morph` project, and returns
|
|
7
|
+
* one {@link TypescriptDocument} per documented top-level declaration.
|
|
8
|
+
*
|
|
9
|
+
* @param file - Source file descriptor with path and optional project root.
|
|
10
|
+
* @param options - Optional parsing configuration.
|
|
11
|
+
* @returns Array of extracted TypeScript documents.
|
|
12
|
+
* @throws {AssertionError} If the file does not have a `.ts` or `.tsx` extension.
|
|
13
|
+
*/
|
|
14
|
+
export declare const parseTsDocFromFileSync: (file: SourceFile, options?: ParseTsDocOptions) => TypescriptDocument[];
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { ParseTsDocOptions, TypescriptDocument } from './types.js';
|
|
2
|
+
/**
|
|
3
|
+
* Parses TSDoc comments from an in-memory TypeScript code string.
|
|
4
|
+
*
|
|
5
|
+
* Creates a temporary `ts-morph` project, analyses the source, and returns
|
|
6
|
+
* one {@link TypescriptDocument} per documented top-level declaration.
|
|
7
|
+
*
|
|
8
|
+
* @param content - TypeScript source code to parse.
|
|
9
|
+
* @param options - Optional parsing configuration.
|
|
10
|
+
* @returns Array of extracted TypeScript documents with TSDoc metadata.
|
|
11
|
+
*/
|
|
12
|
+
export declare const parseTsDocSync: (content: string, options?: ParseTsDocOptions) => TypescriptDocument[];
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { SourceFile as ProjectSourceFile } from 'ts-morph';
|
|
2
|
+
import type { TypescriptDocument, ParseTsDocOptions } from './types.js';
|
|
3
|
+
/**
|
|
4
|
+
* Walks a TypeScript source file and extracts a {@link TypescriptDocument}
|
|
5
|
+
* for every top-level declaration that carries a TSDoc comment.
|
|
6
|
+
*
|
|
7
|
+
* @param sourceFile - The `ts-morph` source file to traverse.
|
|
8
|
+
* @param options - Optional parsing configuration.
|
|
9
|
+
* @returns Array of extracted documents (one per documented declaration).
|
|
10
|
+
*/
|
|
11
|
+
export declare const processSourceFile: (sourceFile: ProjectSourceFile, options?: ParseTsDocOptions) => TypescriptDocument[];
|
package/dist/types/version.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export declare const version = "3.0.
|
|
1
|
+
export declare const version = "3.0.7";
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@equinor/fusion-framework-cli-plugin-ai-index",
|
|
3
|
-
"version": "3.0.
|
|
3
|
+
"version": "3.0.7",
|
|
4
4
|
"description": "AI indexing plugin for Fusion Framework CLI providing document embedding and chunking utilities",
|
|
5
5
|
"main": "dist/esm/index.js",
|
|
6
6
|
"type": "module",
|
|
@@ -53,18 +53,18 @@
|
|
|
53
53
|
"tree-sitter-typescript": "^0.23.2",
|
|
54
54
|
"ts-morph": "^28.0.0",
|
|
55
55
|
"zod": "^4.4.3",
|
|
56
|
-
"@equinor/fusion-framework-cli-plugin-ai-base": "4.0.
|
|
57
|
-
"@equinor/fusion-framework-module
|
|
58
|
-
"@equinor/fusion-
|
|
59
|
-
"@equinor/fusion-
|
|
56
|
+
"@equinor/fusion-framework-cli-plugin-ai-base": "4.0.6",
|
|
57
|
+
"@equinor/fusion-framework-module": "6.1.1",
|
|
58
|
+
"@equinor/fusion-imports": "2.0.1",
|
|
59
|
+
"@equinor/fusion-framework-module-ai": "6.0.2"
|
|
60
60
|
},
|
|
61
61
|
"peerDependencies": {
|
|
62
|
-
"@equinor/fusion-framework-cli": "^15.1
|
|
62
|
+
"@equinor/fusion-framework-cli": "^15.2.1"
|
|
63
63
|
},
|
|
64
64
|
"devDependencies": {
|
|
65
|
-
"typescript": "^
|
|
65
|
+
"typescript": "^7.0.2",
|
|
66
66
|
"vitest": "^4.1.0",
|
|
67
|
-
"@equinor/fusion-framework-cli": "^15.1
|
|
67
|
+
"@equinor/fusion-framework-cli": "^15.2.1"
|
|
68
68
|
},
|
|
69
69
|
"scripts": {
|
|
70
70
|
"build": "tsc -b",
|
|
@@ -3,7 +3,7 @@ import { from, mergeMap, map, tap, toArray } from 'rxjs';
|
|
|
3
3
|
import type { Observable } from 'rxjs';
|
|
4
4
|
import type { VectorStoreDocument } from '@equinor/fusion-framework-module-ai/lib';
|
|
5
5
|
import { extractGitMetadata } from '../utils/git/index.js';
|
|
6
|
-
import { resolvePackage } from '../utils/package
|
|
6
|
+
import { resolvePackage } from '../utils/resolve-package.js';
|
|
7
7
|
import type { DocumentEntry } from './types.js';
|
|
8
8
|
import type { FusionAIConfigWithIndex } from '../config.js';
|
|
9
9
|
|
|
@@ -12,6 +12,11 @@ export type MetadataProgressCallback = (source: string) => void;
|
|
|
12
12
|
|
|
13
13
|
/**
|
|
14
14
|
* Creates a stream that applies metadata to documents.
|
|
15
|
+
*
|
|
16
|
+
* @param document$ - Stream of document entries to enrich with metadata.
|
|
17
|
+
* @param indexConfig - Index configuration controlling git/package metadata resolution.
|
|
18
|
+
* @param onProgress - Optional callback invoked after each document is enriched.
|
|
19
|
+
* @returns A stream emitting enriched documents grouped by source entry.
|
|
15
20
|
* @internal
|
|
16
21
|
*/
|
|
17
22
|
export function applyMetadata(
|
|
@@ -32,62 +37,69 @@ export function applyMetadata(
|
|
|
32
37
|
*/
|
|
33
38
|
const ENTRY_CONCURRENCY = 20;
|
|
34
39
|
|
|
35
|
-
return
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
mergeMap(
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
40
|
+
return (
|
|
41
|
+
document$
|
|
42
|
+
// Enrich each document entry with git/package metadata and re-batch the results
|
|
43
|
+
.pipe(
|
|
44
|
+
mergeMap((entry) => {
|
|
45
|
+
// Extract git metadata concurrently (capped to limit parallel git processes)
|
|
46
|
+
return from(entry.documents).pipe(
|
|
47
|
+
mergeMap(async (document): Promise<VectorStoreDocument> => {
|
|
48
|
+
const rootPath = document.metadata.rootPath ?? process.cwd();
|
|
49
|
+
const sourcePath = path.join(rootPath, document.metadata.source);
|
|
50
|
+
const gitMetadata =
|
|
51
|
+
document.metadata.source && indexConfig?.metadata?.resolveGit !== false
|
|
52
|
+
? await extractGitMetadata(sourcePath)
|
|
53
|
+
: {};
|
|
46
54
|
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
55
|
+
// Resolve package information if enabled
|
|
56
|
+
let packageMetadata = {};
|
|
57
|
+
// Only attempt package resolution when explicitly enabled and a source path is present.
|
|
58
|
+
if (shouldResolvePackage && document.metadata.source) {
|
|
59
|
+
packageMetadata = await resolvePackage(sourcePath)
|
|
60
|
+
.then((pkg) => {
|
|
61
|
+
return {
|
|
62
|
+
pkg_name: pkg?.name,
|
|
63
|
+
pkg_version: pkg?.version,
|
|
64
|
+
pkg_keywords: pkg?.keywords,
|
|
65
|
+
};
|
|
66
|
+
})
|
|
67
|
+
.catch(() => ({}));
|
|
68
|
+
}
|
|
69
|
+
// Merge git and package metadata into the document's existing attributes
|
|
70
|
+
return {
|
|
71
|
+
...document,
|
|
72
|
+
metadata: {
|
|
73
|
+
...document.metadata,
|
|
74
|
+
attributes: {
|
|
75
|
+
...document.metadata.attributes,
|
|
76
|
+
...gitMetadata,
|
|
77
|
+
...packageMetadata,
|
|
78
|
+
},
|
|
79
|
+
},
|
|
80
|
+
};
|
|
81
|
+
}, GIT_CONCURRENCY),
|
|
82
|
+
// Notify caller after each document is enriched
|
|
83
|
+
tap((document) => onProgress?.(document.metadata.source)),
|
|
84
|
+
// Apply custom attribute processor from config
|
|
85
|
+
map((document: VectorStoreDocument) => {
|
|
86
|
+
const attributeProcessor =
|
|
87
|
+
indexConfig?.metadata?.attributeProcessor ||
|
|
88
|
+
((attributes: Record<string, unknown>, _document: VectorStoreDocument) =>
|
|
89
|
+
attributes);
|
|
90
|
+
const attributes = attributeProcessor(document.metadata.attributes ?? {}, document);
|
|
91
|
+
return {
|
|
92
|
+
...document,
|
|
93
|
+
metadata: {
|
|
94
|
+
...document.metadata,
|
|
95
|
+
attributes,
|
|
96
|
+
},
|
|
97
|
+
};
|
|
98
|
+
}),
|
|
99
|
+
// Group back by file for batch deletion in next step
|
|
100
|
+
toArray(),
|
|
101
|
+
);
|
|
102
|
+
}, ENTRY_CONCURRENCY),
|
|
103
|
+
)
|
|
92
104
|
);
|
|
93
105
|
}
|
|
@@ -3,7 +3,7 @@ import { z } from 'zod';
|
|
|
3
3
|
import { of, lastValueFrom } from 'rxjs';
|
|
4
4
|
import type { VectorStoreDocument } from '@equinor/fusion-framework-module-ai/lib';
|
|
5
5
|
|
|
6
|
-
import { defineIndexSchema } from '../schema.js';
|
|
6
|
+
import { defineIndexSchema } from '../define-index-schema.js';
|
|
7
7
|
import { applySchema } from './apply-schema.js';
|
|
8
8
|
|
|
9
9
|
/** Helper to create a minimal VectorStoreDocument for testing. */
|
|
@@ -140,6 +140,7 @@ describe('applySchema', () => {
|
|
|
140
140
|
prepareAttributes: (attrs, doc) => {
|
|
141
141
|
// Type-safe: attrs.tags is string[] | undefined
|
|
142
142
|
attrs.tags ??= [];
|
|
143
|
+
// Only tag documents that live under a package directory
|
|
143
144
|
if (doc.metadata.source.includes('packages/')) {
|
|
144
145
|
attrs.tags.push('package');
|
|
145
146
|
}
|
package/src/bin/apply-schema.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { map } from 'rxjs';
|
|
2
2
|
import type { Observable } from 'rxjs';
|
|
3
3
|
import type { VectorStoreDocument } from '@equinor/fusion-framework-module-ai/lib';
|
|
4
|
-
import type { IndexSchemaConfig } from '../schema.js';
|
|
4
|
+
import type { IndexSchemaConfig } from '../define-index-schema.js';
|
|
5
5
|
|
|
6
6
|
/**
|
|
7
7
|
* Creates an RxJS operator that resolves promoted schema fields for each
|
|
@@ -32,12 +32,15 @@ export function applySchema(
|
|
|
32
32
|
|
|
33
33
|
const promotedKeys = new Set(Object.keys(schema.shape.shape as Record<string, unknown>));
|
|
34
34
|
|
|
35
|
+
// Resolve and validate promoted schema fields for each document batch
|
|
35
36
|
return document$.pipe(
|
|
36
37
|
map((documents) =>
|
|
38
|
+
// Process each document in the batch individually
|
|
37
39
|
documents.map((document) => {
|
|
38
40
|
// Run typed attribute processor before schema resolution so the
|
|
39
41
|
// resolver receives fully enriched attributes
|
|
40
42
|
let enrichedDocument = document;
|
|
43
|
+
// Only run the attribute processor when the schema declares one
|
|
41
44
|
if (schema.prepareAttributes) {
|
|
42
45
|
const enrichedAttributes = schema.prepareAttributes(
|
|
43
46
|
(document.metadata.attributes ?? {}) as Record<string, unknown>,
|
|
@@ -66,7 +69,9 @@ export function applySchema(
|
|
|
66
69
|
unknown
|
|
67
70
|
>;
|
|
68
71
|
const remainingAttributes: Record<string, unknown> = {};
|
|
72
|
+
// Copy over only attributes that were not promoted to schema fields
|
|
69
73
|
for (const [key, value] of Object.entries(currentAttributes)) {
|
|
74
|
+
// Skip keys already promoted to schemaFields to avoid duplication
|
|
70
75
|
if (!promotedKeys.has(key)) {
|
|
71
76
|
remainingAttributes[key] = value;
|
|
72
77
|
}
|
|
@@ -2,10 +2,15 @@ import { map, mergeMap, toArray, filter } from 'rxjs';
|
|
|
2
2
|
import type { Observable } from 'rxjs';
|
|
3
3
|
import type { FrameworkInstance } from '@equinor/fusion-framework-cli-plugin-ai-base';
|
|
4
4
|
import type { ProcessedFile, DeleteRemovedFilesResult } from './types.js';
|
|
5
|
-
import type { CommandOptions } from '../
|
|
5
|
+
import type { CommandOptions } from '../command-options-schema.js';
|
|
6
6
|
|
|
7
7
|
/**
|
|
8
8
|
* Creates a stream that deletes removed files from the vector store.
|
|
9
|
+
*
|
|
10
|
+
* @param removedFiles$ - Stream of files detected as removed from the source tree.
|
|
11
|
+
* @param framework - Framework instance used to access the configured vector store.
|
|
12
|
+
* @param options - Command options controlling target index and dry-run behavior.
|
|
13
|
+
* @returns A stream emitting the result of each batch deletion.
|
|
9
14
|
* @internal
|
|
10
15
|
*/
|
|
11
16
|
export function createDeleteRemovedFilesStream(
|
|
@@ -13,25 +18,31 @@ export function createDeleteRemovedFilesStream(
|
|
|
13
18
|
framework: FrameworkInstance,
|
|
14
19
|
options: CommandOptions,
|
|
15
20
|
): Observable<DeleteRemovedFilesResult> {
|
|
21
|
+
// Batch removed files, then build a single OData filter for deletion
|
|
16
22
|
return removedFiles$.pipe(
|
|
17
23
|
toArray(),
|
|
18
24
|
map((files) => {
|
|
25
|
+
// Nothing to delete when there are no removed files
|
|
19
26
|
if (files.length === 0) {
|
|
20
27
|
return { files: [], filterExpression: null };
|
|
21
28
|
}
|
|
22
29
|
// Build OData filter: "metadata/source eq 'path1' or metadata/source eq 'path2'"
|
|
30
|
+
// Escape each removed file into an OData equality clause
|
|
23
31
|
const filterExpression = files
|
|
24
32
|
.map((file) => `metadata/source eq '${file.relativePath}'`)
|
|
25
33
|
.join(' or ');
|
|
26
34
|
return { files, filterExpression };
|
|
27
35
|
}),
|
|
28
36
|
mergeMap(async ({ files, filterExpression }) => {
|
|
37
|
+
// Skip the deletion call entirely when there is nothing to remove
|
|
29
38
|
if (files.length === 0) {
|
|
30
39
|
return undefined;
|
|
31
40
|
}
|
|
41
|
+
// Log each removed file for visibility before deleting
|
|
32
42
|
for (const file of files) {
|
|
33
43
|
console.log('Removing entry from vector store', file.relativePath);
|
|
34
44
|
}
|
|
45
|
+
// Skip the actual vector store mutation when running in dry-run mode
|
|
35
46
|
if (!options.dryRun) {
|
|
36
47
|
const vectorStoreService = framework.ai.useIndex(options.indexName);
|
|
37
48
|
// Single batch deletion - one file can produce multiple document chunks
|