@equinor/fusion-framework-cli-plugin-ai-index 1.0.0

package/dist/types/bin/apply-metadata.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/types/bin/delete-removed-files.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/types/bin/embed.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/types/bin/execute-pipeline.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/types/bin/file-stream.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/types/bin/get-diff.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/types/bin/index.d.ts

@@ -0,0 +1 @@
+export { embed } from './embed.js';

package/dist/types/bin/types.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/types/command.d.ts

@@ -0,0 +1,2 @@
+export declare const command: import("commander").Command;
+export default command;

package/dist/types/command.options.d.ts

@@ -0,0 +1,62 @@
+import { z } from 'zod';
+/**
+ * Zod schema for validating command options for the embeddings command.
+ *
+ * This schema extends the base AI options schema with embeddings-specific options,
+ * ensuring type safety and runtime validation of command arguments.
+ *
+ * Note: Some optional AI options become required for the embeddings command
+ * (openaiEmbeddingDeployment, azureSearchEndpoint, azureSearchApiKey, azureSearchIndexName)
+ * because the command uses withAiOptions with includeEmbedding and includeSearch set to true.
+ */
+export declare const CommandOptionsSchema: z.ZodObject<{
+    openaiApiKey: z.ZodString;
+    openaiApiVersion: z.ZodString;
+    openaiInstance: z.ZodString;
+    openaiChatDeployment: z.ZodOptional<z.ZodString>;
+} & {
+    openaiEmbeddingDeployment: z.ZodString;
+    azureSearchEndpoint: z.ZodString;
+    azureSearchApiKey: z.ZodString;
+    azureSearchIndexName: z.ZodString;
+    dryRun: z.ZodBoolean;
+    config: z.ZodString;
+    diff: z.ZodBoolean;
+    baseRef: z.ZodOptional<z.ZodString>;
+    clean: z.ZodBoolean;
+}, "strip", z.ZodTypeAny, {
+    openaiEmbeddingDeployment: string;
+    azureSearchEndpoint: string;
+    azureSearchApiKey: string;
+    azureSearchIndexName: string;
+    dryRun: boolean;
+    config: string;
+    diff: boolean;
+    clean: boolean;
+    openaiApiKey: string;
+    openaiApiVersion: string;
+    openaiInstance: string;
+    baseRef?: string | undefined;
+    openaiChatDeployment?: string | undefined;
+}, {
+    openaiEmbeddingDeployment: string;
+    azureSearchEndpoint: string;
+    azureSearchApiKey: string;
+    azureSearchIndexName: string;
+    dryRun: boolean;
+    config: string;
+    diff: boolean;
+    clean: boolean;
+    openaiApiKey: string;
+    openaiApiVersion: string;
+    openaiInstance: string;
+    baseRef?: string | undefined;
+    openaiChatDeployment?: string | undefined;
+}>;
+/**
+ * Type representing the validated command options.
+ *
+ * This type is inferred from the Zod schema and should be used throughout the command
+ * to ensure type safety and consistency with the schema.
+ */
+export type CommandOptions = z.infer<typeof CommandOptionsSchema>;

package/dist/types/config.d.ts

@@ -0,0 +1,33 @@
+import type { VectorStoreDocument } from '@equinor/fusion-framework-module-ai/lib';
+import type { FusionAIConfig } from '@equinor/fusion-framework-cli-plugin-ai-base';
+/**
+ * Index-specific configuration for Fusion AI operations
+ */
+export interface IndexConfig {
+    patterns?: string[];
+    /** Files will be processed as is, without any chunking or transformation */
+    rawPatterns?: string[];
+    /** Globby patterns to ignored, only used when providing paths to the command */
+    ignore?: string[];
+    /** Metadata processing configuration */
+    metadata?: {
+        /** Automatically resolve package information from source file paths */
+        resolvePackage?: boolean;
+        resolveGit?: boolean;
+        /** Custom metadata processors to transform metadata before embedding */
+        attributeProcessor?: (metadata: Record<string, unknown>, document: VectorStoreDocument) => Record<string, unknown>;
+    };
+    /** Embedding generation configuration */
+    embedding?: {
+        /** Size of text chunks for embedding */
+        chunkSize?: number;
+        /** Overlap between chunks */
+        chunkOverlap?: number;
+    };
+}
+/**
+ * Extended Fusion AI configuration with index-specific settings
+ */
+export interface FusionAIConfigWithIndex extends FusionAIConfig {
+    index?: IndexConfig;
+}

package/dist/types/index.d.ts

@@ -0,0 +1,8 @@
+import type { Command } from 'commander';
+/**
+ * Registers the AI index plugin command with the CLI program
+ * @param program - The Commander program instance to register commands with
+ */
+export declare function registerAiPlugin(program: Command): void;
+export default registerAiPlugin;
+export { configureFusionAI, type FusionAIConfig, } from '@equinor/fusion-framework-cli-plugin-ai-base';

package/dist/types/utils/generate-chunk-id.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Generates a unique identifier for a document chunk based on file path
+ * Creates a deterministic, URL-safe hash from the file path for validation and checks
+ * @param filePath - The file path to generate an ID from
+ * @param chunkIndex - Optional chunk index to append for multi-chunk documents
+ * @returns A base64-encoded hash of the file path, optionally suffixed with chunk index
+ */
+export declare const generateChunkId: (filePath: string, chunkIndex?: number) => string;

package/dist/types/utils/git/file-changes.d.ts

@@ -0,0 +1,21 @@
+import type { ChangedFile, GitDiffOptions } from './types.js';
+/**
+ * Get list of changed files using git diff with status
+ * @param options - Git diff configuration options
+ * @returns Array of changed files with their status
+ */
+export declare const getChangedFiles: (options: GitDiffOptions) => Promise<ChangedFile[]>;
+/**
+ * Determine the git status of a file, including handling renames
+ * Returns an array of ChangedFile objects - if the file was renamed, returns both old and new paths
+ * @param filePath - Absolute file path to check
+ * @returns Promise resolving to array of changed files (1 or 2 items if renamed)
+ */
+export declare const getFileStatus: (filePath: string) => Promise<ChangedFile[]>;
+/**
+ * Check if a file path matches any of the changed files
+ * @param filePath - File path to check
+ * @param changedFiles - Array of changed file objects
+ * @returns True if file has changed
+ */
+export declare const isFileChanged: (filePath: string, changedFiles: ChangedFile[]) => boolean;

package/dist/types/utils/git/git-client.d.ts

@@ -0,0 +1,17 @@
+import { type SimpleGit } from 'simple-git';
+/**
+ * Resolve the project root (git repository root) for a given file path
+ * @param filePath - File path to resolve from
+ * @returns Project root path or undefined if not in a git repository
+ */
+export declare const resolveProjectRoot: (filePath: string) => string | undefined;
+/**
+ * Get or create a SimpleGit instance for a given file path
+ * Uses caching to avoid creating multiple instances for the same repository
+ * @param filePath - File path to get git instance for
+ * @returns Git instance and repository path, or undefined if not in a git repository
+ */
+export declare const getGit: (filePath: string) => {
+    git: SimpleGit | undefined;
+    gitRepoPath: string;
+} | undefined;

package/dist/types/utils/git/index.d.ts

@@ -0,0 +1,5 @@
+export type { GitMetadata, GitDiffOptions, FileChangeStatus, ChangedFile, } from './types.js';
+export { resolveProjectRoot, getGit } from './git-client.js';
+export { extractGitMetadata } from './metadata.js';
+export { getChangedFiles, getFileStatus, isFileChanged } from './file-changes.js';
+export { getGitStatus } from './status.js';

package/dist/types/utils/git/metadata.d.ts

@@ -0,0 +1,7 @@
+import type { GitMetadata } from './types.js';
+/**
+ * Extract git metadata for a file
+ * @param filePath - Absolute file path
+ * @returns Git metadata or undefined if not in a git repository
+ */
+export declare const extractGitMetadata: (filePath: string) => Promise<GitMetadata | undefined>;

package/dist/types/utils/git/status.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Get git status information for debugging
+ * @param cwd - Working directory
+ * @returns Git status information
+ */
+export declare const getGitStatus: (cwd?: string) => Promise<{
+    branch: string;
+    commit: string;
+    hasChanges: boolean;
+    stagedFiles: number;
+    unstagedFiles: number;
+}>;

package/dist/types/utils/git/types.d.ts

@@ -0,0 +1,33 @@
+/**
+ * Git metadata extracted from repository
+ */
+export type GitMetadata = Partial<{
+    git_remote_url: string;
+    git_commit_hash: string;
+    git_commit_date: string;
+    git_link: string;
+}>;
+/**
+ * Git diff options for filtering changed files
+ */
+export interface GitDiffOptions {
+    /** Enable diff-based file filtering */
+    diff: boolean;
+    /** Git reference to compare against (default: HEAD~1) */
+    baseRef?: string;
+    /** Working directory for git operations */
+    cwd?: string;
+}
+/**
+ * File change status
+ */
+export type FileChangeStatus = 'new' | 'modified' | 'removed';
+/**
+ * Changed file information
+ */
+export interface ChangedFile {
+    /** Absolute file path */
+    filepath: string;
+    /** Change status: new, modified, or removed */
+    status: FileChangeStatus;
+}

package/dist/types/utils/markdown/index.d.ts

@@ -0,0 +1,2 @@
+export type { MarkdownMetadata, MarkdownDocument } from './types.js';
+export { isMarkdownFile, parseMarkdown, parseMarkdownFile } from './parser.js';

package/dist/types/utils/markdown/parser.d.ts

@@ -0,0 +1,21 @@
+import type { SourceFile } from '../types.js';
+import type { MarkdownDocument } from './types.js';
+/**
+ * Check if a file is a markdown or MDX file
+ * @param filePath - File path to check
+ * @returns True if file has .md or .mdx extension
+ */
+export declare const isMarkdownFile: (filePath: string) => boolean;
+/**
+ * Parse markdown or MDX content into document chunks
+ * @param content - Markdown or MDX content string
+ * @param source - Source file path
+ * @returns Array of markdown documents
+ */
+export declare const parseMarkdown: <T extends Record<string, unknown> = Record<string, unknown>>(content: string, source: string) => Promise<MarkdownDocument<T>[]>;
+/**
+ * Parse a markdown or MDX file into document chunks
+ * @param file - Source file object
+ * @returns Array of markdown documents with root path metadata
+ */
+export declare const parseMarkdownFile: <T extends Record<string, unknown> = Record<string, unknown>>(file: SourceFile) => Promise<MarkdownDocument<T>[]>;

package/dist/types/utils/markdown/types.d.ts

@@ -0,0 +1,11 @@
+import type { VectorStoreDocument, VectorStoreDocumentMetadata } from '@equinor/fusion-framework-module-ai/lib';
+/**
+ * Markdown document metadata
+ */
+export type MarkdownMetadata<T extends Record<string, unknown> = Record<string, unknown>> = VectorStoreDocumentMetadata<T & {
+    type: 'markdown';
+}>;
+/**
+ * Markdown document
+ */
+export type MarkdownDocument<T extends Record<string, unknown> = Record<string, unknown>> = VectorStoreDocument<MarkdownMetadata<T>>;

package/dist/types/utils/package-resolver.d.ts

@@ -0,0 +1,14 @@
+import { type PackageJson } from 'read-package-up';
+/**
+ * Resolves which package a file path belongs to.
+ * First checks the cache map, then uses read-package-up if no match found.
+ *
+ * @param filePath - Absolute or relative file path (e.g., '/path/to/packages/cli/src/index.ts')
+ * @returns Package.json if found, undefined otherwise
+ *
+ * @example
+ * ```ts
+ * const packageJson = await resolvePackage('/path/to/packages/cli/src/index.ts');
+ * ```
+ */
+export declare function resolvePackage(filePath: string): Promise<PackageJson | undefined>;

package/dist/types/utils/ts-doc/constants.d.ts

@@ -0,0 +1,5 @@
+import { SyntaxKind } from 'ts-morph';
+/**
+ * Supported TSDoc node kinds for top-level processing
+ */
+export declare const nodeKinds: SyntaxKind[];

package/dist/types/utils/ts-doc/extractors.d.ts

@@ -0,0 +1,28 @@
+import { type SourceFile as ProjectSourceFile, Node, type ClassDeclaration } from 'ts-morph';
+import type { TypescriptDocument, ParseTsDocOptions } from './types.js';
+/**
+ * Extracts a TSDoc document from a class node, including TSDoc, constructor, and public member signatures.
+ * @param classNode - The ClassDeclaration node to process.
+ * @param sourceFile - The source file containing the node.
+ * @param options - Optional parsing configuration.
+ * @returns A TypeScript document with TSDoc and class interface, or null if no TSDoc is found.
+ */
+export declare const extractDocumentFromClassNode: (classNode: ClassDeclaration, sourceFile: ProjectSourceFile, _options?: ParseTsDocOptions) => TypescriptDocument | null;
+/**
+ * Extracts a TSDoc document from a single node.
+ * @param node - The TypeScript node to process.
+ * @param sourceFile - The source file containing the node.
+ * @param options - Optional parsing configuration.
+ * @param nodeOptions - Optional node-specific configuration (e.g., skipKindCheck for VariableStatement).
+ * @returns A TypeScript document with TSDoc metadata, or null if no TSDoc is found.
+ */
+export declare const extractDocumentFromNode: (node: Node, sourceFile: ProjectSourceFile, options?: ParseTsDocOptions, nodeOptions?: {
+    skipKindCheck?: boolean;
+}) => TypescriptDocument | null;
+/**
+ * Processes a TypeScript source file to extract TSDoc documents.
+ * @param sourceFile - The source file to process.
+ * @param options - Optional parsing configuration.
+ * @returns An array of TypeScript documents with TSDoc metadata.
+ */
+export declare const processSourceFile: (sourceFile: ProjectSourceFile, options?: ParseTsDocOptions) => TypescriptDocument[];

package/dist/types/utils/ts-doc/index.d.ts

@@ -0,0 +1,2 @@
+export type { TypescriptMetadata, TypescriptDocument, ParseTsDocOptions } from './types.js';
+export { isTypescriptFile, parseTsDocSync, parseTsDocFromFileSync } from './parser.js';

package/dist/types/utils/ts-doc/parser.d.ts

@@ -0,0 +1,23 @@
+import type { SourceFile } from '../types.js';
+import type { TypescriptDocument, ParseTsDocOptions } from './types.js';
+/**
+ * Checks if a file is a TypeScript or TSX file based on its extension.
+ * @param filePath - The path to the file.
+ * @returns True if the file ends with .ts or .tsx, false otherwise.
+ */
+export declare const isTypescriptFile: (filePath: string) => boolean;
+/**
+ * Parses TSDoc from a string of TypeScript code.
+ * @param content - The TypeScript code content.
+ * @param options - Optional parsing configuration.
+ * @returns An array of TypeScript documents with TSDoc metadata.
+ */
+export declare const parseTsDocSync: (content: string, options?: ParseTsDocOptions) => TypescriptDocument[];
+/**
+ * Parses TSDoc from a TypeScript file by path.
+ * @param file - The source file object.
+ * @param options - Optional parsing configuration.
+ * @returns An array of TypeScript documents with TSDoc metadata.
+ * @throws If the file is not a TypeScript file.
+ */
+export declare const parseTsDocFromFileSync: (file: SourceFile, options?: ParseTsDocOptions) => TypescriptDocument[];

package/dist/types/utils/ts-doc/types.d.ts

@@ -0,0 +1,20 @@
+import type { VectorStoreDocument, VectorStoreDocumentMetadata } from '@equinor/fusion-framework-module-ai/lib';
+/**
+ * TypeScript document metadata
+ */
+export type TypescriptMetadata = VectorStoreDocumentMetadata<{
+    type: 'tsdoc';
+    ts_kind: string;
+    ts_name: string;
+}>;
+/**
+ * TypeScript document with TSDoc metadata
+ */
+export type TypescriptDocument = VectorStoreDocument<TypescriptMetadata>;
+/**
+ * Options for parsing TypeScript documents
+ */
+export interface ParseTsDocOptions {
+    /** The project root path for generating relative paths */
+    projectRoot?: string;
+}

package/dist/types/utils/types.d.ts

@@ -0,0 +1,17 @@
+/**
+ * File change status in git
+ */
+export type FileStatus = 'new' | 'modified' | 'removed';
+/**
+ * Source file information for processing
+ */
+export type SourceFile = {
+    /** Absolute file path */
+    path: string;
+    /** Project root directory (git repository root) */
+    projectRoot?: string;
+    /** Relative path from project root */
+    relativePath?: string;
+    /** Git change status */
+    status: FileStatus;
+};

package/dist/types/version.d.ts

@@ -0,0 +1 @@
+export declare const version = "1.0.0";

package/package.json

@@ -0,0 +1,72 @@
+{
+  "name": "@equinor/fusion-framework-cli-plugin-ai-index",
+  "version": "1.0.0",
+  "description": "AI indexing plugin for Fusion Framework CLI providing document embedding and chunking utilities",
+  "main": "dist/esm/index.js",
+  "type": "module",
+  "types": "dist/types/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/esm/index.js",
+      "types": "./dist/types/index.d.ts"
+    }
+  },
+  "typesVersions": {
+    "*": {
+      ".": [
+        "dist/types/index.d.ts"
+      ]
+    }
+  },
+  "keywords": [
+    "fusion-framework",
+    "cli",
+    "plugin",
+    "llm",
+    "ai",
+    "index",
+    "embeddings"
+  ],
+  "author": "",
+  "license": "ISC",
+  "publishConfig": {
+    "access": "public"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/equinor/fusion-framework.git",
+    "directory": "packages/cli-plugins/ai-index"
+  },
+  "dependencies": {
+    "@langchain/textsplitters": "^1.0.0",
+    "commander": "^14.0.1",
+    "find-up": "^8.0.0",
+    "globby": "^15.0.0",
+    "gray-matter": "^4.0.3",
+    "multimatch": "^7.0.0",
+    "read-package-up": "^11.0.0",
+    "rxjs": "^7.8.1",
+    "simple-git": "^3.28.0",
+    "tree-sitter": "^0.25.0",
+    "tree-sitter-typescript": "^0.23.2",
+    "ts-morph": "^27.0.2",
+    "zod": "^3.23.8",
+    "@equinor/fusion-framework-module": "5.0.5",
+    "@equinor/fusion-framework-module-ai": "2.0.0",
+    "@equinor/fusion-framework-cli-plugin-ai-base": "1.0.0",
+    "@equinor/fusion-imports": "1.1.8"
+  },
+  "peerDependencies": {
+    "@equinor/fusion-framework-cli": "13.0.0"
+  },
+  "devDependencies": {
+    "typescript": "^5.8.2",
+    "vitest": "^3.2.4"
+  },
+  "scripts": {
+    "build": "tsc -b",
+    "build:types": "tsc -b",
+    "watch": "tsc -b --watch",
+    "test": "vitest"
+  }
+}

package/src/bin/apply-metadata.ts

@@ -0,0 +1,77 @@
+import path from 'node:path';
+import { from, mergeMap, map, toArray } from 'rxjs';
+import type { Observable } from 'rxjs';
+import type { VectorStoreDocument } from '@equinor/fusion-framework-module-ai/lib';
+import { extractGitMetadata } from '../utils/git/index.js';
+import { resolvePackage } from '../utils/package-resolver.js';
+import type { DocumentEntry } from './types.js';
+import type { FusionAIConfigWithIndex } from '../config.js';
+
+/**
+ * Creates a stream that applies metadata to documents.
+ * @internal
+ */
+export function applyMetadata(
+  document$: Observable<DocumentEntry>,
+  indexConfig: FusionAIConfigWithIndex['index'],
+): Observable<VectorStoreDocument[]> {
+  // Resolve packages if enabled
+  const shouldResolvePackage = indexConfig?.metadata?.resolvePackage ?? false;
+
+  return document$.pipe(
+    mergeMap((entry) => {
+      return from(entry.documents).pipe(
+        // Extract git metadata concurrently for all documents
+        mergeMap(async (document): Promise<VectorStoreDocument> => {
+          const rootPath = document.metadata.rootPath ?? process.cwd();
+          const sourcePath = path.join(rootPath, document.metadata.source);
+          const gitMetadata =
+            document.metadata.source && indexConfig?.metadata?.resolveGit !== false
+              ? await extractGitMetadata(sourcePath)
+              : {};
+
+          // Resolve package information if enabled
+          let packageMetadata = {};
+          if (shouldResolvePackage && document.metadata.source) {
+            packageMetadata = await resolvePackage(sourcePath)
+              .then((pkg) => {
+                return {
+                  pkg_name: pkg?.name,
+                  pkg_version: pkg?.version,
+                  pkg_keywords: pkg?.keywords,
+                };
+              })
+              .catch(() => ({}));
+          }
+          return {
+            ...document,
+            metadata: {
+              ...document.metadata,
+              attributes: {
+                ...document.metadata.attributes,
+                ...gitMetadata,
+                ...packageMetadata,
+              },
+            },
+          };
+        }),
+        // Apply custom attribute processor from config
+        map((document: VectorStoreDocument) => {
+          const attributeProcessor =
+            indexConfig?.metadata?.attributeProcessor ||
+            ((attributes: Record<string, unknown>, _document: VectorStoreDocument) => attributes);
+          const attributes = attributeProcessor(document.metadata.attributes ?? {}, document);
+          return {
+            ...document,
+            metadata: {
+              ...document.metadata,
+              attributes,
+            },
+          };
+        }),
+        // Group back by file for batch deletion in next step
+        toArray(),
+      );
+    }),
+  );
+}

package/src/bin/delete-removed-files.ts

@@ -0,0 +1,49 @@
+import { map, mergeMap, toArray, filter } from 'rxjs';
+import type { Observable } from 'rxjs';
+import type { FrameworkInstance } from '@equinor/fusion-framework-cli-plugin-ai-base';
+import type { ProcessedFile, DeleteRemovedFilesResult } from './types.js';
+import type { CommandOptions } from '../command.options.js';
+
+/**
+ * Creates a stream that deletes removed files from the vector store.
+ * @internal
+ */
+export function createDeleteRemovedFilesStream(
+  removedFiles$: Observable<ProcessedFile>,
+  framework: FrameworkInstance,
+  options: CommandOptions,
+): Observable<DeleteRemovedFilesResult> {
+  return removedFiles$.pipe(
+    toArray(),
+    map((files) => {
+      if (files.length === 0) {
+        return { files: [], filterExpression: null };
+      }
+      // Build OData filter: "metadata/source eq 'path1' or metadata/source eq 'path2'"
+      const filterExpression = files
+        .map((file) => `metadata/source eq '${file.relativePath}'`)
+        .join(' or ');
+      return { files, filterExpression };
+    }),
+    mergeMap(async ({ files, filterExpression }) => {
+      if (files.length === 0) {
+        return undefined;
+      }
+      for (const file of files) {
+        console.log('Removing entry from vector store', file.relativePath);
+      }
+      if (!options.dryRun) {
+        const vectorStoreService = framework.ai.getService('search', options.azureSearchIndexName);
+        // Single batch deletion - one file can produce multiple document chunks
+        await vectorStoreService.deleteDocuments({
+          filter: { filterExpression: filterExpression ?? undefined },
+        });
+      }
+      return {
+        status: 'deleted',
+        files: files as { relativePath: string }[],
+      };
+    }),
+    filter((result): result is DeleteRemovedFilesResult => Boolean(result)),
+  );
+}