@equinor/fusion-framework-cli-plugin-ai-index 1.0.5 → 2.0.0

package/dist/types/utils/ts-doc/types.d.ts

@@ -1,20 +1,28 @@
 import type { VectorStoreDocument, VectorStoreDocumentMetadata } from '@equinor/fusion-framework-module-ai/lib';
 /**
- * TypeScript document metadata
+ * Metadata shape for documents generated from TypeScript source files.
+ *
+ * Extends the base vector-store metadata with TSDoc-specific fields.
  */
 export type TypescriptMetadata = VectorStoreDocumentMetadata<{
+    /** Discriminator identifying the document as extracted from TSDoc. */
     type: 'tsdoc';
+    /** The `ts-morph` syntax-kind name (e.g. `'FunctionDeclaration'`). */
     ts_kind: string;
+    /** Name of the TypeScript symbol (function, class, interface, etc.). */
     ts_name: string;
 }>;
 /**
- * TypeScript document with TSDoc metadata
+ * A vector-store document originating from a TypeScript source file.
+ *
+ * Contains the extracted TSDoc comment (and optionally the type signature)
+ * together with {@link TypescriptMetadata}.
  */
 export type TypescriptDocument = VectorStoreDocument<TypescriptMetadata>;
 /**
- * Options for parsing TypeScript documents
+ * Options for controlling TypeScript document parsing behaviour.
  */
 export interface ParseTsDocOptions {
-    /** The project root path for generating relative paths */
+    /** Absolute path to the project root, used to compute relative source paths. */
     projectRoot?: string;
 }

package/dist/types/utils/types.d.ts

@@ -1,17 +1,21 @@
 /**
- * File change status in git
+ * Git-tracked change status of a source file.
+ *
+ * - `'new'` — file is untracked or newly added.
+ * - `'modified'` — file is tracked and has been changed.
+ * - `'removed'` — file has been deleted.
  */
 export type FileStatus = 'new' | 'modified' | 'removed';
 /**
- * Source file information for processing
+ * Represents a source file to be indexed, enriched with path and git status info.
  */
 export type SourceFile = {
-    /** Absolute file path */
+    /** Absolute file system path. */
     path: string;
-    /** Project root directory (git repository root) */
+    /** Absolute path to the git repository root. */
     projectRoot?: string;
-    /** Relative path from project root */
+    /** Path relative to {@link projectRoot}. */
     relativePath?: string;
-    /** Git change status */
+    /** Current git change status. */
     status: FileStatus;
 };

package/dist/types/version.d.ts

@@ -1 +1 @@
-export declare const version = "1.0.5";
+export declare const version = "2.0.0";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@equinor/fusion-framework-cli-plugin-ai-index",
-  "version": "1.0.5",
+  "version": "2.0.0",
   "description": "AI indexing plugin for Fusion Framework CLI providing document embedding and chunking utilities",
   "main": "dist/esm/index.js",
   "type": "module",
@@ -39,30 +39,32 @@
   },
   "dependencies": {
     "@azure/search-documents": "^12.2.0",
+    "@langchain/core": "^1.0.1",
     "@langchain/textsplitters": "^1.0.0",
     "commander": "^14.0.1",
     "find-up": "^8.0.0",
     "globby": "^16.1.0",
     "gray-matter": "^4.0.3",
-    "multimatch": "^7.0.0",
+    "multimatch": "^8.0.0",
     "read-package-up": "^12.0.0",
     "rxjs": "^7.8.1",
-    "simple-git": "^3.28.0",
+    "simple-git": "^3.32.3",
     "tree-sitter": "^0.25.0",
     "tree-sitter-typescript": "^0.23.2",
     "ts-morph": "^27.0.2",
-    "zod": "^4.3.5",
-    "@equinor/fusion-framework-module": "5.0.5",
-    "@equinor/fusion-framework-module-ai": "2.0.1",
-    "@equinor/fusion-imports": "1.1.10",
-    "@equinor/fusion-framework-cli-plugin-ai-base": "1.0.4"
+    "zod": "^4.3.6",
+    "@equinor/fusion-framework-cli-plugin-ai-base": "2.0.0",
+    "@equinor/fusion-framework-module": "6.0.0",
+    "@equinor/fusion-framework-module-ai": "3.0.0",
+    "@equinor/fusion-imports": "2.0.0"
   },
   "peerDependencies": {
-    "@equinor/fusion-framework-cli": "^13.3.3"
+    "@equinor/fusion-framework-cli": "^14.0.0"
   },
   "devDependencies": {
-    "typescript": "^5.8.2",
-    "vitest": "^3.2.4"
+    "typescript": "^5.9.3",
+    "vitest": "^4.1.0",
+    "@equinor/fusion-framework-cli": "^14.0.0"
   },
   "scripts": {
     "build": "tsc -b",

package/src/bin/delete-removed-files.ts

@@ -2,7 +2,7 @@ import { map, mergeMap, toArray, filter } from 'rxjs';
 import type { Observable } from 'rxjs';
 import type { FrameworkInstance } from '@equinor/fusion-framework-cli-plugin-ai-base';
 import type { ProcessedFile, DeleteRemovedFilesResult } from './types.js';
-import type { CommandOptions } from '../command.options.js';
+import type { CommandOptions } from '../embeddings-command.options.js';
 
 /**
  * Creates a stream that deletes removed files from the vector store.

package/src/bin/embed.ts

@@ -1,8 +1,8 @@
 import { globbyStream } from 'globby';
 import { relative } from 'node:path';
 import multimatch from 'multimatch';
-import { concat, from, merge } from 'rxjs';
-import { concatMap, filter, map, mergeMap, shareReplay, toArray } from 'rxjs/operators';
+import { concat, from, merge, timer } from 'rxjs';
+import { concatMap, filter, map, mergeMap, retry, shareReplay, toArray } from 'rxjs/operators';
 
 import { isMarkdownFile, parseMarkdownFile } from '../utils/markdown/index.js';
 import { getFileStatus, resolveProjectRoot } from '../utils/git/index.js';
@@ -36,6 +36,8 @@ const defaultIgnore = ['node_modules', '**/node_modules/**', 'dist', '**/dist/**
 export async function embed(binOptions: EmbeddingsBinOptions): Promise<void> {
   const { framework, options, config, filePatterns } = binOptions;
 
+  console.log(`📇 Index: ${options.azureSearchIndexName}`);
+
   // Handle clean operation (destructive - deletes all existing documents)
   const vectorStoreService = framework.ai.getService('search', options.azureSearchIndexName);
   if (options.clean && !options.dryRun) {
@@ -62,11 +64,14 @@ export async function embed(binOptions: EmbeddingsBinOptions): Promise<void> {
     // to prevent traversing these directories entirely.
     const ignore = config.index?.ignore ?? defaultIgnore;
 
+    // Respect .gitignore by default; configs targeting build artifacts can opt out.
+    const gitignore = config.index?.gitignore ?? true;
+
     return from(
       globbyStream(filePatterns, {
         ignore,
         onlyFiles: true,
-        gitignore: true,
+        gitignore,
         absolute: true,
       }),
     ).pipe(
@@ -165,25 +170,49 @@ export async function embed(binOptions: EmbeddingsBinOptions): Promise<void> {
   // Apply metadata to documents
   const applyMetadata$ = applyMetadata(merge(rawFiles$, markdown$, typescript$), config.index);
 
-  // Generate embeddings
+  // Generate embeddings with concurrency limit and retry on rate-limit (429) errors
   const embeddingService = framework.ai.getService('embeddings', options.openaiEmbeddingDeployment);
+
+  /** Maximum parallel embedding requests to avoid hitting Azure OpenAI TPM limits. */
+  const EMBEDDING_CONCURRENCY = 5;
+
+  /** Maximum retry attempts for transient / rate-limit errors per chunk. */
+  const MAX_RETRIES = 4;
+
   const applyEmbedding$ = applyMetadata$.pipe(
     mergeMap((documents) =>
       from(documents).pipe(
-        mergeMap(async (document) => {
-          console.log('embedding document', document.metadata.source);
-          const embeddings = await embeddingService
-            .embedQuery(document.pageContent)
-            .catch((error) => {
-              console.error(
-                `❌ Error: ${error instanceof Error ? error.message : 'Unknown error'}`,
-              );
-              console.error('document', document);
-              process.exit(1);
-            });
-          const metadata = { ...document.metadata, embedding: embeddings };
-          return { ...document, metadata };
-        }),
+        // Limit concurrency to avoid overwhelming the embedding API
+        mergeMap(
+          (document) =>
+            from(embeddingService.embedQuery(document.pageContent)).pipe(
+              retry({
+                count: MAX_RETRIES,
+                delay: (error, retryIndex) => {
+                  // Parse Retry-After header when available (Azure sends seconds)
+                  const retryAfterSec =
+                    error?.response?.headers?.get?.('retry-after') ??
+                    error?.responseHeaders?.['retry-after'];
+                  const retryAfterMs = retryAfterSec ? Number(retryAfterSec) * 1000 : 0;
+
+                  // Exponential backoff: 2s, 4s, 8s, 16s — or Retry-After if larger
+                  const backoffMs = 2 ** retryIndex * 1000;
+                  const delayMs = Math.max(backoffMs, retryAfterMs);
+
+                  console.warn(
+                    `⏳ Retry ${retryIndex}/${MAX_RETRIES} for "${document.metadata.source}" in ${delayMs}ms`,
+                  );
+                  return timer(delayMs);
+                },
+              }),
+              map((embeddings) => {
+                console.log('embedding document', document.metadata.source);
+                const metadata = { ...document.metadata, embedding: embeddings };
+                return { ...document, metadata };
+              }),
+            ),
+          EMBEDDING_CONCURRENCY,
+        ),
         toArray(),
       ),
     ),

package/src/bin/file-stream.ts

@@ -3,7 +3,7 @@ import { from, mergeMap, concatMap, shareReplay } from 'rxjs';
 import type { Observable } from 'rxjs';
 import { getFileStatus } from '../utils/git/index.js';
 import type { ChangedFile } from '../utils/git/index.js';
-import type { CommandOptions } from '../command.options.js';
+import type { CommandOptions } from '../embeddings-command.options.js';
 
 /**
  * Creates a file stream based on diff mode or glob patterns.

package/src/bin/get-diff.ts

@@ -1,6 +1,6 @@
 import type { ChangedFile } from '../utils/git/index.js';
 import { getChangedFiles, getGitStatus } from '../utils/git/index.js';
-import type { CommandOptions } from '../command.options.js';
+import type { CommandOptions } from '../embeddings-command.options.js';
 
 /**
  * Handles diff-based processing to get changed files from git.

package/src/bin/types.ts

@@ -1,6 +1,6 @@
 import type { VectorStoreDocument } from '@equinor/fusion-framework-module-ai/lib';
 import type { ChangedFile } from '../utils/git/index.js';
-import type { CommandOptions } from '../command.options.js';
+import type { CommandOptions } from '../embeddings-command.options.js';
 import type { FrameworkInstance } from '@equinor/fusion-framework-cli-plugin-ai-base';
 import type { FusionAIConfigWithIndex } from '../config.js';
 

package/src/config.ts

@@ -2,38 +2,80 @@ import type { VectorStoreDocument } from '@equinor/fusion-framework-module-ai/li
 import type { FusionAIConfig } from '@equinor/fusion-framework-cli-plugin-ai-base';
 
 /**
- * Index-specific configuration for Fusion AI operations
+ * Index-specific configuration for Fusion AI document indexing operations.
+ *
+ * Controls which files are collected, how they are chunked, and what metadata
+ * is attached before being sent to the Azure AI Search vector store.
+ *
+ * @example
+ * ```ts
+ * const indexConfig: IndexConfig = {
+ *   patterns: ['src/\**\/*.ts', 'docs/\**\/*.md'],
+ *   ignore: ['dist/\**', 'node_modules/\**'],
+ *   metadata: { resolvePackage: true, resolveGit: true },
+ *   embedding: { chunkSize: 2000, chunkOverlap: 300 },
+ * };
+ * ```
  */
 export interface IndexConfig {
+  /** Azure Cognitive Search index name. Overridden by the `--azure-search-index-name` CLI flag. */
+  name?: string;
+  /** Azure OpenAI embedding deployment name. Overridden by the `--openai-embedding-deployment` CLI flag. */
+  model?: string;
+  // Glob patterns for files to process (defaults to ['**/*.ts', '**/*.md', '**/*.mdx']).
   patterns?: string[];
-  /** Files will be processed as is, without any chunking or transformation */
+  /** Glob patterns for files that should be indexed as-is, without chunking or transformation. */
   rawPatterns?: string[];
-  /** Globby patterns to ignored, only used when providing paths to the command */
+  /** Glob patterns to ignore — only applied when file paths are provided to the command. */
   ignore?: string[];
-  /** Metadata processing configuration */
+  /** Respect `.gitignore` rules when globbing files. Defaults to `true`. Set to `false` for build-output directories that are gitignored. */
+  gitignore?: boolean;
+  /** Metadata processing configuration. */
   metadata?: {
-    /** Automatically resolve package information from source file paths */
+    /** Automatically resolve the nearest `package.json` and attach package name/version/keywords. */
     resolvePackage?: boolean;
+    /** Resolve git metadata (commit hash, date, permalink) for each source file. Defaults to `true`. */
     resolveGit?: boolean;
-    /** Custom metadata processors to transform metadata before embedding */
+    /**
+     * Custom callback to transform document attributes before embedding.
+     *
+     * @param metadata - The current attribute map for the document.
+     * @param document - The full vector-store document being processed.
+     * @returns The transformed attribute map.
+     */
     attributeProcessor?: (
       metadata: Record<string, unknown>,
       document: VectorStoreDocument,
     ) => Record<string, unknown>;
   };
 
-  /** Embedding generation configuration */
+  /** Embedding generation configuration. */
   embedding?: {
-    /** Size of text chunks for embedding */
+    /** Maximum token size of each text chunk sent for embedding generation. */
     chunkSize?: number;
-    /** Overlap between chunks */
+    /** Number of overlapping tokens between consecutive chunks. */
     chunkOverlap?: number;
   };
 }
 
 /**
- * Extended Fusion AI configuration with index-specific settings
+ * Fusion AI configuration extended with {@link IndexConfig | index-specific settings}.
+ *
+ * Used as the return type of `configureFusionAI()` when the `ai index add` or
+ * `ai index remove` commands are configured.
+ *
+ * @example
+ * ```ts
+ * import { configureFusionAI, type FusionAIConfigWithIndex } from '@equinor/fusion-framework-cli-plugin-ai-index';
+ *
+ * export default configureFusionAI((): FusionAIConfigWithIndex => ({
+ *   index: {
+ *     patterns: ['packages/\**\/*.ts', 'packages/\**\/*.md'],
+ *   },
+ * }));
+ * ```
  */
 export interface FusionAIConfigWithIndex extends FusionAIConfig {
+  /** Index-specific configuration for document collection, chunking, and metadata. */
   index?: IndexConfig;
 }

package/src/delete-command.options.ts

@@ -0,0 +1,51 @@
+import { z } from 'zod';
+
+import { AiOptionsSchema } from '@equinor/fusion-framework-cli-plugin-ai-base/command-options';
+
+/**
+ * Zod schema for validating options of the `ai index remove` command.
+ *
+ * Extends the base AI options schema ({@link AiOptionsSchema}) to require
+ * Azure Search credentials and the embedding deployment (needed to initialise
+ * the vector store service for document removal).
+ *
+ * @example
+ * ```ts
+ * const validated = await DeleteOptionsSchema.parseAsync(rawOptions);
+ * // validated.dryRun, validated.filter, validated.azureSearchEndpoint, etc.
+ * ```
+ */
+export const DeleteOptionsSchema = AiOptionsSchema.extend({
+  openaiEmbeddingDeployment: z
+    .string({ message: 'Embedding deployment name is required to initialise the vector store.' })
+    .min(1, 'Embedding deployment name must be a non-empty string.')
+    .describe('Azure OpenAI embedding deployment name'),
+  azureSearchEndpoint: z
+    .string({ message: 'Azure Search endpoint is required for deletion.' })
+    .url('Azure Search endpoint must be a valid URL.')
+    .min(1, 'Azure Search endpoint must be a non-empty string.')
+    .describe('Azure Search endpoint URL'),
+  azureSearchApiKey: z
+    .string({ message: 'Azure Search API key is required for deletion.' })
+    .min(1, 'Azure Search API key must be a non-empty string.')
+    .describe('Azure Search API key'),
+  azureSearchIndexName: z
+    .string({ message: 'Azure Search index name is required for deletion.' })
+    .min(1, 'Azure Search index name must be a non-empty string.')
+    .describe('Azure Search index name'),
+  dryRun: z
+    .boolean({ message: 'dryRun must be a boolean value.' })
+    .describe('Preview what would be deleted without making changes'),
+  filter: z
+    .string()
+    .min(1, 'Filter expression must be a non-empty string.')
+    .optional()
+    .describe('Raw OData filter expression for selecting documents to delete'),
+}).describe('Command options for the delete command');
+
+/**
+ * Validated options for the `ai index remove` command.
+ *
+ * Inferred from {@link DeleteOptionsSchema}.
+ */
+export type DeleteOptions = z.infer<typeof DeleteOptionsSchema>;

package/src/delete-command.ts

@@ -0,0 +1,117 @@
+import { createCommand, createOption } from 'commander';
+
+import { setupFramework } from '@equinor/fusion-framework-cli-plugin-ai-base';
+import { withOptions as withAiOptions } from '@equinor/fusion-framework-cli-plugin-ai-base/command-options';
+
+import { DeleteOptionsSchema, type DeleteOptions } from './delete-command.options.js';
+
+/**
+ * Builds an OData filter expression from source paths and/or a raw filter.
+ *
+ * Source paths are joined with `or`; a raw `--filter` expression is used
+ * directly. When both are supplied, source-path filters take precedence
+ * to prevent unintentional broad deletions.
+ *
+ * @param sources - Relative file paths to match against `metadata/source`.
+ * @param rawFilter - A raw OData filter expression supplied via `--filter`.
+ * @returns The combined OData filter string, or `undefined` when neither
+ *   sources nor a raw filter were provided.
+ */
+function buildFilter(sources: string[], rawFilter?: string): string | undefined {
+  if (sources.length > 0) {
+    return sources.map((s) => `metadata/source eq '${s}'`).join(' or ');
+  }
+  return rawFilter;
+}
+
+/**
+ * CLI command: `ai index remove`
+ *
+ * Removes documents from the Azure AI Search index by source path or OData filter.
+ *
+ * Use this when you need to remove stale, renamed, or noisy documents from the
+ * vector store without running a full re-index.
+ *
+ * Usage:
+ *   $ ffc ai index remove [options] [source-paths...]
+ *
+ * Arguments:
+ *   source-paths    One or more relative file paths whose indexed chunks should
+ *                   be removed (e.g. packages/modules/services/src/foo.ts).
+ *
+ * Options:
+ *   --filter <expr>  Raw OData filter expression for advanced selection
+ *                    (e.g. "metadata/source eq 'src/old-file.ts'").
+ *   --dry-run        Preview matching documents without deleting them.
+ *
+ * Examples:
+ *   # Remove by source paths
+ *   $ ffc ai index remove src/old-module.ts src/legacy/helper.ts
+ *
+ *   # Preview what would be removed (dry-run)
+ *   $ ffc ai index remove --dry-run src/old-module.ts
+ *
+ *   # Remove using a raw OData filter
+ *   $ ffc ai index remove --filter "metadata/source eq 'src/old-module.ts'"
+ *
+ *   # Remove all chunks from a package
+ *   $ ffc ai index remove --filter "metadata/attributes/any(a: a/key eq 'pkg_name' and a/value eq '@equinor/my-pkg')"
+ */
+const _command = createCommand('remove')
+  .description('Remove documents from the search index by source path or OData filter')
+  .addOption(
+    createOption('--dry-run', 'Preview matching documents without deleting them').default(false),
+  )
+  .addOption(
+    createOption(
+      '--filter <expression>',
+      'Raw OData filter expression for selecting documents to delete',
+    ),
+  )
+  .argument('[source-paths...]', 'Relative file paths whose indexed chunks should be removed')
+  .action(async (sources: string[], commandOptions: DeleteOptions) => {
+    const options = await DeleteOptionsSchema.parseAsync(commandOptions);
+    const filterExpression = buildFilter(sources, options.filter);
+
+    if (!filterExpression) {
+      throw new Error(
+        'Nothing to delete. Provide source file paths as arguments or pass a --filter expression.',
+      );
+    }
+
+    if (sources.length > 0) {
+      console.log(`\nTargeting ${sources.length} source path(s):\n`);
+      for (const src of sources.sort()) {
+        console.log(`  ${src}`);
+      }
+    } else {
+      console.log(`\nFilter: ${filterExpression}`);
+    }
+
+    if (options.dryRun) {
+      console.log('\n🔍 Dry run — no documents were deleted.');
+      console.log(`  Would apply filter: ${filterExpression}`);
+      return;
+    }
+
+    const framework = await setupFramework(options);
+    const vectorStoreService = framework.ai.getService('search', options.azureSearchIndexName);
+    await vectorStoreService.deleteDocuments({
+      filter: { filterExpression },
+    });
+
+    console.log(`\n✅ Deleted chunks matching filter.`);
+  });
+
+/**
+ * Configured Commander command for the `ai index remove` subcommand.
+ *
+ * This constant is the fully-configured {@link Command} instance with all
+ * AI-specific options (embedding deployment, Azure Search credentials) applied
+ * via `withAiOptions`. It is registered with the CLI automatically by
+ * {@link registerAiPlugin}.
+ */
+export const deleteCommand = withAiOptions(_command, {
+  includeEmbedding: true,
+  includeSearch: true,
+});

package/src/{command.options.ts → embeddings-command.options.ts}

@@ -3,14 +3,21 @@ import { z } from 'zod';
 import { AiOptionsSchema } from '@equinor/fusion-framework-cli-plugin-ai-base/command-options';
 
 /**
- * Zod schema for validating command options for the embeddings command.
+ * Zod schema for validating command options for the `ai index add` command.
  *
- * This schema extends the base AI options schema with embeddings-specific options,
- * ensuring type safety and runtime validation of command arguments.
+ * Extends the base AI options schema ({@link AiOptionsSchema}) with
+ * add-specific options such as `--dry-run`, `--diff`, `--config`,
+ * `--base-ref`, and `--clean`.
  *
- * Note: Some optional AI options become required for the embeddings command
- * (openaiEmbeddingDeployment, azureSearchEndpoint, azureSearchApiKey, azureSearchIndexName)
- * because the command uses withAiOptions with includeEmbedding and includeSearch set to true.
+ * Azure Search and embedding options that are optional in the base schema
+ * become **required** because the add command always writes to a
+ * vector store.
+ *
+ * @example
+ * ```ts
+ * const validated = await CommandOptionsSchema.parseAsync(rawOptions);
+ * // validated.dryRun, validated.azureSearchEndpoint, etc.
+ * ```
  */
 export const CommandOptionsSchema = AiOptionsSchema.extend({
   // Override optional AI options to make them required for embeddings command
@@ -50,9 +57,9 @@ export const CommandOptionsSchema = AiOptionsSchema.extend({
 }).describe('Command options for the embeddings command');
 
 /**
- * Type representing the validated command options.
+ * Validated options for the `ai index add` command.
  *
- * This type is inferred from the Zod schema and should be used throughout the command
- * to ensure type safety and consistency with the schema.
+ * Inferred from {@link CommandOptionsSchema} and used as the single
+ * source of truth for option types throughout the add/embeddings pipeline.
  */
 export type CommandOptions = z.infer<typeof CommandOptionsSchema>;