@equinor/fusion-framework-cli-plugin-ai-index 1.0.0

package/src/bin/embed.ts

@@ -0,0 +1,262 @@
+import { globbyStream } from 'globby';
+import { relative } from 'node:path';
+import multimatch from 'multimatch';
+import { concat, from, merge } from 'rxjs';
+import { concatMap, filter, map, mergeMap, shareReplay, toArray } from 'rxjs/operators';
+
+import { isMarkdownFile, parseMarkdownFile } from '../utils/markdown/index.js';
+import { getFileStatus, resolveProjectRoot } from '../utils/git/index.js';
+import { isTypescriptFile, parseTsDocFromFileSync } from '../utils/ts-doc/index.js';
+
+import { getDiff } from './get-diff.js';
+import { createDeleteRemovedFilesStream } from './delete-removed-files.js';
+import { applyMetadata } from './apply-metadata.js';
+import type {
+  DocumentEntry,
+  EmbeddingsBinOptions,
+  ProcessedFile,
+  UpdateVectorStoreResult,
+} from './types.js';
+import type { VectorStoreDocument } from '@equinor/fusion-framework-module-ai/lib';
+import { readFileSync } from 'node:fs';
+import { generateChunkId } from '../utils/generate-chunk-id.js';
+
+/**
+ * Default directories to skip before expensive git operations.
+ * These are common build artifacts and dependencies that should be ignored.
+ * @internal
+ */
+const defaultIgnore = ['node_modules', '**/node_modules/**', 'dist', '**/dist/**', '.git'];
+
+/**
+ * Main entry point for the embeddings bin.
+ * Orchestrates the entire embeddings generation pipeline.
+ * @internal
+ */
+export async function embed(binOptions: EmbeddingsBinOptions): Promise<void> {
+  const { framework, options, config, filePatterns } = binOptions;
+
+  // Handle clean operation (destructive - deletes all existing documents)
+  const vectorStoreService = framework.ai.getService('search', options.azureSearchIndexName);
+  if (options.clean && !options.dryRun) {
+    console.log('🧹 Cleaning vector store: deleting all existing documents...');
+    // OData filter: delete all documents with non-empty source (all indexed docs)
+    await vectorStoreService.deleteDocuments({
+      filter: { filterExpression: "metadata/source ne ''" },
+    });
+    console.log('✅ Vector store cleaned successfully');
+  }
+
+  // Handle diff-based processing (workflow mode)
+  const changedFiles = options.diff ? await getDiff(options) : [];
+
+  // Create file stream: diff mode uses git changes, normal mode uses globby
+  const files$ = (() => {
+    if (options.diff) {
+      return from(changedFiles);
+    }
+
+    // Directories to skip before expensive git operations.
+    // Note: Even with gitignore: true, globby still traverses ignored directories when .gitignore
+    // contains negation patterns (like !.yarn/releases), so we add explicit ignore patterns
+    // to prevent traversing these directories entirely.
+    const ignore = config.index?.ignore ?? defaultIgnore;
+
+    return from(
+      globbyStream(filePatterns, {
+        ignore,
+        onlyFiles: true,
+        gitignore: true,
+        absolute: true,
+      }),
+    ).pipe(
+      // Get git status concurrently, then flatten array results
+      mergeMap((path) => getFileStatus(path)),
+      concatMap((files) => from(files)),
+      // Share stream for multiple subscribers (removedFiles$ and indexFiles$)
+      shareReplay({ refCount: true }),
+    );
+  })();
+
+  // Process files: enrich with metadata and filter by allowed patterns
+  const allowedFilePatterns = config.index?.patterns ?? [
+    '**/*.ts',
+    '**/*.tsx',
+    '**/*.md',
+    '**/*.mdx',
+  ];
+
+  // Process files: enrich with metadata and filter by allowed patterns
+  const processedFiles$ = files$.pipe(
+    map((file) => {
+      const { filepath, status } = file;
+      const projectRoot = resolveProjectRoot(filepath);
+      const relativePath = projectRoot ? relative(projectRoot, filepath) : filepath;
+
+      return {
+        path: filepath,
+        status,
+        projectRoot,
+        relativePath,
+      };
+    }),
+    filter((file) => {
+      const matches = multimatch(file.relativePath, allowedFilePatterns);
+      return matches.length > 0;
+    }),
+    // Share for multiple subscribers (removedFiles$, markdown$, typescript$)
+    shareReplay({ refCount: true }),
+  );
+
+  // Split stream: removed files for deletion, new/modified for indexing
+  const removedFiles$ = processedFiles$.pipe(filter((file) => file.status === 'removed'));
+
+  // Create processing streams
+  const delete$ = createDeleteRemovedFilesStream(removedFiles$, framework, options);
+
+  // New/modified files for indexing
+  const indexFiles$ = processedFiles$.pipe(
+    filter((file) => file.status === 'new' || file.status === 'modified'),
+    // Share for markdown$ and typescript$ pipelines
+    shareReplay({ refCount: true }),
+  );
+
+  const isRawFile = (file: ProcessedFile): boolean => {
+    const matches = multimatch(file.relativePath, config.index?.rawPatterns ?? []);
+    if (matches.length > 0) {
+      return true;
+    }
+    return false;
+  };
+
+  const rawFiles$ = indexFiles$.pipe(
+    filter(isRawFile),
+    map((file): DocumentEntry => {
+      const document: VectorStoreDocument = {
+        id: generateChunkId(file.relativePath),
+        pageContent: readFileSync(file.path, 'utf8'),
+        metadata: {
+          source: file.relativePath,
+          type: 'raw',
+        },
+      };
+      return { status: file.status, documents: [document] };
+    }),
+  );
+
+  const markdown$ = indexFiles$.pipe(
+    filter((x) => !isRawFile(x)),
+    filter((file) => isMarkdownFile(file.path)),
+    mergeMap(async (file) => {
+      const documents = await parseMarkdownFile(file);
+      return { status: file.status, documents };
+    }),
+  );
+
+  const typescript$ = indexFiles$.pipe(
+    filter((x) => !isRawFile(x)),
+    filter((file) => isTypescriptFile(file.path)),
+    map((file) => {
+      const documents = parseTsDocFromFileSync(file);
+      return { status: file.status, documents };
+    }),
+  );
+
+  // Apply metadata to documents
+  const applyMetadata$ = applyMetadata(merge(rawFiles$, markdown$, typescript$), config.index);
+
+  // Generate embeddings
+  const embeddingService = framework.ai.getService('embeddings', options.openaiEmbeddingDeployment);
+  const applyEmbedding$ = applyMetadata$.pipe(
+    mergeMap((documents) =>
+      from(documents).pipe(
+        mergeMap(async (document) => {
+          console.log('embedding document', document.metadata.source);
+          const embeddings = await embeddingService
+            .embedQuery(document.pageContent)
+            .catch((error) => {
+              console.error(
+                `❌ Error: ${error instanceof Error ? error.message : 'Unknown error'}`,
+              );
+              console.error('document', document);
+              process.exit(1);
+            });
+          const metadata = { ...document.metadata, embedding: embeddings };
+          return { ...document, metadata };
+        }),
+        toArray(),
+      ),
+    ),
+  );
+
+  // Update vector store
+  const upsert$ = applyEmbedding$.pipe(
+    mergeMap(async (documents) => {
+      const vectorStoreService = framework.ai.getService('search', options.azureSearchIndexName);
+      if (documents.length === 0) {
+        return undefined;
+      }
+      for (const document of documents) {
+        console.log(`Adding entry [${document.id}] to vector store`, document.metadata.source);
+      }
+      if (!options.dryRun) {
+        // For multiple chunks from same file, delete existing chunks first
+        if (documents.length > 1) {
+          const sources = documents
+            .map((document) => document.metadata.source)
+            .reduce((acc, source) => acc.add(source), new Set<string>());
+
+          const filterExpression = Array.from(sources)
+            .map((source) => `metadata/source eq '${source}'`)
+            .join(' or ');
+
+          // Fire-and-forget deletion (not awaited) - brief gap before new docs are indexed
+          vectorStoreService.deleteDocuments({ filter: { filterExpression } });
+        }
+        await vectorStoreService.addDocuments(documents);
+      }
+      return {
+        status: 'added',
+        documents,
+      } as UpdateVectorStoreResult;
+    }),
+    filter((result): result is UpdateVectorStoreResult => Boolean(result)),
+  );
+
+  // Execute pipeline
+  // Track indexing results for reporting: deleted file paths and added document IDs
+  const indexingResults: { deleted: string[]; added: { source: string; id: string }[] } = {
+    deleted: [],
+    added: [],
+  };
+
+  // Execute pipeline: concat ensures deletions happen before additions
+  // This subscription triggers lazy RxJS execution and tracks all results
+  concat(delete$, upsert$).subscribe({
+    next: (result) => {
+      // Track deleted files by relative path
+      if (result.status === 'deleted') {
+        indexingResults.deleted.push(...result.files.map((file) => file.relativePath));
+      }
+      // Track added documents with source and ID (one file can produce multiple IDs)
+      else if (result.status === 'added') {
+        indexingResults.added.push(
+          ...result.documents.map((document) => ({
+            source: document.metadata.source,
+            id: document.id,
+          })),
+        );
+      }
+    },
+    error: (error) => {
+      console.error(`❌ Error: ${error instanceof Error ? error.message : 'Unknown error'}`);
+      process.exit(1);
+    },
+    complete: () => {
+      // Pipeline completed - log results and exit
+      console.log('🗂️ Indexing results:', indexingResults);
+      console.log('✅ Embeddings generation completed!');
+      process.exit(0);
+    },
+  });
+}

package/src/bin/execute-pipeline.ts

@@ -0,0 +1,48 @@
+import { concat } from 'rxjs';
+import type { DeleteRemovedFilesResult, UpdateVectorStoreResult } from './types.js';
+import type { Observable } from 'rxjs';
+
+/**
+ * Executes the pipeline and tracks results.
+ * @internal
+ */
+export function executePipeline(
+  deleteRemovedFiles$: Observable<DeleteRemovedFilesResult>,
+  updateVectorStore$: Observable<UpdateVectorStoreResult>,
+): void {
+  // Track indexing results for reporting: deleted file paths and added document IDs
+  const indexingResults: { deleted: string[]; added: { source: string; id: string }[] } = {
+    deleted: [],
+    added: [],
+  };
+
+  // Execute pipeline: concat ensures deletions happen before additions
+  // This subscription triggers lazy RxJS execution and tracks all results
+  concat(deleteRemovedFiles$, updateVectorStore$).subscribe({
+    next: (result) => {
+      // Track deleted files by relative path
+      if (result.status === 'deleted') {
+        indexingResults.deleted.push(...result.files.map((file) => file.relativePath));
+      }
+      // Track added documents with source and ID (one file can produce multiple IDs)
+      else if (result.status === 'added') {
+        indexingResults.added.push(
+          ...result.documents.map((document) => ({
+            source: document.metadata.source,
+            id: document.id,
+          })),
+        );
+      }
+    },
+    error: (error) => {
+      console.error(`❌ Error: ${error instanceof Error ? error.message : 'Unknown error'}`);
+      process.exit(1);
+    },
+    complete: () => {
+      // Pipeline completed - log results and exit
+      console.log('🗂️ Indexing results:', indexingResults);
+      console.log('✅ Embeddings generation completed!');
+      process.exit(0);
+    },
+  });
+}

package/src/bin/file-stream.ts

@@ -0,0 +1,34 @@
+import { globbyStream } from 'globby';
+import { from, mergeMap, concatMap, shareReplay } from 'rxjs';
+import type { Observable } from 'rxjs';
+import { getFileStatus } from '../utils/git/index.js';
+import type { ChangedFile } from '../utils/git/index.js';
+import type { CommandOptions } from '../command.options.js';
+
+/**
+ * Creates a file stream based on diff mode or glob patterns.
+ * @internal
+ */
+export function createFileStream(
+  options: CommandOptions,
+  changedFiles: ChangedFile[],
+  filePatterns: string[],
+): Observable<ChangedFile> {
+  if (options.diff) {
+    return from(changedFiles);
+  }
+
+  return from(
+    globbyStream(filePatterns, {
+      onlyFiles: true,
+      gitignore: true,
+      absolute: true,
+    }),
+  ).pipe(
+    // Get git status concurrently, then flatten array results
+    mergeMap((path) => getFileStatus(path)),
+    concatMap((files) => from(files)),
+    // Share stream for multiple subscribers (removedFiles$ and indexFiles$)
+    shareReplay({ refCount: true }),
+  );
+}

package/src/bin/get-diff.ts

@@ -0,0 +1,33 @@
+import type { ChangedFile } from '../utils/git/index.js';
+import { getChangedFiles, getGitStatus } from '../utils/git/index.js';
+import type { CommandOptions } from '../command.options.js';
+
+/**
+ * Handles diff-based processing to get changed files from git.
+ * @internal
+ */
+export async function getDiff(options: CommandOptions): Promise<ChangedFile[]> {
+  try {
+    // Get current git status for informational output
+    const gitStatus = await getGitStatus();
+    console.log(`🔍 Git status: ${gitStatus.branch}@${gitStatus.commit}`);
+    console.log(`📊 Changes: ${gitStatus.stagedFiles} staged, ${gitStatus.unstagedFiles} unstaged`);
+
+    // Get changed files compared to base reference (default: HEAD~1)
+    const changedFiles = await getChangedFiles({
+      diff: options.diff,
+      baseRef: options.baseRef,
+    });
+
+    if (changedFiles.length === 0) {
+      console.log('✅ No changed files match the provided patterns. Nothing to process.');
+      process.exit(0);
+    }
+
+    console.log(`📝 Found ${changedFiles.length} changed files matching patterns`);
+    return changedFiles;
+  } catch (error) {
+    console.error(`❌ Git diff error: ${error instanceof Error ? error.message : 'Unknown error'}`);
+    process.exit(1);
+  }
+}

package/src/bin/index.ts

@@ -0,0 +1 @@
+export { embed } from './embed.js';

package/src/bin/types.ts

@@ -0,0 +1,48 @@
+import type { VectorStoreDocument } from '@equinor/fusion-framework-module-ai/lib';
+import type { ChangedFile } from '../utils/git/index.js';
+import type { CommandOptions } from '../command.options.js';
+import type { FrameworkInstance } from '@equinor/fusion-framework-cli-plugin-ai-base';
+import type { FusionAIConfigWithIndex } from '../config.js';
+
+/**
+ * Result of updating the vector store with new documents
+ * @internal
+ */
+export type UpdateVectorStoreResult = { status: 'added'; documents: VectorStoreDocument[] };
+
+/**
+ * Result of deleting removed files from the vector store
+ * @internal
+ */
+export type DeleteRemovedFilesResult = { status: 'deleted'; files: { relativePath: string }[] };
+
+/**
+ * File with enriched metadata for processing
+ * @internal
+ */
+export type ProcessedFile = {
+  path: string;
+  status: ChangedFile['status'];
+  projectRoot: string | undefined;
+  relativePath: string;
+};
+
+/**
+ * Document entry with status for processing
+ * @internal
+ */
+export type DocumentEntry = {
+  status: ChangedFile['status'];
+  documents: VectorStoreDocument[];
+};
+
+/**
+ * Options for the embeddings bin
+ * @internal
+ */
+export interface EmbeddingsBinOptions {
+  framework: FrameworkInstance;
+  options: CommandOptions;
+  config: FusionAIConfigWithIndex;
+  filePatterns: string[];
+}

package/src/command.options.ts

@@ -0,0 +1,58 @@
+import { z } from 'zod';
+
+import { AiOptionsSchema } from '@equinor/fusion-framework-cli-plugin-ai-base/command-options';
+
+/**
+ * Zod schema for validating command options for the embeddings command.
+ *
+ * This schema extends the base AI options schema with embeddings-specific options,
+ * ensuring type safety and runtime validation of command arguments.
+ *
+ * Note: Some optional AI options become required for the embeddings command
+ * (openaiEmbeddingDeployment, azureSearchEndpoint, azureSearchApiKey, azureSearchIndexName)
+ * because the command uses withAiOptions with includeEmbedding and includeSearch set to true.
+ */
+export const CommandOptionsSchema = AiOptionsSchema.extend({
+  // Override optional AI options to make them required for embeddings command
+  openaiEmbeddingDeployment: z
+    .string({ message: 'Embedding deployment name is required for embeddings command.' })
+    .min(1, 'Embedding deployment name must be a non-empty string.')
+    .describe('Azure OpenAI embedding deployment name'),
+  azureSearchEndpoint: z
+    .string({ message: 'Azure Search endpoint is required for embeddings command.' })
+    .url('Azure Search endpoint must be a valid URL.')
+    .min(1, 'Azure Search endpoint must be a non-empty string.')
+    .describe('Azure Search endpoint URL'),
+  azureSearchApiKey: z
+    .string({ message: 'Azure Search API key is required for embeddings command.' })
+    .min(1, 'Azure Search API key must be a non-empty string.')
+    .describe('Azure Search API key'),
+  azureSearchIndexName: z
+    .string({ message: 'Azure Search index name is required for embeddings command.' })
+    .min(1, 'Azure Search index name must be a non-empty string.')
+    .describe('Azure Search index name'),
+
+  // Embeddings-specific options
+  dryRun: z
+    .boolean({ message: 'dryRun must be a boolean value.' })
+    .describe('Show what would be processed without actually doing it'),
+  config: z
+    .string({ message: 'Config file path is required and must be a non-empty string.' })
+    .min(1, 'Config file path must be a non-empty string.')
+    .describe('Path to a config file'),
+  diff: z
+    .boolean({ message: 'diff must be a boolean value.' })
+    .describe('Process only changed files (workflow mode)'),
+  baseRef: z.string().min(1).optional().describe('Git reference to compare against'),
+  clean: z
+    .boolean({ message: 'clean must be a boolean value.' })
+    .describe('Delete all existing documents from the vector store before processing'),
+}).describe('Command options for the embeddings command');
+
+/**
+ * Type representing the validated command options.
+ *
+ * This type is inferred from the Zod schema and should be used throughout the command
+ * to ensure type safety and consistency with the schema.
+ */
+export type CommandOptions = z.infer<typeof CommandOptionsSchema>;

package/src/command.ts

@@ -0,0 +1,100 @@
+import { createCommand, createOption } from 'commander';
+
+import { loadFusionAIConfig, setupFramework } from '@equinor/fusion-framework-cli-plugin-ai-base';
+import { withOptions as withAiOptions } from '@equinor/fusion-framework-cli-plugin-ai-base/command-options';
+
+import { embed } from './bin/embed.js';
+import { CommandOptionsSchema, type CommandOptions } from './command.options.js';
+import type { FusionAIConfigWithIndex } from './config.js';
+
+/**
+ * CLI command: `ai embeddings`
+ *
+ * Document embedding utilities for Large Language Model processing.
+ *
+ * Features:
+ * - Markdown/MDX document chunking with frontmatter extraction
+ * - TypeScript/TSX TSDoc extraction and chunking
+ * - Glob pattern support for file collection
+ * - Git diff-based processing for workflow integration
+ * - Dry-run mode for testing without actual processing
+ * - Configurable file patterns via fusion-ai.config.ts
+ *
+ * Usage:
+ *   $ ffc ai embeddings [options] [glob-patterns...]
+ *
+ * Arguments:
+ *   glob-patterns          Glob patterns to match files (optional when using --diff)
+ *                          Defaults to patterns from fusion-ai.config.ts if not provided
+ *
+ * Options:
+ *   --dry-run              Show what would be processed without actually doing it
+ *   --config <config>      Path to a config file (default: fusion-ai.config)
+ *   --diff                 Process only changed files (workflow mode)
+ *   --base-ref <ref>       Git reference to compare against (default: HEAD~1)
+ *   --clean                Delete all existing documents from the vector store before processing
+ *
+ * AI Options (required):
+ *   --openai-api-key <key>              Azure OpenAI API key (or AZURE_OPENAI_API_KEY env var)
+ *   --openai-api-version <version>      Azure OpenAI API version (default: 2024-02-15-preview)
+ *   --openai-instance <name>            Azure OpenAI instance name (or AZURE_OPENAI_INSTANCE_NAME env var)
+ *   --openai-embedding-deployment <name> Azure OpenAI embedding deployment name (or AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME env var)
+ *   --azure-search-endpoint <url>       Azure Search endpoint URL (or AZURE_SEARCH_ENDPOINT env var)
+ *   --azure-search-api-key <key>        Azure Search API key (or AZURE_SEARCH_API_KEY env var)
+ *   --azure-search-index-name <name>    Azure Search index name (or AZURE_SEARCH_INDEX_NAME env var)
+ *
+ * Examples:
+ *   $ ffc ai embeddings --dry-run ./src
+ *   $ ffc ai embeddings "*.ts" "*.md" "*.mdx"
+ *   $ ffc ai embeddings --diff
+ *   $ ffc ai embeddings --diff --base-ref origin/main
+ *   $ ffc ai embeddings --clean "*.ts"
+ */
+const _command = createCommand('embeddings')
+  .description('Document embedding utilities for Large Language Model processing')
+  .addOption(
+    createOption('--dry-run', 'Show what would be processed without actually doing it').default(
+      false,
+    ),
+  )
+  .addOption(createOption('--config <config>', 'Path to a config file').default('fusion-ai.config'))
+  .addOption(createOption('--diff', 'Process only changed files (workflow mode)').default(false))
+  .addOption(createOption('--base-ref <ref>', 'Git reference to compare against').default('HEAD~1'))
+  .addOption(
+    createOption(
+      '--clean',
+      'Delete all existing documents from the vector store before processing',
+    ).default(false),
+  )
+  .argument('[glob-patterns...]', 'Glob patterns to match files (optional when using --diff)')
+  .action(async (patterns: string[], commandOptions: CommandOptions) => {
+    const options = await CommandOptionsSchema.parseAsync(commandOptions);
+
+    // Load configuration
+    const config = await loadFusionAIConfig<FusionAIConfigWithIndex>(options.config, {
+      baseDir: process.cwd(),
+    });
+
+    // CLI args take precedence over config patterns
+    const indexConfig = config.index ?? {};
+    const allowedFilePatterns = indexConfig.patterns ?? ['**/*.ts', '**/*.md', '**/*.mdx'];
+    const filePatterns = patterns.length ? patterns : allowedFilePatterns;
+
+    // Initialize framework
+    const framework = await setupFramework(options);
+
+    // Execute embeddings bin with framework and options
+    await embed({
+      framework,
+      options,
+      config,
+      filePatterns,
+    });
+  });
+
+export const command = withAiOptions(_command, {
+  includeEmbedding: true,
+  includeSearch: true,
+});
+
+export default command;

package/src/config.ts

@@ -0,0 +1,39 @@
+import type { VectorStoreDocument } from '@equinor/fusion-framework-module-ai/lib';
+import type { FusionAIConfig } from '@equinor/fusion-framework-cli-plugin-ai-base';
+
+/**
+ * Index-specific configuration for Fusion AI operations
+ */
+export interface IndexConfig {
+  patterns?: string[];
+  /** Files will be processed as is, without any chunking or transformation */
+  rawPatterns?: string[];
+  /** Globby patterns to ignored, only used when providing paths to the command */
+  ignore?: string[];
+  /** Metadata processing configuration */
+  metadata?: {
+    /** Automatically resolve package information from source file paths */
+    resolvePackage?: boolean;
+    resolveGit?: boolean;
+    /** Custom metadata processors to transform metadata before embedding */
+    attributeProcessor?: (
+      metadata: Record<string, unknown>,
+      document: VectorStoreDocument,
+    ) => Record<string, unknown>;
+  };
+
+  /** Embedding generation configuration */
+  embedding?: {
+    /** Size of text chunks for embedding */
+    chunkSize?: number;
+    /** Overlap between chunks */
+    chunkOverlap?: number;
+  };
+}
+
+/**
+ * Extended Fusion AI configuration with index-specific settings
+ */
+export interface FusionAIConfigWithIndex extends FusionAIConfig {
+  index?: IndexConfig;
+}

package/src/index.ts

@@ -0,0 +1,19 @@
+import type { Command } from 'commander';
+import { registerAiPlugin as registerAiPluginBase } from '@equinor/fusion-framework-cli-plugin-ai-base';
+import { command as embeddingsCommand } from './command.js';
+
+/**
+ * Registers the AI index plugin command with the CLI program
+ * @param program - The Commander program instance to register commands with
+ */
+export function registerAiPlugin(program: Command): void {
+  registerAiPluginBase(program, embeddingsCommand);
+}
+
+export default registerAiPlugin;
+
+// Re-export config utilities for convenience
+export {
+  configureFusionAI,
+  type FusionAIConfig,
+} from '@equinor/fusion-framework-cli-plugin-ai-base';

package/src/utils/generate-chunk-id.ts

@@ -0,0 +1,17 @@
+/**
+ * Generates a unique identifier for a document chunk based on file path
+ * Creates a deterministic, URL-safe hash from the file path for validation and checks
+ * @param filePath - The file path to generate an ID from
+ * @param chunkIndex - Optional chunk index to append for multi-chunk documents
+ * @returns A base64-encoded hash of the file path, optionally suffixed with chunk index
+ */
+export const generateChunkId = (filePath: string, chunkIndex?: number): string => {
+  // Convert file path to base64 and remove non-alphanumeric characters
+  // This creates a stable, URL-safe identifier from the file path
+  // The deterministic nature allows for validation and duplicate detection
+  const pathHash = Buffer.from(filePath)
+    .toString('base64')
+    .replace(/[^a-zA-Z0-9]/g, '');
+  // Append chunk index if provided to distinguish multiple chunks from the same file
+  return chunkIndex ? `${pathHash}-${chunkIndex}` : pathHash;
+};