@equinor/fusion-framework-cli-plugin-ai-index 2.0.0 → 2.1.0

package/dist/types/bin/apply-metadata.d.ts

@@ -1 +1,2 @@
-export {};
+/** Callback invoked after each document is enriched with metadata. */
+export type MetadataProgressCallback = (source: string) => void;

package/dist/types/bin/apply-schema.d.ts

@@ -0,0 +1,22 @@
+import type { Observable } from 'rxjs';
+import type { VectorStoreDocument } from '@equinor/fusion-framework-module-ai/lib';
+import type { IndexSchemaConfig } from '../schema.js';
+/**
+ * Creates an RxJS operator that resolves promoted schema fields for each
+ * document and separates them from the generic `attributes` bag.
+ *
+ * For each document in the batch:
+ * 1. Runs the optional `prepareAttributes` callback to enrich attributes
+ *    with type-safe access to schema-declared fields
+ * 2. Calls the schema resolver to compute promoted field values
+ * 3. Validates the resolved values against the Zod shape
+ * 4. Stores promoted fields on `metadata.schemaFields`
+ * 5. Removes promoted keys from `metadata.attributes` to avoid duplication
+ *
+ * When no schema is configured, the stream passes through unchanged.
+ *
+ * @param document$ - Stream of document batches from the metadata enrichment step.
+ * @param schema - The index schema config, if defined. When `undefined`, documents pass through unchanged.
+ * @returns Stream of document batches with promoted fields resolved and stored.
+ */
+export declare function applySchema(document$: Observable<VectorStoreDocument[]>, schema: IndexSchemaConfig | undefined): Observable<VectorStoreDocument[]>;

package/dist/types/bin/apply-schema.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/types/config.d.ts

@@ -1,5 +1,6 @@
 import type { VectorStoreDocument } from '@equinor/fusion-framework-module-ai/lib';
 import type { FusionAIConfig } from '@equinor/fusion-framework-cli-plugin-ai-base';
+import type { IndexSchemaConfig } from './schema.js';
 /**
  * Index-specific configuration for Fusion AI document indexing operations.
  *
@@ -49,7 +50,20 @@ export interface IndexConfig {
         chunkSize?: number;
         /** Number of overlapping tokens between consecutive chunks. */
         chunkOverlap?: number;
+        /** Explicit vector dimensions for custom embedding models not in the known model map. */
+        dimensions?: number;
     };
+    /**
+     * Custom index schema that promotes frequently-filtered metadata to
+     * top-level Azure AI Search fields.
+     *
+     * When defined, the schema resolver runs after metadata enrichment and
+     * places resolved values as top-level document fields in Azure Search,
+     * enabling direct OData filters without the `any()` operator.
+     *
+     * @see {@link IndexSchemaConfig} for details and examples.
+     */
+    schema?: IndexSchemaConfig;
 }
 /**
  * Fusion AI configuration extended with {@link IndexConfig | index-specific settings}.

package/dist/types/create-command.d.ts

@@ -0,0 +1,6 @@
+import { type Command } from 'commander';
+/**
+ * The `ai index create` command with inherited AI base options for
+ * authentication and service discovery.
+ */
+export declare const createIndexCommand: Command;

package/dist/types/delete-command.options.d.ts

@@ -1,32 +1,18 @@
 import { z } from 'zod';
 /**
- * Zod schema for validating options of the `ai index remove` command.
+ * Zod schema for the `ai index remove` command.
  *
- * Extends the base AI options schema ({@link AiOptionsSchema}) to require
- * Azure Search credentials and the embedding deployment (needed to initialise
- * the vector store service for document removal).
- *
- * @example
- * ```ts
- * const validated = await DeleteOptionsSchema.parseAsync(rawOptions);
- * // validated.dryRun, validated.filter, validated.azureSearchEndpoint, etc.
- * ```
+ * Extends the base AI options schema making `indexName` required.
  */
 export declare const DeleteOptionsSchema: z.ZodObject<{
-    openaiApiKey: z.ZodString;
-    openaiApiVersion: z.ZodString;
-    openaiInstance: z.ZodString;
-    openaiChatDeployment: z.ZodOptional<z.ZodString>;
-    openaiEmbeddingDeployment: z.ZodString;
-    azureSearchEndpoint: z.ZodString;
-    azureSearchApiKey: z.ZodString;
-    azureSearchIndexName: z.ZodString;
+    env: z.ZodOptional<z.ZodString>;
+    token: z.ZodOptional<z.ZodString>;
+    tenantId: z.ZodOptional<z.ZodString>;
+    clientId: z.ZodOptional<z.ZodString>;
+    chatModel: z.ZodOptional<z.ZodString>;
+    embedModel: z.ZodOptional<z.ZodString>;
+    indexName: z.ZodString;
     dryRun: z.ZodBoolean;
     filter: z.ZodOptional<z.ZodString>;
 }, z.core.$strip>;
-/**
- * Validated options for the `ai index remove` command.
- *
- * Inferred from {@link DeleteOptionsSchema}.
- */
 export type DeleteOptions = z.infer<typeof DeleteOptionsSchema>;

package/dist/types/delete-index-command.d.ts

@@ -0,0 +1,6 @@
+import { type Command } from 'commander';
+/**
+ * The `ai index delete` command with inherited AI base options for
+ * authentication and service discovery.
+ */
+export declare const deleteIndexCommand: Command;

package/dist/types/embed-command.d.ts

@@ -0,0 +1,12 @@
+/**
+ * CLI command: `ai index embed <text>`
+ *
+ * Embeds a single text string and prints the resulting vector.
+ * Useful for verifying the embeddings endpoint and model are reachable.
+ *
+ * @example
+ * ```sh
+ * ffc ai index embed "hello world"
+ * ```
+ */
+export declare const embedCommand: import("commander").Command;

package/dist/types/embeddings-command.options.d.ts

@@ -1,40 +1,21 @@
 import { z } from 'zod';
 /**
- * Zod schema for validating command options for the `ai index add` command.
+ * Zod schema for the `ai index add` command.
  *
- * Extends the base AI options schema ({@link AiOptionsSchema}) with
- * add-specific options such as `--dry-run`, `--diff`, `--config`,
- * `--base-ref`, and `--clean`.
- *
- * Azure Search and embedding options that are optional in the base schema
- * become **required** because the add command always writes to a
- * vector store.
- *
- * @example
- * ```ts
- * const validated = await CommandOptionsSchema.parseAsync(rawOptions);
- * // validated.dryRun, validated.azureSearchEndpoint, etc.
- * ```
+ * Extends the base AI options schema making `embedModel` and `indexName` required.
  */
 export declare const CommandOptionsSchema: z.ZodObject<{
-    openaiApiKey: z.ZodString;
-    openaiApiVersion: z.ZodString;
-    openaiInstance: z.ZodString;
-    openaiChatDeployment: z.ZodOptional<z.ZodString>;
-    openaiEmbeddingDeployment: z.ZodString;
-    azureSearchEndpoint: z.ZodString;
-    azureSearchApiKey: z.ZodString;
-    azureSearchIndexName: z.ZodString;
+    env: z.ZodOptional<z.ZodString>;
+    token: z.ZodOptional<z.ZodString>;
+    tenantId: z.ZodOptional<z.ZodString>;
+    clientId: z.ZodOptional<z.ZodString>;
+    chatModel: z.ZodOptional<z.ZodString>;
+    embedModel: z.ZodString;
+    indexName: z.ZodString;
     dryRun: z.ZodBoolean;
     config: z.ZodString;
     diff: z.ZodBoolean;
     baseRef: z.ZodOptional<z.ZodString>;
     clean: z.ZodBoolean;
 }, z.core.$strip>;
-/**
- * Validated options for the `ai index add` command.
- *
- * Inferred from {@link CommandOptionsSchema} and used as the single
- * source of truth for option types throughout the add/embeddings pipeline.
- */
 export type CommandOptions = z.infer<typeof CommandOptionsSchema>;

package/dist/types/index.d.ts

@@ -1,5 +1,6 @@
 import type { Command } from 'commander';
 export { FusionAIConfigWithIndex, IndexConfig } from './config.js';
+export { defineIndexSchema, IndexSchemaConfig } from './schema.js';
 /**
  * Registers the `ai index` command with the Fusion Framework CLI.
  *

package/dist/types/schema.d.ts

@@ -0,0 +1,137 @@
+import type { z } from 'zod';
+import type { VectorStoreDocument } from '@equinor/fusion-framework-module-ai/lib';
+/**
+ * Attribute map type used by {@link IndexSchemaConfig.prepareAttributes}.
+ *
+ * Combines the schema-declared field types (all optional, since
+ * attributes are built up incrementally) with a `Record<string, unknown>`
+ * base so non-promoted attributes are still accessible.
+ *
+ * @template T - Zod object schema from which attribute types are derived.
+ */
+export type SchemaAttributes<T extends z.ZodObject> = Partial<z.input<T>> & Record<string, unknown>;
+/**
+ * Configuration for a custom Azure AI Search index schema defined via a Zod
+ * object shape.
+ *
+ * Declares which metadata fields should be promoted to top-level Azure AI
+ * Search fields (instead of being stored in the generic `attributes` array)
+ * and how their values are resolved from each document.
+ *
+ * Promoted fields become filterable/facetable at the Azure Search level,
+ * eliminating the need for `any()` OData operators.
+ *
+ * @template T - Zod object schema type that defines the promoted field names and types.
+ *
+ * @example
+ * ```ts
+ * import { z } from 'zod';
+ * import { defineIndexSchema } from '@equinor/fusion-framework-cli-plugin-ai-index';
+ *
+ * const schema = defineIndexSchema({
+ *   shape: z.object({
+ *     pkg_name: z.string().optional(),
+ *     type: z.string(),
+ *     tags: z.array(z.string()).default([]),
+ *     source_dir: z.string(),
+ *   }),
+ *   prepareAttributes: (attrs, doc) => {
+ *     // attrs.tags is typed as string[] | undefined ✅
+ *     attrs.tags ??= [];
+ *     if (doc.metadata.source.includes('packages/')) {
+ *       attrs.tags.push('package');
+ *     }
+ *     return attrs;
+ *   },
+ *   resolve: (doc) => ({
+ *     pkg_name: doc.metadata.attributes?.pkg_name as string | undefined,
+ *     type: (doc.metadata.attributes?.type as string) ?? 'unknown',
+ *     tags: (doc.metadata.attributes?.tags as string[]) ?? [],
+ *     source_dir: doc.metadata.source.split('/')[0],
+ *   }),
+ * });
+ * ```
+ */
+export interface IndexSchemaConfig<T extends z.ZodObject = z.ZodObject> {
+    /**
+     * Zod object schema defining the promoted field names and their types.
+     *
+     * Each key becomes a top-level Azure AI Search field. The Zod type
+     * determines the Azure EDM field type:
+     * - `z.string()` → `Edm.String` (filterable, facetable)
+     * - `z.array(z.string())` → `Collection(Edm.String)` (filterable, facetable)
+     * - `z.number()` → `Edm.Double` (filterable, sortable)
+     * - `z.boolean()` → `Edm.Boolean` (filterable)
+     */
+    shape: T;
+    /**
+     * Type-safe attribute processor that enriches document attributes before
+     * the schema resolver runs.
+     *
+     * Runs in addition to the untyped `metadata.attributeProcessor` callback
+     * when a schema is defined. The `attributes` parameter is typed from the
+     * Zod shape so that schema-declared fields (e.g. `tags`, `pkg_name`)
+     * have proper types while non-schema attributes remain accessible via
+     * the `Record<string, unknown>` base.
+     *
+     * Runs after git and package metadata enrichment and after
+     * `metadata.attributeProcessor`, before
+     * {@link IndexSchemaConfig.resolve | resolve}.
+     *
+     * @param attributes - The accumulated attributes for the document, typed
+     *   from the schema shape. All schema fields are optional since they may
+     *   not be populated yet.
+     * @param document - The vector-store document being processed.
+     * @returns The enriched attributes map.
+     */
+    prepareAttributes?: (attributes: SchemaAttributes<T>, document: VectorStoreDocument) => SchemaAttributes<T>;
+    /**
+     * Per-document resolver that extracts or computes promoted field values.
+     *
+     * Runs after {@link IndexSchemaConfig.prepareAttributes | prepareAttributes}
+     * and metadata enrichment (git, package), so all enriched attributes are
+     * available on the document.
+     *
+     * @param document - The fully enriched vector-store document.
+     * @returns An object matching the Zod shape with resolved field values.
+     */
+    resolve: (document: VectorStoreDocument) => z.output<T>;
+}
+/**
+ * Type-safe factory for creating an {@link IndexSchemaConfig}.
+ *
+ * Infers `T` from the Zod shape and constrains both the
+ * `prepareAttributes` parameter types and the `resolve` return type,
+ * providing compile-time safety that attribute processing and resolution
+ * match the declared schema.
+ *
+ * @template T - Zod object schema type, inferred from `config.shape`.
+ * @param config - Schema configuration with a Zod shape, optional typed
+ *   attribute processor, and a resolver function.
+ * @returns The same config object, narrowed to the inferred generic type.
+ *
+ * @example
+ * ```ts
+ * import { z } from 'zod';
+ * import { defineIndexSchema } from '@equinor/fusion-framework-cli-plugin-ai-index';
+ *
+ * const schema = defineIndexSchema({
+ *   shape: z.object({
+ *     tags: z.array(z.string()).default([]),
+ *     type: z.string(),
+ *   }),
+ *   prepareAttributes: (attrs, doc) => {
+ *     attrs.tags ??= []; // string[] | undefined — type-safe ✅
+ *     if (doc.metadata.source.includes('cookbooks/')) {
+ *       attrs.tags.push('cookbook');
+ *     }
+ *     return attrs;
+ *   },
+ *   resolve: (doc) => ({
+ *     tags: (doc.metadata.attributes?.tags as string[]) ?? [],
+ *     type: (doc.metadata.attributes?.type as string) ?? 'raw',
+ *   }),
+ * });
+ * ```
+ */
+export declare function defineIndexSchema<T extends z.ZodObject>(config: IndexSchemaConfig<T>): IndexSchemaConfig<T>;

package/dist/types/utils/embedding-dimensions.d.ts

@@ -0,0 +1,13 @@
+/**
+ * Resolve the embedding vector dimensions for a given model name.
+ *
+ * Checks the known model→dimensions map first. Falls back to an explicit
+ * `dimensions` override from the config. Throws if neither is available.
+ *
+ * @param model - The embedding model name (e.g. `'text-embedding-3-large'`).
+ * @param configDimensions - Optional explicit dimensions from config, used
+ *   when the model is not in the known map.
+ * @returns The number of dimensions for the embedding vector.
+ * @throws {Error} When the model is unknown and no explicit dimensions are configured.
+ */
+export declare function resolveEmbeddingDimensions(model: string, configDimensions?: number): number;

package/dist/types/utils/zod-to-azure-fields.d.ts

@@ -0,0 +1,61 @@
+import { type z } from 'zod';
+/**
+ * Azure AI Search EDM (Entity Data Model) type identifiers used in
+ * index field definitions.
+ */
+type AzureEdmType = 'Edm.String' | 'Edm.Int32' | 'Edm.Int64' | 'Edm.Double' | 'Edm.Boolean' | 'Collection(Edm.String)';
+/**
+ * Azure AI Search field definition matching the REST API schema for
+ * index creation.
+ *
+ * @see https://learn.microsoft.com/en-us/rest/api/searchservice/indexes/create
+ */
+export interface AzureSearchField {
+    /** Field name as it appears in the index schema. */
+    name: string;
+    /** Azure EDM type for the field. */
+    type: AzureEdmType;
+    /** Whether the field can be used in `$filter` expressions. */
+    filterable: boolean;
+    /** Whether the field can be used in `$orderby` expressions. */
+    sortable: boolean;
+    /** Whether the field supports faceted navigation. */
+    facetable: boolean;
+    /** Whether the field is included in full-text search. */
+    searchable: boolean;
+}
+/**
+ * Convert a Zod object schema into an array of Azure AI Search field
+ * definitions.
+ *
+ * Walks the Zod shape, maps each field to its Azure EDM type, and assigns
+ * default capabilities (filterable, facetable, sortable). Used by the
+ * `ffc ai index create` command to generate the index schema.
+ *
+ * Uses public `instanceof` checks and `unwrap()` methods to avoid
+ * reliance on Zod's private `_zod.def` internals, ensuring compatibility
+ * across Zod versions.
+ *
+ * @param schema - A Zod object schema whose keys define the promoted fields.
+ * @returns An array of Azure AI Search field definitions.
+ * @throws {Error} When a field type cannot be mapped to an Azure EDM type.
+ *
+ * @example
+ * ```ts
+ * import { z } from 'zod';
+ * import { zodToAzureFields } from './zod-to-azure-fields.js';
+ *
+ * const fields = zodToAzureFields(
+ *   z.object({
+ *     pkg_name: z.string().optional(),
+ *     tags: z.array(z.string()).default([]),
+ *   }),
+ * );
+ * // [
+ * //   { name: 'pkg_name', type: 'Edm.String', filterable: true, facetable: true, ... },
+ * //   { name: 'tags', type: 'Collection(Edm.String)', filterable: true, facetable: true, ... },
+ * // ]
+ * ```
+ */
+export declare function zodToAzureFields(schema: z.ZodObject): AzureSearchField[];
+export {};

package/dist/types/utils/zod-to-azure-fields.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/types/version.d.ts

@@ -1 +1 @@
-export declare const version = "2.0.0";
+export declare const version = "2.1.0";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@equinor/fusion-framework-cli-plugin-ai-index",
-  "version": "2.0.0",
+  "version": "2.1.0",
   "description": "AI indexing plugin for Fusion Framework CLI providing document embedding and chunking utilities",
   "main": "dist/esm/index.js",
   "type": "module",
@@ -51,20 +51,20 @@
     "simple-git": "^3.32.3",
     "tree-sitter": "^0.25.0",
     "tree-sitter-typescript": "^0.23.2",
-    "ts-morph": "^27.0.2",
+    "ts-morph": "^28.0.0",
     "zod": "^4.3.6",
-    "@equinor/fusion-framework-cli-plugin-ai-base": "2.0.0",
     "@equinor/fusion-framework-module": "6.0.0",
-    "@equinor/fusion-framework-module-ai": "3.0.0",
-    "@equinor/fusion-imports": "2.0.0"
+    "@equinor/fusion-imports": "2.0.0",
+    "@equinor/fusion-framework-cli-plugin-ai-base": "3.0.0",
+    "@equinor/fusion-framework-module-ai": "4.0.0"
   },
   "peerDependencies": {
-    "@equinor/fusion-framework-cli": "^14.0.0"
+    "@equinor/fusion-framework-cli": "^14.2.7"
   },
   "devDependencies": {
     "typescript": "^5.9.3",
     "vitest": "^4.1.0",
-    "@equinor/fusion-framework-cli": "^14.0.0"
+    "@equinor/fusion-framework-cli": "^14.2.7"
   },
   "scripts": {
     "build": "tsc -b",

package/src/bin/apply-metadata.ts

@@ -1,5 +1,5 @@
 import path from 'node:path';
-import { from, mergeMap, map, toArray } from 'rxjs';
+import { from, mergeMap, map, tap, toArray } from 'rxjs';
 import type { Observable } from 'rxjs';
 import type { VectorStoreDocument } from '@equinor/fusion-framework-module-ai/lib';
 import { extractGitMetadata } from '../utils/git/index.js';
@@ -7,6 +7,9 @@ import { resolvePackage } from '../utils/package-resolver.js';
 import type { DocumentEntry } from './types.js';
 import type { FusionAIConfigWithIndex } from '../config.js';
 
+/** Callback invoked after each document is enriched with metadata. */
+export type MetadataProgressCallback = (source: string) => void;
+
 /**
  * Creates a stream that applies metadata to documents.
  * @internal
@@ -14,14 +17,25 @@ import type { FusionAIConfigWithIndex } from '../config.js';
 export function applyMetadata(
   document$: Observable<DocumentEntry>,
   indexConfig: FusionAIConfigWithIndex['index'],
+  onProgress?: MetadataProgressCallback,
 ): Observable<VectorStoreDocument[]> {
   // Resolve packages if enabled
   const shouldResolvePackage = indexConfig?.metadata?.resolvePackage ?? false;
 
+  /** Cap concurrent git subprocess calls to avoid overwhelming the OS process table. */
+  const GIT_CONCURRENCY = 20;
+
+  /**
+   * Cap the number of file entries processed in parallel.
+   * Each entry fans out to GIT_CONCURRENCY inner git calls, so
+   * total concurrent git processes ≤ ENTRY_CONCURRENCY × GIT_CONCURRENCY.
+   */
+  const ENTRY_CONCURRENCY = 20;
+
   return document$.pipe(
     mergeMap((entry) => {
       return from(entry.documents).pipe(
-        // Extract git metadata concurrently for all documents
+        // Extract git metadata concurrently (capped to limit parallel git processes)
         mergeMap(async (document): Promise<VectorStoreDocument> => {
           const rootPath = document.metadata.rootPath ?? process.cwd();
           const sourcePath = path.join(rootPath, document.metadata.source);
@@ -54,7 +68,9 @@ export function applyMetadata(
               },
             },
           };
-        }),
+        }, GIT_CONCURRENCY),
+        // Notify caller after each document is enriched
+        tap((document) => onProgress?.(document.metadata.source)),
         // Apply custom attribute processor from config
         map((document: VectorStoreDocument) => {
           const attributeProcessor =
@@ -72,6 +88,6 @@ export function applyMetadata(
         // Group back by file for batch deletion in next step
         toArray(),
       );
-    }),
+    }, ENTRY_CONCURRENCY),
   );
 }