@equinor/fusion-framework-cli-plugin-ai-index 2.0.0 → 2.1.0

package/CHANGELOG.md

@@ -1,5 +1,69 @@
 # @equinor/fusion-framework-cli-plugin-ai-index
 
+## 2.1.0
+
+### Minor Changes
+
+- ece8f42: Add Zod-based schema promotion for Azure AI Search indexes.
+
+  New `defineIndexSchema()` factory lets config authors declare which metadata fields should be promoted to top-level Azure AI Search fields, enabling direct OData filtering without `any()` operators.
+
+  Features:
+  - `defineIndexSchema({ shape, prepareAttributes, resolve })` — type-safe schema config with Zod object shape, typed attribute processor, and field resolver
+  - `zodToAzureFields()` — converts Zod schemas to Azure EDM field definitions with appropriate capabilities
+  - `applySchema` pipeline step — validates resolved values via Zod and moves promoted fields to `metadata.schemaFields`
+  - `ffc ai index create --dry-run` — previews the full index schema including promoted fields
+  - `resolveEmbeddingDimensions()` — maps known embedding models to vector dimensions, with fallback to `embedding.dimensions` config
+
+  ```typescript
+  import { z } from "zod";
+  import { defineIndexSchema } from "@equinor/fusion-framework-cli-plugin-ai-index";
+
+  // Note: `zod` must be a direct dependency of your project.
+  // The CLI plugin uses zod internally but does not re-export it.
+  const schema = defineIndexSchema({
+    shape: z.object({
+      type: z.string(),
+      tags: z.array(z.string()).default([]),
+    }),
+    prepareAttributes: (attrs, doc) => {
+      attrs.tags ??= [];
+      if (doc.metadata.source.includes("packages/")) attrs.tags.push("package");
+      return attrs;
+    },
+    resolve: (doc) => ({
+      type: (doc.metadata.attributes?.type as string) ?? "unknown",
+      tags: (doc.metadata.attributes?.tags as string[]) ?? [],
+    }),
+  });
+  ```
+
+  ref: equinor/fusion-core-tasks#1011
+
+- ece8f42: Internal: update index plugin to use Fusion service discovery and strategy-based AI module. Add `ai index embed` sub-command for standalone embedding operations.
+
+  Ref: https://github.com/equinor/fusion-framework/issues/1008
+
+### Patch Changes
+
+- Updated dependencies [ece8f42]
+- Updated dependencies [ece8f42]
+- Updated dependencies [ece8f42]
+  - @equinor/fusion-framework-cli-plugin-ai-base@3.0.0
+  - @equinor/fusion-framework-module-ai@4.0.0
+
+## 2.0.1
+
+### Patch Changes
+
+- cb8bfe2: Internal: Bump `ts-morph` from 27.0.2 to 28.0.0 (adds TypeScript 6 support).
+- 4f71408: Internal: Bump `simple-git` from 3.35.2 to 3.36.0.
+- 4f71408: Internal: Bump LangChain ecosystem dependencies (`langchain`, `@langchain/core`, `@langchain/community`, `@langchain/textsplitters`).
+- Updated dependencies [4f71408]
+- Updated dependencies [ee9c669]
+  - @equinor/fusion-framework-module-ai@3.0.1
+  - @equinor/fusion-framework-cli-plugin-ai-base@2.0.1
+
 ## 2.0.0
 
 ### Major Changes

package/dist/esm/bin/apply-metadata.js

@@ -1,17 +1,25 @@
 import path from 'node:path';
-import { from, mergeMap, map, toArray } from 'rxjs';
+import { from, mergeMap, map, tap, toArray } from 'rxjs';
 import { extractGitMetadata } from '../utils/git/index.js';
 import { resolvePackage } from '../utils/package-resolver.js';
 /**
  * Creates a stream that applies metadata to documents.
  * @internal
  */
-export function applyMetadata(document$, indexConfig) {
+export function applyMetadata(document$, indexConfig, onProgress) {
     // Resolve packages if enabled
     const shouldResolvePackage = indexConfig?.metadata?.resolvePackage ?? false;
+    /** Cap concurrent git subprocess calls to avoid overwhelming the OS process table. */
+    const GIT_CONCURRENCY = 20;
+    /**
+     * Cap the number of file entries processed in parallel.
+     * Each entry fans out to GIT_CONCURRENCY inner git calls, so
+     * total concurrent git processes ≤ ENTRY_CONCURRENCY × GIT_CONCURRENCY.
+     */
+    const ENTRY_CONCURRENCY = 20;
     return document$.pipe(mergeMap((entry) => {
         return from(entry.documents).pipe(
-        // Extract git metadata concurrently for all documents
+        // Extract git metadata concurrently (capped to limit parallel git processes)
         mergeMap(async (document) => {
             const rootPath = document.metadata.rootPath ?? process.cwd();
             const sourcePath = path.join(rootPath, document.metadata.source);
@@ -42,7 +50,9 @@ export function applyMetadata(document$, indexConfig) {
                     },
                 },
             };
-        }), 
+        }, GIT_CONCURRENCY), 
+        // Notify caller after each document is enriched
+        tap((document) => onProgress?.(document.metadata.source)), 
         // Apply custom attribute processor from config
         map((document) => {
             const attributeProcessor = indexConfig?.metadata?.attributeProcessor ||
@@ -58,6 +68,6 @@ export function applyMetadata(document$, indexConfig) {
         }), 
         // Group back by file for batch deletion in next step
         toArray());
-    }));
+    }, ENTRY_CONCURRENCY));
 }
 //# sourceMappingURL=apply-metadata.js.map

package/dist/esm/bin/apply-metadata.js.map

@@ -1 +1 @@
-{"version":3,"file":"apply-metadata.js","sourceRoot":"","sources":["../../../src/bin/apply-metadata.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AAGpD,OAAO,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AAC3D,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAI9D;;;GAGG;AACH,MAAM,UAAU,aAAa,CAC3B,SAAoC,EACpC,WAA6C;IAE7C,8BAA8B;IAC9B,MAAM,oBAAoB,GAAG,WAAW,EAAE,QAAQ,EAAE,cAAc,IAAI,KAAK,CAAC;IAE5E,OAAO,SAAS,CAAC,IAAI,CACnB,QAAQ,CAAC,CAAC,KAAK,EAAE,EAAE;QACjB,OAAO,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,IAAI;QAC/B,sDAAsD;QACtD,QAAQ,CAAC,KAAK,EAAE,QAAQ,EAAgC,EAAE;YACxD,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,QAAQ,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;YAC7D,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YACjE,MAAM,WAAW,GACf,QAAQ,CAAC,QAAQ,CAAC,MAAM,IAAI,WAAW,EAAE,QAAQ,EAAE,UAAU,KAAK,KAAK;gBACrE,CAAC,CAAC,MAAM,kBAAkB,CAAC,UAAU,CAAC;gBACtC,CAAC,CAAC,EAAE,CAAC;YAET,yCAAyC;YACzC,IAAI,eAAe,GAAG,EAAE,CAAC;YACzB,IAAI,oBAAoB,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;gBACrD,eAAe,GAAG,MAAM,cAAc,CAAC,UAAU,CAAC;qBAC/C,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE;oBACZ,OAAO;wBACL,QAAQ,EAAE,GAAG,EAAE,IAAI;wBACnB,WAAW,EAAE,GAAG,EAAE,OAAO;wBACzB,YAAY,EAAE,GAAG,EAAE,QAAQ;qBAC5B,CAAC;gBACJ,CAAC,CAAC;qBACD,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YACvB,CAAC;YACD,OAAO;gBACL,GAAG,QAAQ;gBACX,QAAQ,EAAE;oBACR,GAAG,QAAQ,CAAC,QAAQ;oBACpB,UAAU,EAAE;wBACV,GAAG,QAAQ,CAAC,QAAQ,CAAC,UAAU;wBAC/B,GAAG,WAAW;wBACd,GAAG,eAAe;qBACnB;iBACF;aACF,CAAC;QACJ,CAAC,CAAC;QACF,+CAA+C;QAC/C,GAAG,CAAC,CAAC,QAA6B,EAAE,EAAE;YACpC,MAAM,kBAAkB,GACtB,WAAW,EAAE,QAAQ,EAAE,kBAAkB;gBACzC,CAAC,CAAC,UAAmC,EAAE,SAA8B,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC;YACxF,MAAM,UAAU,GAAG,kBAAkB,CAAC,QAAQ,CAAC,QAAQ,CAAC,UAAU,IAAI,EAAE,EAAE,QAAQ,CAAC,CAAC;YACpF,OAAO;gBACL,GAAG,QAAQ;gBACX,QAAQ,EAAE;oBACR,GAAG,QAAQ,CAAC,QAAQ;oBACpB,UAAU;iBACX;aACF,CAAC;QACJ,CAAC,CAAC;QACF,qDAAqD;QACrD,OAAO,EAAE,CACV,CAAC;IACJ,CAAC,CAAC,CACH,CAAC;AACJ,CAAC"}
+{"version":3,"file":"apply-metadata.js","sourceRoot":"","sources":["../../../src/bin/apply-metadata.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AAGzD,OAAO,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AAC3D,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAO9D;;;GAGG;AACH,MAAM,UAAU,aAAa,CAC3B,SAAoC,EACpC,WAA6C,EAC7C,UAAqC;IAErC,8BAA8B;IAC9B,MAAM,oBAAoB,GAAG,WAAW,EAAE,QAAQ,EAAE,cAAc,IAAI,KAAK,CAAC;IAE5E,sFAAsF;IACtF,MAAM,eAAe,GAAG,EAAE,CAAC;IAE3B;;;;OAIG;IACH,MAAM,iBAAiB,GAAG,EAAE,CAAC;IAE7B,OAAO,SAAS,CAAC,IAAI,CACnB,QAAQ,CAAC,CAAC,KAAK,EAAE,EAAE;QACjB,OAAO,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,IAAI;QAC/B,6EAA6E;QAC7E,QAAQ,CAAC,KAAK,EAAE,QAAQ,EAAgC,EAAE;YACxD,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,QAAQ,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;YAC7D,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YACjE,MAAM,WAAW,GACf,QAAQ,CAAC,QAAQ,CAAC,MAAM,IAAI,WAAW,EAAE,QAAQ,EAAE,UAAU,KAAK,KAAK;gBACrE,CAAC,CAAC,MAAM,kBAAkB,CAAC,UAAU,CAAC;gBACtC,CAAC,CAAC,EAAE,CAAC;YAET,yCAAyC;YACzC,IAAI,eAAe,GAAG,EAAE,CAAC;YACzB,IAAI,oBAAoB,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;gBACrD,eAAe,GAAG,MAAM,cAAc,CAAC,UAAU,CAAC;qBAC/C,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE;oBACZ,OAAO;wBACL,QAAQ,EAAE,GAAG,EAAE,IAAI;wBACnB,WAAW,EAAE,GAAG,EAAE,OAAO;wBACzB,YAAY,EAAE,GAAG,EAAE,QAAQ;qBAC5B,CAAC;gBACJ,CAAC,CAAC;qBACD,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YACvB,CAAC;YACD,OAAO;gBACL,GAAG,QAAQ;gBACX,QAAQ,EAAE;oBACR,GAAG,QAAQ,CAAC,QAAQ;oBACpB,UAAU,EAAE;wBACV,GAAG,QAAQ,CAAC,QAAQ,CAAC,UAAU;wBAC/B,GAAG,WAAW;wBACd,GAAG,eAAe;qBACnB;iBACF;aACF,CAAC;QACJ,CAAC,EAAE,eAAe,CAAC;QACnB,gDAAgD;QAChD,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,UAAU,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACzD,+CAA+C;QAC/C,GAAG,CAAC,CAAC,QAA6B,EAAE,EAAE;YACpC,MAAM,kBAAkB,GACtB,WAAW,EAAE,QAAQ,EAAE,kBAAkB;gBACzC,CAAC,CAAC,UAAmC,EAAE,SAA8B,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC;YACxF,MAAM,UAAU,GAAG,kBAAkB,CAAC,QAAQ,CAAC,QAAQ,CAAC,UAAU,IAAI,EAAE,EAAE,QAAQ,CAAC,CAAC;YACpF,OAAO;gBACL,GAAG,QAAQ;gBACX,QAAQ,EAAE;oBACR,GAAG,QAAQ,CAAC,QAAQ;oBACpB,UAAU;iBACX;aACF,CAAC;QACJ,CAAC,CAAC;QACF,qDAAqD;QACrD,OAAO,EAAE,CACV,CAAC;IACJ,CAAC,EAAE,iBAAiB,CAAC,CACtB,CAAC;AACJ,CAAC"}

package/dist/esm/bin/apply-schema.js

@@ -0,0 +1,64 @@
+import { map } from 'rxjs';
+/**
+ * Creates an RxJS operator that resolves promoted schema fields for each
+ * document and separates them from the generic `attributes` bag.
+ *
+ * For each document in the batch:
+ * 1. Runs the optional `prepareAttributes` callback to enrich attributes
+ *    with type-safe access to schema-declared fields
+ * 2. Calls the schema resolver to compute promoted field values
+ * 3. Validates the resolved values against the Zod shape
+ * 4. Stores promoted fields on `metadata.schemaFields`
+ * 5. Removes promoted keys from `metadata.attributes` to avoid duplication
+ *
+ * When no schema is configured, the stream passes through unchanged.
+ *
+ * @param document$ - Stream of document batches from the metadata enrichment step.
+ * @param schema - The index schema config, if defined. When `undefined`, documents pass through unchanged.
+ * @returns Stream of document batches with promoted fields resolved and stored.
+ */
+export function applySchema(document$, schema) {
+    // No schema configured — pass through unchanged (backward compatible)
+    if (!schema) {
+        return document$;
+    }
+    const promotedKeys = new Set(Object.keys(schema.shape.shape));
+    return document$.pipe(map((documents) => documents.map((document) => {
+        // Run typed attribute processor before schema resolution so the
+        // resolver receives fully enriched attributes
+        let enrichedDocument = document;
+        if (schema.prepareAttributes) {
+            const enrichedAttributes = schema.prepareAttributes((document.metadata.attributes ?? {}), document);
+            enrichedDocument = {
+                ...document,
+                metadata: {
+                    ...document.metadata,
+                    attributes: enrichedAttributes,
+                },
+            };
+        }
+        // Resolve promoted field values from the fully enriched document
+        const resolved = schema.resolve(enrichedDocument);
+        // Validate against the Zod shape — throws on invalid data with
+        // a clear error message pointing to the offending field
+        const validated = schema.shape.parse(resolved);
+        // Remove promoted keys from attributes to avoid storing them
+        // in both top-level fields and the generic attributes array
+        const currentAttributes = (enrichedDocument.metadata.attributes ?? {});
+        const remainingAttributes = {};
+        for (const [key, value] of Object.entries(currentAttributes)) {
+            if (!promotedKeys.has(key)) {
+                remainingAttributes[key] = value;
+            }
+        }
+        return {
+            ...enrichedDocument,
+            metadata: {
+                ...enrichedDocument.metadata,
+                attributes: remainingAttributes,
+                schemaFields: validated,
+            },
+        };
+    })));
+}
+//# sourceMappingURL=apply-schema.js.map

package/dist/esm/bin/apply-schema.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"apply-schema.js","sourceRoot":"","sources":["../../../src/bin/apply-schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,MAAM,MAAM,CAAC;AAK3B;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,UAAU,WAAW,CACzB,SAA4C,EAC5C,MAAqC;IAErC,sEAAsE;IACtE,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAgC,CAAC,CAAC,CAAC;IAEzF,OAAO,SAAS,CAAC,IAAI,CACnB,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAChB,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE;QACzB,gEAAgE;QAChE,8CAA8C;QAC9C,IAAI,gBAAgB,GAAG,QAAQ,CAAC;QAChC,IAAI,MAAM,CAAC,iBAAiB,EAAE,CAAC;YAC7B,MAAM,kBAAkB,GAAG,MAAM,CAAC,iBAAiB,CACjD,CAAC,QAAQ,CAAC,QAAQ,CAAC,UAAU,IAAI,EAAE,CAA4B,EAC/D,QAAQ,CACT,CAAC;YACF,gBAAgB,GAAG;gBACjB,GAAG,QAAQ;gBACX,QAAQ,EAAE;oBACR,GAAG,QAAQ,CAAC,QAAQ;oBACpB,UAAU,EAAE,kBAA6C;iBAC1D;aACF,CAAC;QACJ,CAAC;QAED,iEAAiE;QACjE,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC;QAElD,+DAA+D;QAC/D,wDAAwD;QACxD,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,CAA4B,CAAC;QAE1E,6DAA6D;QAC7D,4DAA4D;QAC5D,MAAM,iBAAiB,GAAG,CAAC,gBAAgB,CAAC,QAAQ,CAAC,UAAU,IAAI,EAAE,CAGpE,CAAC;QACF,MAAM,mBAAmB,GAA4B,EAAE,CAAC;QACxD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,iBAAiB,CAAC,EAAE,CAAC;YAC7D,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC3B,mBAAmB,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;YACnC,CAAC;QACH,CAAC;QAED,OAAO;YACL,GAAG,gBAAgB;YACnB,QAAQ,EAAE;gBACR,GAAG,gBAAgB,CAAC,QAAQ;gBAC5B,UAAU,EAAE,mBAAmB;gBAC/B,YAAY,EAAE,SAAS;aACxB;SACF,CAAC;IACJ,CAAC,CAAC,CACH,CACF,CAAC;AACJ,CAAC"}

package/dist/esm/bin/apply-schema.test.js

@@ -0,0 +1,143 @@
+import { describe, it, expect } from 'vitest';
+import { z } from 'zod';
+import { of, lastValueFrom } from 'rxjs';
+import { defineIndexSchema } from '../schema.js';
+import { applySchema } from './apply-schema.js';
+/** Helper to create a minimal VectorStoreDocument for testing. */
+function makeDocument(overrides) {
+    return {
+        id: 'test-id',
+        pageContent: 'test content',
+        ...overrides,
+    };
+}
+describe('defineIndexSchema', () => {
+    it('returns the same config object (type-narrowing only)', () => {
+        const shape = z.object({ type: z.string() });
+        const resolve = () => ({ type: 'tsdoc' });
+        const schema = defineIndexSchema({ shape, resolve });
+        expect(schema.shape).toBe(shape);
+        expect(schema.resolve).toBe(resolve);
+    });
+});
+describe('applySchema', () => {
+    const schema = defineIndexSchema({
+        shape: z.object({
+            pkg_name: z.string().optional(),
+            type: z.string(),
+            tags: z.array(z.string()).default([]),
+            source_dir: z.string(),
+        }),
+        resolve: (doc) => ({
+            pkg_name: doc.metadata.attributes?.pkg_name,
+            type: doc.metadata.attributes?.type ?? 'unknown',
+            tags: doc.metadata.attributes?.tags ?? [],
+            source_dir: doc.metadata.source.split('/')[0],
+        }),
+    });
+    it('passes through unchanged when schema is undefined', async () => {
+        const doc = makeDocument({
+            metadata: { source: 'packages/foo/src/index.ts', attributes: { type: 'tsdoc' } },
+        });
+        const docs$ = of([doc]);
+        const result = await lastValueFrom(applySchema(docs$, undefined));
+        expect(result).toEqual([doc]);
+    });
+    it('resolves promoted fields and stores them on metadata.schemaFields', async () => {
+        const doc = makeDocument({
+            metadata: {
+                source: 'packages/foo/src/index.ts',
+                attributes: {
+                    type: 'tsdoc',
+                    pkg_name: '@equinor/fusion-framework',
+                    tags: ['package', 'react'],
+                    other_attr: 'keep-me',
+                },
+            },
+        });
+        const docs$ = of([doc]);
+        const result = await lastValueFrom(applySchema(docs$, schema));
+        expect(result[0].metadata.schemaFields).toEqual({
+            pkg_name: '@equinor/fusion-framework',
+            type: 'tsdoc',
+            tags: ['package', 'react'],
+            source_dir: 'packages',
+        });
+    });
+    it('removes promoted keys from attributes to avoid duplication', async () => {
+        const doc = makeDocument({
+            metadata: {
+                source: 'packages/foo/src/index.ts',
+                attributes: {
+                    type: 'tsdoc',
+                    pkg_name: '@equinor/fusion-framework',
+                    tags: ['package'],
+                    git_commit_hash: 'abc123',
+                },
+            },
+        });
+        const docs$ = of([doc]);
+        const result = await lastValueFrom(applySchema(docs$, schema));
+        // Promoted keys removed, non-promoted keys preserved
+        expect(result[0].metadata.attributes).toEqual({ git_commit_hash: 'abc123' });
+    });
+    it('handles documents with no attributes gracefully', async () => {
+        const doc = makeDocument({
+            metadata: { source: 'cookbooks/app-react/src/App.tsx' },
+        });
+        const docs$ = of([doc]);
+        const result = await lastValueFrom(applySchema(docs$, schema));
+        expect(result[0].metadata.schemaFields).toEqual({
+            pkg_name: undefined,
+            type: 'unknown',
+            tags: [],
+            source_dir: 'cookbooks',
+        });
+        expect(result[0].metadata.attributes).toEqual({});
+    });
+    it('throws when resolved values fail Zod validation', async () => {
+        const badSchema = defineIndexSchema({
+            shape: z.object({ type: z.string().min(1) }),
+            resolve: () => ({ type: '' }), // Empty string fails min(1)
+        });
+        const doc = makeDocument({
+            metadata: { source: 'test.ts', attributes: {} },
+        });
+        const docs$ = of([doc]);
+        await expect(lastValueFrom(applySchema(docs$, badSchema))).rejects.toThrow();
+    });
+    it('runs prepareAttributes before resolve to enrich attributes', async () => {
+        const schemaWithPrepare = defineIndexSchema({
+            shape: z.object({
+                tags: z.array(z.string()).default([]),
+                type: z.string(),
+            }),
+            prepareAttributes: (attrs, doc) => {
+                // Type-safe: attrs.tags is string[] | undefined
+                attrs.tags ??= [];
+                if (doc.metadata.source.includes('packages/')) {
+                    attrs.tags.push('package');
+                }
+                return attrs;
+            },
+            resolve: (doc) => ({
+                tags: doc.metadata.attributes?.tags ?? [],
+                type: doc.metadata.attributes?.type ?? 'unknown',
+            }),
+        });
+        const doc = makeDocument({
+            metadata: {
+                source: 'packages/framework/src/index.ts',
+                attributes: { type: 'tsdoc' },
+            },
+        });
+        const docs$ = of([doc]);
+        const result = await lastValueFrom(applySchema(docs$, schemaWithPrepare));
+        // prepareAttributes added 'package' tag before resolve consumed it
+        expect(result[0].metadata.schemaFields).toEqual({
+            tags: ['package'],
+            type: 'tsdoc',
+        });
+    });
+});
+//# sourceMappingURL=apply-schema.test.js.map

package/dist/esm/bin/apply-schema.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"apply-schema.test.js","sourceRoot":"","sources":["../../../src/bin/apply-schema.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,EAAE,EAAE,aAAa,EAAE,MAAM,MAAM,CAAC;AAGzC,OAAO,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,kEAAkE;AAClE,SAAS,YAAY,CACnB,SAAuF;IAEvF,OAAO;QACL,EAAE,EAAE,SAAS;QACb,WAAW,EAAE,cAAc;QAC3B,GAAG,SAAS;KACb,CAAC;AACJ,CAAC;AAED,QAAQ,CAAC,mBAAmB,EAAE,GAAG,EAAE;IACjC,EAAE,CAAC,sDAAsD,EAAE,GAAG,EAAE;QAC9D,MAAM,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAC7C,MAAM,OAAO,GAAG,GAAG,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;QAE1C,MAAM,MAAM,GAAG,iBAAiB,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;QAErD,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACjC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACvC,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;IAC3B,MAAM,MAAM,GAAG,iBAAiB,CAAC;QAC/B,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC;YACd,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;YAC/B,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;YAChB,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;YACrC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE;SACvB,CAAC;QACF,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACjB,QAAQ,EAAE,GAAG,CAAC,QAAQ,CAAC,UAAU,EAAE,QAA8B;YACjE,IAAI,EAAG,GAAG,CAAC,QAAQ,CAAC,UAAU,EAAE,IAAe,IAAI,SAAS;YAC5D,IAAI,EAAG,GAAG,CAAC,QAAQ,CAAC,UAAU,EAAE,IAAiB,IAAI,EAAE;YACvD,UAAU,EAAE,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;SAC9C,CAAC;KACH,CAAC,CAAC;IAEH,EAAE,CAAC,mDAAmD,EAAE,KAAK,IAAI,EAAE;QACjE,MAAM,GAAG,GAAG,YAAY,CAAC;YACvB,QAAQ,EAAE,EAAE,MAAM,EAAE,2BAA2B,EAAE,UAAU,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE;SACjF,CAAC,CAAC;QACH,MAAM,KAAK,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAExB,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,WAAW,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC;QAElE,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAChC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mEAAmE,EAAE,KAAK,IAAI,EAAE;QACjF,MAAM,GAAG,GAAG,YAAY,CAAC;YACvB,QAAQ,EAAE;gBACR,MAAM,EAAE,2BAA2B;gBACnC,UAAU,EAAE;oBACV,IAAI,EAAE,OAAO;oBACb,QAAQ,EAAE,2BAA2B;oBACrC,IAAI,EAAE,CAAC,SAAS,EAAE,OAAO,CAAC;oBAC1B,UAAU,EAAE,SAAS;iBACtB;aACF;SACF,CAAC,CAAC;QACH,MAAM,KAAK,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAExB,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,WAAW,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;QAE/D,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,OAAO,CAAC;YAC9C,QAAQ,EAAE,2BAA2B;YACrC,IAAI,EAAE,OAAO;YACb,IAAI,EAAE,CAAC,SAAS,EAAE,OAAO,CAAC;YAC1B,UAAU,EAAE,UAAU;SACvB,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4DAA4D,EAAE,KAAK,IAAI,EAAE;QAC1E,MAAM,GAAG,GAAG,YAAY,CAAC;YACvB,QAAQ,EAAE;gBACR,MAAM,EAAE,2BAA2B;gBACnC,UAAU,EAAE;oBACV,IAAI,EAAE,OAAO;oBACb,QAAQ,EAAE,2BAA2B;oBACrC,IAAI,EAAE,CAAC,SAAS,CAAC;oBACjB,eAAe,EAAE,QAAQ;iBAC1B;aACF;SACF,CAAC,CAAC;QACH,MAAM,KAAK,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAExB,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,WAAW,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;QAE/D,qDAAqD;QACrD,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,OAAO,CAAC,EAAE,eAAe,EAAE,QAAQ,EAAE,CAAC,CAAC;IAC/E,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iDAAiD,EAAE,KAAK,IAAI,EAAE;QAC/D,MAAM,GAAG,GAAG,YAAY,CAAC;YACvB,QAAQ,EAAE,EAAE,MAAM,EAAE,iCAAiC,EAAE;SACxD,CAAC,CAAC;QACH,MAAM,KAAK,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAExB,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,WAAW,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;QAE/D,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,OAAO,CAAC;YAC9C,QAAQ,EAAE,SAAS;YACnB,IAAI,EAAE,SAAS;YACf,IAAI,EAAE,EAAE;YACR,UAAU,EAAE,WAAW;SACxB,CAAC,CAAC;QACH,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IACpD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iDAAiD,EAAE,KAAK,IAAI,EAAE;QAC/D,MAAM,SAAS,GAAG,iBAAiB,CAAC;YAClC,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;YAC5C,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,EAAE,4BAA4B;SAC5D,CAAC,CAAC;QAEH,MAAM,GAAG,GAAG,YAAY,CAAC;YACvB,QAAQ,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,EAAE,EAAE;SAChD,CAAC,CAAC;QACH,MAAM,KAAK,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAExB,MAAM,MAAM,CAAC,aAAa,CAAC,WAAW,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;IAC/E,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4DAA4D,EAAE,KAAK,IAAI,EAAE;QAC1E,MAAM,iBAAiB,GAAG,iBAAiB,CAAC;YAC1C,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC;gBACd,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;gBACrC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;aACjB,CAAC;YACF,iBAAiB,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;gBAChC,gDAAgD;gBAChD,KAAK,CAAC,IAAI,KAAK,EAAE,CAAC;gBAClB,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;oBAC9C,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBAC7B,CAAC;gBACD,OAAO,KAAK,CAAC;YACf,CAAC;YACD,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;gBACjB,IAAI,EAAG,GAAG,CAAC,QAAQ,CAAC,UAAU,EAAE,IAAiB,IAAI,EAAE;gBACvD,IAAI,EAAG,GAAG,CAAC,QAAQ,CAAC,UAAU,EAAE,IAAe,IAAI,SAAS;aAC7D,CAAC;SACH,CAAC,CAAC;QAEH,MAAM,GAAG,GAAG,YAAY,CAAC;YACvB,QAAQ,EAAE;gBACR,MAAM,EAAE,iCAAiC;gBACzC,UAAU,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE;aAC9B;SACF,CAAC,CAAC;QACH,MAAM,KAAK,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAExB,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,WAAW,CAAC,KAAK,EAAE,iBAAiB,CAAC,CAAC,CAAC;QAE1E,mEAAmE;QACnE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,OAAO,CAAC;YAC9C,IAAI,EAAE,CAAC,SAAS,CAAC;YACjB,IAAI,EAAE,OAAO;SACd,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/esm/bin/delete-removed-files.js

@@ -21,7 +21,7 @@ export function createDeleteRemovedFilesStream(removedFiles$, framework, options
             console.log('Removing entry from vector store', file.relativePath);
         }
         if (!options.dryRun) {
-            const vectorStoreService = framework.ai.getService('search', options.azureSearchIndexName);
+            const vectorStoreService = framework.ai.useIndex(options.indexName);
             // Single batch deletion - one file can produce multiple document chunks
             await vectorStoreService.deleteDocuments({
                 filter: { filterExpression: filterExpression ?? undefined },

package/dist/esm/bin/delete-removed-files.js.map

@@ -1 +1 @@
-{"version":3,"file":"delete-removed-files.js","sourceRoot":"","sources":["../../../src/bin/delete-removed-files.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AAMtD;;;GAGG;AACH,MAAM,UAAU,8BAA8B,CAC5C,aAAwC,EACxC,SAA4B,EAC5B,OAAuB;IAEvB,OAAO,aAAa,CAAC,IAAI,CACvB,OAAO,EAAE,EACT,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QACZ,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAAC;QAC/C,CAAC;QACD,iFAAiF;QACjF,MAAM,gBAAgB,GAAG,KAAK;aAC3B,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,uBAAuB,IAAI,CAAC,YAAY,GAAG,CAAC;aAC1D,IAAI,CAAC,MAAM,CAAC,CAAC;QAChB,OAAO,EAAE,KAAK,EAAE,gBAAgB,EAAE,CAAC;IACrC,CAAC,CAAC,EACF,QAAQ,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,gBAAgB,EAAE,EAAE,EAAE;QAC7C,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,OAAO,CAAC,GAAG,CAAC,kCAAkC,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;QACrE,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;YACpB,MAAM,kBAAkB,GAAG,SAAS,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,EAAE,OAAO,CAAC,oBAAoB,CAAC,CAAC;YAC3F,wEAAwE;YACxE,MAAM,kBAAkB,CAAC,eAAe,CAAC;gBACvC,MAAM,EAAE,EAAE,gBAAgB,EAAE,gBAAgB,IAAI,SAAS,EAAE;aAC5D,CAAC,CAAC;QACL,CAAC;QACD,OAAO;YACL,MAAM,EAAE,SAAS;YACjB,KAAK,EAAE,KAAmC;SAC3C,CAAC;IACJ,CAAC,CAAC,EACF,MAAM,CAAC,CAAC,MAAM,EAAsC,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CACxE,CAAC;AACJ,CAAC"}
+{"version":3,"file":"delete-removed-files.js","sourceRoot":"","sources":["../../../src/bin/delete-removed-files.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC;AAMtD;;;GAGG;AACH,MAAM,UAAU,8BAA8B,CAC5C,aAAwC,EACxC,SAA4B,EAC5B,OAAuB;IAEvB,OAAO,aAAa,CAAC,IAAI,CACvB,OAAO,EAAE,EACT,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QACZ,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAAC;QAC/C,CAAC;QACD,iFAAiF;QACjF,MAAM,gBAAgB,GAAG,KAAK;aAC3B,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,uBAAuB,IAAI,CAAC,YAAY,GAAG,CAAC;aAC1D,IAAI,CAAC,MAAM,CAAC,CAAC;QAChB,OAAO,EAAE,KAAK,EAAE,gBAAgB,EAAE,CAAC;IACrC,CAAC,CAAC,EACF,QAAQ,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,gBAAgB,EAAE,EAAE,EAAE;QAC7C,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,OAAO,CAAC,GAAG,CAAC,kCAAkC,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;QACrE,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;YACpB,MAAM,kBAAkB,GAAG,SAAS,CAAC,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;YACpE,wEAAwE;YACxE,MAAM,kBAAkB,CAAC,eAAe,CAAC;gBACvC,MAAM,EAAE,EAAE,gBAAgB,EAAE,gBAAgB,IAAI,SAAS,EAAE;aAC5D,CAAC,CAAC;QACL,CAAC;QACD,OAAO;YACL,MAAM,EAAE,SAAS;YACjB,KAAK,EAAE,KAAmC;SAC3C,CAAC;IACJ,CAAC,CAAC,EACF,MAAM,CAAC,CAAC,MAAM,EAAsC,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CACxE,CAAC;AACJ,CAAC"}