@equinor/fusion-framework-cli-plugin-ai-index 2.1.0 → 3.0.1

package/dist/types/delete-command.options.d.ts

@@ -11,6 +11,7 @@ export declare const DeleteOptionsSchema: z.ZodObject<{
     clientId: z.ZodOptional<z.ZodString>;
     chatModel: z.ZodOptional<z.ZodString>;
     embedModel: z.ZodOptional<z.ZodString>;
+    debug: z.ZodDefault<z.ZodCoercedBoolean<unknown>>;
     indexName: z.ZodString;
     dryRun: z.ZodBoolean;
     filter: z.ZodOptional<z.ZodString>;

package/dist/types/embeddings-command.options.d.ts

@@ -10,6 +10,7 @@ export declare const CommandOptionsSchema: z.ZodObject<{
     tenantId: z.ZodOptional<z.ZodString>;
     clientId: z.ZodOptional<z.ZodString>;
     chatModel: z.ZodOptional<z.ZodString>;
+    debug: z.ZodDefault<z.ZodCoercedBoolean<unknown>>;
     embedModel: z.ZodString;
     indexName: z.ZodString;
     dryRun: z.ZodBoolean;

package/dist/types/version.d.ts

@@ -1 +1 @@
-export declare const version = "2.1.0";
+export declare const version = "3.0.1";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@equinor/fusion-framework-cli-plugin-ai-index",
-  "version": "2.1.0",
+  "version": "3.0.1",
   "description": "AI indexing plugin for Fusion Framework CLI providing document embedding and chunking utilities",
   "main": "dist/esm/index.js",
   "type": "module",
@@ -53,18 +53,18 @@
     "tree-sitter-typescript": "^0.23.2",
     "ts-morph": "^28.0.0",
     "zod": "^4.3.6",
+    "@equinor/fusion-framework-cli-plugin-ai-base": "4.0.0",
     "@equinor/fusion-framework-module": "6.0.0",
-    "@equinor/fusion-imports": "2.0.0",
-    "@equinor/fusion-framework-cli-plugin-ai-base": "3.0.0",
-    "@equinor/fusion-framework-module-ai": "4.0.0"
+    "@equinor/fusion-framework-module-ai": "4.0.0",
+    "@equinor/fusion-imports": "2.0.0"
   },
   "peerDependencies": {
-    "@equinor/fusion-framework-cli": "^14.2.7"
+    "@equinor/fusion-framework-cli": "^15.0.1"
   },
   "devDependencies": {
     "typescript": "^5.9.3",
     "vitest": "^4.1.0",
-    "@equinor/fusion-framework-cli": "^14.2.7"
+    "@equinor/fusion-framework-cli": "^15.0.1"
   },
   "scripts": {
     "build": "tsc -b",

package/src/bin/embed.ts

@@ -36,9 +36,16 @@ import { generateChunkId } from '../utils/generate-chunk-id.js';
 /** Braille spinner frames (same as ora's default). */
 const SPINNER_FRAMES = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'];
 
+/** Whether the process is running in a non-interactive environment (CI). */
+const IS_CI = !process.stdout.isTTY || Boolean(process.env.CI);
+
 /**
  * Manages a fixed block of sticky progress lines with per-line spinners.
  * Each line can be updated independently without overwriting the others.
+ *
+ * In non-interactive environments (CI) the ANSI cursor-movement dance is
+ * replaced with simple `console.log` lines so the output is readable in
+ * plain-text log viewers.
  * @internal
  */
 class ProgressDisplay {
@@ -48,17 +55,27 @@ class ProgressDisplay {
   private frame = 0;
   private timer: ReturnType<typeof setInterval> | undefined;
 
+  /** Tracks last CI log time per line to throttle output. */
+  private lastCiLog: number[] = [];
+
+  /** Minimum interval (ms) between CI progress lines for the same line slot. */
+  private static CI_LOG_INTERVAL_MS = 15_000;
+
   /** Register the line labels up front and print empty placeholders. */
   start(count: number): void {
     this.lines = new Array<string>(count).fill('');
     this.spinning = new Array<boolean>(count).fill(false);
-    // Print placeholder lines so the cursor block exists
-    for (let i = 0; i < count; i++) {
-      process.stdout.write('\n');
+    this.lastCiLog = new Array<number>(count).fill(0);
+
+    if (!IS_CI) {
+      // Print placeholder lines so the cursor block exists
+      for (let i = 0; i < count; i++) {
+        process.stdout.write('\n');
+      }
+      // Tick spinner at 80ms (same cadence as ora)
+      this.timer = setInterval(() => this.tick(), 80);
     }
     this.started = true;
-    // Tick spinner at 80ms (same cadence as ora)
-    this.timer = setInterval(() => this.tick(), 80);
   }
 
   /** Update a specific line (0-indexed) without touching the others. */
@@ -66,14 +83,27 @@ class ProgressDisplay {
     if (!this.started) return;
     this.lines[line] = message;
     this.spinning[line] = true;
+    if (IS_CI) {
+      const now = Date.now();
+      if (now - this.lastCiLog[line] >= ProgressDisplay.CI_LOG_INTERVAL_MS) {
+        this.lastCiLog[line] = now;
+        console.log(`⏳ ${message}`);
+      }
+      return;
+    }
     this.render(line);
   }
 
   /** Mark a line as completed — stops its spinner and shows a checkmark. */
   succeed(line: number, message: string): void {
     if (!this.started) return;
-    this.lines[line] = `✅ ${message}`;
+    const text = `✅ ${message}`;
+    this.lines[line] = text;
     this.spinning[line] = false;
+    if (IS_CI) {
+      console.log(text);
+      return;
+    }
     this.render(line);
   }
 
@@ -81,13 +111,15 @@ class ProgressDisplay {
   clear(): void {
     if (!this.started) return;
     if (this.timer) clearInterval(this.timer);
-    // Move up to the first progress line and clear each one
-    for (let i = 0; i < this.lines.length; i++) {
-      const linesUp = this.lines.length - i;
-      process.stdout.write(`\x1b[${linesUp}A\x1b[2K\r\x1b[${linesUp}B\r`);
+    if (!IS_CI) {
+      // Move up to the first progress line and clear each one
+      for (let i = 0; i < this.lines.length; i++) {
+        const linesUp = this.lines.length - i;
+        process.stdout.write(`\x1b[${linesUp}A\x1b[2K\r\x1b[${linesUp}B\r`);
+      }
+      // Move cursor up past the now-empty block
+      process.stdout.write(`\x1b[${this.lines.length}A\r`);
     }
-    // Move cursor up past the now-empty block
-    process.stdout.write(`\x1b[${this.lines.length}A\r`);
     this.started = false;
   }
 
@@ -129,18 +161,38 @@ const GIT_CONCURRENCY = 20;
 /** Maximum parallel upsert requests to the vector store. */
 const UPSERT_CONCURRENCY = 10;
 
-/** Number of texts to embed per API request. */
-const EMBED_BATCH_SIZE = 20;
+/**
+ * Number of texts to embed per API request.
+ *
+ * Azure OpenAI accepts up to 2 048 inputs per embedding call.
+ * LangChain's `batchSize` default is 1 (!) so we also set it on the
+ * client constructor.  This outer batch controls how many documents
+ * are grouped before handing them to the embed client.
+ */
+const EMBED_BATCH_SIZE = 500;
 
-/** Number of concurrent batch requests in flight. */
-const EMBED_BATCH_CONCURRENCY = 4;
+/**
+ * Number of concurrent embedding API requests in flight.
+ *
+ * Each request now carries EMBED_BATCH_SIZE texts in a single HTTP call
+ * (LangChain batchSize is aligned), so 3 concurrent requests should
+ * saturate most Azure OpenAI TPM quotas without triggering rate limits.
+ */
+const EMBED_BATCH_CONCURRENCY = 3;
 
 /**
  * Maximum time (ms) to wait before flushing a partial embedding batch.
- * Without this, `bufferCount` waits indefinitely for a full batch, which
- * starves `mergeMap` concurrency when upstream document throughput is slow.
+ *
+ * A longer window lets more documents accumulate before triggering an
+ * HTTP call, which drastically cuts the number of round-trips when
+ * upstream (metadata enrichment) feeds documents slowly.
+ *
+ * Full batches (EMBED_BATCH_SIZE) are emitted immediately regardless of
+ * the timer, and `bufferTime` flushes any remainder the instant the
+ * source stream completes — so a large value only affects mid-stream
+ * partial batches, not tail latency.
  */
-const EMBED_BUFFER_FLUSH_MS = 250;
+const EMBED_BUFFER_FLUSH_MS = 10_000;
 
 /** Maximum retry attempts for transient / rate-limit errors per chunk. */
 const MAX_RETRIES = 4;
@@ -152,9 +204,24 @@ const MAX_RETRIES = 4;
  */
 export async function embed(binOptions: EmbeddingsBinOptions): Promise<void> {
   const { framework, options, config, filePatterns } = binOptions;
+  const debug = options.debug ?? false;
 
   console.log(`📇 Index: ${options.indexName}`);
 
+  if (debug) {
+    console.debug('[debug] Embed model:', options.embedModel);
+    console.debug('[debug] File patterns:', filePatterns);
+    console.debug(
+      '[debug] Allowed patterns:',
+      config.index?.patterns ?? ['**/*.ts', '**/*.tsx', '**/*.md', '**/*.mdx'],
+    );
+    console.debug('[debug] Raw patterns:', config.index?.rawPatterns ?? []);
+    console.debug('[debug] Ignore patterns:', config.index?.ignore ?? defaultIgnore);
+    console.debug('[debug] Diff mode:', options.diff);
+    console.debug('[debug] Dry run:', options.dryRun);
+    console.debug('[debug] Clean:', options.clean);
+  }
+
   const progress = new ProgressDisplay();
 
   // Handle clean operation (destructive - deletes all existing documents)
@@ -227,6 +294,9 @@ export async function embed(binOptions: EmbeddingsBinOptions): Promise<void> {
     }),
     filter((file) => {
       const matches = multimatch(file.relativePath, allowedFilePatterns);
+      if (debug && matches.length === 0) {
+        console.debug('[debug] Skipped (no pattern match):', file.relativePath);
+      }
       return matches.length > 0;
     }),
     tap((file) => {
@@ -283,6 +353,9 @@ export async function embed(binOptions: EmbeddingsBinOptions): Promise<void> {
     mergeMap(async (file) => {
       const documents = await parseMarkdownFile(file);
       docCount++;
+      if (debug) {
+        console.debug(`[debug] Markdown ${file.relativePath} → ${documents.length} chunk(s)`);
+      }
       progress.update(LINE_PARSE, `📄 Parsing [${docCount}] ${file.relativePath}`);
       return { status: file.status, documents };
     }),
@@ -294,6 +367,9 @@ export async function embed(binOptions: EmbeddingsBinOptions): Promise<void> {
     map((file) => {
       const documents = parseTsDocFromFileSync(file);
       docCount++;
+      if (debug) {
+        console.debug(`[debug] TypeScript ${file.relativePath} → ${documents.length} chunk(s)`);
+      }
       progress.update(LINE_PARSE, `📄 Parsing [${docCount}] ${file.relativePath}`);
       return { status: file.status, documents };
     }),
@@ -336,51 +412,52 @@ export async function embed(binOptions: EmbeddingsBinOptions): Promise<void> {
     // whichever comes first — prevents upstream starvation from blocking concurrency
     bufferTime(EMBED_BUFFER_FLUSH_MS, null, EMBED_BATCH_SIZE),
     filter((batch) => batch.length > 0),
-    mergeMap(
-      (batch) =>
-        from(embeddingService.embedDocuments(batch.map((d) => d.pageContent))).pipe(
-          retry({
-            count: MAX_RETRIES,
-            delay: (error, retryIndex) => {
-              // Auth errors are terminal — abort immediately with actionable message
-              if (error?.name === 'NoAccountsError') {
-                console.error(
-                  '\n🔒 Authentication expired. Run `ffc auth login` then retry with `--diff`.',
-                );
-                throw error;
-              }
-
-              const retryAfterSec =
-                error?.response?.headers?.get?.('retry-after') ??
-                error?.responseHeaders?.['retry-after'];
-              const retryAfterMs = retryAfterSec ? Number(retryAfterSec) * 1000 : 0;
-
-              const backoffMs = 2 ** retryIndex * 1000;
-              const delayMs = Math.max(backoffMs, retryAfterMs);
-
-              console.warn(
-                `\n⏳ Retry ${retryIndex}/${MAX_RETRIES} for batch of ${batch.length} in ${delayMs}ms`,
-              );
-              return timer(delayMs);
-            },
-          }),
-          map((allEmbeddings) => {
-            return batch.map((document, i) => {
-              embeddedCount++;
-              const total = metadataDone ? metadataCount : 0;
-              const pct = total > 0 ? ` ${Math.round((embeddedCount / total) * 100)}%` : '';
-              const denominator = total > 0 ? `/${total}` : '';
-              progress.update(
-                LINE_EMBED,
-                `🧠 Embedding [${embeddedCount}${denominator}]${pct} — ${document.metadata.source}`,
+    mergeMap((batch) => {
+      if (debug) {
+        console.debug(`[debug] Embedding batch of ${batch.length} documents`);
+      }
+      return from(embeddingService.embedDocuments(batch.map((d) => d.pageContent))).pipe(
+        retry({
+          count: MAX_RETRIES,
+          delay: (error, retryIndex) => {
+            // Auth errors are terminal — abort immediately with actionable message
+            if (error?.name === 'NoAccountsError') {
+              console.error(
+                '\n🔒 Authentication expired. Run `ffc auth login` then retry with `--diff`.',
               );
-              const metadata = { ...document.metadata, embedding: allEmbeddings[i] };
-              return { ...document, metadata };
-            });
-          }),
-        ),
-      EMBED_BATCH_CONCURRENCY,
-    ),
+              throw error;
+            }
+
+            const retryAfterSec =
+              error?.response?.headers?.get?.('retry-after') ??
+              error?.responseHeaders?.['retry-after'];
+            const retryAfterMs = retryAfterSec ? Number(retryAfterSec) * 1000 : 0;
+
+            const backoffMs = 2 ** retryIndex * 1000;
+            const delayMs = Math.max(backoffMs, retryAfterMs);
+
+            console.warn(
+              `\n⏳ Retry ${retryIndex}/${MAX_RETRIES} for batch of ${batch.length} in ${delayMs}ms`,
+            );
+            return timer(delayMs);
+          },
+        }),
+        map((allEmbeddings) => {
+          return batch.map((document, i) => {
+            embeddedCount++;
+            const total = metadataDone ? metadataCount : 0;
+            const pct = total > 0 ? ` ${Math.round((embeddedCount / total) * 100)}%` : '';
+            const denominator = total > 0 ? `/${total}` : '';
+            progress.update(
+              LINE_EMBED,
+              `🧠 Embedding [${embeddedCount}${denominator}]${pct} — ${document.metadata.source}`,
+            );
+            const metadata = { ...document.metadata, embedding: allEmbeddings[i] };
+            return { ...document, metadata };
+          });
+        }),
+      );
+    }, EMBED_BATCH_CONCURRENCY),
     finalize(() => {
       embeddingDone = true;
       progress.succeed(LINE_EMBED, `🧠 Embedded ${embeddedCount} documents`);
@@ -398,6 +475,12 @@ export async function embed(binOptions: EmbeddingsBinOptions): Promise<void> {
         return undefined;
       }
       if (!options.dryRun) {
+        if (debug) {
+          console.debug(
+            `[debug] Upserting batch of ${documents.length} documents:`,
+            documents.map((d) => d.id),
+          );
+        }
         await vectorStoreService.addDocuments(documents);
       }
       return {

package/src/bin/get-diff.ts

@@ -25,6 +25,11 @@ export async function getDiff(options: CommandOptions): Promise<ChangedFile[]> {
     }
 
     console.log(`📝 Found ${changedFiles.length} changed files matching patterns`);
+    if (options.debug) {
+      for (const file of changedFiles) {
+        console.debug(`[debug] ${file.status}: ${file.filepath}`);
+      }
+    }
     return changedFiles;
   } catch (error) {
     console.error(`❌ Git diff error: ${error instanceof Error ? error.message : 'Unknown error'}`);

package/src/version.ts

@@ -1,2 +1,2 @@
 // Generated by genversion.
-export const version = '2.1.0';
+export const version = '3.0.1';