@copilotkit/pathfinder 1.4.0 → 1.5.0

package/dist/indexing/distiller.js

@@ -0,0 +1,104 @@
+// LLM thread distiller — extracts Q&A pairs from conversation threads.
+// Source-agnostic: takes structured messages, returns structured Q&A pairs.
+import OpenAI from 'openai';
+// ── Constants ────────────────────────────────────────────────────────────────
+const DEFAULT_MODEL = 'gpt-4o-mini';
+const DEFAULT_MAX_MESSAGES = 100;
+const SYSTEM_PROMPT = `You are a Q&A extraction engine. Given a conversation thread, identify distinct question-answer pairs.
+
+For each pair:
+1. Extract the core question (rephrase if needed for clarity)
+2. Extract the best answer (synthesize from multiple replies if needed)
+3. Score confidence from 0.0 to 1.0 based on:
+   - Answer completeness (does it fully address the question?)
+   - Questioner satisfaction signals ("thanks", "that worked", etc.)
+   - Community validation (reactions like thumbsup, check marks)
+   - Answer specificity (concrete steps vs vague suggestions)
+
+Return JSON with this exact structure:
+{
+  "pairs": [
+    {
+      "question": "How do I configure X?",
+      "answer": "You can configure X by...",
+      "confidence": 0.85
+    }
+  ]
+}
+
+Rules:
+- A thread may contain multiple Q&A pairs (follow-up questions)
+- Skip greetings, pleasantries, and off-topic tangents
+- If no clear Q&A exists, return {"pairs": []}
+- Keep answers concise but complete (aim for 1-3 paragraphs)
+- Preserve code blocks, URLs, and technical details from answers
+- Confidence below 0.3 means the answer is likely incomplete or wrong`;
+// ── Distiller ────────────────────────────────────────────────────────────────
+/**
+ * Distill a conversation thread into Q&A pairs using an LLM.
+ */
+export async function distillThread(messages, options) {
+    const model = options?.model ?? DEFAULT_MODEL;
+    const maxMessages = options?.maxMessages ?? DEFAULT_MAX_MESSAGES;
+    if (messages.length === 0) {
+        return { pairs: [] };
+    }
+    // Truncate to max messages
+    const truncated = messages.slice(0, maxMessages);
+    // Format as conversation transcript
+    const transcript = truncated.map(msg => {
+        const reactions = msg.reactions && msg.reactions.length > 0
+            ? ` [reactions: ${msg.reactions.map(r => `:${r.name}: x${r.count}`).join(', ')}]`
+            : '';
+        return `[${msg.timestamp}] ${msg.author}: ${msg.content}${reactions}`;
+    }).join('\n\n');
+    const client = options?.client ?? new OpenAI({ apiKey: options?.apiKey });
+    try {
+        const response = await client.chat.completions.create({
+            model,
+            messages: [
+                { role: 'system', content: SYSTEM_PROMPT },
+                { role: 'user', content: transcript },
+            ],
+            response_format: { type: 'json_object' },
+            temperature: 0.1,
+        });
+        const content = response.choices[0]?.message?.content;
+        if (!content) {
+            console.warn('[distiller] Empty response from LLM');
+            return { pairs: [] };
+        }
+        const parsed = JSON.parse(content);
+        // Validate structure
+        if (!Array.isArray(parsed.pairs)) {
+            console.warn('[distiller] Invalid response structure — missing pairs array');
+            return { pairs: [] };
+        }
+        // Validate and filter each pair
+        const validPairs = [];
+        for (const pair of parsed.pairs) {
+            if (typeof pair.question === 'string' && pair.question.trim() &&
+                typeof pair.answer === 'string' && pair.answer.trim() &&
+                typeof pair.confidence === 'number' &&
+                pair.confidence >= 0 && pair.confidence <= 1) {
+                validPairs.push({
+                    question: pair.question.trim(),
+                    answer: pair.answer.trim(),
+                    confidence: pair.confidence,
+                });
+            }
+            else {
+                console.warn('[distiller] Skipping malformed pair:', JSON.stringify(pair).slice(0, 200));
+            }
+        }
+        return { pairs: validPairs };
+    }
+    catch (error) {
+        if (error instanceof SyntaxError) {
+            console.error('[distiller] Failed to parse LLM JSON response:', error.message);
+            return { pairs: [] };
+        }
+        throw error; // Re-throw API errors for caller to handle
+    }
+}
+//# sourceMappingURL=distiller.js.map

package/dist/indexing/distiller.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"distiller.js","sourceRoot":"","sources":["../../src/indexing/distiller.ts"],"names":[],"mappings":"AAAA,uEAAuE;AACvE,4EAA4E;AAE5E,OAAO,MAAM,MAAM,QAAQ,CAAC;AA4B5B,gFAAgF;AAEhF,MAAM,aAAa,GAAG,aAAa,CAAC;AACpC,MAAM,oBAAoB,GAAG,GAAG,CAAC;AAEjC,MAAM,aAAa,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;sEA4BgD,CAAC;AAEvE,gFAAgF;AAEhF;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAC/B,QAAyB,EACzB,OAA0B;IAE1B,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,IAAI,aAAa,CAAC;IAC9C,MAAM,WAAW,GAAG,OAAO,EAAE,WAAW,IAAI,oBAAoB,CAAC;IAEjE,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;IACzB,CAAC;IAED,2BAA2B;IAC3B,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC;IAEjD,oCAAoC;IACpC,MAAM,UAAU,GAAG,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;QACnC,MAAM,SAAS,GAAG,GAAG,CAAC,SAAS,IAAI,GAAG,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC;YACvD,CAAC,CAAC,gBAAgB,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG;YACjF,CAAC,CAAC,EAAE,CAAC;QACT,OAAO,IAAI,GAAG,CAAC,SAAS,KAAK,GAAG,CAAC,MAAM,KAAK,GAAG,CAAC,OAAO,GAAG,SAAS,EAAE,CAAC;IAC1E,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAEhB,MAAM,MAAM,GAAG,OAAO,EAAE,MAAM,IAAI,IAAI,MAAM,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;IAE1E,IAAI,CAAC;QACD,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC;YAClD,KAAK;YACL,QAAQ,EAAE;gBACN,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,aAAa,EAAE;gBAC1C,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE;aACxC;YACD,eAAe,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE;YACxC,WAAW,EAAE,GAAG;SACnB,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC;QACtD,IAAI,CAAC,OAAO,EAAE,CAAC;YACX,OAAO,CAAC,IAAI,CAAC,qCAAqC,CAAC,CAAC;YACpD,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;QACzB,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAEnC,qBAAqB;QACrB,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;YAC/B,OAAO,CAAC,IAAI,CAAC,8DAA8D,CAAC,CAAC;YAC7E,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;QACzB,CAAC;QAED,gCAAgC;QAChC,MAAM,UAAU,GAAoB,EAAE,CAAC;QACvC,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YAC9B,IACI,OAAO,IAAI,CAAC,QAAQ,KAAK,QAAQ,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE;gBACzD,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE;gBACrD,OAAO,IAAI,CAAC,UAAU,KAAK,QAAQ;gBACnC,IAAI,CAAC,UAAU,IAAI,CAAC,IAAI,IAAI,CAAC,UAAU,IAAI,CAAC,EAC9C,CAAC;gBACC,UAAU,CAAC,IAAI,CAAC;oBACZ,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE;oBAC9B,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE;oBAC1B,UAAU,EAAE,IAAI,CAAC,UAAU;iBAC9B,CAAC,CAAC;YACP,CAAC;iBAAM,CAAC;gBACJ,OAAO,CAAC,IAAI,CAAC,sCAAsC,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;YAC7F,CAAC;QACL,CAAC;QAED,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC;IACjC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACb,IAAI,KAAK,YAAY,WAAW,EAAE,CAAC;YAC/B,OAAO,CAAC,KAAK,CAAC,gDAAgD,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;YAC/E,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;QACzB,CAAC;QACD,MAAM,KAAK,CAAC,CAAC,2CAA2C;IAC5D,CAAC;AACL,CAAC"}

package/dist/indexing/orchestrator.d.ts

@@ -13,10 +13,10 @@ export declare class IndexingOrchestrator {
      */
     checkAndIndex(): Promise<void>;
     /**
-     * Get the HEAD SHA of a remote repo without cloning.
-     * Uses `git ls-remote` which only fetches refs.
+     * Get the current state token for a source without acquiring items.
+     * Returns null if the source is unavailable.
      */
-    private getRemoteHead;
+    private getSourceStateToken;
     /**
      * Queue a full re-index of all sources. Returns immediately.
      */
@@ -25,6 +25,11 @@ export declare class IndexingOrchestrator {
      * Queue an incremental re-index for a specific repo. Returns immediately.
      */
     queueIncrementalReindex(repoUrl: string): void;
+    /**
+     * Queue a reindex for a single named source. Returns immediately.
+     * Used by webhook handlers to trigger reindexing of specific sources.
+     */
+    queueSourceReindex(sourceName: string): void;
     /**
      * Returns true if any indexing job is currently running.
      */

package/dist/indexing/orchestrator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"orchestrator.d.ts","sourceRoot":"","sources":["../../src/indexing/orchestrator.ts"],"names":[],"mappings":"AAiCA,qBAAa,oBAAoB;IAC7B,OAAO,CAAC,KAAK,CAAa;IAC1B,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,UAAU,CAAS;IAG3B,OAAO,CAAC,aAAa,CAAqB;IAG1C,OAAO,CAAC,eAAe,CAAuB;IAG9C,iBAAiB,CAAC,EAAE,CAAC,WAAW,EAAE,MAAM,EAAE,KAAK,IAAI,CAAC;;IAMpD;;;;OAIG;IACG,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC;IA4GpC;;;OAGG;YACW,aAAa;IAa3B;;OAEG;IACH,gBAAgB,IAAI,IAAI;IAQxB;;OAEG;IACH,uBAAuB,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI;IAU9C;;OAEG;IACH,UAAU,IAAI,OAAO;IAIrB;;;OAGG;IACH,mBAAmB,IAAI,IAAI;IA4B3B;;OAEG;IACH,OAAO,CAAC,OAAO;IASf;;OAEG;YACW,KAAK;IAwBnB;;OAEG;YACW,UAAU;IAkDxB;;OAEG;YACW,cAAc;IAqB5B;;OAEG;YACW,qBAAqB;IAoDnC;;OAEG;YACW,oBAAoB;IAgDlC;;OAEG;YACW,cAAc;IAmB5B;;OAEG;YACW,cAAc;CAgB/B"}
+{"version":3,"file":"orchestrator.d.ts","sourceRoot":"","sources":["../../src/indexing/orchestrator.ts"],"names":[],"mappings":"AAqCA,qBAAa,oBAAoB;IAC7B,OAAO,CAAC,KAAK,CAAa;IAC1B,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,UAAU,CAAS;IAG3B,OAAO,CAAC,aAAa,CAAqB;IAG1C,OAAO,CAAC,eAAe,CAAuB;IAG9C,iBAAiB,CAAC,EAAE,CAAC,WAAW,EAAE,MAAM,EAAE,KAAK,IAAI,CAAC;;IAMpD;;;;OAIG;IACG,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC;IA+GpC;;;OAGG;YACW,mBAAmB;IAYjC;;OAEG;IACH,gBAAgB,IAAI,IAAI;IAQxB;;OAEG;IACH,uBAAuB,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI;IAU9C;;;OAGG;IACH,kBAAkB,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IAQ5C;;OAEG;IACH,UAAU,IAAI,OAAO;IAIrB;;;OAGG;IACH,mBAAmB,IAAI,IAAI;IA4B3B;;OAEG;IACH,OAAO,CAAC,OAAO;IASf;;OAEG;YACW,KAAK;IAwBnB;;OAEG;YACW,UAAU;IA8DxB;;OAEG;YACW,cAAc;IAqB5B;;OAEG;YACW,qBAAqB;IAiBnC;;OAEG;YACW,oBAAoB;IAsDlC;;OAEG;YACW,cAAc;IAmB5B;;OAEG;YACW,cAAc;CAgB/B"}

package/dist/indexing/orchestrator.js

@@ -1,17 +1,18 @@
 // Job queue and coordination for indexing pipelines.
 // Fully config-driven: indexes sources referenced by search tools in mcp-docs.yaml.
-import fs from 'fs';
-import path from 'path';
-import { simpleGit } from 'simple-git';
+import fs from 'node:fs';
+import path from 'node:path';
 import { getConfig, getServerConfig, getIndexableSourceNames } from '../config.js';
 import { EmbeddingClient } from './embeddings.js';
-import { SourceIndexer } from './source-indexer.js';
+import { getProvider } from './providers/index.js';
+import { IndexingPipeline } from './pipeline.js';
 import { getIndexState, upsertIndexState, } from '../db/queries.js';
+import { isFileSourceConfig } from '../types.js';
 /**
  * Find all source configs that reference a given repo URL.
  */
 function getSourcesByRepo(repoUrl) {
-    return getServerConfig().sources.filter(s => s.repo === repoUrl);
+    return getServerConfig().sources.filter(s => isFileSourceConfig(s) && s.repo === repoUrl);
 }
 function getStaleThresholdMs() {
     const serverCfg = getServerConfig();
@@ -67,89 +68,93 @@ export class IndexingOrchestrator {
                 return;
             }
             // Queue incremental reindexes for each affected git-backed repo
-            const reposToReindex = new Set(sourcesNeedingFullReindex.filter(s => s.repo).map(s => s.repo));
+            const reposToReindex = new Set();
+            for (const s of sourcesNeedingFullReindex) {
+                if (isFileSourceConfig(s) && s.repo)
+                    reposToReindex.add(s.repo);
+            }
             for (const repoUrl of reposToReindex) {
                 this.queueIncrementalReindex(repoUrl);
             }
             // Local sources (no repo) get queued as a full reindex of just those sources
-            const localSources = sourcesNeedingFullReindex.filter(s => !s.repo);
+            const localSources = sourcesNeedingFullReindex.filter(s => isFileSourceConfig(s) && !s.repo);
             if (localSources.length > 0) {
                 this.queue.push({ type: 'full-reindex-local', sources: localSources });
                 this.drain().catch(err => console.error('[orchestrator] drain() failed:', err));
             }
+            // Non-file sources (e.g., Slack) that need reindexing
+            const nonFileSources = sourcesNeedingFullReindex.filter(s => !isFileSourceConfig(s));
+            for (const source of nonFileSources) {
+                this.queueSourceReindex(source.name);
+            }
         }
         if (sourcesOk.length === 0)
             return;
         // Local sources in sourcesOk have no remote to check — always reindex on startup
-        const localSourcesOk = sourcesOk.filter(s => !s.repo);
+        const localSourcesOk = sourcesOk.filter(s => isFileSourceConfig(s) && !s.repo);
         if (localSourcesOk.length > 0) {
             console.log(`[orchestrator] Queuing reindex for ${localSourcesOk.length} local source(s)`);
             this.queue.push({ type: 'full-reindex-local', sources: localSourcesOk });
             this.drain().catch(err => console.error('[orchestrator] drain() failed:', err));
         }
         console.log('[orchestrator] Checking remotes for changes on indexed sources...');
-        // Only check remotes for git-backed sources
-        const repos = [...new Set(sourcesOk.filter(s => s.repo).map(s => s.repo))];
-        for (const repoUrl of repos) {
+        // Check each git-backed source for changes
+        const gitSourcesOk = sourcesOk.filter(s => isFileSourceConfig(s) && s.repo);
+        for (const source of gitSourcesOk) {
             try {
-                const remoteHead = await this.getRemoteHead(repoUrl);
-                const sources = getSourcesByRepo(repoUrl).filter(s => indexableNames.has(s.name));
-                let anyChanged = false;
-                for (const source of sources) {
-                    const state = await getIndexState(source.type, source.name);
-                    if (state?.last_commit_sha !== remoteHead) {
-                        anyChanged = true;
-                        break;
+                const currentToken = await this.getSourceStateToken(source);
+                const state = await getIndexState(source.type, source.name);
+                if (currentToken === null || state?.last_commit_sha !== currentToken) {
+                    const reason = currentToken === null
+                        ? 'source unavailable (clone missing?)'
+                        : `remote ${currentToken.slice(0, 8)} differs from indexed`;
+                    console.log(`[orchestrator] ${reason} for ${source.name} — queuing reindex`);
+                    if (isFileSourceConfig(source) && source.repo) {
+                        this.queueIncrementalReindex(source.repo);
                     }
                 }
-                // Even if DB says current, verify clone dir exists (fresh container = empty /tmp)
-                const repoName = repoUrl.split('/').pop()?.replace(/\.git$/, '') ?? '';
-                const cloneDir = getConfig().cloneDir;
-                const repoDir = path.join(cloneDir, repoName);
-                const cloneMissing = !fs.existsSync(repoDir);
-                if (anyChanged || cloneMissing) {
-                    const reason = cloneMissing
-                        ? `clone dir missing at ${repoDir}`
-                        : `remote HEAD ${remoteHead.slice(0, 8)} differs from indexed`;
-                    console.log(`[orchestrator] ${reason} for ${repoUrl} — queuing incremental reindex`);
-                    this.queueIncrementalReindex(repoUrl);
-                }
                 else {
-                    console.log(`[orchestrator] Index current at ${remoteHead.slice(0, 8)}`);
+                    console.log(`[orchestrator] ${source.name} index current at ${currentToken.slice(0, 8)}`);
                 }
             }
             catch (err) {
-                // If we can't check remote, fall back to age-based staleness
-                console.warn(`[orchestrator] Failed to check remote HEAD for ${repoUrl}, falling back to age check:`, err);
-                const repoSources = getSourcesByRepo(repoUrl);
-                const firstState = await getIndexState(repoSources[0].type, repoSources[0].name);
-                if (this.isStale(firstState)) {
-                    const thresholdHours = getServerConfig().indexing?.stale_threshold_hours ?? 24;
-                    console.log(`[orchestrator] Index for ${repoUrl} is stale (>${thresholdHours}h) — queuing full reindex`);
+                console.warn(`[orchestrator] Failed to check state for ${source.name}, falling back to age check:`, err);
+                const state = await getIndexState(source.type, source.name);
+                if (this.isStale(state)) {
+                    console.log(`[orchestrator] Index for ${source.name} is stale — queuing full reindex`);
                     this.queueFullReindex();
                 }
-                else {
-                    console.log(`[orchestrator] Index for ${repoUrl} appears fresh, skipping`);
-                }
+            }
+        }
+        // Ensure git repos are cloned even when index is current.
+        // On fresh deploys, the container has no local clones but the DB may have valid state.
+        // Bash tools need the clone directories to build their filesystem.
+        const cloneDir = getConfig().cloneDir;
+        for (const source of gitSourcesOk) {
+            if (!isFileSourceConfig(source) || !source.repo)
+                continue;
+            const repoName = source.repo.replace(/\.git$/, '').split('/').pop();
+            const repoDir = path.join(cloneDir, repoName);
+            if (!fs.existsSync(repoDir)) {
+                console.log(`[orchestrator] Clone directory missing for ${source.name}, queuing reindex to populate`);
+                this.queueIncrementalReindex(source.repo);
             }
         }
     }
     /**
-     * Get the HEAD SHA of a remote repo without cloning.
-     * Uses `git ls-remote` which only fetches refs.
+     * Get the current state token for a source without acquiring items.
+     * Returns null if the source is unavailable.
      */
-    async getRemoteHead(repoUrl) {
+    async getSourceStateToken(source) {
         const config = getConfig();
-        let url = repoUrl;
-        if (config.githubToken) {
-            url = repoUrl.replace('https://github.com/', `https://x-access-token:${config.githubToken}@github.com/`);
-        }
-        const git = simpleGit();
-        const result = await git.listRemote([url, 'HEAD']);
-        const sha = result.split('\t')[0]?.trim();
-        if (!sha)
-            throw new Error(`Could not resolve HEAD for ${repoUrl}`);
-        return sha;
+        const providerOptions = {
+            cloneDir: config.cloneDir,
+            githubToken: config.githubToken,
+            slackBotToken: config.slackBotToken,
+            discordBotToken: config.discordBotToken,
+        };
+        const provider = getProvider(source.type)(source, providerOptions);
+        return provider.getCurrentStateToken();
     }
     /**
      * Queue a full re-index of all sources. Returns immediately.
@@ -171,6 +176,17 @@ export class IndexingOrchestrator {
             console.error('[orchestrator] drain() failed:', err);
         });
     }
+    /**
+     * Queue a reindex for a single named source. Returns immediately.
+     * Used by webhook handlers to trigger reindexing of specific sources.
+     */
+    queueSourceReindex(sourceName) {
+        this.queue.push({ type: 'source-reindex', sourceName });
+        console.log(`[orchestrator] Source re-index queued for ${sourceName}`);
+        this.drain().catch((err) => {
+            console.error('[orchestrator] drain() failed:', err);
+        });
+    }
     /**
      * Returns true if any indexing job is currently running.
      */
@@ -276,6 +292,19 @@ export class IndexingOrchestrator {
             await this.runIncrementalReindex(embeddingClient, config.cloneDir, config.githubToken, job.repoUrl);
             affectedSourceNames = getSourcesByRepo(job.repoUrl).map(s => s.name);
         }
+        else if (job.type === 'source-reindex') {
+            if (!job.sourceName) {
+                console.warn('[orchestrator] source-reindex job has no sourceName, skipping');
+                return;
+            }
+            const sourceConfig = serverCfg2.sources.find(s => s.name === job.sourceName);
+            if (!sourceConfig) {
+                console.warn(`[orchestrator] source-reindex: source "${job.sourceName}" not found in config`);
+                return;
+            }
+            await this.indexSourceWithState(sourceConfig, embeddingClient, config.cloneDir);
+            affectedSourceNames = [job.sourceName];
+        }
         if (affectedSourceNames.length > 0 && this.onReindexComplete) {
             try {
                 this.onReindexComplete(affectedSourceNames);
@@ -305,38 +334,7 @@ export class IndexingOrchestrator {
         const indexableNames = getIndexableSourceNames();
         const sources = getSourcesByRepo(repoUrl).filter(s => indexableNames.has(s.name));
         for (const sourceConfig of sources) {
-            const state = await getIndexState(sourceConfig.type, sourceConfig.name);
-            if (state?.last_commit_sha) {
-                await this.withSourceLock(`${sourceConfig.type}:${sourceConfig.name}`, async () => {
-                    await this.setIndexStatus(sourceConfig.type, sourceConfig.name, 'indexing');
-                    try {
-                        const indexer = new SourceIndexer(sourceConfig, embeddingClient, cloneDir, githubToken);
-                        await indexer.incrementalIndex(state.last_commit_sha);
-                        const headSha = await indexer.getHeadSha();
-                        await upsertIndexState({
-                            source_type: sourceConfig.type,
-                            source_key: sourceConfig.name,
-                            last_commit_sha: headSha,
-                            last_indexed_at: new Date(),
-                            status: 'idle',
-                        });
-                    }
-                    catch (err) {
-                        console.error(`[orchestrator] Incremental reindex failed for ${sourceConfig.name}:`, err);
-                        try {
-                            await this.setIndexStatus(sourceConfig.type, sourceConfig.name, 'error', err instanceof Error ? err.message : String(err));
-                        }
-                        catch (statusErr) {
-                            console.error('[orchestrator] Failed to update index status:', statusErr);
-                        }
-                        // Don't rethrow — continue with remaining sources
-                    }
-                });
-            }
-            else {
-                // No previous state — do a full index for this source
-                await this.indexSourceWithState(sourceConfig, embeddingClient, cloneDir, githubToken);
-            }
+            await this.indexSourceWithState(sourceConfig, embeddingClient, cloneDir, githubToken);
         }
         console.log(`[orchestrator] Incremental re-index complete for ${repoUrl}`);
     }
@@ -346,21 +344,29 @@ export class IndexingOrchestrator {
     async indexSourceWithState(sourceConfig, embeddingClient, cloneDir, githubToken) {
         const lockKey = `${sourceConfig.type}:${sourceConfig.name}`;
         await this.withSourceLock(lockKey, async () => {
-            const indexer = new SourceIndexer(sourceConfig, embeddingClient, cloneDir, githubToken);
+            const providerOptions = { cloneDir, githubToken, slackBotToken: getConfig().slackBotToken, discordBotToken: getConfig().discordBotToken };
+            const provider = getProvider(sourceConfig.type)(sourceConfig, providerOptions);
+            const pipeline = new IndexingPipeline(embeddingClient, sourceConfig);
             await this.setIndexStatus(sourceConfig.type, sourceConfig.name, 'indexing');
             try {
                 const state = await getIndexState(sourceConfig.type, sourceConfig.name);
+                let result;
                 if (state?.last_commit_sha) {
-                    await indexer.incrementalIndex(state.last_commit_sha);
+                    result = await provider.incrementalAcquire(state.last_commit_sha);
                 }
                 else {
-                    await indexer.fullIndex();
+                    result = await provider.fullAcquire();
+                }
+                if (result.removedIds.length > 0) {
+                    await pipeline.removeItems(result.removedIds);
+                }
+                if (result.items.length > 0) {
+                    await pipeline.indexItems(result.items, result.stateToken);
                 }
-                const headSha = await indexer.getHeadSha();
                 await upsertIndexState({
                     source_type: sourceConfig.type,
                     source_key: sourceConfig.name,
-                    last_commit_sha: headSha,
+                    last_commit_sha: result.stateToken,
                     last_indexed_at: new Date(),
                     status: 'idle',
                 });
@@ -374,7 +380,6 @@ export class IndexingOrchestrator {
                 catch (statusErr) {
                     console.error('[orchestrator] Failed to update index status:', statusErr);
                 }
-                // Don't rethrow — continue with remaining sources
             }
         });
     }

package/dist/indexing/orchestrator.js.map

@@ -1 +1 @@
-{"version":3,"file":"orchestrator.js","sourceRoot":"","sources":["../../src/indexing/orchestrator.ts"],"names":[],"mappings":"AAAA,qDAAqD;AACrD,oFAAoF;AAEpF,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,uBAAuB,EAAE,MAAM,cAAc,CAAC;AACnF,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAClD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EACH,aAAa,EACb,gBAAgB,GACnB,MAAM,kBAAkB,CAAC;AAG1B;;GAEG;AACH,SAAS,gBAAgB,CAAC,OAAe;IACrC,OAAO,eAAe,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC;AACrE,CAAC;AAED,SAAS,mBAAmB;IACxB,MAAM,SAAS,GAAG,eAAe,EAAE,CAAC;IACpC,OAAO,CAAC,SAAS,CAAC,QAAQ,EAAE,qBAAqB,IAAI,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAC9E,CAAC;AAQD,MAAM,OAAO,oBAAoB;IACrB,KAAK,GAAU,EAAE,CAAC;IAClB,OAAO,GAAG,KAAK,CAAC;IAChB,UAAU,GAAG,KAAK,CAAC;IAE3B,oEAAoE;IAC5D,aAAa,GAAG,IAAI,GAAG,EAAU,CAAC;IAE1C,4EAA4E;IACpE,eAAe,GAAkB,IAAI,CAAC;IAE9C,6EAA6E;IAC7E,iBAAiB,CAAmC;IAEpD;QACI,mDAAmD;IACvD,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,aAAa;QACf,MAAM,cAAc,GAAG,uBAAuB,EAAE,CAAC;QACjD,IAAI,cAAc,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;YAC5B,OAAO,CAAC,GAAG,CAAC,+DAA+D,CAAC,CAAC;YAC7E,OAAO;QACX,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,wCAAwC,CAAC,CAAC;QAEtD,MAAM,SAAS,GAAG,eAAe,EAAE,CAAC;QACpC,MAAM,gBAAgB,GAAG,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAEnF,qDAAqD;QACrD,qGAAqG;QACrG,MAAM,yBAAyB,GAAmB,EAAE,CAAC;QACrD,MAAM,SAAS,GAAmB,EAAE,CAAC;QACrC,KAAK,MAAM,MAAM,IAAI,gBAAgB,EAAE,CAAC;YACpC,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;YAC5D,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,eAAe,IAAI,KAAK,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;gBAC/D,OAAO,CAAC,GAAG,CAAC,kBAAkB,MAAM,CAAC,IAAI,4DAA4D,CAAC,CAAC;gBACvG,yBAAyB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC3C,CAAC;iBAAM,CAAC;gBACJ,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC3B,CAAC;QACL,CAAC;QAED,4DAA4D;QAC5D,IAAI,yBAAyB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvC,sDAAsD;YACtD,IAAI,yBAAyB,CAAC,MAAM,KAAK,gBAAgB,CAAC,MAAM,EAAE,CAAC;gBAC/D,OAAO,CAAC,GAAG,CAAC,6EAA6E,CAAC,CAAC;gBAC3F,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBACxB,OAAO;YACX,CAAC;YACD,gEAAgE;YAChE,MAAM,cAAc,GAAG,IAAI,GAAG,CAC1B,yBAAyB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAK,CAAC,CAClE,CAAC;YACF,KAAK,MAAM,OAAO,IAAI,cAAc,EAAE,CAAC;gBACnC,IAAI,CAAC,uBAAuB,CAAC,OAAO,CAAC,CAAC;YAC1C,CAAC;YACD,6EAA6E;YAC7E,MAAM,YAAY,GAAG,yBAAyB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YACpE,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,oBAAoB,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC,CAAC;gBACvE,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,gCAAgC,EAAE,GAAG,CAAC,CAAC,CAAC;YACpF,CAAC;QACL,CAAC;QAED,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAEnC,iFAAiF;QACjF,MAAM,cAAc,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACtD,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,OAAO,CAAC,GAAG,CAAC,sCAAsC,cAAc,CAAC,MAAM,kBAAkB,CAAC,CAAC;YAC3F,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,oBAAoB,EAAE,OAAO,EAAE,cAAc,EAAE,CAAC,CAAC;YACzE,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,gCAAgC,EAAE,GAAG,CAAC,CAAC,CAAC;QACpF,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,mEAAmE,CAAC,CAAC;QAEjF,4CAA4C;QAC5C,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAK,CAAC,CAAC,CAAC,CAAC;QAC5E,KAAK,MAAM,OAAO,IAAI,KAAK,EAAE,CAAC;YAC1B,IAAI,CAAC;gBACD,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;gBACrD,MAAM,OAAO,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;gBAElF,IAAI,UAAU,GAAG,KAAK,CAAC;gBACvB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;oBAC3B,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;oBAC5D,IAAI,KAAK,EAAE,eAAe,KAAK,UAAU,EAAE,CAAC;wBACxC,UAAU,GAAG,IAAI,CAAC;wBAClB,MAAM;oBACV,CAAC;gBACL,CAAC;gBAED,kFAAkF;gBAClF,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,IAAI,EAAE,CAAC;gBACvE,MAAM,QAAQ,GAAG,SAAS,EAAE,CAAC,QAAQ,CAAC;gBACtC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;gBAC9C,MAAM,YAAY,GAAG,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;gBAE7C,IAAI,UAAU,IAAI,YAAY,EAAE,CAAC;oBAC7B,MAAM,MAAM,GAAG,YAAY;wBACvB,CAAC,CAAC,wBAAwB,OAAO,EAAE;wBACnC,CAAC,CAAC,eAAe,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,uBAAuB,CAAC;oBACnE,OAAO,CAAC,GAAG,CAAC,kBAAkB,MAAM,QAAQ,OAAO,gCAAgC,CAAC,CAAC;oBACrF,IAAI,CAAC,uBAAuB,CAAC,OAAO,CAAC,CAAC;gBAC1C,CAAC;qBAAM,CAAC;oBACJ,OAAO,CAAC,GAAG,CAAC,mCAAmC,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;gBAC7E,CAAC;YACL,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,6DAA6D;gBAC7D,OAAO,CAAC,IAAI,CAAC,kDAAkD,OAAO,8BAA8B,EAAE,GAAG,CAAC,CAAC;gBAC3G,MAAM,WAAW,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;gBAC9C,MAAM,UAAU,GAAG,MAAM,aAAa,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;gBACjF,IAAI,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;oBAC3B,MAAM,cAAc,GAAG,eAAe,EAAE,CAAC,QAAQ,EAAE,qBAAqB,IAAI,EAAE,CAAC;oBAC/E,OAAO,CAAC,GAAG,CAAC,4BAA4B,OAAO,eAAe,cAAc,2BAA2B,CAAC,CAAC;oBACzG,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBAC5B,CAAC;qBAAM,CAAC;oBACJ,OAAO,CAAC,GAAG,CAAC,4BAA4B,OAAO,0BAA0B,CAAC,CAAC;gBAC/E,CAAC;YACL,CAAC;QACL,CAAC;IACL,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,aAAa,CAAC,OAAe;QACvC,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;QAC3B,IAAI,GAAG,GAAG,OAAO,CAAC;QAClB,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;YACrB,GAAG,GAAG,OAAO,CAAC,OAAO,CAAC,qBAAqB,EAAE,0BAA0B,MAAM,CAAC,WAAW,cAAc,CAAC,CAAC;QAC7G,CAAC;QACD,MAAM,GAAG,GAAG,SAAS,EAAE,CAAC;QACxB,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,UAAU,CAAC,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC;QACnD,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC;QAC1C,IAAI,CAAC,GAAG;YAAE,MAAM,IAAI,KAAK,CAAC,8BAA8B,OAAO,EAAE,CAAC,CAAC;QACnE,OAAO,GAAG,CAAC;IACf,CAAC;IAED;;OAEG;IACH,gBAAgB;QACZ,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,CAAC,CAAC;QAC1C,OAAO,CAAC,GAAG,CAAC,qCAAqC,CAAC,CAAC;QACnD,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YACvB,OAAO,CAAC,KAAK,CAAC,gCAAgC,EAAE,GAAG,CAAC,CAAC;QACzD,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACH,uBAAuB,CAAC,OAAe;QACnC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,qBAAqB,EAAE,OAAO,EAAE,CAAC,CAAC;QAC1D,OAAO,CAAC,GAAG,CACP,kDAAkD,OAAO,EAAE,CAC9D,CAAC;QACF,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YACvB,OAAO,CAAC,KAAK,CAAC,gCAAgC,EAAE,GAAG,CAAC,CAAC;QACzD,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACH,UAAU;QACN,OAAO,IAAI,CAAC,OAAO,CAAC;IACxB,CAAC;IAED;;;OAGG;IACH,mBAAmB;QACf,MAAM,SAAS,GAAG,eAAe,EAAE,CAAC;QACpC,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,YAAY,EAAE,CAAC;YACpC,OAAO,CAAC,GAAG,CAAC,yCAAyC,CAAC,CAAC;YACvD,OAAO;QACX,CAAC;QAED,MAAM,IAAI,GAAG,SAAS,CAAC,QAAQ,CAAC,gBAAgB,CAAC;QACjD,OAAO,CAAC,GAAG,CAAC,+CAA+C,IAAI,SAAS,CAAC,CAAC;QAE1E,6CAA6C;QAC7C,WAAW,CAAC,GAAG,EAAE;YACb,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,KAAK,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC7C,IAAI,GAAG,CAAC,WAAW,EAAE,KAAK,IAAI,IAAI,IAAI,CAAC,eAAe,KAAK,KAAK,EAAE,CAAC;gBAC/D,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,EAAE,CAAC;oBACrB,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC;oBAC7B,OAAO,CAAC,GAAG,CAAC,yCAAyC,CAAC,CAAC;oBACvD,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBAC5B,CAAC;YACL,CAAC;QACL,CAAC,EAAE,MAAM,CAAC,CAAC;IACf,CAAC;IAED,0EAA0E;IAC1E,UAAU;IACV,0EAA0E;IAE1E;;OAEG;IACK,OAAO,CAAC,KAAwB;QACpC,IAAI,CAAC,KAAK;YAAE,OAAO,IAAI,CAAC;QACxB,IAAI,CAAC,KAAK,CAAC,eAAe;YAAE,OAAO,IAAI,CAAC;QACxC,IAAI,KAAK,CAAC,MAAM,KAAK,OAAO;YAAE,OAAO,IAAI,CAAC;QAE1C,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,OAAO,EAAE,CAAC;QACnE,OAAO,GAAG,GAAG,mBAAmB,EAAE,CAAC;IACvC,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,KAAK;QACf,IAAI,IAAI,CAAC,UAAU;YAAE,OAAO,CAAC,mBAAmB;QAChD,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QAEvB,IAAI,CAAC;YACD,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAG,CAAC;gBAChC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;gBAEpB,IAAI,CAAC;oBACD,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;gBAC/B,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACX,OAAO,CAAC,KAAK,CACT,4BAA4B,EAC5B,GAAG,CACN,CAAC;gBACN,CAAC;YACL,CAAC;QACL,CAAC;gBAAS,CAAC;YACP,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;YACrB,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;QAC5B,CAAC;IACL,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,UAAU,CAAC,GAAQ;QAC7B,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;QAC3B,MAAM,SAAS,GAAG,eAAe,EAAE,CAAC;QACpC,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;QACjE,CAAC;QACD,MAAM,eAAe,GAAG,IAAI,eAAe,CACvC,MAAM,CAAC,YAAY,EACnB,SAAS,CAAC,SAAS,CAAC,KAAK,EACzB,SAAS,CAAC,SAAS,CAAC,UAAU,CACjC,CAAC;QAEF,MAAM,UAAU,GAAG,eAAe,EAAE,CAAC;QACrC,IAAI,mBAAmB,GAAa,EAAE,CAAC;QAEvC,IAAI,GAAG,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;YAC9B,MAAM,IAAI,CAAC,cAAc,CAAC,eAAe,EAAE,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC;YAChF,mBAAmB,GAAG,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC9D,CAAC;aAAM,IAAI,GAAG,CAAC,IAAI,KAAK,oBAAoB,EAAE,CAAC;YAC3C,IAAI,CAAC,GAAG,CAAC,OAAO,IAAI,GAAG,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC3C,OAAO,CAAC,IAAI,CAAC,gEAAgE,CAAC,CAAC;gBAC/E,OAAO;YACX,CAAC;YACD,KAAK,MAAM,YAAY,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC;gBACrC,MAAM,IAAI,CAAC,oBAAoB,CAAC,YAAY,EAAE,eAAe,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;YACpF,CAAC;YACD,mBAAmB,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACvD,CAAC;aAAM,IAAI,GAAG,CAAC,IAAI,KAAK,qBAAqB,EAAE,CAAC;YAC5C,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;gBACf,OAAO,CAAC,IAAI,CAAC,iEAAiE,CAAC,CAAC;gBAChF,OAAO;YACX,CAAC;YACD,MAAM,IAAI,CAAC,qBAAqB,CAC5B,eAAe,EACf,MAAM,CAAC,QAAQ,EACf,MAAM,CAAC,WAAW,EAClB,GAAG,CAAC,OAAO,CACd,CAAC;YACF,mBAAmB,GAAG,gBAAgB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACzE,CAAC;QAED,IAAI,mBAAmB,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC3D,IAAI,CAAC;gBACD,IAAI,CAAC,iBAAiB,CAAC,mBAAmB,CAAC,CAAC;YAChD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,OAAO,CAAC,KAAK,CAAC,mDAAmD,EAAE,GAAG,CAAC,CAAC;YAC5E,CAAC;QACL,CAAC;IACL,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,cAAc,CACxB,eAAgC,EAChC,QAAgB,EAChB,WAAoB;QAEpB,OAAO,CAAC,GAAG,CAAC,uCAAuC,CAAC,CAAC;QAErD,MAAM,SAAS,GAAG,eAAe,EAAE,CAAC;QACpC,MAAM,cAAc,GAAG,uBAAuB,EAAE,CAAC;QACjD,KAAK,MAAM,YAAY,IAAI,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;YACnF,MAAM,IAAI,CAAC,oBAAoB,CAC3B,YAAY,EACZ,eAAe,EACf,QAAQ,EACR,WAAW,CACd,CAAC;QACN,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,uCAAuC,CAAC,CAAC;IACzD,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,qBAAqB,CAC/B,eAAgC,EAChC,QAAgB,EAChB,WAA+B,EAC/B,OAAe;QAEf,OAAO,CAAC,GAAG,CACP,oDAAoD,OAAO,EAAE,CAChE,CAAC;QAEF,MAAM,cAAc,GAAG,uBAAuB,EAAE,CAAC;QACjD,MAAM,OAAO,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAClF,KAAK,MAAM,YAAY,IAAI,OAAO,EAAE,CAAC;YACjC,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,YAAY,CAAC,IAAI,EAAE,YAAY,CAAC,IAAI,CAAC,CAAC;YACxE,IAAI,KAAK,EAAE,eAAe,EAAE,CAAC;gBACzB,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,YAAY,CAAC,IAAI,IAAI,YAAY,CAAC,IAAI,EAAE,EAAE,KAAK,IAAI,EAAE;oBAC9E,MAAM,IAAI,CAAC,cAAc,CAAC,YAAY,CAAC,IAAI,EAAE,YAAY,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;oBAC5E,IAAI,CAAC;wBACD,MAAM,OAAO,GAAG,IAAI,aAAa,CAAC,YAAY,EAAE,eAAe,EAAE,QAAQ,EAAE,WAAW,CAAC,CAAC;wBACxF,MAAM,OAAO,CAAC,gBAAgB,CAAC,KAAK,CAAC,eAAgB,CAAC,CAAC;wBACvD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;wBAC3C,MAAM,gBAAgB,CAAC;4BACnB,WAAW,EAAE,YAAY,CAAC,IAAI;4BAC9B,UAAU,EAAE,YAAY,CAAC,IAAI;4BAC7B,eAAe,EAAE,OAAO;4BACxB,eAAe,EAAE,IAAI,IAAI,EAAE;4BAC3B,MAAM,EAAE,MAAM;yBACjB,CAAC,CAAC;oBACP,CAAC;oBAAC,OAAO,GAAG,EAAE,CAAC;wBACX,OAAO,CAAC,KAAK,CAAC,iDAAiD,YAAY,CAAC,IAAI,GAAG,EAAE,GAAG,CAAC,CAAC;wBAC1F,IAAI,CAAC;4BACD,MAAM,IAAI,CAAC,cAAc,CACrB,YAAY,CAAC,IAAI,EACjB,YAAY,CAAC,IAAI,EACjB,OAAO,EACP,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CACnD,CAAC;wBACN,CAAC;wBAAC,OAAO,SAAS,EAAE,CAAC;4BACjB,OAAO,CAAC,KAAK,CAAC,+CAA+C,EAAE,SAAS,CAAC,CAAC;wBAC9E,CAAC;wBACD,kDAAkD;oBACtD,CAAC;gBACL,CAAC,CAAC,CAAC;YACP,CAAC;iBAAM,CAAC;gBACJ,sDAAsD;gBACtD,MAAM,IAAI,CAAC,oBAAoB,CAAC,YAAY,EAAE,eAAe,EAAE,QAAQ,EAAE,WAAW,CAAC,CAAC;YAC1F,CAAC;QACL,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,oDAAoD,OAAO,EAAE,CAAC,CAAC;IAC/E,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,oBAAoB,CAC9B,YAA0B,EAC1B,eAAgC,EAChC,QAAgB,EAChB,WAAoB;QAEpB,MAAM,OAAO,GAAG,GAAG,YAAY,CAAC,IAAI,IAAI,YAAY,CAAC,IAAI,EAAE,CAAC;QAC5D,MAAM,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,KAAK,IAAI,EAAE;YAC1C,MAAM,OAAO,GAAG,IAAI,aAAa,CAAC,YAAY,EAAE,eAAe,EAAE,QAAQ,EAAE,WAAW,CAAC,CAAC;YACxF,MAAM,IAAI,CAAC,cAAc,CAAC,YAAY,CAAC,IAAI,EAAE,YAAY,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;YAE5E,IAAI,CAAC;gBACD,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,YAAY,CAAC,IAAI,EAAE,YAAY,CAAC,IAAI,CAAC,CAAC;gBACxE,IAAI,KAAK,EAAE,eAAe,EAAE,CAAC;oBACzB,MAAM,OAAO,CAAC,gBAAgB,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;gBAC1D,CAAC;qBAAM,CAAC;oBACJ,MAAM,OAAO,CAAC,SAAS,EAAE,CAAC;gBAC9B,CAAC;gBAED,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;gBAC3C,MAAM,gBAAgB,CAAC;oBACnB,WAAW,EAAE,YAAY,CAAC,IAAI;oBAC9B,UAAU,EAAE,YAAY,CAAC,IAAI;oBAC7B,eAAe,EAAE,OAAO;oBACxB,eAAe,EAAE,IAAI,IAAI,EAAE;oBAC3B,MAAM,EAAE,MAAM;iBACjB,CAAC,CAAC;gBACH,OAAO,CAAC,GAAG,CAAC,wCAAwC,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC;YAC7E,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,OAAO,CAAC,KAAK,CACT,sCAAsC,YAAY,CAAC,IAAI,GAAG,EAC1D,GAAG,CACN,CAAC;gBACF,IAAI,CAAC;oBACD,MAAM,IAAI,CAAC,cAAc,CACrB,YAAY,CAAC,IAAI,EACjB,YAAY,CAAC,IAAI,EACjB,OAAO,EACP,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CACnD,CAAC;gBACN,CAAC;gBAAC,OAAO,SAAS,EAAE,CAAC;oBACjB,OAAO,CAAC,KAAK,CAAC,+CAA+C,EAAE,SAAS,CAAC,CAAC;gBAC9E,CAAC;gBACD,kDAAkD;YACtD,CAAC;QACL,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,cAAc,CACxB,SAAiB,EACjB,EAAuB;QAEvB,IAAI,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;YACpC,OAAO,CAAC,GAAG,CACP,2BAA2B,SAAS,0BAA0B,CACjE,CAAC;YACF,OAAO;QACX,CAAC;QAED,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAClC,IAAI,CAAC;YACD,MAAM,EAAE,EAAE,CAAC;QACf,CAAC;gBAAS,CAAC;YACP,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QACzC,CAAC;IACL,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,cAAc,CACxB,UAAkB,EAClB,SAAiB,EACjB,MAAmB,EACnB,YAAqB;QAErB,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;QAC5D,MAAM,gBAAgB,CAAC;YACnB,WAAW,EAAE,UAAU;YACvB,UAAU,EAAE,SAAS;YACrB,eAAe,EAAE,QAAQ,EAAE,eAAe,IAAI,IAAI;YAClD,eAAe,EAAE,QAAQ,EAAE,eAAe,IAAI,IAAI;YAClD,MAAM;YACN,aAAa,EAAE,YAAY,IAAI,IAAI;SACtC,CAAC,CAAC;IACP,CAAC;CACJ"}
+{"version":3,"file":"orchestrator.js","sourceRoot":"","sources":["../../src/indexing/orchestrator.ts"],"names":[],"mappings":"AAAA,qDAAqD;AACrD,oFAAoF;AAEpF,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,uBAAuB,EAAE,MAAM,cAAc,CAAC;AACnF,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAClD,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AACjD,OAAO,EACH,aAAa,EACb,gBAAgB,GACnB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAIjD;;GAEG;AACH,SAAS,gBAAgB,CAAC,OAAe;IACrC,OAAO,eAAe,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC;AAC9F,CAAC;AAED,SAAS,mBAAmB;IACxB,MAAM,SAAS,GAAG,eAAe,EAAE,CAAC;IACpC,OAAO,CAAC,SAAS,CAAC,QAAQ,EAAE,qBAAqB,IAAI,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAC9E,CAAC;AASD,MAAM,OAAO,oBAAoB;IACrB,KAAK,GAAU,EAAE,CAAC;IAClB,OAAO,GAAG,KAAK,CAAC;IAChB,UAAU,GAAG,KAAK,CAAC;IAE3B,oEAAoE;IAC5D,aAAa,GAAG,IAAI,GAAG,EAAU,CAAC;IAE1C,4EAA4E;IACpE,eAAe,GAAkB,IAAI,CAAC;IAE9C,6EAA6E;IAC7E,iBAAiB,CAAmC;IAEpD;QACI,mDAAmD;IACvD,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,aAAa;QACf,MAAM,cAAc,GAAG,uBAAuB,EAAE,CAAC;QACjD,IAAI,cAAc,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;YAC5B,OAAO,CAAC,GAAG,CAAC,+DAA+D,CAAC,CAAC;YAC7E,OAAO;QACX,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,wCAAwC,CAAC,CAAC;QAEtD,MAAM,SAAS,GAAG,eAAe,EAAE,CAAC;QACpC,MAAM,gBAAgB,GAAG,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAEnF,qDAAqD;QACrD,qGAAqG;QACrG,MAAM,yBAAyB,GAAmB,EAAE,CAAC;QACrD,MAAM,SAAS,GAAmB,EAAE,CAAC;QACrC,KAAK,MAAM,MAAM,IAAI,gBAAgB,EAAE,CAAC;YACpC,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;YAC5D,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,eAAe,IAAI,KAAK,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;gBAC/D,OAAO,CAAC,GAAG,CAAC,kBAAkB,MAAM,CAAC,IAAI,4DAA4D,CAAC,CAAC;gBACvG,yBAAyB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC3C,CAAC;iBAAM,CAAC;gBACJ,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC3B,CAAC;QACL,CAAC;QAED,4DAA4D;QAC5D,IAAI,yBAAyB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvC,sDAAsD;YACtD,IAAI,yBAAyB,CAAC,MAAM,KAAK,gBAAgB,CAAC,MAAM,EAAE,CAAC;gBAC/D,OAAO,CAAC,GAAG,CAAC,6EAA6E,CAAC,CAAC;gBAC3F,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBACxB,OAAO;YACX,CAAC;YACD,gEAAgE;YAChE,MAAM,cAAc,GAAG,IAAI,GAAG,EAAU,CAAC;YACzC,KAAK,MAAM,CAAC,IAAI,yBAAyB,EAAE,CAAC;gBACxC,IAAI,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI;oBAAE,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YACpE,CAAC;YACD,KAAK,MAAM,OAAO,IAAI,cAAc,EAAE,CAAC;gBACnC,IAAI,CAAC,uBAAuB,CAAC,OAAO,CAAC,CAAC;YAC1C,CAAC;YACD,6EAA6E;YAC7E,MAAM,YAAY,GAAG,yBAAyB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAC7F,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,oBAAoB,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC,CAAC;gBACvE,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,gCAAgC,EAAE,GAAG,CAAC,CAAC,CAAC;YACpF,CAAC;YAED,sDAAsD;YACtD,MAAM,cAAc,GAAG,yBAAyB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC;YACrF,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE,CAAC;gBAClC,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YACzC,CAAC;QACL,CAAC;QAED,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAEnC,iFAAiF;QACjF,MAAM,cAAc,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC/E,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,OAAO,CAAC,GAAG,CAAC,sCAAsC,cAAc,CAAC,MAAM,kBAAkB,CAAC,CAAC;YAC3F,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,oBAAoB,EAAE,OAAO,EAAE,cAAc,EAAE,CAAC,CAAC;YACzE,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,gCAAgC,EAAE,GAAG,CAAC,CAAC,CAAC;QACpF,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,mEAAmE,CAAC,CAAC;QAEjF,2CAA2C;QAC3C,MAAM,YAAY,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;QAC5E,KAAK,MAAM,MAAM,IAAI,YAAY,EAAE,CAAC;YAChC,IAAI,CAAC;gBACD,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC;gBAC5D,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;gBAE5D,IAAI,YAAY,KAAK,IAAI,IAAI,KAAK,EAAE,eAAe,KAAK,YAAY,EAAE,CAAC;oBACnE,MAAM,MAAM,GAAG,YAAY,KAAK,IAAI;wBAChC,CAAC,CAAC,qCAAqC;wBACvC,CAAC,CAAC,UAAU,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,uBAAuB,CAAC;oBAChE,OAAO,CAAC,GAAG,CAAC,kBAAkB,MAAM,QAAQ,MAAM,CAAC,IAAI,oBAAoB,CAAC,CAAC;oBAC7E,IAAI,kBAAkB,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;wBAC5C,IAAI,CAAC,uBAAuB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;oBAC9C,CAAC;gBACL,CAAC;qBAAM,CAAC;oBACJ,OAAO,CAAC,GAAG,CAAC,kBAAkB,MAAM,CAAC,IAAI,qBAAqB,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;gBAC9F,CAAC;YACL,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,OAAO,CAAC,IAAI,CAAC,4CAA4C,MAAM,CAAC,IAAI,8BAA8B,EAAE,GAAG,CAAC,CAAC;gBACzG,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;gBAC5D,IAAI,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;oBACtB,OAAO,CAAC,GAAG,CAAC,4BAA4B,MAAM,CAAC,IAAI,kCAAkC,CAAC,CAAC;oBACvF,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBAC5B,CAAC;YACL,CAAC;QACL,CAAC;QAED,0DAA0D;QAC1D,uFAAuF;QACvF,mEAAmE;QACnE,MAAM,QAAQ,GAAG,SAAS,EAAE,CAAC,QAAQ,CAAC;QACtC,KAAK,MAAM,MAAM,IAAI,YAAY,EAAE,CAAC;YAChC,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI;gBAAE,SAAS;YAC1D,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAG,CAAC;YACrE,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;YAC9C,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC1B,OAAO,CAAC,GAAG,CAAC,8CAA8C,MAAM,CAAC,IAAI,+BAA+B,CAAC,CAAC;gBACtG,IAAI,CAAC,uBAAuB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAC9C,CAAC;QACL,CAAC;IACL,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,mBAAmB,CAAC,MAAoB;QAClD,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;QAC3B,MAAM,eAAe,GAAoB;YACrC,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,aAAa,EAAE,MAAM,CAAC,aAAa;YACnC,eAAe,EAAE,MAAM,CAAC,eAAe;SAC1C,CAAC;QACF,MAAM,QAAQ,GAAG,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;QACnE,OAAO,QAAQ,CAAC,oBAAoB,EAAE,CAAC;IAC3C,CAAC;IAED;;OAEG;IACH,gBAAgB;QACZ,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,CAAC,CAAC;QAC1C,OAAO,CAAC,GAAG,CAAC,qCAAqC,CAAC,CAAC;QACnD,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YACvB,OAAO,CAAC,KAAK,CAAC,gCAAgC,EAAE,GAAG,CAAC,CAAC;QACzD,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACH,uBAAuB,CAAC,OAAe;QACnC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,qBAAqB,EAAE,OAAO,EAAE,CAAC,CAAC;QAC1D,OAAO,CAAC,GAAG,CACP,kDAAkD,OAAO,EAAE,CAC9D,CAAC;QACF,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YACvB,OAAO,CAAC,KAAK,CAAC,gCAAgC,EAAE,GAAG,CAAC,CAAC;QACzD,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;OAGG;IACH,kBAAkB,CAAC,UAAkB;QACjC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,UAAU,EAAE,CAAC,CAAC;QACxD,OAAO,CAAC,GAAG,CAAC,6CAA6C,UAAU,EAAE,CAAC,CAAC;QACvE,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YACvB,OAAO,CAAC,KAAK,CAAC,gCAAgC,EAAE,GAAG,CAAC,CAAC;QACzD,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACH,UAAU;QACN,OAAO,IAAI,CAAC,OAAO,CAAC;IACxB,CAAC;IAED;;;OAGG;IACH,mBAAmB;QACf,MAAM,SAAS,GAAG,eAAe,EAAE,CAAC;QACpC,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,YAAY,EAAE,CAAC;YACpC,OAAO,CAAC,GAAG,CAAC,yCAAyC,CAAC,CAAC;YACvD,OAAO;QACX,CAAC;QAED,MAAM,IAAI,GAAG,SAAS,CAAC,QAAQ,CAAC,gBAAgB,CAAC;QACjD,OAAO,CAAC,GAAG,CAAC,+CAA+C,IAAI,SAAS,CAAC,CAAC;QAE1E,6CAA6C;QAC7C,WAAW,CAAC,GAAG,EAAE;YACb,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,KAAK,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC7C,IAAI,GAAG,CAAC,WAAW,EAAE,KAAK,IAAI,IAAI,IAAI,CAAC,eAAe,KAAK,KAAK,EAAE,CAAC;gBAC/D,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,EAAE,CAAC;oBACrB,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC;oBAC7B,OAAO,CAAC,GAAG,CAAC,yCAAyC,CAAC,CAAC;oBACvD,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBAC5B,CAAC;YACL,CAAC;QACL,CAAC,EAAE,MAAM,CAAC,CAAC;IACf,CAAC;IAED,0EAA0E;IAC1E,UAAU;IACV,0EAA0E;IAE1E;;OAEG;IACK,OAAO,CAAC,KAAwB;QACpC,IAAI,CAAC,KAAK;YAAE,OAAO,IAAI,CAAC;QACxB,IAAI,CAAC,KAAK,CAAC,eAAe;YAAE,OAAO,IAAI,CAAC;QACxC,IAAI,KAAK,CAAC,MAAM,KAAK,OAAO;YAAE,OAAO,IAAI,CAAC;QAE1C,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,OAAO,EAAE,CAAC;QACnE,OAAO,GAAG,GAAG,mBAAmB,EAAE,CAAC;IACvC,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,KAAK;QACf,IAAI,IAAI,CAAC,UAAU;YAAE,OAAO,CAAC,mBAAmB;QAChD,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QAEvB,IAAI,CAAC;YACD,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAG,CAAC;gBAChC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;gBAEpB,IAAI,CAAC;oBACD,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;gBAC/B,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACX,OAAO,CAAC,KAAK,CACT,4BAA4B,EAC5B,GAAG,CACN,CAAC;gBACN,CAAC;YACL,CAAC;QACL,CAAC;gBAAS,CAAC;YACP,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;YACrB,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;QAC5B,CAAC;IACL,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,UAAU,CAAC,GAAQ;QAC7B,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;QAC3B,MAAM,SAAS,GAAG,eAAe,EAAE,CAAC;QACpC,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;QACjE,CAAC;QACD,MAAM,eAAe,GAAG,IAAI,eAAe,CACvC,MAAM,CAAC,YAAY,EACnB,SAAS,CAAC,SAAS,CAAC,KAAK,EACzB,SAAS,CAAC,SAAS,CAAC,UAAU,CACjC,CAAC;QAEF,MAAM,UAAU,GAAG,eAAe,EAAE,CAAC;QACrC,IAAI,mBAAmB,GAAa,EAAE,CAAC;QAEvC,IAAI,GAAG,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;YAC9B,MAAM,IAAI,CAAC,cAAc,CAAC,eAAe,EAAE,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC;YAChF,mBAAmB,GAAG,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC9D,CAAC;aAAM,IAAI,GAAG,CAAC,IAAI,KAAK,oBAAoB,EAAE,CAAC;YAC3C,IAAI,CAAC,GAAG,CAAC,OAAO,IAAI,GAAG,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC3C,OAAO,CAAC,IAAI,CAAC,gEAAgE,CAAC,CAAC;gBAC/E,OAAO;YACX,CAAC;YACD,KAAK,MAAM,YAAY,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC;gBACrC,MAAM,IAAI,CAAC,oBAAoB,CAAC,YAAY,EAAE,eAAe,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;YACpF,CAAC;YACD,mBAAmB,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACvD,CAAC;aAAM,IAAI,GAAG,CAAC,IAAI,KAAK,qBAAqB,EAAE,CAAC;YAC5C,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;gBACf,OAAO,CAAC,IAAI,CAAC,iEAAiE,CAAC,CAAC;gBAChF,OAAO;YACX,CAAC;YACD,MAAM,IAAI,CAAC,qBAAqB,CAC5B,eAAe,EACf,MAAM,CAAC,QAAQ,EACf,MAAM,CAAC,WAAW,EAClB,GAAG,CAAC,OAAO,CACd,CAAC;YACF,mBAAmB,GAAG,gBAAgB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACzE,CAAC;aAAM,IAAI,GAAG,CAAC,IAAI,KAAK,gBAAgB,EAAE,CAAC;YACvC,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC;gBAClB,OAAO,CAAC,IAAI,CAAC,+DAA+D,CAAC,CAAC;gBAC9E,OAAO;YACX,CAAC;YACD,MAAM,YAAY,GAAG,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,UAAU,CAAC,CAAC;YAC7E,IAAI,CAAC,YAAY,EAAE,CAAC;gBAChB,OAAO,CAAC,IAAI,CAAC,0CAA0C,GAAG,CAAC,UAAU,uBAAuB,CAAC,CAAC;gBAC9F,OAAO;YACX,CAAC;YACD,MAAM,IAAI,CAAC,oBAAoB,CAAC,YAAY,EAAE,eAAe,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;YAChF,mBAAmB,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAC3C,CAAC;QAED,IAAI,mBAAmB,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC3D,IAAI,CAAC;gBACD,IAAI,CAAC,iBAAiB,CAAC,mBAAmB,CAAC,CAAC;YAChD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,OAAO,CAAC,KAAK,CAAC,mDAAmD,EAAE,GAAG,CAAC,CAAC;YAC5E,CAAC;QACL,CAAC;IACL,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,cAAc,CACxB,eAAgC,EAChC,QAAgB,EAChB,WAAoB;QAEpB,OAAO,CAAC,GAAG,CAAC,uCAAuC,CAAC,CAAC;QAErD,MAAM,SAAS,GAAG,eAAe,EAAE,CAAC;QACpC,MAAM,cAAc,GAAG,uBAAuB,EAAE,CAAC;QACjD,KAAK,MAAM,YAAY,IAAI,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;YACnF,MAAM,IAAI,CAAC,oBAAoB,CAC3B,YAAY,EACZ,eAAe,EACf,QAAQ,EACR,WAAW,CACd,CAAC;QACN,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,uCAAuC,CAAC,CAAC;IACzD,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,qBAAqB,CAC/B,eAAgC,EAChC,QAAgB,EAChB,WAA+B,EAC/B,OAAe;QAEf,OAAO,CAAC,GAAG,CAAC,oDAAoD,OAAO,EAAE,CAAC,CAAC;QAE3E,MAAM,cAAc,GAAG,uBAAuB,EAAE,CAAC;QACjD,MAAM,OAAO,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAClF,KAAK,MAAM,YAAY,IAAI,OAAO,EAAE,CAAC;YACjC,MAAM,IAAI,CAAC,oBAAoB,CAAC,YAAY,EAAE,eAAe,EAAE,QAAQ,EAAE,WAAW,CAAC,CAAC;QAC1F,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,oDAAoD,OAAO,EAAE,CAAC,CAAC;IAC/E,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,oBAAoB,CAC9B,YAA0B,EAC1B,eAAgC,EAChC,QAAgB,EAChB,WAAoB;QAEpB,MAAM,OAAO,GAAG,GAAG,YAAY,CAAC,IAAI,IAAI,YAAY,CAAC,IAAI,EAAE,CAAC;QAC5D,MAAM,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,KAAK,IAAI,EAAE;YAC1C,MAAM,eAAe,GAAoB,EAAE,QAAQ,EAAE,WAAW,EAAE,aAAa,EAAE,SAAS,EAAE,CAAC,aAAa,EAAE,eAAe,EAAE,SAAS,EAAE,CAAC,eAAe,EAAE,CAAC;YAC3J,MAAM,QAAQ,GAAG,WAAW,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,YAAY,EAAE,eAAe,CAAC,CAAC;YAC/E,MAAM,QAAQ,GAAG,IAAI,gBAAgB,CAAC,eAAe,EAAE,YAAY,CAAC,CAAC;YAErE,MAAM,IAAI,CAAC,cAAc,CAAC,YAAY,CAAC,IAAI,EAAE,YAAY,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;YAE5E,IAAI,CAAC;gBACD,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,YAAY,CAAC,IAAI,EAAE,YAAY,CAAC,IAAI,CAAC,CAAC;gBACxE,IAAI,MAAM,CAAC;gBACX,IAAI,KAAK,EAAE,eAAe,EAAE,CAAC;oBACzB,MAAM,GAAG,MAAM,QAAQ,CAAC,kBAAkB,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;gBACtE,CAAC;qBAAM,CAAC;oBACJ,MAAM,GAAG,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC;gBAC1C,CAAC;gBAED,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC/B,MAAM,QAAQ,CAAC,WAAW,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;gBAClD,CAAC;gBACD,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC1B,MAAM,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;gBAC/D,CAAC;gBAED,MAAM,gBAAgB,CAAC;oBACnB,WAAW,EAAE,YAAY,CAAC,IAAI;oBAC9B,UAAU,EAAE,YAAY,CAAC,IAAI;oBAC7B,eAAe,EAAE,MAAM,CAAC,UAAU;oBAClC,eAAe,EAAE,IAAI,IAAI,EAAE;oBAC3B,MAAM,EAAE,MAAM;iBACjB,CAAC,CAAC;gBACH,OAAO,CAAC,GAAG,CAAC,wCAAwC,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC;YAC7E,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,OAAO,CAAC,KAAK,CAAC,sCAAsC,YAAY,CAAC,IAAI,GAAG,EAAE,GAAG,CAAC,CAAC;gBAC/E,IAAI,CAAC;oBACD,MAAM,IAAI,CAAC,cAAc,CACrB,YAAY,CAAC,IAAI,EACjB,YAAY,CAAC,IAAI,EACjB,OAAO,EACP,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CACnD,CAAC;gBACN,CAAC;gBAAC,OAAO,SAAS,EAAE,CAAC;oBACjB,OAAO,CAAC,KAAK,CAAC,+CAA+C,EAAE,SAAS,CAAC,CAAC;gBAC9E,CAAC;YACL,CAAC;QACL,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,cAAc,CACxB,SAAiB,EACjB,EAAuB;QAEvB,IAAI,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;YACpC,OAAO,CAAC,GAAG,CACP,2BAA2B,SAAS,0BAA0B,CACjE,CAAC;YACF,OAAO;QACX,CAAC;QAED,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAClC,IAAI,CAAC;YACD,MAAM,EAAE,EAAE,CAAC;QACf,CAAC;gBAAS,CAAC;YACP,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QACzC,CAAC;IACL,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,cAAc,CACxB,UAAkB,EAClB,SAAiB,EACjB,MAAmB,EACnB,YAAqB;QAErB,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;QAC5D,MAAM,gBAAgB,CAAC;YACnB,WAAW,EAAE,UAAU;YACvB,UAAU,EAAE,SAAS;YACrB,eAAe,EAAE,QAAQ,EAAE,eAAe,IAAI,IAAI;YAClD,eAAe,EAAE,QAAQ,EAAE,eAAe,IAAI,IAAI;YAClD,MAAM;YACN,aAAa,EAAE,YAAY,IAAI,IAAI;SACtC,CAAC,CAAC;IACP,CAAC;CACJ"}

package/dist/indexing/pipeline.d.ts

@@ -0,0 +1,18 @@
+import { EmbeddingClient } from './embeddings.js';
+import type { SourceConfig } from '../types.js';
+import type { ContentItem } from './providers/types.js';
+export declare class IndexingPipeline {
+    private sourceConfig;
+    private embeddingClient;
+    private logPrefix;
+    constructor(embeddingClient: EmbeddingClient, sourceConfig: SourceConfig);
+    /**
+     * Index a batch of content items: chunk → embed → upsert.
+     * Each item's existing chunks are deleted first to handle shrinkage.
+     */
+    indexItems(items: ContentItem[], stateToken: string): Promise<void>;
+    /** Remove items from the index by ID. */
+    removeItems(ids: string[]): Promise<void>;
+    private indexItem;
+}
+//# sourceMappingURL=pipeline.d.ts.map

package/dist/indexing/pipeline.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../src/indexing/pipeline.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAGlD,OAAO,KAAK,EAAS,YAAY,EAAE,MAAM,aAAa,CAAC;AACvD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAExD,qBAAa,gBAAgB;IACzB,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,eAAe,CAAkB;IACzC,OAAO,CAAC,SAAS,CAAS;gBAEd,eAAe,EAAE,eAAe,EAAE,YAAY,EAAE,YAAY;IAMxE;;;OAGG;IACG,UAAU,CAAC,KAAK,EAAE,WAAW,EAAE,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAWzE,yCAAyC;IACnC,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;YAMjC,SAAS;CAmC1B"}

package/dist/indexing/pipeline.js

@@ -0,0 +1,68 @@
+// IndexingPipeline — source-agnostic chunk → embed → upsert logic.
+import { getChunker } from './chunking/index.js';
+import { deriveUrl } from './url-derivation.js';
+import { upsertChunks, deleteChunksByFile } from '../db/queries.js';
+import { isFileSourceConfig } from '../types.js';
+export class IndexingPipeline {
+    sourceConfig;
+    embeddingClient;
+    logPrefix;
+    constructor(embeddingClient, sourceConfig) {
+        this.embeddingClient = embeddingClient;
+        this.sourceConfig = sourceConfig;
+        this.logPrefix = `[pipeline:${sourceConfig.name}]`;
+    }
+    /**
+     * Index a batch of content items: chunk → embed → upsert.
+     * Each item's existing chunks are deleted first to handle shrinkage.
+     */
+    async indexItems(items, stateToken) {
+        for (const item of items) {
+            try {
+                await this.indexItem(item, stateToken);
+            }
+            catch (err) {
+                const msg = err instanceof Error ? err.message : String(err);
+                console.error(`${this.logPrefix} Failed to index ${item.id}: ${msg}`);
+            }
+        }
+    }
+    /** Remove items from the index by ID. */
+    async removeItems(ids) {
+        for (const id of ids) {
+            await deleteChunksByFile(this.sourceConfig.name, id);
+        }
+    }
+    async indexItem(item, stateToken) {
+        const chunker = getChunker(this.sourceConfig.type);
+        const chunkOutputs = chunker(item.content, item.id, this.sourceConfig);
+        if (chunkOutputs.length === 0) {
+            return;
+        }
+        const texts = chunkOutputs.map(c => c.content);
+        const embeddings = await this.embeddingClient.embedBatch(texts);
+        const sourceUrl = item.sourceUrl ?? (isFileSourceConfig(this.sourceConfig) ? deriveUrl(item.id, this.sourceConfig) : null);
+        const chunks = chunkOutputs.map((chunk, i) => ({
+            source_name: this.sourceConfig.name,
+            source_url: sourceUrl,
+            title: chunk.title ?? item.title ?? null,
+            content: chunk.content,
+            embedding: embeddings[i],
+            repo_url: isFileSourceConfig(this.sourceConfig) ? (this.sourceConfig.repo ?? null) : null,
+            file_path: item.id,
+            start_line: chunk.startLine ?? null,
+            end_line: chunk.endLine ?? null,
+            language: chunk.language ?? null,
+            chunk_index: chunk.chunkIndex,
+            metadata: {
+                ...(chunk.headingPath ? { headingPath: chunk.headingPath } : {}),
+                ...(item.metadata ?? {}),
+            },
+            commit_sha: stateToken,
+            version: this.sourceConfig.version ?? null,
+        }));
+        await deleteChunksByFile(this.sourceConfig.name, item.id);
+        await upsertChunks(chunks);
+    }
+}
+//# sourceMappingURL=pipeline.js.map

package/dist/indexing/pipeline.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pipeline.js","sourceRoot":"","sources":["../../src/indexing/pipeline.ts"],"names":[],"mappings":"AAAA,mEAAmE;AAEnE,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACjD,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAEhD,OAAO,EAAE,YAAY,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACpE,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAIjD,MAAM,OAAO,gBAAgB;IACjB,YAAY,CAAe;IAC3B,eAAe,CAAkB;IACjC,SAAS,CAAS;IAE1B,YAAY,eAAgC,EAAE,YAA0B;QACpE,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,SAAS,GAAG,aAAa,YAAY,CAAC,IAAI,GAAG,CAAC;IACvD,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,UAAU,CAAC,KAAoB,EAAE,UAAkB;QACrD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACvB,IAAI,CAAC;gBACD,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;YAC3C,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC7D,OAAO,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,oBAAoB,IAAI,CAAC,EAAE,KAAK,GAAG,EAAE,CAAC,CAAC;YAC1E,CAAC;QACL,CAAC;IACL,CAAC;IAED,yCAAyC;IACzC,KAAK,CAAC,WAAW,CAAC,GAAa;QAC3B,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;YACnB,MAAM,kBAAkB,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QACzD,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,SAAS,CAAC,IAAiB,EAAE,UAAkB;QACzD,MAAM,OAAO,GAAG,UAAU,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;QACnD,MAAM,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;QAEvE,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,OAAO;QACX,CAAC;QAED,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QAC/C,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QAChE,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAE3H,MAAM,MAAM,GAAY,YAAY,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YACpD,WAAW,EAAE,IAAI,CAAC,YAAY,CAAC,IAAI;YACnC,UAAU,EAAE,SAAS;YACrB,KAAK,EAAE,KAAK,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI;YACxC,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,SAAS,EAAE,UAAU,CAAC,CAAC,CAAC;YACxB,QAAQ,EAAE,kBAAkB,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI;YACzF,SAAS,EAAE,IAAI,CAAC,EAAE;YAClB,UAAU,EAAE,KAAK,CAAC,SAAS,IAAI,IAAI;YACnC,QAAQ,EAAE,KAAK,CAAC,OAAO,IAAI,IAAI;YAC/B,QAAQ,EAAE,KAAK,CAAC,QAAQ,IAAI,IAAI;YAChC,WAAW,EAAE,KAAK,CAAC,UAAU;YAC7B,QAAQ,EAAE;gBACN,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAChE,GAAG,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC;aAC3B;YACD,UAAU,EAAE,UAAU;YACtB,OAAO,EAAE,IAAI,CAAC,YAAY,CAAC,OAAO,IAAI,IAAI;SAC7C,CAAC,CAAC,CAAC;QAEJ,MAAM,kBAAkB,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;QAC1D,MAAM,YAAY,CAAC,MAAM,CAAC,CAAC;IAC/B,CAAC;CACJ"}