@copilotkit/pathfinder 1.4.0 → 1.6.0

package/dist/faq-txt.js

@@ -0,0 +1,37 @@
+/**
+ * Generate the /faq.txt content from FAQ chunks.
+ * Groups Q&A pairs by source, with source headings.
+ */
+export function generateFaqTxt(chunks, serverName, faqSources) {
+    const lines = [
+        `# ${serverName} — Frequently Asked Questions`,
+        '',
+    ];
+    // Group chunks by source_name
+    const bySource = new Map();
+    for (const chunk of chunks) {
+        if (!bySource.has(chunk.source_name))
+            bySource.set(chunk.source_name, []);
+        bySource.get(chunk.source_name).push(chunk);
+    }
+    // Emit each source section in the order of faqSources config
+    let hasContent = false;
+    for (const source of faqSources) {
+        const sourceChunks = bySource.get(source.name);
+        if (!sourceChunks || sourceChunks.length === 0)
+            continue;
+        hasContent = true;
+        lines.push(`## ${source.name}`, '');
+        for (const chunk of sourceChunks) {
+            // Content is stored as "Q: ...\n\nA: ..."
+            lines.push(chunk.content);
+            lines.push('');
+        }
+    }
+    if (!hasContent) {
+        lines.push('No FAQ content available yet.');
+        lines.push('');
+    }
+    return lines.join('\n');
+}
+//# sourceMappingURL=faq-txt.js.map

package/dist/faq-txt.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"faq-txt.js","sourceRoot":"","sources":["../src/faq-txt.ts"],"names":[],"mappings":"AAOA;;;GAGG;AACH,MAAM,UAAU,cAAc,CAC1B,MAAwB,EACxB,UAAkB,EAClB,UAAuB;IAEvB,MAAM,KAAK,GAAa;QACpB,KAAK,UAAU,+BAA+B;QAC9C,EAAE;KACL,CAAC;IAEF,8BAA8B;IAC9B,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA4B,CAAC;IACrD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QACzB,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC;YAAE,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;QAC1E,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,CAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACjD,CAAC;IAED,6DAA6D;IAC7D,IAAI,UAAU,GAAG,KAAK,CAAC;IACvB,KAAK,MAAM,MAAM,IAAI,UAAU,EAAE,CAAC;QAC9B,MAAM,YAAY,GAAG,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAC/C,IAAI,CAAC,YAAY,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAEzD,UAAU,GAAG,IAAI,CAAC;QAClB,KAAK,CAAC,IAAI,CAAC,MAAM,MAAM,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;QAEpC,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;YAC/B,0CAA0C;YAC1C,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAC1B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACnB,CAAC;IACL,CAAC;IAED,IAAI,CAAC,UAAU,EAAE,CAAC;QACd,KAAK,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;QAC5C,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACnB,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC5B,CAAC"}

package/dist/indexing/chunking/index.js

@@ -18,4 +18,8 @@ registerChunker('markdown', chunkMarkdown);
 registerChunker('code', chunkCode);
 registerChunker('raw-text', chunkRawText);
 registerChunker('html', chunkHtml);
+import { chunkQa } from './qa.js';
+registerChunker('slack', chunkQa);
+registerChunker('discord', chunkQa);
+registerChunker('notion', chunkMarkdown);
 //# sourceMappingURL=index.js.map

package/dist/indexing/chunking/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/indexing/chunking/index.ts"],"names":[],"mappings":"AAAA,8DAA8D;AAM9D,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAqB,CAAC;AAE9C,MAAM,UAAU,eAAe,CAAC,IAAY,EAAE,EAAa;IACvD,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;AAC3B,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,IAAY;IACnC,MAAM,EAAE,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAC9B,IAAI,CAAC,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,0BAA0B,IAAI,iBAAiB,CAAC,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC3G,OAAO,EAAE,CAAC;AACd,CAAC;AAED,+BAA+B;AAC/B,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAEtC,eAAe,CAAC,UAAU,EAAE,aAAa,CAAC,CAAC;AAC3C,eAAe,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;AACnC,eAAe,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;AAC1C,eAAe,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/indexing/chunking/index.ts"],"names":[],"mappings":"AAAA,8DAA8D;AAM9D,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAqB,CAAC;AAE9C,MAAM,UAAU,eAAe,CAAC,IAAY,EAAE,EAAa;IACvD,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;AAC3B,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,IAAY;IACnC,MAAM,EAAE,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAC9B,IAAI,CAAC,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,0BAA0B,IAAI,iBAAiB,CAAC,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC3G,OAAO,EAAE,CAAC;AACd,CAAC;AAED,+BAA+B;AAC/B,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAEtC,eAAe,CAAC,UAAU,EAAE,aAAa,CAAC,CAAC;AAC3C,eAAe,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;AACnC,eAAe,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;AAC1C,eAAe,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;AAEnC,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAElC,eAAe,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;AAClC,eAAe,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;AACpC,eAAe,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC"}

package/dist/indexing/chunking/qa.d.ts

@@ -0,0 +1,8 @@
+import type { ChunkOutput, SourceConfig } from '../../types.js';
+/**
+ * Chunk Q&A content. Each content item from a FAQ-category provider
+ * is a single Q&A pair, already sized appropriately.
+ * The chunker formats it and returns a single ChunkOutput.
+ */
+export declare function chunkQa(content: string, filePath: string, config: SourceConfig): ChunkOutput[];
+//# sourceMappingURL=qa.d.ts.map

package/dist/indexing/chunking/qa.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"qa.d.ts","sourceRoot":"","sources":["../../../src/indexing/chunking/qa.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAEhE;;;;GAIG;AACH,wBAAgB,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,GAAG,WAAW,EAAE,CAe9F"}

package/dist/indexing/chunking/qa.js

@@ -0,0 +1,22 @@
+// Q&A chunker — formats distilled Q&A pairs for embedding.
+// Source-agnostic: used by any source that produces Q&A-formatted content.
+/**
+ * Chunk Q&A content. Each content item from a FAQ-category provider
+ * is a single Q&A pair, already sized appropriately.
+ * The chunker formats it and returns a single ChunkOutput.
+ */
+export function chunkQa(content, filePath, config) {
+    if (!content || !content.trim()) {
+        return [];
+    }
+    // The content is already formatted as "Q: ...\n\nA: ..." by the provider.
+    // Extract the question for use as title.
+    const questionMatch = content.match(/^Q:\s*(.+?)(?:\n|$)/);
+    const title = questionMatch ? questionMatch[1].trim() : undefined;
+    return [{
+            content: content.trim(),
+            title,
+            chunkIndex: 0,
+        }];
+}
+//# sourceMappingURL=qa.js.map

package/dist/indexing/chunking/qa.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"qa.js","sourceRoot":"","sources":["../../../src/indexing/chunking/qa.ts"],"names":[],"mappings":"AAAA,2DAA2D;AAC3D,2EAA2E;AAI3E;;;;GAIG;AACH,MAAM,UAAU,OAAO,CAAC,OAAe,EAAE,QAAgB,EAAE,MAAoB;IAC3E,IAAI,CAAC,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;QAC9B,OAAO,EAAE,CAAC;IACd,CAAC;IAED,0EAA0E;IAC1E,yCAAyC;IACzC,MAAM,aAAa,GAAG,OAAO,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;IAC3D,MAAM,KAAK,GAAG,aAAa,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;IAElE,OAAO,CAAC;YACJ,OAAO,EAAE,OAAO,CAAC,IAAI,EAAE;YACvB,KAAK;YACL,UAAU,EAAE,CAAC;SAChB,CAAC,CAAC;AACP,CAAC"}

package/dist/indexing/distiller.d.ts

@@ -0,0 +1,29 @@
+import OpenAI from 'openai';
+export interface ThreadMessage {
+    author: string;
+    content: string;
+    timestamp: string;
+    reactions?: Array<{
+        name: string;
+        count: number;
+    }>;
+}
+export interface DistilledPair {
+    question: string;
+    answer: string;
+    confidence: number;
+}
+export interface DistillerResult {
+    pairs: DistilledPair[];
+}
+export interface DistillerOptions {
+    model?: string;
+    maxMessages?: number;
+    apiKey?: string;
+    client?: OpenAI;
+}
+/**
+ * Distill a conversation thread into Q&A pairs using an LLM.
+ */
+export declare function distillThread(messages: ThreadMessage[], options?: DistillerOptions): Promise<DistillerResult>;
+//# sourceMappingURL=distiller.d.ts.map

package/dist/indexing/distiller.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"distiller.d.ts","sourceRoot":"","sources":["../../src/indexing/distiller.ts"],"names":[],"mappings":"AAGA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAI5B,MAAM,WAAW,aAAa;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACtD;AAED,MAAM,WAAW,aAAa;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,eAAe;IAC5B,KAAK,EAAE,aAAa,EAAE,CAAC;CAC1B;AAED,MAAM,WAAW,gBAAgB;IAC7B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACnB;AAuCD;;GAEG;AACH,wBAAsB,aAAa,CAC/B,QAAQ,EAAE,aAAa,EAAE,EACzB,OAAO,CAAC,EAAE,gBAAgB,GAC3B,OAAO,CAAC,eAAe,CAAC,CAyE1B"}

package/dist/indexing/distiller.js

@@ -0,0 +1,104 @@
+// LLM thread distiller — extracts Q&A pairs from conversation threads.
+// Source-agnostic: takes structured messages, returns structured Q&A pairs.
+import OpenAI from 'openai';
+// ── Constants ────────────────────────────────────────────────────────────────
+const DEFAULT_MODEL = 'gpt-4o-mini';
+const DEFAULT_MAX_MESSAGES = 100;
+const SYSTEM_PROMPT = `You are a Q&A extraction engine. Given a conversation thread, identify distinct question-answer pairs.
+
+For each pair:
+1. Extract the core question (rephrase if needed for clarity)
+2. Extract the best answer (synthesize from multiple replies if needed)
+3. Score confidence from 0.0 to 1.0 based on:
+   - Answer completeness (does it fully address the question?)
+   - Questioner satisfaction signals ("thanks", "that worked", etc.)
+   - Community validation (reactions like thumbsup, check marks)
+   - Answer specificity (concrete steps vs vague suggestions)
+
+Return JSON with this exact structure:
+{
+  "pairs": [
+    {
+      "question": "How do I configure X?",
+      "answer": "You can configure X by...",
+      "confidence": 0.85
+    }
+  ]
+}
+
+Rules:
+- A thread may contain multiple Q&A pairs (follow-up questions)
+- Skip greetings, pleasantries, and off-topic tangents
+- If no clear Q&A exists, return {"pairs": []}
+- Keep answers concise but complete (aim for 1-3 paragraphs)
+- Preserve code blocks, URLs, and technical details from answers
+- Confidence below 0.3 means the answer is likely incomplete or wrong`;
+// ── Distiller ────────────────────────────────────────────────────────────────
+/**
+ * Distill a conversation thread into Q&A pairs using an LLM.
+ */
+export async function distillThread(messages, options) {
+    const model = options?.model ?? DEFAULT_MODEL;
+    const maxMessages = options?.maxMessages ?? DEFAULT_MAX_MESSAGES;
+    if (messages.length === 0) {
+        return { pairs: [] };
+    }
+    // Truncate to max messages
+    const truncated = messages.slice(0, maxMessages);
+    // Format as conversation transcript
+    const transcript = truncated.map(msg => {
+        const reactions = msg.reactions && msg.reactions.length > 0
+            ? ` [reactions: ${msg.reactions.map(r => `:${r.name}: x${r.count}`).join(', ')}]`
+            : '';
+        return `[${msg.timestamp}] ${msg.author}: ${msg.content}${reactions}`;
+    }).join('\n\n');
+    const client = options?.client ?? new OpenAI({ apiKey: options?.apiKey });
+    try {
+        const response = await client.chat.completions.create({
+            model,
+            messages: [
+                { role: 'system', content: SYSTEM_PROMPT },
+                { role: 'user', content: transcript },
+            ],
+            response_format: { type: 'json_object' },
+            temperature: 0.1,
+        });
+        const content = response.choices[0]?.message?.content;
+        if (!content) {
+            console.warn('[distiller] Empty response from LLM');
+            return { pairs: [] };
+        }
+        const parsed = JSON.parse(content);
+        // Validate structure
+        if (!Array.isArray(parsed.pairs)) {
+            console.warn('[distiller] Invalid response structure — missing pairs array');
+            return { pairs: [] };
+        }
+        // Validate and filter each pair
+        const validPairs = [];
+        for (const pair of parsed.pairs) {
+            if (typeof pair.question === 'string' && pair.question.trim() &&
+                typeof pair.answer === 'string' && pair.answer.trim() &&
+                typeof pair.confidence === 'number' &&
+                pair.confidence >= 0 && pair.confidence <= 1) {
+                validPairs.push({
+                    question: pair.question.trim(),
+                    answer: pair.answer.trim(),
+                    confidence: pair.confidence,
+                });
+            }
+            else {
+                console.warn('[distiller] Skipping malformed pair:', JSON.stringify(pair).slice(0, 200));
+            }
+        }
+        return { pairs: validPairs };
+    }
+    catch (error) {
+        if (error instanceof SyntaxError) {
+            console.error('[distiller] Failed to parse LLM JSON response:', error.message);
+            return { pairs: [] };
+        }
+        throw error; // Re-throw API errors for caller to handle
+    }
+}
+//# sourceMappingURL=distiller.js.map

package/dist/indexing/distiller.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"distiller.js","sourceRoot":"","sources":["../../src/indexing/distiller.ts"],"names":[],"mappings":"AAAA,uEAAuE;AACvE,4EAA4E;AAE5E,OAAO,MAAM,MAAM,QAAQ,CAAC;AA4B5B,gFAAgF;AAEhF,MAAM,aAAa,GAAG,aAAa,CAAC;AACpC,MAAM,oBAAoB,GAAG,GAAG,CAAC;AAEjC,MAAM,aAAa,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;sEA4BgD,CAAC;AAEvE,gFAAgF;AAEhF;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAC/B,QAAyB,EACzB,OAA0B;IAE1B,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,IAAI,aAAa,CAAC;IAC9C,MAAM,WAAW,GAAG,OAAO,EAAE,WAAW,IAAI,oBAAoB,CAAC;IAEjE,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;IACzB,CAAC;IAED,2BAA2B;IAC3B,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC;IAEjD,oCAAoC;IACpC,MAAM,UAAU,GAAG,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;QACnC,MAAM,SAAS,GAAG,GAAG,CAAC,SAAS,IAAI,GAAG,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC;YACvD,CAAC,CAAC,gBAAgB,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG;YACjF,CAAC,CAAC,EAAE,CAAC;QACT,OAAO,IAAI,GAAG,CAAC,SAAS,KAAK,GAAG,CAAC,MAAM,KAAK,GAAG,CAAC,OAAO,GAAG,SAAS,EAAE,CAAC;IAC1E,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAEhB,MAAM,MAAM,GAAG,OAAO,EAAE,MAAM,IAAI,IAAI,MAAM,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;IAE1E,IAAI,CAAC;QACD,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC;YAClD,KAAK;YACL,QAAQ,EAAE;gBACN,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,aAAa,EAAE;gBAC1C,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE;aACxC;YACD,eAAe,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE;YACxC,WAAW,EAAE,GAAG;SACnB,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC;QACtD,IAAI,CAAC,OAAO,EAAE,CAAC;YACX,OAAO,CAAC,IAAI,CAAC,qCAAqC,CAAC,CAAC;YACpD,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;QACzB,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAEnC,qBAAqB;QACrB,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;YAC/B,OAAO,CAAC,IAAI,CAAC,8DAA8D,CAAC,CAAC;YAC7E,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;QACzB,CAAC;QAED,gCAAgC;QAChC,MAAM,UAAU,GAAoB,EAAE,CAAC;QACvC,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YAC9B,IACI,OAAO,IAAI,CAAC,QAAQ,KAAK,QAAQ,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE;gBACzD,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE;gBACrD,OAAO,IAAI,CAAC,UAAU,KAAK,QAAQ;gBACnC,IAAI,CAAC,UAAU,IAAI,CAAC,IAAI,IAAI,CAAC,UAAU,IAAI,CAAC,EAC9C,CAAC;gBACC,UAAU,CAAC,IAAI,CAAC;oBACZ,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE;oBAC9B,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE;oBAC1B,UAAU,EAAE,IAAI,CAAC,UAAU;iBAC9B,CAAC,CAAC;YACP,CAAC;iBAAM,CAAC;gBACJ,OAAO,CAAC,IAAI,CAAC,sCAAsC,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;YAC7F,CAAC;QACL,CAAC;QAED,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC;IACjC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACb,IAAI,KAAK,YAAY,WAAW,EAAE,CAAC;YAC/B,OAAO,CAAC,KAAK,CAAC,gDAAgD,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;YAC/E,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;QACzB,CAAC;QACD,MAAM,KAAK,CAAC,CAAC,2CAA2C;IAC5D,CAAC;AACL,CAAC"}

package/dist/indexing/orchestrator.d.ts

@@ -13,10 +13,10 @@ export declare class IndexingOrchestrator {
      */
     checkAndIndex(): Promise<void>;
     /**
-     * Get the HEAD SHA of a remote repo without cloning.
-     * Uses `git ls-remote` which only fetches refs.
+     * Get the current state token for a source without acquiring items.
+     * Returns null if the source is unavailable.
      */
-    private getRemoteHead;
+    private getSourceStateToken;
     /**
      * Queue a full re-index of all sources. Returns immediately.
      */
@@ -25,6 +25,11 @@ export declare class IndexingOrchestrator {
      * Queue an incremental re-index for a specific repo. Returns immediately.
      */
     queueIncrementalReindex(repoUrl: string): void;
+    /**
+     * Queue a reindex for a single named source. Returns immediately.
+     * Used by webhook handlers to trigger reindexing of specific sources.
+     */
+    queueSourceReindex(sourceName: string): void;
     /**
      * Returns true if any indexing job is currently running.
      */

package/dist/indexing/orchestrator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"orchestrator.d.ts","sourceRoot":"","sources":["../../src/indexing/orchestrator.ts"],"names":[],"mappings":"AAiCA,qBAAa,oBAAoB;IAC7B,OAAO,CAAC,KAAK,CAAa;IAC1B,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,UAAU,CAAS;IAG3B,OAAO,CAAC,aAAa,CAAqB;IAG1C,OAAO,CAAC,eAAe,CAAuB;IAG9C,iBAAiB,CAAC,EAAE,CAAC,WAAW,EAAE,MAAM,EAAE,KAAK,IAAI,CAAC;;IAMpD;;;;OAIG;IACG,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC;IA4GpC;;;OAGG;YACW,aAAa;IAa3B;;OAEG;IACH,gBAAgB,IAAI,IAAI;IAQxB;;OAEG;IACH,uBAAuB,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI;IAU9C;;OAEG;IACH,UAAU,IAAI,OAAO;IAIrB;;;OAGG;IACH,mBAAmB,IAAI,IAAI;IA4B3B;;OAEG;IACH,OAAO,CAAC,OAAO;IASf;;OAEG;YACW,KAAK;IAwBnB;;OAEG;YACW,UAAU;IAkDxB;;OAEG;YACW,cAAc;IAqB5B;;OAEG;YACW,qBAAqB;IAoDnC;;OAEG;YACW,oBAAoB;IAgDlC;;OAEG;YACW,cAAc;IAmB5B;;OAEG;YACW,cAAc;CAgB/B"}
+{"version":3,"file":"orchestrator.d.ts","sourceRoot":"","sources":["../../src/indexing/orchestrator.ts"],"names":[],"mappings":"AAqCA,qBAAa,oBAAoB;IAC7B,OAAO,CAAC,KAAK,CAAa;IAC1B,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,UAAU,CAAS;IAG3B,OAAO,CAAC,aAAa,CAAqB;IAG1C,OAAO,CAAC,eAAe,CAAuB;IAG9C,iBAAiB,CAAC,EAAE,CAAC,WAAW,EAAE,MAAM,EAAE,KAAK,IAAI,CAAC;;IAMpD;;;;OAIG;IACG,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC;IA+GpC;;;OAGG;YACW,mBAAmB;IAajC;;OAEG;IACH,gBAAgB,IAAI,IAAI;IAQxB;;OAEG;IACH,uBAAuB,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI;IAU9C;;;OAGG;IACH,kBAAkB,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IAQ5C;;OAEG;IACH,UAAU,IAAI,OAAO;IAIrB;;;OAGG;IACH,mBAAmB,IAAI,IAAI;IA4B3B;;OAEG;IACH,OAAO,CAAC,OAAO;IASf;;OAEG;YACW,KAAK;IAwBnB;;OAEG;YACW,UAAU;IA8DxB;;OAEG;YACW,cAAc;IAqB5B;;OAEG;YACW,qBAAqB;IAiBnC;;OAEG;YACW,oBAAoB;IAsDlC;;OAEG;YACW,cAAc;IAmB5B;;OAEG;YACW,cAAc;CAgB/B"}

package/dist/indexing/orchestrator.js

@@ -1,17 +1,18 @@
 // Job queue and coordination for indexing pipelines.
 // Fully config-driven: indexes sources referenced by search tools in mcp-docs.yaml.
-import fs from 'fs';
-import path from 'path';
-import { simpleGit } from 'simple-git';
+import fs from 'node:fs';
+import path from 'node:path';
 import { getConfig, getServerConfig, getIndexableSourceNames } from '../config.js';
 import { EmbeddingClient } from './embeddings.js';
-import { SourceIndexer } from './source-indexer.js';
+import { getProvider } from './providers/index.js';
+import { IndexingPipeline } from './pipeline.js';
 import { getIndexState, upsertIndexState, } from '../db/queries.js';
+import { isFileSourceConfig } from '../types.js';
 /**
  * Find all source configs that reference a given repo URL.
  */
 function getSourcesByRepo(repoUrl) {
-    return getServerConfig().sources.filter(s => s.repo === repoUrl);
+    return getServerConfig().sources.filter(s => isFileSourceConfig(s) && s.repo === repoUrl);
 }
 function getStaleThresholdMs() {
     const serverCfg = getServerConfig();
@@ -67,89 +68,94 @@ export class IndexingOrchestrator {
                 return;
             }
             // Queue incremental reindexes for each affected git-backed repo
-            const reposToReindex = new Set(sourcesNeedingFullReindex.filter(s => s.repo).map(s => s.repo));
+            const reposToReindex = new Set();
+            for (const s of sourcesNeedingFullReindex) {
+                if (isFileSourceConfig(s) && s.repo)
+                    reposToReindex.add(s.repo);
+            }
             for (const repoUrl of reposToReindex) {
                 this.queueIncrementalReindex(repoUrl);
             }
             // Local sources (no repo) get queued as a full reindex of just those sources
-            const localSources = sourcesNeedingFullReindex.filter(s => !s.repo);
+            const localSources = sourcesNeedingFullReindex.filter(s => isFileSourceConfig(s) && !s.repo);
             if (localSources.length > 0) {
                 this.queue.push({ type: 'full-reindex-local', sources: localSources });
                 this.drain().catch(err => console.error('[orchestrator] drain() failed:', err));
             }
+            // Non-file sources (e.g., Slack) that need reindexing
+            const nonFileSources = sourcesNeedingFullReindex.filter(s => !isFileSourceConfig(s));
+            for (const source of nonFileSources) {
+                this.queueSourceReindex(source.name);
+            }
         }
         if (sourcesOk.length === 0)
             return;
         // Local sources in sourcesOk have no remote to check — always reindex on startup
-        const localSourcesOk = sourcesOk.filter(s => !s.repo);
+        const localSourcesOk = sourcesOk.filter(s => isFileSourceConfig(s) && !s.repo);
         if (localSourcesOk.length > 0) {
             console.log(`[orchestrator] Queuing reindex for ${localSourcesOk.length} local source(s)`);
             this.queue.push({ type: 'full-reindex-local', sources: localSourcesOk });
             this.drain().catch(err => console.error('[orchestrator] drain() failed:', err));
         }
         console.log('[orchestrator] Checking remotes for changes on indexed sources...');
-        // Only check remotes for git-backed sources
-        const repos = [...new Set(sourcesOk.filter(s => s.repo).map(s => s.repo))];
-        for (const repoUrl of repos) {
+        // Check each git-backed source for changes
+        const gitSourcesOk = sourcesOk.filter(s => isFileSourceConfig(s) && s.repo);
+        for (const source of gitSourcesOk) {
             try {
-                const remoteHead = await this.getRemoteHead(repoUrl);
-                const sources = getSourcesByRepo(repoUrl).filter(s => indexableNames.has(s.name));
-                let anyChanged = false;
-                for (const source of sources) {
-                    const state = await getIndexState(source.type, source.name);
-                    if (state?.last_commit_sha !== remoteHead) {
-                        anyChanged = true;
-                        break;
+                const currentToken = await this.getSourceStateToken(source);
+                const state = await getIndexState(source.type, source.name);
+                if (currentToken === null || state?.last_commit_sha !== currentToken) {
+                    const reason = currentToken === null
+                        ? 'source unavailable (clone missing?)'
+                        : `remote ${currentToken.slice(0, 8)} differs from indexed`;
+                    console.log(`[orchestrator] ${reason} for ${source.name} — queuing reindex`);
+                    if (isFileSourceConfig(source) && source.repo) {
+                        this.queueIncrementalReindex(source.repo);
                     }
                 }
-                // Even if DB says current, verify clone dir exists (fresh container = empty /tmp)
-                const repoName = repoUrl.split('/').pop()?.replace(/\.git$/, '') ?? '';
-                const cloneDir = getConfig().cloneDir;
-                const repoDir = path.join(cloneDir, repoName);
-                const cloneMissing = !fs.existsSync(repoDir);
-                if (anyChanged || cloneMissing) {
-                    const reason = cloneMissing
-                        ? `clone dir missing at ${repoDir}`
-                        : `remote HEAD ${remoteHead.slice(0, 8)} differs from indexed`;
-                    console.log(`[orchestrator] ${reason} for ${repoUrl} — queuing incremental reindex`);
-                    this.queueIncrementalReindex(repoUrl);
-                }
                 else {
-                    console.log(`[orchestrator] Index current at ${remoteHead.slice(0, 8)}`);
+                    console.log(`[orchestrator] ${source.name} index current at ${currentToken.slice(0, 8)}`);
                 }
             }
             catch (err) {
-                // If we can't check remote, fall back to age-based staleness
-                console.warn(`[orchestrator] Failed to check remote HEAD for ${repoUrl}, falling back to age check:`, err);
-                const repoSources = getSourcesByRepo(repoUrl);
-                const firstState = await getIndexState(repoSources[0].type, repoSources[0].name);
-                if (this.isStale(firstState)) {
-                    const thresholdHours = getServerConfig().indexing?.stale_threshold_hours ?? 24;
-                    console.log(`[orchestrator] Index for ${repoUrl} is stale (>${thresholdHours}h) — queuing full reindex`);
+                console.warn(`[orchestrator] Failed to check state for ${source.name}, falling back to age check:`, err);
+                const state = await getIndexState(source.type, source.name);
+                if (this.isStale(state)) {
+                    console.log(`[orchestrator] Index for ${source.name} is stale — queuing full reindex`);
                     this.queueFullReindex();
                 }
-                else {
-                    console.log(`[orchestrator] Index for ${repoUrl} appears fresh, skipping`);
-                }
+            }
+        }
+        // Ensure git repos are cloned even when index is current.
+        // On fresh deploys, the container has no local clones but the DB may have valid state.
+        // Bash tools need the clone directories to build their filesystem.
+        const cloneDir = getConfig().cloneDir;
+        for (const source of gitSourcesOk) {
+            if (!isFileSourceConfig(source) || !source.repo)
+                continue;
+            const repoName = source.repo.replace(/\.git$/, '').split('/').pop();
+            const repoDir = path.join(cloneDir, repoName);
+            if (!fs.existsSync(repoDir)) {
+                console.log(`[orchestrator] Clone directory missing for ${source.name}, queuing reindex to populate`);
+                this.queueIncrementalReindex(source.repo);
             }
         }
     }
     /**
-     * Get the HEAD SHA of a remote repo without cloning.
-     * Uses `git ls-remote` which only fetches refs.
+     * Get the current state token for a source without acquiring items.
+     * Returns null if the source is unavailable.
      */
-    async getRemoteHead(repoUrl) {
+    async getSourceStateToken(source) {
         const config = getConfig();
-        let url = repoUrl;
-        if (config.githubToken) {
-            url = repoUrl.replace('https://github.com/', `https://x-access-token:${config.githubToken}@github.com/`);
-        }
-        const git = simpleGit();
-        const result = await git.listRemote([url, 'HEAD']);
-        const sha = result.split('\t')[0]?.trim();
-        if (!sha)
-            throw new Error(`Could not resolve HEAD for ${repoUrl}`);
-        return sha;
+        const providerOptions = {
+            cloneDir: config.cloneDir,
+            githubToken: config.githubToken,
+            slackBotToken: config.slackBotToken,
+            discordBotToken: config.discordBotToken,
+            notionToken: config.notionToken,
+        };
+        const provider = getProvider(source.type)(source, providerOptions);
+        return provider.getCurrentStateToken();
     }
     /**
      * Queue a full re-index of all sources. Returns immediately.
@@ -171,6 +177,17 @@ export class IndexingOrchestrator {
             console.error('[orchestrator] drain() failed:', err);
         });
     }
+    /**
+     * Queue a reindex for a single named source. Returns immediately.
+     * Used by webhook handlers to trigger reindexing of specific sources.
+     */
+    queueSourceReindex(sourceName) {
+        this.queue.push({ type: 'source-reindex', sourceName });
+        console.log(`[orchestrator] Source re-index queued for ${sourceName}`);
+        this.drain().catch((err) => {
+            console.error('[orchestrator] drain() failed:', err);
+        });
+    }
     /**
      * Returns true if any indexing job is currently running.
      */
@@ -276,6 +293,19 @@ export class IndexingOrchestrator {
             await this.runIncrementalReindex(embeddingClient, config.cloneDir, config.githubToken, job.repoUrl);
             affectedSourceNames = getSourcesByRepo(job.repoUrl).map(s => s.name);
         }
+        else if (job.type === 'source-reindex') {
+            if (!job.sourceName) {
+                console.warn('[orchestrator] source-reindex job has no sourceName, skipping');
+                return;
+            }
+            const sourceConfig = serverCfg2.sources.find(s => s.name === job.sourceName);
+            if (!sourceConfig) {
+                console.warn(`[orchestrator] source-reindex: source "${job.sourceName}" not found in config`);
+                return;
+            }
+            await this.indexSourceWithState(sourceConfig, embeddingClient, config.cloneDir);
+            affectedSourceNames = [job.sourceName];
+        }
         if (affectedSourceNames.length > 0 && this.onReindexComplete) {
             try {
                 this.onReindexComplete(affectedSourceNames);
@@ -305,38 +335,7 @@ export class IndexingOrchestrator {
         const indexableNames = getIndexableSourceNames();
         const sources = getSourcesByRepo(repoUrl).filter(s => indexableNames.has(s.name));
         for (const sourceConfig of sources) {
-            const state = await getIndexState(sourceConfig.type, sourceConfig.name);
-            if (state?.last_commit_sha) {
-                await this.withSourceLock(`${sourceConfig.type}:${sourceConfig.name}`, async () => {
-                    await this.setIndexStatus(sourceConfig.type, sourceConfig.name, 'indexing');
-                    try {
-                        const indexer = new SourceIndexer(sourceConfig, embeddingClient, cloneDir, githubToken);
-                        await indexer.incrementalIndex(state.last_commit_sha);
-                        const headSha = await indexer.getHeadSha();
-                        await upsertIndexState({
-                            source_type: sourceConfig.type,
-                            source_key: sourceConfig.name,
-                            last_commit_sha: headSha,
-                            last_indexed_at: new Date(),
-                            status: 'idle',
-                        });
-                    }
-                    catch (err) {
-                        console.error(`[orchestrator] Incremental reindex failed for ${sourceConfig.name}:`, err);
-                        try {
-                            await this.setIndexStatus(sourceConfig.type, sourceConfig.name, 'error', err instanceof Error ? err.message : String(err));
-                        }
-                        catch (statusErr) {
-                            console.error('[orchestrator] Failed to update index status:', statusErr);
-                        }
-                        // Don't rethrow — continue with remaining sources
-                    }
-                });
-            }
-            else {
-                // No previous state — do a full index for this source
-                await this.indexSourceWithState(sourceConfig, embeddingClient, cloneDir, githubToken);
-            }
+            await this.indexSourceWithState(sourceConfig, embeddingClient, cloneDir, githubToken);
         }
         console.log(`[orchestrator] Incremental re-index complete for ${repoUrl}`);
     }
@@ -346,21 +345,29 @@ export class IndexingOrchestrator {
     async indexSourceWithState(sourceConfig, embeddingClient, cloneDir, githubToken) {
         const lockKey = `${sourceConfig.type}:${sourceConfig.name}`;
         await this.withSourceLock(lockKey, async () => {
-            const indexer = new SourceIndexer(sourceConfig, embeddingClient, cloneDir, githubToken);
+            const providerOptions = { cloneDir, githubToken, slackBotToken: getConfig().slackBotToken, discordBotToken: getConfig().discordBotToken, notionToken: getConfig().notionToken };
+            const provider = getProvider(sourceConfig.type)(sourceConfig, providerOptions);
+            const pipeline = new IndexingPipeline(embeddingClient, sourceConfig);
             await this.setIndexStatus(sourceConfig.type, sourceConfig.name, 'indexing');
             try {
                 const state = await getIndexState(sourceConfig.type, sourceConfig.name);
+                let result;
                 if (state?.last_commit_sha) {
-                    await indexer.incrementalIndex(state.last_commit_sha);
+                    result = await provider.incrementalAcquire(state.last_commit_sha);
                 }
                 else {
-                    await indexer.fullIndex();
+                    result = await provider.fullAcquire();
+                }
+                if (result.removedIds.length > 0) {
+                    await pipeline.removeItems(result.removedIds);
+                }
+                if (result.items.length > 0) {
+                    await pipeline.indexItems(result.items, result.stateToken);
                 }
-                const headSha = await indexer.getHeadSha();
                 await upsertIndexState({
                     source_type: sourceConfig.type,
                     source_key: sourceConfig.name,
-                    last_commit_sha: headSha,
+                    last_commit_sha: result.stateToken,
                     last_indexed_at: new Date(),
                     status: 'idle',
                 });
@@ -374,7 +381,6 @@ export class IndexingOrchestrator {
                 catch (statusErr) {
                     console.error('[orchestrator] Failed to update index status:', statusErr);
                 }
-                // Don't rethrow — continue with remaining sources
             }
         });
     }