@copilotkit/pathfinder 1.1.0 → 1.5.0

package/dist/indexing/providers/discord-api.js

@@ -0,0 +1,167 @@
+// Discord REST API client wrapper — handles pagination, rate limiting, user caching.
+// Uses @discordjs/rest (REST-only, no gateway/WebSocket).
+import { REST } from '@discordjs/rest';
+import { Routes } from 'discord-api-types/v10';
+// ── Rate limit helpers ───────────────────────────────────────────────────────
+function sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+}
+// ── Client ───────────────────────────────────────────────────────────────────
+const MAX_PAGES = 100;
+const DEFAULT_PAGE_LIMIT = 100;
+export class DiscordApiClient {
+    rest;
+    maxRetries;
+    userCache = new Map();
+    logPrefix = '[discord-api]';
+    constructor(options) {
+        this.rest = new REST({ version: '10', rejectOnRateLimit: ['/'] }).setToken(options.token);
+        this.maxRetries = options.maxRetries ?? 5;
+    }
+    /**
+     * Fetch messages from a text channel, paginated using `after` parameter.
+     * If `after` is provided, only messages after that snowflake ID are returned.
+     */
+    async fetchChannelMessages(channelId, after, limit) {
+        const messages = [];
+        let lastId = after;
+        const pageLimit = limit ?? DEFAULT_PAGE_LIMIT;
+        let pages = 0;
+        do {
+            pages++;
+            const query = new URLSearchParams({ limit: String(pageLimit) });
+            if (lastId)
+                query.set('after', lastId);
+            const result = await this.callWithRetry(() => this.rest.get(Routes.channelMessages(channelId), { query }));
+            if (!result || result.length === 0)
+                break;
+            messages.push(...result);
+            lastId = result[result.length - 1].id;
+            // If we got fewer than the page limit, there are no more pages
+            if (result.length < pageLimit)
+                break;
+            if (pages >= MAX_PAGES) {
+                console.warn(`${this.logPrefix} fetchChannelMessages hit max pages (${MAX_PAGES}) for channel ${channelId}`);
+                break;
+            }
+        } while (true);
+        return messages;
+    }
+    /**
+     * Fetch all threads in a forum channel (active + archived).
+     * Active threads: GET /guilds/{guildId}/threads/active, filtered by parent_id.
+     * Archived threads: paginated via before parameter.
+     */
+    async fetchForumThreads(channelId, guildId) {
+        const threadMap = new Map();
+        // 1. Active threads (guild-level, filter by parent_id)
+        try {
+            const activeResult = await this.callWithRetry(() => this.rest.get(Routes.guildActiveThreads(guildId)));
+            for (const thread of activeResult.threads ?? []) {
+                if (thread.parent_id === channelId) {
+                    threadMap.set(thread.id, thread);
+                }
+            }
+        }
+        catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            console.warn(`${this.logPrefix} Failed to fetch active threads for guild ${guildId}: ${msg}`);
+        }
+        // 2. Public archived threads
+        await this.fetchArchivedThreads(channelId, 'public', threadMap);
+        // 3. Private archived threads
+        await this.fetchArchivedThreads(channelId, 'private', threadMap);
+        return [...threadMap.values()];
+    }
+    /**
+     * Fetch messages within a thread.
+     */
+    async fetchThreadMessages(threadId, limit) {
+        return this.fetchChannelMessages(threadId, undefined, limit);
+    }
+    /**
+     * Fetch user info with caching.
+     */
+    async fetchUser(userId) {
+        const cached = this.userCache.get(userId);
+        if (cached)
+            return cached;
+        const result = await this.callWithRetry(() => this.rest.get(Routes.user(userId)));
+        const user = {
+            id: result.id,
+            displayName: result.global_name ?? result.username,
+        };
+        this.userCache.set(userId, user);
+        return user;
+    }
+    /**
+     * Get a message URL (deterministic, no API call).
+     */
+    getMessageUrl(guildId, channelId, messageId) {
+        return `https://discord.com/channels/${guildId}/${channelId}/${messageId}`;
+    }
+    // -----------------------------------------------------------------------
+    // Private helpers
+    // -----------------------------------------------------------------------
+    /**
+     * Fetch archived threads (public or private) with pagination.
+     */
+    async fetchArchivedThreads(channelId, type, threadMap) {
+        let before;
+        let pages = 0;
+        do {
+            pages++;
+            const query = new URLSearchParams();
+            if (before)
+                query.set('before', before);
+            try {
+                const result = await this.callWithRetry(() => this.rest.get(`/channels/${channelId}/threads/archived/${type}`, { query }));
+                for (const thread of result.threads ?? []) {
+                    if (!threadMap.has(thread.id)) {
+                        threadMap.set(thread.id, thread);
+                    }
+                }
+                if (!result.has_more || !result.threads?.length)
+                    break;
+                // Use the last thread's ID as the before cursor
+                before = result.threads[result.threads.length - 1].id;
+            }
+            catch (err) {
+                const msg = err instanceof Error ? err.message : String(err);
+                console.warn(`${this.logPrefix} Failed to fetch ${type} archived threads for channel ${channelId}: ${msg}`);
+                break;
+            }
+            if (pages >= MAX_PAGES) {
+                console.warn(`${this.logPrefix} fetchArchivedThreads hit max pages (${MAX_PAGES})`);
+                break;
+            }
+        } while (true);
+    }
+    /**
+     * Call a Discord API method with retry on rate limit (429) responses.
+     * Respects retry_after with exponential backoff fallback.
+     */
+    async callWithRetry(fn, attempt = 1) {
+        try {
+            return await fn();
+        }
+        catch (error) {
+            const status = error?.status ?? error?.statusCode;
+            const isRateLimited = status === 429;
+            if (!isRateLimited || attempt > this.maxRetries) {
+                throw error;
+            }
+            const retryAfter = error?.retryAfter
+                ?? error?.retry_after;
+            // retryAfter from @discordjs/rest is in ms; fallback to exponential backoff in ms
+            const delayMs = retryAfter != null
+                ? Math.max(retryAfter, 1000)
+                : Math.max(Math.pow(2, attempt) * 1000, 1000);
+            console.warn(`${this.logPrefix} Rate limited (attempt ${attempt}/${this.maxRetries}), ` +
+                `retrying in ${delayMs}ms`);
+            await sleep(delayMs);
+            return this.callWithRetry(fn, attempt + 1);
+        }
+    }
+}
+//# sourceMappingURL=discord-api.js.map

package/dist/indexing/providers/discord-api.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"discord-api.js","sourceRoot":"","sources":["../../../src/indexing/providers/discord-api.ts"],"names":[],"mappings":"AAAA,qFAAqF;AACrF,0DAA0D;AAE1D,OAAO,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AACvC,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAmC/C,gFAAgF;AAEhF,SAAS,KAAK,CAAC,EAAU;IACrB,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED,gFAAgF;AAEhF,MAAM,SAAS,GAAG,GAAG,CAAC;AACtB,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAE/B,MAAM,OAAO,gBAAgB;IACjB,IAAI,CAAO;IACX,UAAU,CAAS;IACnB,SAAS,GAAG,IAAI,GAAG,EAAuB,CAAC;IAC3C,SAAS,GAAG,eAAe,CAAC;IAEpC,YAAY,OAAgC;QACxC,IAAI,CAAC,IAAI,GAAG,IAAI,IAAI,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,iBAAiB,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAC1F,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,CAAC,CAAC;IAC9C,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,oBAAoB,CACtB,SAAiB,EACjB,KAAc,EACd,KAAc;QAEd,MAAM,QAAQ,GAAqB,EAAE,CAAC;QACtC,IAAI,MAAM,GAAG,KAAK,CAAC;QACnB,MAAM,SAAS,GAAG,KAAK,IAAI,kBAAkB,CAAC;QAC9C,IAAI,KAAK,GAAG,CAAC,CAAC;QAEd,GAAG,CAAC;YACA,KAAK,EAAE,CAAC;YACR,MAAM,KAAK,GAAG,IAAI,eAAe,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;YAChE,IAAI,MAAM;gBAAE,KAAK,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YAEvC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,CACzC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,eAAe,CAAC,SAAS,CAAC,EAAE,EAAE,KAAK,EAAE,CAA8B,CAC3F,CAAC;YAEF,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;gBAAE,MAAM;YAE1C,QAAQ,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC;YACzB,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;YAEtC,+DAA+D;YAC/D,IAAI,MAAM,CAAC,MAAM,GAAG,SAAS;gBAAE,MAAM;YAErC,IAAI,KAAK,IAAI,SAAS,EAAE,CAAC;gBACrB,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,SAAS,wCAAwC,SAAS,iBAAiB,SAAS,EAAE,CAAC,CAAC;gBAC7G,MAAM;YACV,CAAC;QACL,CAAC,QAAQ,IAAI,EAAE;QAEf,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,iBAAiB,CAAC,SAAiB,EAAE,OAAe;QACtD,MAAM,SAAS,GAAG,IAAI,GAAG,EAAyB,CAAC;QAEnD,uDAAuD;QACvD,IAAI,CAAC;YACD,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,CAC/C,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAA0C,CAC7F,CAAC;YACF,KAAK,MAAM,MAAM,IAAI,YAAY,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC;gBAC9C,IAAI,MAAM,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;oBACjC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;gBACrC,CAAC;YACL,CAAC;QACL,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC7D,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,SAAS,6CAA6C,OAAO,KAAK,GAAG,EAAE,CAAC,CAAC;QAClG,CAAC;QAED,6BAA6B;QAC7B,MAAM,IAAI,CAAC,oBAAoB,CAAC,SAAS,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC;QAEhE,8BAA8B;QAC9B,MAAM,IAAI,CAAC,oBAAoB,CAAC,SAAS,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;QAEjE,OAAO,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC;IACnC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,mBAAmB,CAAC,QAAgB,EAAE,KAAc;QACtD,OAAO,IAAI,CAAC,oBAAoB,CAAC,QAAQ,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;IACjE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,SAAS,CAAC,MAAc;QAC1B,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC1C,IAAI,MAAM;YAAE,OAAO,MAAM,CAAC;QAE1B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,CACzC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAoE,CACxG,CAAC;QAEF,MAAM,IAAI,GAAgB;YACtB,EAAE,EAAE,MAAM,CAAC,EAAE;YACb,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,MAAM,CAAC,QAAQ;SACrD,CAAC;QAEF,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QACjC,OAAO,IAAI,CAAC;IAChB,CAAC;IAED;;OAEG;IACH,aAAa,CAAC,OAAe,EAAE,SAAiB,EAAE,SAAiB;QAC/D,OAAO,gCAAgC,OAAO,IAAI,SAAS,IAAI,SAAS,EAAE,CAAC;IAC/E,CAAC;IAED,0EAA0E;IAC1E,kBAAkB;IAClB,0EAA0E;IAE1E;;OAEG;IACK,KAAK,CAAC,oBAAoB,CAC9B,SAAiB,EACjB,IAA0B,EAC1B,SAAqC;QAErC,IAAI,MAA0B,CAAC;QAC/B,IAAI,KAAK,GAAG,CAAC,CAAC;QAEd,GAAG,CAAC;YACA,KAAK,EAAE,CAAC;YACR,MAAM,KAAK,GAAG,IAAI,eAAe,EAAE,CAAC;YACpC,IAAI,MAAM;gBAAE,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;YAExC,IAAI,CAAC;gBACD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,CACzC,IAAI,CAAC,IAAI,CAAC,GAAG,CACT,aAAa,SAAS,qBAAqB,IAAI,EAAE,EACjD,EAAE,KAAK,EAAE,CACgD,CAChE,CAAC;gBAEF,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC;oBACxC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC;wBAC5B,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;oBACrC,CAAC;gBACL,CAAC;gBAED,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM;oBAAE,MAAM;gBAEvD,gDAAgD;gBAChD,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;YAC1D,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC7D,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,SAAS,oBAAoB,IAAI,iCAAiC,SAAS,KAAK,GAAG,EAAE,CAAC,CAAC;gBAC5G,MAAM;YACV,CAAC;YAED,IAAI,KAAK,IAAI,SAAS,EAAE,CAAC;gBACrB,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,SAAS,wCAAwC,SAAS,GAAG,CAAC,CAAC;gBACpF,MAAM;YACV,CAAC;QACL,CAAC,QAAQ,IAAI,EAAE;IACnB,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,aAAa,CACvB,EAAoB,EACpB,UAAkB,CAAC;QAEnB,IAAI,CAAC;YACD,OAAO,MAAM,EAAE,EAAE,CAAC;QACtB,CAAC;QAAC,OAAO,KAAc,EAAE,CAAC;YACtB,MAAM,MAAM,GAAI,KAAa,EAAE,MAAM,IAAK,KAAa,EAAE,UAAU,CAAC;YACpE,MAAM,aAAa,GAAG,MAAM,KAAK,GAAG,CAAC;YAErC,IAAI,CAAC,aAAa,IAAI,OAAO,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;gBAC9C,MAAM,KAAK,CAAC;YAChB,CAAC;YAED,MAAM,UAAU,GAAI,KAAa,EAAE,UAAU;mBACrC,KAAa,EAAE,WAAW,CAAC;YACnC,kFAAkF;YAClF,MAAM,OAAO,GAAG,UAAU,IAAI,IAAI;gBAC9B,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,IAAI,CAAC;gBAC5B,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,GAAG,IAAI,EAAE,IAAI,CAAC,CAAC;YAElD,OAAO,CAAC,IAAI,CACR,GAAG,IAAI,CAAC,SAAS,0BAA0B,OAAO,IAAI,IAAI,CAAC,UAAU,KAAK;gBAC1E,eAAe,OAAO,IAAI,CAC7B,CAAC;YAEF,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;YACrB,OAAO,IAAI,CAAC,aAAa,CAAC,EAAE,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC;QAC/C,CAAC;IACL,CAAC;CACJ"}

package/dist/indexing/providers/discord.d.ts

@@ -0,0 +1,25 @@
+import type { SourceConfig } from '../../types.js';
+import type { DataProvider, AcquisitionResult, ProviderOptions } from './types.js';
+export declare class DiscordDataProvider implements DataProvider {
+    private config;
+    private apiClient;
+    private openaiClient;
+    private logPrefix;
+    constructor(config: SourceConfig, options: ProviderOptions);
+    fullAcquire(): Promise<AcquisitionResult>;
+    incrementalAcquire(lastStateToken: string): Promise<AcquisitionResult>;
+    getCurrentStateToken(): Promise<string | null>;
+    /**
+     * Process a text channel: fetch messages with threads, distill via LLM.
+     */
+    private processTextChannel;
+    /**
+     * Process a single text channel thread: fetch replies, resolve users, distill.
+     */
+    private processTextThread;
+    /**
+     * Process a forum channel: fetch threads, extract Q&A directly.
+     */
+    private processForumChannel;
+}
+//# sourceMappingURL=discord.d.ts.map

package/dist/indexing/providers/discord.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"discord.d.ts","sourceRoot":"","sources":["../../../src/indexing/providers/discord.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,YAAY,EAA6C,MAAM,gBAAgB,CAAC;AAC9F,OAAO,KAAK,EAAE,YAAY,EAAE,iBAAiB,EAAe,eAAe,EAAE,MAAM,YAAY,CAAC;AAIhG,qBAAa,mBAAoB,YAAW,YAAY;IACpD,OAAO,CAAC,MAAM,CAAsB;IACpC,OAAO,CAAC,SAAS,CAAmB;IACpC,OAAO,CAAC,YAAY,CAAuB;IAC3C,OAAO,CAAC,SAAS,CAAS;gBAEd,MAAM,EAAE,YAAY,EAAE,OAAO,EAAE,eAAe;IAoBpD,WAAW,IAAI,OAAO,CAAC,iBAAiB,CAAC;IAkCzC,kBAAkB,CAAC,cAAc,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAkCtE,oBAAoB,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAgCpD;;OAEG;YACW,kBAAkB;IA8BhC;;OAEG;YACW,iBAAiB;IA4D/B;;OAEG;YACW,mBAAmB;CA0DpC"}

package/dist/indexing/providers/discord.js

@@ -0,0 +1,282 @@
+// DiscordDataProvider — Discord thread acquisition via REST API + LLM distillation (text)
+// and direct Q&A extraction (forum). Implements DataProvider.
+import OpenAI from 'openai';
+import { DiscordApiClient } from './discord-api.js';
+import { distillThread } from '../distiller.js';
+import { getConfig } from '../../config.js';
+const MAX_ANSWER_CHARS = 8000;
+export class DiscordDataProvider {
+    config;
+    apiClient;
+    openaiClient = null;
+    logPrefix;
+    constructor(config, options) {
+        if (config.type !== 'discord') {
+            throw new Error('DiscordDataProvider requires a discord source config');
+        }
+        this.config = config;
+        const token = options.discordBotToken;
+        if (!token) {
+            throw new Error('DiscordDataProvider requires a discordBotToken in provider options');
+        }
+        this.apiClient = new DiscordApiClient({ token });
+        // Only create OpenAI client if text channels are configured
+        const hasTextChannels = this.config.channels.some(c => c.type === 'text');
+        if (hasTextChannels) {
+            this.openaiClient = new OpenAI({ apiKey: getConfig().openaiApiKey });
+        }
+        this.logPrefix = `[discord-provider:${config.name}]`;
+    }
+    async fullAcquire() {
+        console.log(`${this.logPrefix} Starting full acquire for ${this.config.channels.length} channel(s)`);
+        const allItems = [];
+        let maxSnowflake = '0';
+        let failedChannels = 0;
+        for (const channel of this.config.channels) {
+            try {
+                const { items, latestId } = channel.type === 'text'
+                    ? await this.processTextChannel(channel.id)
+                    : await this.processForumChannel(channel.id);
+                allItems.push(...items);
+                if (BigInt(latestId) > BigInt(maxSnowflake))
+                    maxSnowflake = latestId;
+            }
+            catch (err) {
+                failedChannels++;
+                const msg = err instanceof Error ? err.message : String(err);
+                console.error(`${this.logPrefix} Failed to process channel ${channel.id}: ${msg}`);
+            }
+        }
+        if (failedChannels === this.config.channels.length) {
+            throw new Error(`All ${failedChannels} channel(s) failed during acquire`);
+        }
+        console.log(`${this.logPrefix} Full acquire complete: ${allItems.length} Q&A pairs from ${this.config.channels.length} channel(s)`);
+        return {
+            items: allItems,
+            removedIds: [],
+            stateToken: maxSnowflake,
+        };
+    }
+    async incrementalAcquire(lastStateToken) {
+        console.log(`${this.logPrefix} Starting incremental acquire since ${lastStateToken}`);
+        const allItems = [];
+        let maxSnowflake = lastStateToken;
+        let failedChannels = 0;
+        for (const channel of this.config.channels) {
+            try {
+                const { items, latestId } = channel.type === 'text'
+                    ? await this.processTextChannel(channel.id, lastStateToken)
+                    : await this.processForumChannel(channel.id, lastStateToken);
+                allItems.push(...items);
+                if (BigInt(latestId) > BigInt(maxSnowflake))
+                    maxSnowflake = latestId;
+            }
+            catch (err) {
+                failedChannels++;
+                const msg = err instanceof Error ? err.message : String(err);
+                console.error(`${this.logPrefix} Failed to process channel ${channel.id}: ${msg}`);
+            }
+        }
+        if (failedChannels === this.config.channels.length) {
+            throw new Error(`All ${failedChannels} channel(s) failed during acquire`);
+        }
+        console.log(`${this.logPrefix} Incremental acquire complete: ${allItems.length} Q&A pairs`);
+        return {
+            items: allItems,
+            removedIds: [],
+            stateToken: maxSnowflake,
+        };
+    }
+    async getCurrentStateToken() {
+        let maxSnowflake = '0';
+        for (const channel of this.config.channels) {
+            try {
+                if (channel.type === 'text') {
+                    const messages = await this.apiClient.fetchChannelMessages(channel.id, undefined, 1);
+                    if (messages.length > 0) {
+                        const latest = messages[0].id;
+                        if (BigInt(latest) > BigInt(maxSnowflake))
+                            maxSnowflake = latest;
+                    }
+                }
+                else {
+                    const threads = await this.apiClient.fetchForumThreads(channel.id, this.config.guild_id);
+                    for (const thread of threads) {
+                        if (BigInt(thread.last_message_id) > BigInt(maxSnowflake)) {
+                            maxSnowflake = thread.last_message_id;
+                        }
+                    }
+                }
+            }
+            catch (err) {
+                const msg = err instanceof Error ? err.message : String(err);
+                console.warn(`${this.logPrefix} Failed to check channel ${channel.id}: ${msg}`);
+            }
+        }
+        return maxSnowflake === '0' ? null : maxSnowflake;
+    }
+    // -----------------------------------------------------------------------
+    // Private helpers
+    // -----------------------------------------------------------------------
+    /**
+     * Process a text channel: fetch messages with threads, distill via LLM.
+     */
+    async processTextChannel(channelId, after) {
+        const messages = await this.apiClient.fetchChannelMessages(channelId, after);
+        const items = [];
+        let latestId = after ?? '0';
+        // Filter to messages with threads having >= min_thread_replies
+        const threads = messages.filter(m => m.thread && m.thread.message_count >= this.config.min_thread_replies);
+        console.log(`${this.logPrefix} Channel ${channelId}: ${messages.length} messages, ${threads.length} qualifying threads`);
+        for (const message of threads) {
+            try {
+                const threadItems = await this.processTextThread(channelId, message);
+                items.push(...threadItems);
+                if (BigInt(message.id) > BigInt(latestId))
+                    latestId = message.id;
+            }
+            catch (err) {
+                const msg = err instanceof Error ? err.message : String(err);
+                console.error(`${this.logPrefix} Failed to distill thread ${message.id}: ${msg}`);
+            }
+        }
+        return { items, latestId };
+    }
+    /**
+     * Process a single text channel thread: fetch replies, resolve users, distill.
+     */
+    async processTextThread(channelId, parentMessage) {
+        const threadId = parentMessage.thread.id;
+        const replies = await this.apiClient.fetchThreadMessages(threadId);
+        // Resolve user display names
+        const threadMessages = [];
+        for (const reply of replies) {
+            let author = reply.author.global_name ?? reply.author.username;
+            try {
+                const user = await this.apiClient.fetchUser(reply.author.id);
+                author = user.displayName;
+            }
+            catch (err) {
+                const msg = err instanceof Error ? err.message : String(err);
+                console.warn(`${this.logPrefix} Failed to resolve user ${reply.author.id}: ${msg}`);
+            }
+            threadMessages.push({
+                author,
+                content: reply.content,
+                timestamp: reply.timestamp,
+                reactions: reply.reactions?.map(r => ({ name: r.emoji.name, count: r.count })),
+            });
+        }
+        // Distill the thread
+        const distillerResult = await distillThread(threadMessages, {
+            model: this.config.distiller_model,
+            client: this.openaiClient,
+        });
+        const items = [];
+        const sourceUrl = this.apiClient.getMessageUrl(this.config.guild_id, channelId, parentMessage.id);
+        const participants = [...new Set(threadMessages.map(m => m.author))];
+        for (let i = 0; i < distillerResult.pairs.length; i++) {
+            const pair = distillerResult.pairs[i];
+            items.push({
+                id: `${channelId}:${parentMessage.id}:${i}`,
+                content: `Q: ${pair.question}\n\nA: ${pair.answer}`,
+                title: pair.question,
+                sourceUrl,
+                metadata: {
+                    channel: channelId,
+                    participants,
+                    confidence: pair.confidence,
+                },
+            });
+        }
+        return items;
+    }
+    /**
+     * Process a forum channel: fetch threads, extract Q&A directly.
+     */
+    async processForumChannel(channelId, afterToken) {
+        const allThreads = await this.apiClient.fetchForumThreads(channelId, this.config.guild_id);
+        let latestId = afterToken ?? '0';
+        const items = [];
+        // Filter by min_thread_replies
+        let threads = allThreads.filter(t => t.message_count >= this.config.min_thread_replies);
+        // For incremental, filter threads with new activity
+        if (afterToken) {
+            threads = threads.filter(t => BigInt(t.last_message_id) > BigInt(afterToken));
+        }
+        console.log(`${this.logPrefix} Forum ${channelId}: ${allThreads.length} total threads, ${threads.length} qualifying`);
+        for (const thread of threads) {
+            try {
+                const messages = await this.apiClient.fetchThreadMessages(thread.id);
+                const answer = synthesizeForumAnswer(messages, thread.name);
+                if (!answer) {
+                    console.log(`${this.logPrefix} Skipping forum thread ${thread.id} — empty synthesized answer`);
+                    continue;
+                }
+                const truncatedAnswer = truncateAnswer(answer);
+                const sourceUrl = this.apiClient.getMessageUrl(this.config.guild_id, thread.id, messages[0]?.id ?? thread.id);
+                items.push({
+                    id: `${channelId}:${thread.id}`,
+                    content: `Q: ${thread.name}\n\nA: ${truncatedAnswer}`,
+                    title: thread.name,
+                    sourceUrl,
+                    metadata: {
+                        channel: channelId,
+                        confidence: 1.0,
+                        forumThread: true,
+                    },
+                });
+                if (BigInt(thread.last_message_id) > BigInt(latestId)) {
+                    latestId = thread.last_message_id;
+                }
+            }
+            catch (err) {
+                const msg = err instanceof Error ? err.message : String(err);
+                console.error(`${this.logPrefix} Failed to process forum thread ${thread.id}: ${msg}`);
+            }
+        }
+        return { items, latestId };
+    }
+}
+/**
+ * Synthesize an answer from forum thread reply messages.
+ */
+function synthesizeForumAnswer(messages, threadName) {
+    if (messages.length === 0)
+        return '';
+    const firstContent = messages[0]?.content.trim() ?? '';
+    const titleLower = threadName.trim().toLowerCase();
+    let processedMessages = messages;
+    // If the first message starts with or matches the title, strip the title portion
+    // but keep any additional content the OP may have added
+    if (firstContent.toLowerCase().startsWith(titleLower)) {
+        const remainder = firstContent.slice(threadName.trim().length).trim();
+        if (!remainder) {
+            // First message was just the title restated — skip it
+            processedMessages = messages.slice(1);
+        }
+        // If there's a remainder, keep the full message (OP added context beyond the title)
+    }
+    else if (titleLower.startsWith(firstContent.toLowerCase()) && firstContent.length > 0) {
+        // First message is a truncated version of the title — skip it
+        processedMessages = messages.slice(1);
+    }
+    // Filter out messages with empty/whitespace-only content
+    const withContent = processedMessages.filter(m => m.content.trim().length > 0);
+    if (withContent.length === 0)
+        return '';
+    return withContent
+        .map(m => `${m.author.global_name ?? m.author.username}: ${m.content}`)
+        .join('\n\n');
+}
+/**
+ * Truncate an answer if it exceeds MAX_ANSWER_CHARS.
+ */
+function truncateAnswer(answer) {
+    if (answer.length <= MAX_ANSWER_CHARS)
+        return answer;
+    const truncated = answer.slice(0, MAX_ANSWER_CHARS);
+    const lastBreak = truncated.lastIndexOf('\n\n');
+    return (lastBreak > 0 ? truncated.slice(0, lastBreak) : truncated) + '\n\n[truncated]';
+}
+//# sourceMappingURL=discord.js.map

package/dist/indexing/providers/discord.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"discord.js","sourceRoot":"","sources":["../../../src/indexing/providers/discord.ts"],"names":[],"mappings":"AAAA,0FAA0F;AAC1F,8DAA8D;AAE9D,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,gBAAgB,EAA2C,MAAM,kBAAkB,CAAC;AAC7F,OAAO,EAAE,aAAa,EAAsB,MAAM,iBAAiB,CAAC;AACpE,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAI5C,MAAM,gBAAgB,GAAG,IAAI,CAAC;AAE9B,MAAM,OAAO,mBAAmB;IACpB,MAAM,CAAsB;IAC5B,SAAS,CAAmB;IAC5B,YAAY,GAAkB,IAAI,CAAC;IACnC,SAAS,CAAS;IAE1B,YAAY,MAAoB,EAAE,OAAwB;QACtD,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;QAC5E,CAAC;QACD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,MAAM,KAAK,GAAG,OAAO,CAAC,eAAe,CAAC;QACtC,IAAI,CAAC,KAAK,EAAE,CAAC;YACT,MAAM,IAAI,KAAK,CAAC,oEAAoE,CAAC,CAAC;QAC1F,CAAC;QACD,IAAI,CAAC,SAAS,GAAG,IAAI,gBAAgB,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;QAEjD,4DAA4D;QAC5D,MAAM,eAAe,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;QAC1E,IAAI,eAAe,EAAE,CAAC;YAClB,IAAI,CAAC,YAAY,GAAG,IAAI,MAAM,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,YAAY,EAAE,CAAC,CAAC;QACzE,CAAC;QAED,IAAI,CAAC,SAAS,GAAG,qBAAqB,MAAM,CAAC,IAAI,GAAG,CAAC;IACzD,CAAC;IAED,KAAK,CAAC,WAAW;QACb,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,SAAS,8BAA8B,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,aAAa,CAAC,CAAC;QACrG,MAAM,QAAQ,GAAkB,EAAE,CAAC;QACnC,IAAI,YAAY,GAAG,GAAG,CAAC;QACvB,IAAI,cAAc,GAAG,CAAC,CAAC;QAEvB,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;YACzC,IAAI,CAAC;gBACD,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,IAAI,KAAK,MAAM;oBAC/C,CAAC,CAAC,MAAM,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,EAAE,CAAC;oBAC3C,CAAC,CAAC,MAAM,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;gBAEjD,QAAQ,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC;gBACxB,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAM,CAAC,YAAY,CAAC;oBAAE,YAAY,GAAG,QAAQ,CAAC;YACzE,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,cAAc,EAAE,CAAC;gBACjB,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC7D,OAAO,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,8BAA8B,OAAO,CAAC,EAAE,KAAK,GAAG,EAAE,CAAC,CAAC;YACvF,CAAC;QACL,CAAC;QAED,IAAI,cAAc,KAAK,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;YACjD,MAAM,IAAI,KAAK,CAAC,OAAO,cAAc,mCAAmC,CAAC,CAAC;QAC9E,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,SAAS,2BAA2B,QAAQ,CAAC,MAAM,mBAAmB,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,aAAa,CAAC,CAAC;QAEpI,OAAO;YACH,KAAK,EAAE,QAAQ;YACf,UAAU,EAAE,EAAE;YACd,UAAU,EAAE,YAAY;SAC3B,CAAC;IACN,CAAC;IAED,KAAK,CAAC,kBAAkB,CAAC,cAAsB;QAC3C,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,SAAS,uCAAuC,cAAc,EAAE,CAAC,CAAC;QACtF,MAAM,QAAQ,GAAkB,EAAE,CAAC;QACnC,IAAI,YAAY,GAAG,cAAc,CAAC;QAClC,IAAI,cAAc,GAAG,CAAC,CAAC;QAEvB,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;YACzC,IAAI,CAAC;gBACD,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,IAAI,KAAK,MAAM;oBAC/C,CAAC,CAAC,MAAM,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,EAAE,EAAE,cAAc,CAAC;oBAC3D,CAAC,CAAC,MAAM,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,EAAE,EAAE,cAAc,CAAC,CAAC;gBAEjE,QAAQ,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC;gBACxB,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAM,CAAC,YAAY,CAAC;oBAAE,YAAY,GAAG,QAAQ,CAAC;YACzE,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,cAAc,EAAE,CAAC;gBACjB,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC7D,OAAO,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,8BAA8B,OAAO,CAAC,EAAE,KAAK,GAAG,EAAE,CAAC,CAAC;YACvF,CAAC;QACL,CAAC;QAED,IAAI,cAAc,KAAK,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;YACjD,MAAM,IAAI,KAAK,CAAC,OAAO,cAAc,mCAAmC,CAAC,CAAC;QAC9E,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,SAAS,kCAAkC,QAAQ,CAAC,MAAM,YAAY,CAAC,CAAC;QAE5F,OAAO;YACH,KAAK,EAAE,QAAQ;YACf,UAAU,EAAE,EAAE;YACd,UAAU,EAAE,YAAY;SAC3B,CAAC;IACN,CAAC;IAED,KAAK,CAAC,oBAAoB;QACtB,IAAI,YAAY,GAAG,GAAG,CAAC;QAEvB,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;YACzC,IAAI,CAAC;gBACD,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;oBAC1B,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,oBAAoB,CAAC,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC;oBACrF,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBACtB,MAAM,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;wBAC9B,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC,YAAY,CAAC;4BAAE,YAAY,GAAG,MAAM,CAAC;oBACrE,CAAC;gBACL,CAAC;qBAAM,CAAC;oBACJ,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAAC,OAAO,CAAC,EAAE,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;oBACzF,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;wBAC3B,IAAI,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,GAAG,MAAM,CAAC,YAAY,CAAC,EAAE,CAAC;4BACxD,YAAY,GAAG,MAAM,CAAC,eAAe,CAAC;wBAC1C,CAAC;oBACL,CAAC;gBACL,CAAC;YACL,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC7D,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,SAAS,4BAA4B,OAAO,CAAC,EAAE,KAAK,GAAG,EAAE,CAAC,CAAC;YACpF,CAAC;QACL,CAAC;QAED,OAAO,YAAY,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,YAAY,CAAC;IACtD,CAAC;IAED,0EAA0E;IAC1E,kBAAkB;IAClB,0EAA0E;IAE1E;;OAEG;IACK,KAAK,CAAC,kBAAkB,CAC5B,SAAiB,EACjB,KAAc;QAEd,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,oBAAoB,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;QAC7E,MAAM,KAAK,GAAkB,EAAE,CAAC;QAChC,IAAI,QAAQ,GAAG,KAAK,IAAI,GAAG,CAAC;QAE5B,+DAA+D;QAC/D,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,CAC3B,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC,aAAa,IAAI,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAC5E,CAAC;QAEF,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,SAAS,YAAY,SAAS,KAAK,QAAQ,CAAC,MAAM,cAAc,OAAO,CAAC,MAAM,qBAAqB,CAAC,CAAC;QAEzH,KAAK,MAAM,OAAO,IAAI,OAAO,EAAE,CAAC;YAC5B,IAAI,CAAC;gBACD,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;gBACrE,KAAK,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,CAAC;gBAE3B,IAAI,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC;oBAAE,QAAQ,GAAG,OAAO,CAAC,EAAE,CAAC;YACrE,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC7D,OAAO,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,6BAA6B,OAAO,CAAC,EAAE,KAAK,GAAG,EAAE,CAAC,CAAC;YACtF,CAAC;QACL,CAAC;QAED,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;IAC/B,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,iBAAiB,CAC3B,SAAiB,EACjB,aAA6B;QAE7B,MAAM,QAAQ,GAAG,aAAa,CAAC,MAAO,CAAC,EAAE,CAAC;QAC1C,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,mBAAmB,CAAC,QAAQ,CAAC,CAAC;QAEnE,6BAA6B;QAC7B,MAAM,cAAc,GAAoB,EAAE,CAAC;QAC3C,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC1B,IAAI,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,WAAW,IAAI,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC;YAC/D,IAAI,CAAC;gBACD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;gBAC7D,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC;YAC9B,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC7D,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,SAAS,2BAA2B,KAAK,CAAC,MAAM,CAAC,EAAE,KAAK,GAAG,EAAE,CAAC,CAAC;YACxF,CAAC;YAED,cAAc,CAAC,IAAI,CAAC;gBAChB,MAAM;gBACN,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,SAAS,EAAE,KAAK,CAAC,SAAS;gBAC1B,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;aACjF,CAAC,CAAC;QACP,CAAC;QAED,qBAAqB;QACrB,MAAM,eAAe,GAAG,MAAM,aAAa,CAAC,cAAc,EAAE;YACxD,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,eAAe;YAClC,MAAM,EAAE,IAAI,CAAC,YAAa;SAC7B,CAAC,CAAC;QAEH,MAAM,KAAK,GAAkB,EAAE,CAAC;QAChC,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,aAAa,CAC1C,IAAI,CAAC,MAAM,CAAC,QAAQ,EACpB,SAAS,EACT,aAAa,CAAC,EAAE,CACnB,CAAC;QACF,MAAM,YAAY,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAErE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,eAAe,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACpD,MAAM,IAAI,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAEtC,KAAK,CAAC,IAAI,CAAC;gBACP,EAAE,EAAE,GAAG,SAAS,IAAI,aAAa,CAAC,EAAE,IAAI,CAAC,EAAE;gBAC3C,OAAO,EAAE,MAAM,IAAI,CAAC,QAAQ,UAAU,IAAI,CAAC,MAAM,EAAE;gBACnD,KAAK,EAAE,IAAI,CAAC,QAAQ;gBACpB,SAAS;gBACT,QAAQ,EAAE;oBACN,OAAO,EAAE,SAAS;oBAClB,YAAY;oBACZ,UAAU,EAAE,IAAI,CAAC,UAAU;iBAC9B;aACJ,CAAC,CAAC;QACP,CAAC;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,mBAAmB,CAC7B,SAAiB,EACjB,UAAmB;QAEnB,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC3F,IAAI,QAAQ,GAAG,UAAU,IAAI,GAAG,CAAC;QACjC,MAAM,KAAK,GAAkB,EAAE,CAAC;QAEhC,+BAA+B;QAC/B,IAAI,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,aAAa,IAAI,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC;QAExF,oDAAoD;QACpD,IAAI,UAAU,EAAE,CAAC;YACb,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,eAAe,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC;QAClF,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,SAAS,UAAU,SAAS,KAAK,UAAU,CAAC,MAAM,mBAAmB,OAAO,CAAC,MAAM,aAAa,CAAC,CAAC;QAEtH,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC3B,IAAI,CAAC;gBACD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,mBAAmB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;gBACrE,MAAM,MAAM,GAAG,qBAAqB,CAAC,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;gBAE5D,IAAI,CAAC,MAAM,EAAE,CAAC;oBACV,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,SAAS,0BAA0B,MAAM,CAAC,EAAE,6BAA6B,CAAC,CAAC;oBAC/F,SAAS;gBACb,CAAC;gBAED,MAAM,eAAe,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;gBAC/C,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,aAAa,CAC1C,IAAI,CAAC,MAAM,CAAC,QAAQ,EACpB,MAAM,CAAC,EAAE,EACT,QAAQ,CAAC,CAAC,CAAC,EAAE,EAAE,IAAI,MAAM,CAAC,EAAE,CAC/B,CAAC;gBAEF,KAAK,CAAC,IAAI,CAAC;oBACP,EAAE,EAAE,GAAG,SAAS,IAAI,MAAM,CAAC,EAAE,EAAE;oBAC/B,OAAO,EAAE,MAAM,MAAM,CAAC,IAAI,UAAU,eAAe,EAAE;oBACrD,KAAK,EAAE,MAAM,CAAC,IAAI;oBAClB,SAAS;oBACT,QAAQ,EAAE;wBACN,OAAO,EAAE,SAAS;wBAClB,UAAU,EAAE,GAAG;wBACf,WAAW,EAAE,IAAI;qBACpB;iBACJ,CAAC,CAAC;gBAEH,IAAI,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;oBACpD,QAAQ,GAAG,MAAM,CAAC,eAAe,CAAC;gBACtC,CAAC;YACL,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC7D,OAAO,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,mCAAmC,MAAM,CAAC,EAAE,KAAK,GAAG,EAAE,CAAC,CAAC;YAC3F,CAAC;QACL,CAAC;QAED,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;IAC/B,CAAC;CACJ;AAED;;GAEG;AACH,SAAS,qBAAqB,CAAC,QAA0B,EAAE,UAAkB;IACzE,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAErC,MAAM,YAAY,GAAG,QAAQ,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC;IACvD,MAAM,UAAU,GAAG,UAAU,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAEnD,IAAI,iBAAiB,GAAG,QAAQ,CAAC;IAEjC,iFAAiF;IACjF,wDAAwD;IACxD,IAAI,YAAY,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QACpD,MAAM,SAAS,GAAG,YAAY,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;QACtE,IAAI,CAAC,SAAS,EAAE,CAAC;YACb,sDAAsD;YACtD,iBAAiB,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC1C,CAAC;QACD,oFAAoF;IACxF,CAAC;SAAM,IAAI,UAAU,CAAC,UAAU,CAAC,YAAY,CAAC,WAAW,EAAE,CAAC,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtF,8DAA8D;QAC9D,iBAAiB,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAC1C,CAAC;IAED,yDAAyD;IACzD,MAAM,WAAW,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC/E,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAExC,OAAO,WAAW;SACb,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC,MAAM,CAAC,QAAQ,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC;SACtE,IAAI,CAAC,MAAM,CAAC,CAAC;AACtB,CAAC;AAED;;GAEG;AACH,SAAS,cAAc,CAAC,MAAc;IAClC,IAAI,MAAM,CAAC,MAAM,IAAI,gBAAgB;QAAE,OAAO,MAAM,CAAC;IACrD,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,gBAAgB,CAAC,CAAC;IACpD,MAAM,SAAS,GAAG,SAAS,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;IAChD,OAAO,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,iBAAiB,CAAC;AAC3F,CAAC"}

package/dist/indexing/providers/file.d.ts

@@ -0,0 +1,18 @@
+import type { SourceConfig } from '../../types.js';
+import type { DataProvider, AcquisitionResult, ProviderOptions } from './types.js';
+export declare class FileDataProvider implements DataProvider {
+    private config;
+    private options;
+    private logPrefix;
+    private skipDirs;
+    private maxFileSize;
+    constructor(config: SourceConfig, options: ProviderOptions);
+    private isLocal;
+    fullAcquire(): Promise<AcquisitionResult>;
+    incrementalAcquire(lastStateToken: string): Promise<AcquisitionResult>;
+    getCurrentStateToken(): Promise<string | null>;
+    private ensureRepo;
+    private walkFiles;
+    private computeLocalSha;
+}
+//# sourceMappingURL=file.d.ts.map

package/dist/indexing/providers/file.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"file.d.ts","sourceRoot":"","sources":["../../../src/indexing/providers/file.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EAAE,YAAY,EAAoB,MAAM,gBAAgB,CAAC;AACrE,OAAO,KAAK,EAAE,YAAY,EAAE,iBAAiB,EAAe,eAAe,EAAE,MAAM,YAAY,CAAC;AAoBhG,qBAAa,gBAAiB,YAAW,YAAY;IACjD,OAAO,CAAC,MAAM,CAAmB;IACjC,OAAO,CAAC,OAAO,CAAkB;IACjC,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,QAAQ,CAAc;IAC9B,OAAO,CAAC,WAAW,CAAS;gBAEhB,MAAM,EAAE,YAAY,EAAE,OAAO,EAAE,eAAe;IAW1D,OAAO,CAAC,OAAO;IAIT,WAAW,IAAI,OAAO,CAAC,iBAAiB,CAAC;IAsDzC,kBAAkB,CAAC,cAAc,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAwEtE,oBAAoB,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;YAuCtC,UAAU;YA2BV,SAAS;YAiCT,eAAe;CAShC"}