@copilotkit/pathfinder 1.1.0 → 1.5.0

package/dist/indexing/providers/file.js

@@ -0,0 +1,262 @@
+// FileDataProvider — git-backed and local file data acquisition.
+// Handles clone/pull, file walking, pattern matching, content reading.
+import fs from 'node:fs';
+import path from 'node:path';
+import { createHash } from 'node:crypto';
+import { simpleGit } from 'simple-git';
+import { matchesPatterns, hasLowSemanticValue } from '../utils.js';
+import { isFileSourceConfig } from '../../types.js';
+const DEFAULT_SKIP_DIRS = new Set(['node_modules', 'dist', 'build', '.git']);
+const DEFAULT_MAX_FILE_SIZE = 102400; // 100KB
+function repoNameFromUrl(repoUrl) {
+    const last = repoUrl.split('/').pop() ?? '';
+    return last.replace(/\.git$/, '');
+}
+function authenticatedUrl(repoUrl, githubToken) {
+    if (githubToken) {
+        return repoUrl.replace('https://github.com/', `https://x-access-token:${githubToken}@github.com/`);
+    }
+    return repoUrl;
+}
+export class FileDataProvider {
+    config;
+    options;
+    logPrefix;
+    skipDirs;
+    maxFileSize;
+    constructor(config, options) {
+        if (!isFileSourceConfig(config)) {
+            throw new Error('FileDataProvider cannot handle slack source type');
+        }
+        this.config = config;
+        this.options = options;
+        this.logPrefix = `[file-provider:${config.name}]`;
+        this.skipDirs = new Set([...DEFAULT_SKIP_DIRS, ...(config.skip_dirs ?? [])]);
+        this.maxFileSize = config.max_file_size ?? DEFAULT_MAX_FILE_SIZE;
+    }
+    isLocal() {
+        return !this.config.repo;
+    }
+    async fullAcquire() {
+        let repoDir;
+        let stateToken;
+        if (this.isLocal()) {
+            repoDir = path.resolve(this.config.path);
+            if (!fs.existsSync(repoDir)) {
+                throw new Error(`Local source path does not exist: ${repoDir}`);
+            }
+            stateToken = await this.computeLocalSha(repoDir);
+        }
+        else {
+            const repoName = repoNameFromUrl(this.config.repo);
+            repoDir = path.join(this.options.cloneDir, repoName);
+            const git = await this.ensureRepo(repoDir, repoName);
+            stateToken = await git.revparse(['HEAD']);
+        }
+        const walkRoot = this.isLocal()
+            ? repoDir
+            : path.join(repoDir, this.config.path);
+        if (!fs.existsSync(walkRoot)) {
+            console.warn(`${this.logPrefix} Walk root not found at ${walkRoot}, skipping`);
+            return { items: [], removedIds: [], stateToken };
+        }
+        const allFiles = await this.walkFiles(walkRoot);
+        const matchingFiles = allFiles.filter(absPath => {
+            const relPath = path.relative(repoDir, absPath);
+            return matchesPatterns(relPath, this.config);
+        });
+        const skipped = allFiles.length - matchingFiles.length;
+        console.log(`${this.logPrefix} Found ${matchingFiles.length} files for full acquire` +
+            (skipped > 0 ? ` (${skipped} excluded by patterns)` : ''));
+        const items = [];
+        for (const absPath of matchingFiles) {
+            const relPath = path.relative(repoDir, absPath);
+            try {
+                const content = await fs.promises.readFile(absPath, 'utf-8');
+                if (hasLowSemanticValue(content))
+                    continue;
+                items.push({ id: relPath, content });
+            }
+            catch (err) {
+                const msg = err instanceof Error ? err.message : String(err);
+                console.error(`${this.logPrefix} Failed to read ${relPath}: ${msg}`);
+            }
+        }
+        return { items, removedIds: [], stateToken };
+    }
+    async incrementalAcquire(lastStateToken) {
+        if (this.isLocal()) {
+            console.log(`${this.logPrefix} Local source — falling back to full acquire`);
+            return this.fullAcquire();
+        }
+        const repoName = repoNameFromUrl(this.config.repo);
+        const repoDir = path.join(this.options.cloneDir, repoName);
+        const git = await this.ensureRepo(repoDir, repoName);
+        const headSha = await git.revparse(['HEAD']);
+        if (headSha === lastStateToken) {
+            console.log(`${this.logPrefix} No new commits, skipping`);
+            return { items: [], removedIds: [], stateToken: headSha };
+        }
+        // Unshallow for diff
+        try {
+            await git.fetch(['--unshallow']);
+        }
+        catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            if (!msg.includes('unshallow') && !msg.includes('does not make sense')) {
+                console.warn(`${this.logPrefix} git fetch --unshallow failed:`, msg);
+            }
+        }
+        let diffOutput;
+        try {
+            diffOutput = await git.diff(['--name-only', `${lastStateToken}..HEAD`]);
+        }
+        catch (err) {
+            console.warn(`${this.logPrefix} git diff failed, falling back to full acquire:`, err);
+            return this.fullAcquire();
+        }
+        const changedFiles = diffOutput.split('\n').map(f => f.trim()).filter(f => f.length > 0);
+        const matchingChanged = changedFiles.filter(f => matchesPatterns(f, this.config));
+        if (matchingChanged.length === 0) {
+            console.log(`${this.logPrefix} No matching changes detected`);
+            return { items: [], removedIds: [], stateToken: headSha };
+        }
+        console.log(`${this.logPrefix} Incremental acquire: ${matchingChanged.length} changed files`);
+        // Find deleted files
+        const diffStatusOutput = await git.diff(['--name-status', `${lastStateToken}..HEAD`]);
+        const deletedFiles = diffStatusOutput.split('\n')
+            .filter(line => line.startsWith('D\t'))
+            .map(line => line.slice(2).trim())
+            .filter(f => matchesPatterns(f, this.config));
+        // Read changed (non-deleted) files
+        const filesToRead = matchingChanged.filter(f => !deletedFiles.includes(f));
+        const items = [];
+        for (const relPath of filesToRead) {
+            const absPath = path.join(repoDir, relPath);
+            if (!fs.existsSync(absPath))
+                continue;
+            try {
+                const stat = await fs.promises.stat(absPath);
+                if (stat.size > this.maxFileSize)
+                    continue;
+                const content = await fs.promises.readFile(absPath, 'utf-8');
+                if (hasLowSemanticValue(content))
+                    continue;
+                items.push({ id: relPath, content });
+            }
+            catch (err) {
+                const msg = err instanceof Error ? err.message : String(err);
+                console.error(`${this.logPrefix} Failed to read ${relPath}: ${msg}`);
+            }
+        }
+        return { items, removedIds: deletedFiles, stateToken: headSha };
+    }
+    async getCurrentStateToken() {
+        if (this.isLocal()) {
+            const walkRoot = path.resolve(this.config.path);
+            if (!fs.existsSync(walkRoot))
+                return null;
+            return this.computeLocalSha(walkRoot);
+        }
+        // For git sources: try ls-remote first (no clone needed)
+        try {
+            let url = this.config.repo;
+            if (this.options.githubToken) {
+                url = authenticatedUrl(url, this.options.githubToken);
+            }
+            const git = simpleGit();
+            const result = await git.listRemote([url, 'HEAD']);
+            const sha = result.split('\t')[0]?.trim();
+            return sha || null;
+        }
+        catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            console.warn(`${this.logPrefix} ls-remote failed for ${this.config.repo}: ${msg}`);
+            // Fall back to local HEAD if clone dir exists
+            const repoName = repoNameFromUrl(this.config.repo);
+            const repoDir = path.join(this.options.cloneDir, repoName);
+            if (!fs.existsSync(repoDir))
+                return null;
+            try {
+                const git = simpleGit(repoDir);
+                return await git.revparse(['HEAD']);
+            }
+            catch (innerErr) {
+                const innerMsg = innerErr instanceof Error ? innerErr.message : String(innerErr);
+                console.warn(`${this.logPrefix} local HEAD lookup also failed for ${repoDir}: ${innerMsg}`);
+                return null;
+            }
+        }
+    }
+    // -----------------------------------------------------------------------
+    // Private helpers
+    // -----------------------------------------------------------------------
+    async ensureRepo(repoDir, repoName) {
+        await fs.promises.mkdir(this.options.cloneDir, { recursive: true });
+        const gitDir = path.join(repoDir, '.git');
+        if (fs.existsSync(gitDir)) {
+            console.log(`${this.logPrefix} Pulling latest changes for ${repoName}`);
+            const git = simpleGit(repoDir);
+            try {
+                await git.pull();
+                return git;
+            }
+            catch (err) {
+                console.warn(`${this.logPrefix} Corrupted repo at ${repoDir}, re-cloning:`, err);
+                await fs.promises.rm(repoDir, { recursive: true, force: true });
+            }
+        }
+        const authUrl = authenticatedUrl(this.config.repo, this.options.githubToken);
+        console.log(`${this.logPrefix} Cloning ${this.config.repo} into ${repoDir}`);
+        const git = simpleGit(this.options.cloneDir);
+        const cloneOpts = ['--depth=1'];
+        if (this.config.branch) {
+            cloneOpts.push('--branch', this.config.branch);
+        }
+        await git.clone(authUrl, repoName, cloneOpts);
+        return simpleGit(repoDir);
+    }
+    async walkFiles(dir) {
+        const results = [];
+        let entries;
+        try {
+            entries = await fs.promises.readdir(dir, { withFileTypes: true });
+        }
+        catch (err) {
+            console.warn(`${this.logPrefix} Unable to read directory ${dir}:`, err);
+            return results;
+        }
+        for (const entry of entries) {
+            if (this.skipDirs.has(entry.name))
+                continue;
+            const fullPath = path.join(dir, entry.name);
+            if (entry.isDirectory()) {
+                const nested = await this.walkFiles(fullPath);
+                results.push(...nested);
+            }
+            else if (entry.isFile()) {
+                try {
+                    const stat = await fs.promises.stat(fullPath);
+                    if (stat.size > this.maxFileSize)
+                        continue;
+                }
+                catch (err) {
+                    console.warn(`${this.logPrefix} Unable to stat ${fullPath}:`, err);
+                    continue;
+                }
+                results.push(fullPath);
+            }
+        }
+        return results;
+    }
+    async computeLocalSha(walkRoot) {
+        const files = await this.walkFiles(walkRoot);
+        const hash = createHash('sha256');
+        for (const f of files.sort()) {
+            const stat = await fs.promises.stat(f);
+            hash.update(`${f}:${stat.mtimeMs}\n`);
+        }
+        return `local-${hash.digest('hex').slice(0, 12)}`;
+    }
+}
+//# sourceMappingURL=file.js.map

package/dist/indexing/providers/file.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"file.js","sourceRoot":"","sources":["../../../src/indexing/providers/file.ts"],"names":[],"mappings":"AAAA,iEAAiE;AACjE,uEAAuE;AAEvE,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,SAAS,EAAkB,MAAM,YAAY,CAAC;AACvD,OAAO,EAAE,eAAe,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AACnE,OAAO,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AAIpD,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,CAAC,cAAc,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC;AAC7E,MAAM,qBAAqB,GAAG,MAAM,CAAC,CAAC,QAAQ;AAE9C,SAAS,eAAe,CAAC,OAAe;IACpC,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC;IAC5C,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;AACtC,CAAC;AAED,SAAS,gBAAgB,CAAC,OAAe,EAAE,WAAoB;IAC3D,IAAI,WAAW,EAAE,CAAC;QACd,OAAO,OAAO,CAAC,OAAO,CAClB,qBAAqB,EACrB,0BAA0B,WAAW,cAAc,CACtD,CAAC;IACN,CAAC;IACD,OAAO,OAAO,CAAC;AACnB,CAAC;AAED,MAAM,OAAO,gBAAgB;IACjB,MAAM,CAAmB;IACzB,OAAO,CAAkB;IACzB,SAAS,CAAS;IAClB,QAAQ,CAAc;IACtB,WAAW,CAAS;IAE5B,YAAY,MAAoB,EAAE,OAAwB;QACtD,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;QACxE,CAAC;QACD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,SAAS,GAAG,kBAAkB,MAAM,CAAC,IAAI,GAAG,CAAC;QAClD,IAAI,CAAC,QAAQ,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,iBAAiB,EAAE,GAAG,CAAC,MAAM,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7E,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,aAAa,IAAI,qBAAqB,CAAC;IACrE,CAAC;IAEO,OAAO;QACX,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;IAC7B,CAAC;IAED,KAAK,CAAC,WAAW;QACb,IAAI,OAAe,CAAC;QACpB,IAAI,UAAkB,CAAC;QAEvB,IAAI,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;YACjB,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YACzC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC1B,MAAM,IAAI,KAAK,CAAC,qCAAqC,OAAO,EAAE,CAAC,CAAC;YACpE,CAAC;YACD,UAAU,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;QACrD,CAAC;aAAM,CAAC;YACJ,MAAM,QAAQ,GAAG,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,IAAK,CAAC,CAAC;YACpD,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;YACrD,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;YACrD,UAAU,GAAG,MAAM,GAAG,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;QAC9C,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE;YAC3B,CAAC,CAAC,OAAO;YACT,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAE3C,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC3B,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,SAAS,2BAA2B,QAAQ,YAAY,CAAC,CAAC;YAC/E,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE,UAAU,EAAE,CAAC;QACrD,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;QAChD,MAAM,aAAa,GAAG,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE;YAC5C,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAChD,OAAO,eAAe,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QACjD,CAAC,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC;QACvD,OAAO,CAAC,GAAG,CACP,GAAG,IAAI,CAAC,SAAS,UAAU,aAAa,CAAC,MAAM,yBAAyB;YACxE,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,OAAO,wBAAwB,CAAC,CAAC,CAAC,EAAE,CAAC,CAC5D,CAAC;QAEF,MAAM,KAAK,GAAkB,EAAE,CAAC;QAChC,KAAK,MAAM,OAAO,IAAI,aAAa,EAAE,CAAC;YAClC,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAChD,IAAI,CAAC;gBACD,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;gBAC7D,IAAI,mBAAmB,CAAC,OAAO,CAAC;oBAAE,SAAS;gBAC3C,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;YACzC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC7D,OAAO,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,mBAAmB,OAAO,KAAK,GAAG,EAAE,CAAC,CAAC;YACzE,CAAC;QACL,CAAC;QAED,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,EAAE,EAAE,UAAU,EAAE,CAAC;IACjD,CAAC;IAED,KAAK,CAAC,kBAAkB,CAAC,cAAsB;QAC3C,IAAI,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;YACjB,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,SAAS,8CAA8C,CAAC,CAAC;YAC7E,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC;QAC9B,CAAC;QAED,MAAM,QAAQ,GAAG,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,IAAK,CAAC,CAAC;QACpD,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QAC3D,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;QACrD,MAAM,OAAO,GAAG,MAAM,GAAG,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;QAE7C,IAAI,OAAO,KAAK,cAAc,EAAE,CAAC;YAC7B,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,SAAS,2BAA2B,CAAC,CAAC;YAC1D,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE,UAAU,EAAE,OAAO,EAAE,CAAC;QAC9D,CAAC;QAED,qBAAqB;QACrB,IAAI,CAAC;YACD,MAAM,GAAG,CAAC,KAAK,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC;QACrC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC7D,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,qBAAqB,CAAC,EAAE,CAAC;gBACrE,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,SAAS,gCAAgC,EAAE,GAAG,CAAC,CAAC;YACzE,CAAC;QACL,CAAC;QAED,IAAI,UAAkB,CAAC;QACvB,IAAI,CAAC;YACD,UAAU,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,CAAC,aAAa,EAAE,GAAG,cAAc,QAAQ,CAAC,CAAC,CAAC;QAC5E,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,SAAS,iDAAiD,EAAE,GAAG,CAAC,CAAC;YACtF,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC;QAC9B,CAAC;QAED,MAAM,YAAY,GAAG,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACzF,MAAM,eAAe,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,eAAe,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;QAElF,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/B,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,SAAS,+BAA+B,CAAC,CAAC;YAC9D,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE,UAAU,EAAE,OAAO,EAAE,CAAC;QAC9D,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,SAAS,yBAAyB,eAAe,CAAC,MAAM,gBAAgB,CAAC,CAAC;QAE9F,qBAAqB;QACrB,MAAM,gBAAgB,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,CAAC,eAAe,EAAE,GAAG,cAAc,QAAQ,CAAC,CAAC,CAAC;QACtF,MAAM,YAAY,GAAG,gBAAgB,CAAC,KAAK,CAAC,IAAI,CAAC;aAC5C,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;aACtC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;aACjC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,eAAe,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;QAElD,mCAAmC;QACnC,MAAM,WAAW,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3E,MAAM,KAAK,GAAkB,EAAE,CAAC;QAChC,KAAK,MAAM,OAAO,IAAI,WAAW,EAAE,CAAC;YAChC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAC5C,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC;gBAAE,SAAS;YACtC,IAAI,CAAC;gBACD,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAC7C,IAAI,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,WAAW;oBAAE,SAAS;gBAC3C,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;gBAC7D,IAAI,mBAAmB,CAAC,OAAO,CAAC;oBAAE,SAAS;gBAC3C,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;YACzC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC7D,OAAO,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,mBAAmB,OAAO,KAAK,GAAG,EAAE,CAAC,CAAC;YACzE,CAAC;QACL,CAAC;QAED,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,UAAU,EAAE,OAAO,EAAE,CAAC;IACpE,CAAC;IAED,KAAK,CAAC,oBAAoB;QACtB,IAAI,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;YACjB,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAChD,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC;gBAAE,OAAO,IAAI,CAAC;YAC1C,OAAO,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC;QAC1C,CAAC;QAED,yDAAyD;QACzD,IAAI,CAAC;YACD,IAAI,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,IAAK,CAAC;YAC5B,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;gBAC3B,GAAG,GAAG,gBAAgB,CAAC,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;YAC1D,CAAC;YACD,MAAM,GAAG,GAAG,SAAS,EAAE,CAAC;YACxB,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,UAAU,CAAC,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC;YACnD,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC;YAC1C,OAAO,GAAG,IAAI,IAAI,CAAC;QACvB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC7D,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,SAAS,yBAAyB,IAAI,CAAC,MAAM,CAAC,IAAI,KAAK,GAAG,EAAE,CAAC,CAAC;YACnF,8CAA8C;YAC9C,MAAM,QAAQ,GAAG,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,IAAK,CAAC,CAAC;YACpD,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;YAC3D,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC;gBAAE,OAAO,IAAI,CAAC;YACzC,IAAI,CAAC;gBACD,MAAM,GAAG,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;gBAC/B,OAAO,MAAM,GAAG,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;YACxC,CAAC;YAAC,OAAO,QAAQ,EAAE,CAAC;gBAChB,MAAM,QAAQ,GAAG,QAAQ,YAAY,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;gBACjF,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,SAAS,sCAAsC,OAAO,KAAK,QAAQ,EAAE,CAAC,CAAC;gBAC5F,OAAO,IAAI,CAAC;YAChB,CAAC;QACL,CAAC;IACL,CAAC;IAED,0EAA0E;IAC1E,kBAAkB;IAClB,0EAA0E;IAElE,KAAK,CAAC,UAAU,CAAC,OAAe,EAAE,QAAgB;QACtD,MAAM,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAEpE,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC1C,IAAI,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YACxB,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,SAAS,+BAA+B,QAAQ,EAAE,CAAC,CAAC;YACxE,MAAM,GAAG,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;YAC/B,IAAI,CAAC;gBACD,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;gBACjB,OAAO,GAAG,CAAC;YACf,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,SAAS,sBAAsB,OAAO,eAAe,EAAE,GAAG,CAAC,CAAC;gBACjF,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YACpE,CAAC;QACL,CAAC;QAED,MAAM,OAAO,GAAG,gBAAgB,CAAC,IAAI,CAAC,MAAM,CAAC,IAAK,EAAE,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QAC9E,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,SAAS,YAAY,IAAI,CAAC,MAAM,CAAC,IAAK,SAAS,OAAO,EAAE,CAAC,CAAC;QAC9E,MAAM,GAAG,GAAG,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC7C,MAAM,SAAS,GAAG,CAAC,WAAW,CAAC,CAAC;QAChC,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACrB,SAAS,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACnD,CAAC;QACD,MAAM,GAAG,CAAC,KAAK,CAAC,OAAO,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC;QAC9C,OAAO,SAAS,CAAC,OAAO,CAAC,CAAC;IAC9B,CAAC;IAEO,KAAK,CAAC,SAAS,CAAC,GAAW;QAC/B,MAAM,OAAO,GAAa,EAAE,CAAC;QAE7B,IAAI,OAAoB,CAAC;QACzB,IAAI,CAAC;YACD,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;QACtE,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,SAAS,6BAA6B,GAAG,GAAG,EAAE,GAAG,CAAC,CAAC;YACxE,OAAO,OAAO,CAAC;QACnB,CAAC;QAED,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC1B,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC;gBAAE,SAAS;YAC5C,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YAE5C,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;gBACtB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;gBAC9C,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC;YAC5B,CAAC;iBAAM,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;gBACxB,IAAI,CAAC;oBACD,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;oBAC9C,IAAI,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,WAAW;wBAAE,SAAS;gBAC/C,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACX,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,SAAS,mBAAmB,QAAQ,GAAG,EAAE,GAAG,CAAC,CAAC;oBACnE,SAAS;gBACb,CAAC;gBACD,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC3B,CAAC;QACL,CAAC;QAED,OAAO,OAAO,CAAC;IACnB,CAAC;IAEO,KAAK,CAAC,eAAe,CAAC,QAAgB;QAC1C,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;QAC7C,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;QAClC,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;YAC3B,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACvC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC;QAC1C,CAAC;QACD,OAAO,SAAS,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;IACtD,CAAC;CACJ"}

package/dist/indexing/providers/index.d.ts

@@ -0,0 +1,5 @@
+import type { DataProviderFactory } from './types.js';
+export declare function registerProvider(type: string, factory: DataProviderFactory): void;
+export declare function getProvider(type: string): DataProviderFactory;
+export type { DataProvider, DataProviderFactory, ProviderOptions, ContentItem, AcquisitionResult } from './types.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/indexing/providers/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/indexing/providers/index.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AAItD,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,mBAAmB,GAAG,IAAI,CAEjF;AAED,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,mBAAmB,CAM7D;AAED,YAAY,EAAE,YAAY,EAAE,mBAAmB,EAAE,eAAe,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC"}

package/dist/indexing/providers/index.js

@@ -0,0 +1,22 @@
+// Provider registry — maps source type to a DataProvider factory.
+const registry = new Map();
+export function registerProvider(type, factory) {
+    registry.set(type, factory);
+}
+export function getProvider(type) {
+    const factory = registry.get(type);
+    if (!factory) {
+        throw new Error(`Unknown provider type: "${type}". Available: ${[...registry.keys()].join(', ')}`);
+    }
+    return factory;
+}
+// Register built-in providers
+import { FileDataProvider } from './file.js';
+for (const type of ['markdown', 'code', 'raw-text', 'html']) {
+    registerProvider(type, (config, options) => new FileDataProvider(config, options));
+}
+import { SlackDataProvider } from './slack.js';
+registerProvider('slack', (config, options) => new SlackDataProvider(config, options));
+import { DiscordDataProvider } from './discord.js';
+registerProvider('discord', (config, options) => new DiscordDataProvider(config, options));
+//# sourceMappingURL=index.js.map

package/dist/indexing/providers/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/indexing/providers/index.ts"],"names":[],"mappings":"AAAA,kEAAkE;AAIlE,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA+B,CAAC;AAExD,MAAM,UAAU,gBAAgB,CAAC,IAAY,EAAE,OAA4B;IACvE,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,IAAY;IACpC,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACnC,IAAI,CAAC,OAAO,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,iBAAiB,CAAC,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACvG,CAAC;IACD,OAAO,OAAO,CAAC;AACnB,CAAC;AAID,8BAA8B;AAC9B,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAE7C,KAAK,MAAM,IAAI,IAAI,CAAC,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,CAAC,EAAE,CAAC;IAC1D,gBAAgB,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC,IAAI,gBAAgB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;AACvF,CAAC;AAED,OAAO,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAE/C,gBAAgB,CAAC,OAAO,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC,IAAI,iBAAiB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;AAEvF,OAAO,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAEnD,gBAAgB,CAAC,SAAS,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC,IAAI,mBAAmB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC"}

package/dist/indexing/providers/slack-api.d.ts

@@ -0,0 +1,62 @@
+import { WebClient } from '@slack/web-api';
+export interface SlackMessage {
+    ts: string;
+    user?: string;
+    text?: string;
+    thread_ts?: string;
+    reply_count?: number;
+    reactions?: Array<{
+        name: string;
+        count: number;
+    }>;
+    team?: string;
+}
+export interface SlackUser {
+    id: string;
+    displayName: string;
+    teamId?: string;
+}
+export interface SlackApiClientOptions {
+    token: string;
+    /** Max retries on rate-limited requests (default: 5) */
+    maxRetries?: number;
+}
+export declare class SlackApiClient {
+    private client;
+    private maxRetries;
+    private userCache;
+    private teamNameCache;
+    private logPrefix;
+    constructor(options: SlackApiClientOptions);
+    /**
+     * Fetch channel message history, paginated. Returns all top-level messages.
+     * If `oldest` is provided, only messages after that timestamp are returned.
+     */
+    fetchChannelHistory(channelId: string, oldest?: string, limit?: number): Promise<SlackMessage[]>;
+    /**
+     * Fetch all replies in a thread, paginated.
+     */
+    fetchThreadReplies(channelId: string, threadTs: string): Promise<SlackMessage[]>;
+    /**
+     * Fetch user info with caching. Returns display name with workspace suffix
+     * for external users in Slack Connect channels.
+     */
+    fetchUserInfo(userId: string): Promise<SlackUser>;
+    /**
+     * Get a permalink URL for a specific message.
+     */
+    getChannelPermalink(channelId: string, messageTs: string): Promise<string>;
+    /**
+     * Resolve a team ID to a team name (for Slack Connect workspace suffix).
+     * Caches results. Returns null if lookup fails (e.g., no permission).
+     */
+    private resolveTeamName;
+    /**
+     * Call a Slack API method with retry on rate limit (429) responses.
+     * Respects Retry-After header with exponential backoff fallback.
+     */
+    private callWithRetry;
+    /** Expose the underlying WebClient for testing/advanced usage. */
+    get webClient(): WebClient;
+}
+//# sourceMappingURL=slack-api.d.ts.map

package/dist/indexing/providers/slack-api.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"slack-api.d.ts","sourceRoot":"","sources":["../../../src/indexing/providers/slack-api.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,SAAS,EAAyB,MAAM,gBAAgB,CAAC;AAIlE,MAAM,WAAW,YAAY;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACnD,IAAI,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,SAAS;IACtB,EAAE,EAAE,MAAM,CAAC;IACX,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,qBAAqB;IAClC,KAAK,EAAE,MAAM,CAAC;IACd,wDAAwD;IACxD,UAAU,CAAC,EAAE,MAAM,CAAC;CACvB;AAYD,qBAAa,cAAc;IACvB,OAAO,CAAC,MAAM,CAAY;IAC1B,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,SAAS,CAAgC;IACjD,OAAO,CAAC,aAAa,CAA6B;IAClD,OAAO,CAAC,SAAS,CAAiB;gBAEtB,OAAO,EAAE,qBAAqB;IAQ1C;;;OAGG;IACG,mBAAmB,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IAoCtG;;OAEG;IACG,kBAAkB,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IAgCtF;;;OAGG;IACG,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC;IA+BvD;;OAEG;IACG,mBAAmB,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAWhF;;;OAGG;YACW,eAAe;IAiB7B;;;OAGG;YACW,aAAa;IA6B3B,kEAAkE;IAClE,IAAI,SAAS,IAAI,SAAS,CAEzB;CACJ"}

package/dist/indexing/providers/slack-api.js

@@ -0,0 +1,167 @@
+// Slack Web API client wrapper — handles pagination, rate limiting, user caching.
+import { WebClient } from '@slack/web-api';
+// ── Rate limit helpers ───────────────────────────────────────────────────────
+function sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+}
+// ── Client ───────────────────────────────────────────────────────────────────
+const MAX_PAGES = 100;
+export class SlackApiClient {
+    client;
+    maxRetries;
+    userCache = new Map();
+    teamNameCache = new Map();
+    logPrefix = '[slack-api]';
+    constructor(options) {
+        this.client = new WebClient(options.token, {
+            // Disable built-in retry so we handle it ourselves
+            retryConfig: { retries: 0 },
+        });
+        this.maxRetries = options.maxRetries ?? 5;
+    }
+    /**
+     * Fetch channel message history, paginated. Returns all top-level messages.
+     * If `oldest` is provided, only messages after that timestamp are returned.
+     */
+    async fetchChannelHistory(channelId, oldest, limit) {
+        const messages = [];
+        let cursor;
+        const pageLimit = limit ?? 200;
+        let pages = 0;
+        do {
+            pages++;
+            const result = await this.callWithRetry(() => this.client.conversations.history({
+                channel: channelId,
+                oldest,
+                cursor,
+                limit: pageLimit,
+            }));
+            const rawMessages = result.messages;
+            if (rawMessages) {
+                messages.push(...rawMessages);
+            }
+            // When an explicit limit is provided, don't paginate — return just the first page
+            if (limit !== undefined)
+                break;
+            cursor = result.response_metadata?.next_cursor;
+            if (pages >= MAX_PAGES) {
+                console.warn(`${this.logPrefix} fetchChannelHistory hit max pages (${MAX_PAGES}) for channel ${channelId}`);
+                break;
+            }
+        } while (cursor);
+        return messages;
+    }
+    /**
+     * Fetch all replies in a thread, paginated.
+     */
+    async fetchThreadReplies(channelId, threadTs) {
+        const messages = [];
+        let cursor;
+        let pages = 0;
+        do {
+            pages++;
+            const result = await this.callWithRetry(() => this.client.conversations.replies({
+                channel: channelId,
+                ts: threadTs,
+                cursor,
+                limit: 200,
+            }));
+            const rawMessages = result.messages;
+            if (rawMessages) {
+                messages.push(...rawMessages);
+            }
+            cursor = result.response_metadata?.next_cursor;
+            if (pages >= MAX_PAGES) {
+                console.warn(`${this.logPrefix} fetchThreadReplies hit max pages (${MAX_PAGES}) for thread ${threadTs}`);
+                break;
+            }
+        } while (cursor);
+        return messages;
+    }
+    /**
+     * Fetch user info with caching. Returns display name with workspace suffix
+     * for external users in Slack Connect channels.
+     */
+    async fetchUserInfo(userId) {
+        const cached = this.userCache.get(userId);
+        if (cached)
+            return cached;
+        const result = await this.callWithRetry(() => this.client.users.info({ user: userId }));
+        const user = result.user;
+        const profile = user?.profile;
+        const displayName = profile?.display_name || profile?.real_name || userId;
+        const teamId = user?.team_id;
+        let fullName = displayName;
+        if (teamId) {
+            const teamName = await this.resolveTeamName(teamId);
+            if (teamName) {
+                fullName = `${displayName} (${teamName})`;
+            }
+        }
+        const slackUser = {
+            id: userId,
+            displayName: fullName,
+            teamId,
+        };
+        this.userCache.set(userId, slackUser);
+        return slackUser;
+    }
+    /**
+     * Get a permalink URL for a specific message.
+     */
+    async getChannelPermalink(channelId, messageTs) {
+        const result = await this.callWithRetry(() => this.client.chat.getPermalink({
+            channel: channelId,
+            message_ts: messageTs,
+        }));
+        return result.permalink;
+    }
+    /**
+     * Resolve a team ID to a team name (for Slack Connect workspace suffix).
+     * Caches results. Returns null if lookup fails (e.g., no permission).
+     */
+    async resolveTeamName(teamId) {
+        const cached = this.teamNameCache.get(teamId);
+        if (cached !== undefined)
+            return cached || null;
+        try {
+            const result = await this.callWithRetry(() => this.client.team.info({ team: teamId }));
+            const name = result.team?.name ?? null;
+            this.teamNameCache.set(teamId, name ?? '');
+            return name;
+        }
+        catch {
+            this.teamNameCache.set(teamId, '');
+            return null;
+        }
+    }
+    /**
+     * Call a Slack API method with retry on rate limit (429) responses.
+     * Respects Retry-After header with exponential backoff fallback.
+     */
+    async callWithRetry(fn, attempt = 1) {
+        try {
+            return await fn();
+        }
+        catch (error) {
+            const isRateLimited = error?.code === 'slack_webapi_rate_limited_error'
+                || error?.data?.error === 'ratelimited';
+            if (!isRateLimited || attempt > this.maxRetries) {
+                throw error;
+            }
+            const retryAfter = error?.data?.retryAfter
+                || error?.retryAfter
+                || Math.pow(2, attempt);
+            const delayMs = Math.max(retryAfter * 1000, 1000);
+            console.warn(`${this.logPrefix} Rate limited (attempt ${attempt}/${this.maxRetries}), ` +
+                `retrying in ${retryAfter}s`);
+            await sleep(delayMs);
+            return this.callWithRetry(fn, attempt + 1);
+        }
+    }
+    /** Expose the underlying WebClient for testing/advanced usage. */
+    get webClient() {
+        return this.client;
+    }
+}
+//# sourceMappingURL=slack-api.js.map

package/dist/indexing/providers/slack-api.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"slack-api.js","sourceRoot":"","sources":["../../../src/indexing/providers/slack-api.ts"],"names":[],"mappings":"AAAA,kFAAkF;AAElF,OAAO,EAAE,SAAS,EAAyB,MAAM,gBAAgB,CAAC;AA0BlE,gFAAgF;AAEhF,SAAS,KAAK,CAAC,EAAU;IACrB,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED,gFAAgF;AAEhF,MAAM,SAAS,GAAG,GAAG,CAAC;AAEtB,MAAM,OAAO,cAAc;IACf,MAAM,CAAY;IAClB,UAAU,CAAS;IACnB,SAAS,GAAG,IAAI,GAAG,EAAqB,CAAC;IACzC,aAAa,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC1C,SAAS,GAAG,aAAa,CAAC;IAElC,YAAY,OAA8B;QACtC,IAAI,CAAC,MAAM,GAAG,IAAI,SAAS,CAAC,OAAO,CAAC,KAAK,EAAE;YACvC,mDAAmD;YACnD,WAAW,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE;SAC9B,CAAC,CAAC;QACH,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,CAAC,CAAC;IAC9C,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,mBAAmB,CAAC,SAAiB,EAAE,MAAe,EAAE,KAAc;QACxE,MAAM,QAAQ,GAAmB,EAAE,CAAC;QACpC,IAAI,MAA0B,CAAC;QAC/B,MAAM,SAAS,GAAG,KAAK,IAAI,GAAG,CAAC;QAC/B,IAAI,KAAK,GAAG,CAAC,CAAC;QAEd,GAAG,CAAC;YACA,KAAK,EAAE,CAAC;YACR,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,CACzC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,OAAO,CAAC;gBAC9B,OAAO,EAAE,SAAS;gBAClB,MAAM;gBACN,MAAM;gBACN,KAAK,EAAE,SAAS;aACnB,CAAC,CACL,CAAC;YAEF,MAAM,WAAW,GAAI,MAAc,CAAC,QAAsC,CAAC;YAC3E,IAAI,WAAW,EAAE,CAAC;gBACd,QAAQ,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,CAAC;YAClC,CAAC;YAED,kFAAkF;YAClF,IAAI,KAAK,KAAK,SAAS;gBAAE,MAAM;YAE/B,MAAM,GAAI,MAAc,CAAC,iBAAiB,EAAE,WAAW,CAAC;YAExD,IAAI,KAAK,IAAI,SAAS,EAAE,CAAC;gBACrB,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,SAAS,uCAAuC,SAAS,iBAAiB,SAAS,EAAE,CAAC,CAAC;gBAC5G,MAAM;YACV,CAAC;QACL,CAAC,QAAQ,MAAM,EAAE;QAEjB,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,kBAAkB,CAAC,SAAiB,EAAE,QAAgB;QACxD,MAAM,QAAQ,GAAmB,EAAE,CAAC;QACpC,IAAI,MAA0B,CAAC;QAC/B,IAAI,KAAK,GAAG,CAAC,CAAC;QAEd,GAAG,CAAC;YACA,KAAK,EAAE,CAAC;YACR,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,CACzC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,OAAO,CAAC;gBAC9B,OAAO,EAAE,SAAS;gBAClB,EAAE,EAAE,QAAQ;gBACZ,MAAM;gBACN,KAAK,EAAE,GAAG;aACb,CAAC,CACL,CAAC;YAEF,MAAM,WAAW,GAAI,MAAc,CAAC,QAAsC,CAAC;YAC3E,IAAI,WAAW,EAAE,CAAC;gBACd,QAAQ,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,CAAC;YAClC,CAAC;YAED,MAAM,GAAI,MAAc,CAAC,iBAAiB,EAAE,WAAW,CAAC;YAExD,IAAI,KAAK,IAAI,SAAS,EAAE,CAAC;gBACrB,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,SAAS,sCAAsC,SAAS,gBAAgB,QAAQ,EAAE,CAAC,CAAC;gBACzG,MAAM;YACV,CAAC;QACL,CAAC,QAAQ,MAAM,EAAE;QAEjB,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,aAAa,CAAC,MAAc;QAC9B,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC1C,IAAI,MAAM;YAAE,OAAO,MAAM,CAAC;QAE1B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,CACzC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAC3C,CAAC;QAEF,MAAM,IAAI,GAAI,MAAc,CAAC,IAAI,CAAC;QAClC,MAAM,OAAO,GAAG,IAAI,EAAE,OAAO,CAAC;QAC9B,MAAM,WAAW,GAAG,OAAO,EAAE,YAAY,IAAI,OAAO,EAAE,SAAS,IAAI,MAAM,CAAC;QAC1E,MAAM,MAAM,GAAG,IAAI,EAAE,OAAO,CAAC;QAE7B,IAAI,QAAQ,GAAG,WAAW,CAAC;QAC3B,IAAI,MAAM,EAAE,CAAC;YACT,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;YACpD,IAAI,QAAQ,EAAE,CAAC;gBACX,QAAQ,GAAG,GAAG,WAAW,KAAK,QAAQ,GAAG,CAAC;YAC9C,CAAC;QACL,CAAC;QAED,MAAM,SAAS,GAAc;YACzB,EAAE,EAAE,MAAM;YACV,WAAW,EAAE,QAAQ;YACrB,MAAM;SACT,CAAC;QAEF,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QACtC,OAAO,SAAS,CAAC;IACrB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,mBAAmB,CAAC,SAAiB,EAAE,SAAiB;QAC1D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,CACzC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC;YAC1B,OAAO,EAAE,SAAS;YAClB,UAAU,EAAE,SAAS;SACxB,CAAC,CACL,CAAC;QAEF,OAAQ,MAAc,CAAC,SAAmB,CAAC;IAC/C,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,eAAe,CAAC,MAAc;QACxC,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC9C,IAAI,MAAM,KAAK,SAAS;YAAE,OAAO,MAAM,IAAI,IAAI,CAAC;QAEhD,IAAI,CAAC;YACD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,CACzC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAC1C,CAAC;YACF,MAAM,IAAI,GAAI,MAAc,CAAC,IAAI,EAAE,IAAI,IAAI,IAAI,CAAC;YAChD,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC;YAC3C,OAAO,IAAI,CAAC;QAChB,CAAC;QAAC,MAAM,CAAC;YACL,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YACnC,OAAO,IAAI,CAAC;QAChB,CAAC;IACL,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,aAAa,CACvB,EAAoB,EACpB,UAAkB,CAAC;QAEnB,IAAI,CAAC;YACD,OAAO,MAAM,EAAE,EAAE,CAAC;QACtB,CAAC;QAAC,OAAO,KAAc,EAAE,CAAC;YACtB,MAAM,aAAa,GAAI,KAAa,EAAE,IAAI,KAAK,iCAAiC;mBACxE,KAAa,EAAE,IAAI,EAAE,KAAK,KAAK,aAAa,CAAC;YAErD,IAAI,CAAC,aAAa,IAAI,OAAO,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;gBAC9C,MAAM,KAAK,CAAC;YAChB,CAAC;YAED,MAAM,UAAU,GAAI,KAAa,EAAE,IAAI,EAAE,UAAU;mBAC3C,KAAa,EAAE,UAAU;mBAC1B,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;YAC5B,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,GAAG,IAAI,EAAE,IAAI,CAAC,CAAC;YAElD,OAAO,CAAC,IAAI,CACR,GAAG,IAAI,CAAC,SAAS,0BAA0B,OAAO,IAAI,IAAI,CAAC,UAAU,KAAK;gBAC1E,eAAe,UAAU,GAAG,CAC/B,CAAC;YAEF,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;YACrB,OAAO,IAAI,CAAC,aAAa,CAAC,EAAE,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC;QAC/C,CAAC;IACL,CAAC;IAED,kEAAkE;IAClE,IAAI,SAAS;QACT,OAAO,IAAI,CAAC,MAAM,CAAC;IACvB,CAAC;CACJ"}

package/dist/indexing/providers/slack.d.ts

@@ -0,0 +1,21 @@
+import type { SourceConfig } from '../../types.js';
+import type { DataProvider, AcquisitionResult, ProviderOptions } from './types.js';
+export declare class SlackDataProvider implements DataProvider {
+    private config;
+    private apiClient;
+    private openaiClient;
+    private logPrefix;
+    constructor(config: SourceConfig, options: ProviderOptions);
+    fullAcquire(): Promise<AcquisitionResult>;
+    incrementalAcquire(lastStateToken: string): Promise<AcquisitionResult>;
+    getCurrentStateToken(): Promise<string | null>;
+    /**
+     * Process a single channel: fetch threads, distill, return items.
+     */
+    private processChannel;
+    /**
+     * Process a single thread: fetch replies, resolve users, distill, create items.
+     */
+    private processThread;
+}
+//# sourceMappingURL=slack.d.ts.map

package/dist/indexing/providers/slack.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"slack.d.ts","sourceRoot":"","sources":["../../../src/indexing/providers/slack.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAAE,YAAY,EAAqB,MAAM,gBAAgB,CAAC;AACtE,OAAO,KAAK,EAAE,YAAY,EAAE,iBAAiB,EAAe,eAAe,EAAE,MAAM,YAAY,CAAC;AAEhG,qBAAa,iBAAkB,YAAW,YAAY;IAClD,OAAO,CAAC,MAAM,CAAoB;IAClC,OAAO,CAAC,SAAS,CAAiB;IAClC,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,SAAS,CAAS;gBAEd,MAAM,EAAE,YAAY,EAAE,OAAO,EAAE,eAAe;IAcpD,WAAW,IAAI,OAAO,CAAC,iBAAiB,CAAC;IA+BzC,kBAAkB,CAAC,cAAc,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,CAAC;IA+BtE,oBAAoB,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAwBpD;;OAEG;YACW,cAAc;IA+B5B;;OAEG;YACW,aAAa;CAqE9B"}