@crowdlisten/harness 1.0.0

package/dist/insights/browser/BrowserPool.js

@@ -0,0 +1,266 @@
+/**
+ * BrowserPool — manages browser sessions for visual extraction.
+ *
+ * Providers:
+ *   - local  → Playwright launches Chromium directly (default, zero-config)
+ *   - remote → Connects to any CDP endpoint (Browserbase, E2B, etc.)
+ *
+ * Each platform gets its own BrowserContext with isolated cookies but shared browser.
+ * Session persistence via cookies saved to ~/.crowdlisten/sessions/{platform}/
+ */
+import { chromium } from 'playwright';
+import * as fs from 'fs';
+import * as path from 'path';
+import * as os from 'os';
+const PLATFORM_PROFILES = {
+    twitter: {
+        viewport: { width: 1280, height: 900 },
+        userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+    },
+    tiktok: {
+        viewport: { width: 1280, height: 900 },
+        userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+    },
+    instagram: {
+        viewport: { width: 1280, height: 900 },
+        userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+    },
+    xiaohongshu: {
+        viewport: { width: 390, height: 844 },
+        userAgent: 'Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1',
+        locale: 'zh-CN',
+        isMobile: true,
+    },
+};
+export class BrowserPool {
+    browser = null;
+    contexts = new Map();
+    activePages = new Set();
+    maxContexts;
+    sessionDir;
+    headless;
+    provider;
+    cdpUrl;
+    constructor(options = {}) {
+        this.maxContexts = options.maxContexts ?? parseInt(process.env.VISUAL_MAX_CONCURRENCY || '5');
+        this.sessionDir = options.sessionDir ?? process.env.VISUAL_SESSION_DIR
+            ?? path.join(os.homedir(), '.crowdlisten', 'sessions');
+        this.headless = options.headless ?? (process.env.VISUAL_HEADLESS !== 'false');
+        this.provider = options.provider
+            ?? process.env.BROWSER_PROVIDER ?? 'local';
+        this.cdpUrl = options.cdpUrl ?? process.env.BROWSER_CDP_URL ?? null;
+    }
+    /**
+     * Launch or connect to a browser based on the configured provider.
+     */
+    async ensureBrowser() {
+        if (this.browser && this.browser.isConnected()) {
+            return this.browser;
+        }
+        this.browser = this.provider === 'remote'
+            ? await this.connectViaRemote()
+            : await this.launchLocal();
+        console.log(`[BrowserPool] Connected via provider: ${this.provider}`);
+        return this.browser;
+    }
+    /**
+     * Local provider: launch Chromium directly via Playwright (default).
+     */
+    async launchLocal() {
+        return chromium.launch({
+            headless: this.headless,
+            args: [
+                '--no-sandbox',
+                '--disable-blink-features=AutomationControlled',
+                '--disable-dev-shm-usage',
+            ],
+        });
+    }
+    /**
+     * Remote provider: connect to an existing CDP endpoint.
+     * Set BROWSER_CDP_URL to your endpoint (Browserbase, E2B, etc.)
+     */
+    async connectViaRemote() {
+        if (!this.cdpUrl) {
+            throw new Error('[BrowserPool] Remote provider requires BROWSER_CDP_URL. Set it to your CDP endpoint.\n' +
+                'Examples:\n' +
+                '  BROWSER_CDP_URL=ws://localhost:9222\n' +
+                '  BROWSER_CDP_URL=wss://connect.browserbase.com?apiKey=YOUR_KEY');
+        }
+        console.log(`[BrowserPool] Connecting to remote browser: ${this.cdpUrl.substring(0, 50)}...`);
+        return chromium.connectOverCDP(this.cdpUrl);
+    }
+    getSessionPath(platform) {
+        return path.join(this.sessionDir, platform);
+    }
+    getCookiePath(platform) {
+        return path.join(this.getSessionPath(platform), 'cookies.json');
+    }
+    /**
+     * Acquire a Page for a specific platform.
+     * Creates a BrowserContext if one doesn't exist for this platform.
+     * Loads persisted cookies if available.
+     */
+    async acquire(platform) {
+        const browser = await this.ensureBrowser();
+        if (!this.contexts.has(platform)) {
+            if (this.contexts.size >= this.maxContexts) {
+                // Evict least recently used context
+                const oldest = this.contexts.keys().next().value;
+                if (oldest) {
+                    await this.releaseContext(oldest);
+                }
+            }
+            const profile = PLATFORM_PROFILES[platform] ?? PLATFORM_PROFILES.twitter;
+            const context = await browser.newContext({
+                viewport: profile.viewport,
+                userAgent: profile.userAgent,
+                locale: profile.locale,
+                isMobile: profile.isMobile,
+                ignoreHTTPSErrors: true,
+            });
+            // Restore persisted cookies
+            await this.loadCookies(context, platform);
+            this.contexts.set(platform, context);
+        }
+        const context = this.contexts.get(platform);
+        const page = await context.newPage();
+        this.activePages.add(page);
+        return page;
+    }
+    /**
+     * Release a page back to the pool. Saves cookies and closes the page.
+     */
+    async release(page) {
+        this.activePages.delete(page);
+        // Find which platform context this page belongs to
+        for (const [platform, context] of this.contexts) {
+            if (context.pages().includes(page)) {
+                await this.saveCookies(context, platform);
+                break;
+            }
+        }
+        if (!page.isClosed()) {
+            await page.close();
+        }
+    }
+    /**
+     * Get a persistent context for a platform (reuses existing chrome profile).
+     * Used for platforms that need login persistence (Twitter, XHS).
+     * Only available with local provider — remote/docker use cookie persistence instead.
+     */
+    async acquirePersistent(platform) {
+        if (this.provider !== 'local') {
+            // Persistent contexts only work locally — fall back to cookie-based persistence
+            console.log(`[BrowserPool] Persistent context unavailable for ${this.provider} provider, using cookie persistence`);
+            const page = await this.acquire(platform);
+            const context = this.contexts.get(platform);
+            return { context, page };
+        }
+        const profilePath = this.getPersistentProfilePath(platform);
+        fs.mkdirSync(profilePath, { recursive: true });
+        const profile = PLATFORM_PROFILES[platform] ?? PLATFORM_PROFILES.twitter;
+        const context = await chromium.launchPersistentContext(profilePath, {
+            headless: this.headless,
+            viewport: profile.viewport,
+            userAgent: profile.userAgent,
+            locale: profile.locale,
+            args: [
+                '--no-sandbox',
+                '--disable-blink-features=AutomationControlled',
+            ],
+        });
+        this.contexts.set(`persistent_${platform}`, context);
+        const pages = context.pages();
+        const page = pages.length > 0 ? pages[0] : await context.newPage();
+        this.activePages.add(page);
+        return { context, page };
+    }
+    getPersistentProfilePath(platform) {
+        const envMap = {
+            twitter: process.env.TWITTER_CHROME_PROFILE_PATH,
+            tiktok: process.env.TIKTOK_CHROME_PROFILE_PATH,
+            xiaohongshu: process.env.XHS_CHROME_PROFILE_PATH,
+            instagram: process.env.INSTAGRAM_CHROME_PROFILE_PATH,
+        };
+        return envMap[platform] ?? path.join(this.sessionDir, `${platform}-profile`);
+    }
+    async loadCookies(context, platform) {
+        const cookiePath = this.getCookiePath(platform);
+        try {
+            if (fs.existsSync(cookiePath)) {
+                const cookies = JSON.parse(fs.readFileSync(cookiePath, 'utf-8'));
+                if (Array.isArray(cookies) && cookies.length > 0) {
+                    await context.addCookies(cookies);
+                    console.log(`[BrowserPool] Loaded ${cookies.length} cookies for ${platform}`);
+                }
+            }
+        }
+        catch (err) {
+            console.warn(`[BrowserPool] Failed to load cookies for ${platform}:`, err);
+        }
+    }
+    async saveCookies(context, platform) {
+        const cookiePath = this.getCookiePath(platform);
+        try {
+            const sessionPath = this.getSessionPath(platform);
+            fs.mkdirSync(sessionPath, { recursive: true });
+            const cookies = await context.cookies();
+            fs.writeFileSync(cookiePath, JSON.stringify(cookies, null, 2));
+            console.log(`[BrowserPool] Saved ${cookies.length} cookies for ${platform}`);
+        }
+        catch (err) {
+            console.warn(`[BrowserPool] Failed to save cookies for ${platform}:`, err);
+        }
+    }
+    async releaseContext(platform) {
+        const context = this.contexts.get(platform);
+        if (context) {
+            await this.saveCookies(context, platform);
+            // Close all pages in this context
+            for (const page of context.pages()) {
+                this.activePages.delete(page);
+            }
+            await context.close();
+            this.contexts.delete(platform);
+        }
+    }
+    /**
+     * Close all contexts and the browser.
+     */
+    async cleanup() {
+        for (const [platform, context] of this.contexts) {
+            try {
+                await this.saveCookies(context, platform);
+                await context.close();
+            }
+            catch (err) {
+                console.warn(`[BrowserPool] Error cleaning up ${platform}:`, err);
+            }
+        }
+        this.contexts.clear();
+        this.activePages.clear();
+        if (this.browser) {
+            await this.browser.close();
+            this.browser = null;
+        }
+        console.log('[BrowserPool] Cleanup complete');
+    }
+    get activeContextCount() {
+        return this.contexts.size;
+    }
+    get activePageCount() {
+        return this.activePages.size;
+    }
+    get currentProvider() {
+        return this.provider;
+    }
+}
+// Singleton pool instance
+let _poolInstance = null;
+export function getBrowserPool(options) {
+    if (!_poolInstance) {
+        _poolInstance = new BrowserPool(options);
+    }
+    return _poolInstance;
+}

package/dist/insights/browser/RequestInterceptor.d.ts

@@ -0,0 +1,46 @@
+/**
+ * RequestInterceptor — captures JSON responses from internal platform API calls.
+ * Registers response listeners on a Playwright Page, filters by URL patterns,
+ * and stores structured JSON for later retrieval.
+ */
+import { Page } from 'playwright';
+export interface InterceptedResponse {
+    url: string;
+    status: number;
+    data: any;
+    timestamp: number;
+    matchedPattern: string;
+}
+export declare class RequestInterceptor {
+    private intercepted;
+    private patterns;
+    private seenHashes;
+    private listening;
+    /**
+     * Set up response interception on a page for the given URL patterns.
+     * Patterns are substring-matched against response URLs.
+     */
+    setup(page: Page, urlPatterns: string[]): Promise<void>;
+    /**
+     * Get all intercepted responses matching a specific pattern.
+     */
+    getIntercepted(pattern?: string): InterceptedResponse[];
+    /**
+     * Get all intercepted data payloads (just the JSON bodies).
+     */
+    getAllData(pattern?: string): any[];
+    /**
+     * Wait for a response matching a specific pattern, with timeout.
+     */
+    waitForResponse(page: Page, pattern: string, timeout?: number): Promise<InterceptedResponse | null>;
+    /**
+     * Clear all captured data.
+     */
+    clear(): void;
+    /**
+     * Stop listening for responses.
+     */
+    stop(): void;
+    get count(): number;
+    private computeHash;
+}

package/dist/insights/browser/RequestInterceptor.js

@@ -0,0 +1,115 @@
+/**
+ * RequestInterceptor — captures JSON responses from internal platform API calls.
+ * Registers response listeners on a Playwright Page, filters by URL patterns,
+ * and stores structured JSON for later retrieval.
+ */
+export class RequestInterceptor {
+    intercepted = [];
+    patterns = [];
+    seenHashes = new Set();
+    listening = false;
+    /**
+     * Set up response interception on a page for the given URL patterns.
+     * Patterns are substring-matched against response URLs.
+     */
+    async setup(page, urlPatterns) {
+        this.patterns = urlPatterns;
+        this.intercepted = [];
+        this.seenHashes.clear();
+        this.listening = true;
+        page.on('response', async (response) => {
+            if (!this.listening)
+                return;
+            const url = response.url();
+            const matchedPattern = this.patterns.find(p => url.includes(p));
+            if (!matchedPattern)
+                return;
+            try {
+                const contentType = response.headers()['content-type'] || '';
+                if (!contentType.includes('json') && !contentType.includes('javascript')) {
+                    return;
+                }
+                const status = response.status();
+                if (status < 200 || status >= 400)
+                    return;
+                const body = await response.json().catch(() => null);
+                if (!body)
+                    return;
+                // Deduplicate by hashing URL + stringified body
+                const hash = this.computeHash(url, body);
+                if (this.seenHashes.has(hash))
+                    return;
+                this.seenHashes.add(hash);
+                this.intercepted.push({
+                    url,
+                    status,
+                    data: body,
+                    timestamp: Date.now(),
+                    matchedPattern,
+                });
+            }
+            catch {
+                // Non-JSON response or body read failure — skip silently
+            }
+        });
+    }
+    /**
+     * Get all intercepted responses matching a specific pattern.
+     */
+    getIntercepted(pattern) {
+        if (!pattern)
+            return [...this.intercepted];
+        return this.intercepted.filter(r => r.matchedPattern === pattern || r.url.includes(pattern));
+    }
+    /**
+     * Get all intercepted data payloads (just the JSON bodies).
+     */
+    getAllData(pattern) {
+        return this.getIntercepted(pattern).map(r => r.data);
+    }
+    /**
+     * Wait for a response matching a specific pattern, with timeout.
+     */
+    async waitForResponse(page, pattern, timeout = 15000) {
+        // Check if we already have it
+        const existing = this.intercepted.find(r => r.url.includes(pattern));
+        if (existing)
+            return existing;
+        return new Promise((resolve) => {
+            const startTime = Date.now();
+            const check = setInterval(() => {
+                const match = this.intercepted.find(r => r.url.includes(pattern) && r.timestamp > startTime);
+                if (match) {
+                    clearInterval(check);
+                    resolve(match);
+                }
+                else if (Date.now() - startTime > timeout) {
+                    clearInterval(check);
+                    resolve(null);
+                }
+            }, 200);
+        });
+    }
+    /**
+     * Clear all captured data.
+     */
+    clear() {
+        this.intercepted = [];
+        this.seenHashes.clear();
+    }
+    /**
+     * Stop listening for responses.
+     */
+    stop() {
+        this.listening = false;
+    }
+    get count() {
+        return this.intercepted.length;
+    }
+    computeHash(url, body) {
+        // Simple hash for deduplication — URL path + first 200 chars of stringified body
+        const urlPath = new URL(url).pathname;
+        const bodyStr = JSON.stringify(body).substring(0, 200);
+        return `${urlPath}::${bodyStr}`;
+    }
+}

package/dist/insights/cli.d.ts

@@ -0,0 +1,8 @@
+#!/usr/bin/env node
+/**
+ * CrowdListen CLI
+ * Cross-channel feedback analysis for AI agents.
+ * Extracts audience signal (pain points, feature requests, sentiment) from
+ * social platforms into structured JSON. stdout = data, stderr = errors.
+ */
+export {};

package/dist/insights/cli.js

@@ -0,0 +1,206 @@
+#!/usr/bin/env node
+/**
+ * CrowdListen CLI
+ * Cross-channel feedback analysis for AI agents.
+ * Extracts audience signal (pain points, feature requests, sentiment) from
+ * social platforms into structured JSON. stdout = data, stderr = errors.
+ */
+import { Command } from 'commander';
+import { createService } from './service-config.js';
+import { searchContent, getContentComments, analyzeContent, clusterOpinions, getTrendingContent, getUserContent, getPlatformStatus, healthCheck, deepAnalyze, extractInsights, researchSynthesis, extractWithVision, } from './handlers.js';
+const program = new Command();
+program
+    .name('crowdlisten')
+    .description('Cross-channel feedback analysis for AI agents')
+    .version('2.0.0');
+// Shared service instance — initialized lazily
+let service = null;
+let initialized = false;
+async function getService() {
+    if (!service) {
+        service = createService();
+    }
+    if (!initialized) {
+        const results = await service.initialize();
+        const ok = Object.entries(results).filter(([, s]) => s).map(([p]) => p);
+        if (ok.length === 0) {
+            console.error('Error: No platforms initialized successfully');
+            process.exit(1);
+        }
+        console.error(`[crowdlisten] Platforms: ${ok.join(', ')}`);
+        initialized = true;
+    }
+    return service;
+}
+function output(data) {
+    console.log(JSON.stringify(data, null, 2));
+}
+async function run(fn) {
+    try {
+        const result = await fn();
+        output(result);
+        process.exit(0);
+    }
+    catch (err) {
+        console.error(`Error: ${err.message || err}`);
+        process.exit(1);
+    }
+}
+// --- Commands ---
+program
+    .command('search <platform> <query>')
+    .description('Search social media for audience conversations')
+    .option('-l, --limit <n>', 'Max results', '10')
+    .option('--vision', 'Force vision extraction mode')
+    .action(async (platform, query, opts) => {
+    await run(async () => {
+        const svc = await getService();
+        return searchContent(svc, {
+            platform,
+            query,
+            limit: parseInt(opts.limit),
+            useVision: opts.vision || false,
+        });
+    });
+});
+program
+    .command('comments <platform> <contentId>')
+    .description('Get comments for a specific post/video')
+    .option('-l, --limit <n>', 'Max comments', '20')
+    .option('--vision', 'Force vision extraction mode')
+    .action(async (platform, contentId, opts) => {
+    await run(async () => {
+        const svc = await getService();
+        return getContentComments(svc, {
+            platform,
+            contentId,
+            limit: parseInt(opts.limit),
+            useVision: opts.vision || false,
+        });
+    });
+});
+program
+    .command('analyze <platform> <contentId>')
+    .description('Full analysis pipeline: comments + clustering + sentiment')
+    .option('-d, --depth <level>', 'Analysis depth (surface|standard|deep|comprehensive)', 'standard')
+    .option('--no-clustering', 'Disable opinion clustering')
+    .action(async (platform, contentId, opts) => {
+    const depth = opts.depth;
+    // Route deep/comprehensive to paid agent API
+    if (depth === 'deep' || depth === 'comprehensive') {
+        await run(async () => deepAnalyze({ platform, contentId, analysisDepth: depth }));
+    }
+    else {
+        await run(async () => {
+            const svc = await getService();
+            return analyzeContent(svc, {
+                platform,
+                contentId,
+                analysisDepth: depth,
+            });
+        });
+    }
+});
+program
+    .command('cluster <platform> <contentId>')
+    .description('Cluster opinions from comments using embeddings')
+    .option('-n, --clusters <n>', 'Number of clusters', '5')
+    .option('--no-examples', 'Exclude example comments')
+    .action(async (platform, contentId, opts) => {
+    await run(async () => {
+        const svc = await getService();
+        return clusterOpinions(svc, {
+            platform,
+            contentId,
+            clusterCount: parseInt(opts.clusters),
+            includeExamples: opts.examples !== false,
+            weightByEngagement: true,
+        });
+    });
+});
+program
+    .command('trending <platform>')
+    .description('Get trending content from a platform')
+    .option('-l, --limit <n>', 'Max results', '10')
+    .action(async (platform, opts) => {
+    await run(async () => {
+        const svc = await getService();
+        return getTrendingContent(svc, { platform, limit: parseInt(opts.limit) });
+    });
+});
+program
+    .command('user <platform> <userId>')
+    .description('Get content from a specific user')
+    .option('-l, --limit <n>', 'Max results', '10')
+    .action(async (platform, userId, opts) => {
+    await run(async () => {
+        const svc = await getService();
+        return getUserContent(svc, { platform, userId, limit: parseInt(opts.limit) });
+    });
+});
+program
+    .command('vision <url>')
+    .description('Extract content from any URL using vision (LLM screenshot analysis)')
+    .option('-m, --mode <mode>', 'Extraction mode (posts|comments|raw)', 'posts')
+    .option('-l, --limit <n>', 'Max results', '10')
+    .action(async (url, opts) => {
+    await run(async () => {
+        return extractWithVision({
+            url,
+            mode: opts.mode,
+            limit: parseInt(opts.limit),
+        });
+    });
+});
+program
+    .command('status')
+    .description('Show available platforms and capabilities')
+    .action(async () => {
+    await run(async () => {
+        const svc = await getService();
+        return getPlatformStatus(svc);
+    });
+});
+program
+    .command('health')
+    .description('Check health of all platforms')
+    .action(async () => {
+    await run(async () => {
+        const svc = await getService();
+        return healthCheck(svc);
+    });
+});
+// --- Paid Commands (require CROWDLISTEN_API_KEY) ---
+program
+    .command('insights <platform> <contentId>')
+    .description('Extract structured insights with categorization and confidence (paid)')
+    .option('-c, --categories <list>', 'Comma-separated insight categories', '')
+    .action(async (platform, contentId, opts) => {
+    await run(async () => extractInsights({
+        platform,
+        contentId,
+        categories: opts.categories ? opts.categories.split(',') : undefined,
+    }));
+});
+program
+    .command('research <query>')
+    .description('Multi-source research synthesis with AI analysis (paid)')
+    .option('-p, --platforms <list>', 'Comma-separated platforms', 'reddit,twitter,youtube')
+    .option('-d, --depth <level>', 'Research depth (quick|standard|deep)', 'standard')
+    .action(async (query, opts) => {
+    await run(async () => researchSynthesis({
+        query,
+        platforms: opts.platforms.split(','),
+        depth: opts.depth,
+    }));
+});
+// CLI only — MCP server is now unified in the parent package's index.ts
+// If called with no args via piped stdin, print help
+if (process.argv.length <= 2 && !process.stdin.isTTY) {
+    console.error('CrowdListen insights CLI. Use the unified MCP server instead:');
+    console.error('  npx @crowdlisten/harness');
+    process.exit(0);
+}
+else {
+    program.parse();
+}

package/dist/insights/core/base/BaseAdapter.d.ts

@@ -0,0 +1,37 @@
+/**
+ * Base adapter class providing common functionality for all social media platforms
+ * Implements shared features like rate limiting, error handling, and logging
+ *
+ * Analysis (clustering, enrichment) has been moved to the CrowdListen API.
+ * This class now only handles data retrieval.
+ */
+import { SocialMediaPlatform, PlatformType, PlatformConfig, PlatformCapabilities, Post, Comment, ContentAnalysis } from '../interfaces/SocialMediaPlatform.js';
+export declare abstract class BaseAdapter implements SocialMediaPlatform {
+    protected config: PlatformConfig;
+    protected isInitialized: boolean;
+    protected lastRequestTime: number;
+    protected requestCount: number;
+    protected rateLimitWindow: number;
+    protected maxRequestsPerWindow: number;
+    constructor(config: PlatformConfig);
+    protected enforceRateLimit(): Promise<void>;
+    protected sleep(ms: number): Promise<void>;
+    protected handleError(error: any, operation: string): never;
+    protected isRateLimitError(error: any): boolean;
+    protected isAuthError(error: any): boolean;
+    protected isNotFoundError(error: any): boolean;
+    protected log(message: string, level?: 'info' | 'warn' | 'error'): void;
+    protected validateUserId(userId: string): void;
+    protected validateContentId(contentId: string): void;
+    protected validateLimit(limit: number): void;
+    analyzeContent(contentId: string, enableClustering?: boolean): Promise<ContentAnalysis>;
+    protected ensureInitialized(): void;
+    abstract getTrendingContent(limit?: number): Promise<Post[]>;
+    abstract getUserContent(userId: string, limit?: number): Promise<Post[]>;
+    abstract searchContent(query: string, limit?: number): Promise<Post[]>;
+    abstract getContentComments(contentId: string, limit?: number): Promise<Comment[]>;
+    abstract getPlatformName(): PlatformType;
+    abstract getSupportedFeatures(): PlatformCapabilities;
+    abstract initialize(): Promise<boolean>;
+    cleanup(): Promise<void>;
+}