@crowdlisten/harness 1.0.0

package/dist/insights/platforms/youtube/YouTubeAdapter.js

@@ -0,0 +1,254 @@
+/**
+ * YouTube Platform Adapter
+ * Uses YouTube Data API v3 for video search, trending, comments, and channel content
+ */
+import { BaseAdapter } from '../../core/base/BaseAdapter.js';
+import { DataNormalizer } from '../../core/utils/DataNormalizer.js';
+import { NotFoundError } from '../../core/interfaces/SocialMediaPlatform.js';
+import axios from 'axios';
+export class YouTubeAdapter extends BaseAdapter {
+    client = null;
+    apiKey = '';
+    constructor(config) {
+        super(config);
+        // YouTube API quota is 10,000 units/day; keep requests conservative
+        this.maxRequestsPerWindow = 20;
+    }
+    async initialize() {
+        try {
+            this.apiKey = this.config.credentials?.apiKey || process.env.YOUTUBE_API_KEY || '';
+            if (!this.apiKey) {
+                this.log('No YouTube API key provided', 'warn');
+                return false;
+            }
+            this.client = axios.create({
+                baseURL: 'https://www.googleapis.com/youtube/v3',
+                timeout: 15000
+            });
+            this.isInitialized = true;
+            this.log('YouTube adapter initialized successfully');
+            return true;
+        }
+        catch (error) {
+            this.log('Failed to initialize YouTube adapter', 'error');
+            this.isInitialized = false;
+            return false;
+        }
+    }
+    async getTrendingContent(limit = 10) {
+        this.ensureInitialized();
+        this.validateLimit(limit);
+        try {
+            await this.enforceRateLimit();
+            const response = await this.client.get('/videos', {
+                params: {
+                    part: 'snippet,statistics',
+                    chart: 'mostPopular',
+                    regionCode: 'US',
+                    maxResults: Math.min(limit, 50),
+                    key: this.apiKey
+                }
+            });
+            const posts = [];
+            for (const item of (response.data.items || []).slice(0, limit)) {
+                posts.push(DataNormalizer.normalizePost(item, 'youtube'));
+            }
+            this.log(`Retrieved ${posts.length} trending YouTube videos`);
+            return posts;
+        }
+        catch (error) {
+            this.handleError(error, 'getTrendingContent');
+        }
+    }
+    async getUserContent(userId, limit = 10) {
+        this.ensureInitialized();
+        this.validateUserId(userId);
+        this.validateLimit(limit);
+        try {
+            await this.enforceRateLimit();
+            // First resolve the channel ID — userId could be a channel ID or a handle
+            let channelId = userId;
+            if (!userId.startsWith('UC')) {
+                const handle = userId.startsWith('@') ? userId : `@${userId}`;
+                const channelRes = await this.client.get('/channels', {
+                    params: {
+                        part: 'id',
+                        forHandle: handle,
+                        key: this.apiKey
+                    }
+                });
+                channelId = channelRes.data.items?.[0]?.id;
+                if (!channelId) {
+                    throw new NotFoundError('youtube', `Channel ${userId}`);
+                }
+            }
+            await this.enforceRateLimit();
+            // Search for videos on this channel
+            const response = await this.client.get('/search', {
+                params: {
+                    part: 'snippet',
+                    channelId,
+                    type: 'video',
+                    order: 'date',
+                    maxResults: Math.min(limit, 50),
+                    key: this.apiKey
+                }
+            });
+            // Get video statistics in a separate call
+            const videoIds = (response.data.items || [])
+                .map((item) => item.id?.videoId)
+                .filter(Boolean)
+                .join(',');
+            let statsMap = {};
+            if (videoIds) {
+                await this.enforceRateLimit();
+                const statsRes = await this.client.get('/videos', {
+                    params: {
+                        part: 'statistics',
+                        id: videoIds,
+                        key: this.apiKey
+                    }
+                });
+                for (const v of statsRes.data.items || []) {
+                    statsMap[v.id] = v.statistics;
+                }
+            }
+            const posts = [];
+            for (const item of (response.data.items || []).slice(0, limit)) {
+                const videoId = item.id?.videoId;
+                const merged = {
+                    id: videoId,
+                    snippet: item.snippet,
+                    statistics: statsMap[videoId] || {}
+                };
+                posts.push(DataNormalizer.normalizePost(merged, 'youtube'));
+            }
+            this.log(`Retrieved ${posts.length} videos from YouTube channel ${userId}`);
+            return posts;
+        }
+        catch (error) {
+            if (error.response?.status === 404) {
+                throw new NotFoundError('youtube', `Channel ${userId}`, error);
+            }
+            this.handleError(error, 'getUserContent');
+        }
+    }
+    async searchContent(query, limit = 10) {
+        this.ensureInitialized();
+        this.validateLimit(limit);
+        if (!query || query.trim().length === 0) {
+            throw new Error('Search query cannot be empty');
+        }
+        try {
+            await this.enforceRateLimit();
+            const response = await this.client.get('/search', {
+                params: {
+                    part: 'snippet',
+                    q: query.trim(),
+                    type: 'video',
+                    order: 'relevance',
+                    maxResults: Math.min(limit, 50),
+                    key: this.apiKey
+                }
+            });
+            // Get video statistics
+            const videoIds = (response.data.items || [])
+                .map((item) => item.id?.videoId)
+                .filter(Boolean)
+                .join(',');
+            let statsMap = {};
+            if (videoIds) {
+                await this.enforceRateLimit();
+                const statsRes = await this.client.get('/videos', {
+                    params: {
+                        part: 'statistics',
+                        id: videoIds,
+                        key: this.apiKey
+                    }
+                });
+                for (const v of statsRes.data.items || []) {
+                    statsMap[v.id] = v.statistics;
+                }
+            }
+            const posts = [];
+            for (const item of (response.data.items || []).slice(0, limit)) {
+                const videoId = item.id?.videoId;
+                const merged = {
+                    id: videoId,
+                    snippet: item.snippet,
+                    statistics: statsMap[videoId] || {}
+                };
+                posts.push(DataNormalizer.normalizePost(merged, 'youtube'));
+            }
+            this.log(`Found ${posts.length} YouTube videos for query: ${query}`);
+            return posts;
+        }
+        catch (error) {
+            this.handleError(error, 'searchContent');
+        }
+    }
+    async getContentComments(contentId, limit = 20) {
+        this.ensureInitialized();
+        this.validateContentId(contentId);
+        this.validateLimit(limit);
+        try {
+            await this.enforceRateLimit();
+            const response = await this.client.get('/commentThreads', {
+                params: {
+                    part: 'snippet,replies',
+                    videoId: contentId,
+                    maxResults: Math.min(limit, 100),
+                    order: 'relevance',
+                    key: this.apiKey
+                }
+            });
+            const comments = [];
+            for (const item of (response.data.items || []).slice(0, limit)) {
+                comments.push(DataNormalizer.normalizeComment(item, 'youtube'));
+            }
+            this.log(`Retrieved ${comments.length} comments for YouTube video ${contentId}`);
+            return comments;
+        }
+        catch (error) {
+            if (error.response?.status === 403 &&
+                error.response?.data?.error?.errors?.[0]?.reason === 'commentsDisabled') {
+                this.log(`Comments are disabled for video ${contentId}`, 'warn');
+                return [];
+            }
+            this.handleError(error, 'getContentComments');
+        }
+    }
+    getPlatformName() {
+        return 'youtube';
+    }
+    getSupportedFeatures() {
+        return {
+            supportsTrending: true,
+            supportsUserContent: true,
+            supportsSearch: true,
+            supportsComments: true,
+            supportsAnalysis: true
+        };
+    }
+    isRateLimitError(error) {
+        return error.response?.status === 429 ||
+            error.response?.data?.error?.errors?.[0]?.reason === 'quotaExceeded' ||
+            error.response?.data?.error?.errors?.[0]?.reason === 'rateLimitExceeded';
+    }
+    isAuthError(error) {
+        return error.response?.status === 401 ||
+            error.response?.status === 403;
+    }
+    isNotFoundError(error) {
+        return error.response?.status === 404;
+    }
+    async cleanup() {
+        try {
+            this.client = null;
+            await super.cleanup();
+        }
+        catch (error) {
+            this.log('Error during YouTube cleanup', 'warn');
+        }
+    }
+}

package/dist/insights/service-config.d.ts

@@ -0,0 +1,7 @@
+/**
+ * CrowdListen Service Configuration
+ * Shared service factory used by MCP and CLI entry points.
+ */
+import { UnifiedSocialMediaService, UnifiedServiceConfig } from './services/UnifiedSocialMediaService.js';
+export declare function createServiceConfig(): UnifiedServiceConfig;
+export declare function createService(): UnifiedSocialMediaService;

package/dist/insights/service-config.js

@@ -0,0 +1,60 @@
+/**
+ * CrowdListen Service Configuration
+ * Shared service factory used by MCP and CLI entry points.
+ */
+import * as dotenv from 'dotenv';
+import { UnifiedSocialMediaService } from './services/UnifiedSocialMediaService.js';
+dotenv.config();
+export function createServiceConfig() {
+    const config = {
+        platforms: {},
+        globalOptions: {
+            timeout: 30000,
+            retries: 3,
+            fallbackStrategy: 'continue',
+        },
+    };
+    // Twitter — uses twitter-scraper, needs TWITTER_USERNAME + TWITTER_PASSWORD
+    config.platforms.twitter = {
+        platform: 'twitter',
+        credentials: {},
+    };
+    // TikTok — browser adapter
+    config.platforms.tiktok = {
+        platform: 'tiktok',
+        credentials: {},
+    };
+    // Instagram — browser adapter
+    config.platforms.instagram = {
+        platform: 'instagram',
+        credentials: {},
+    };
+    // Xiaohongshu — browser adapter
+    config.platforms.xiaohongshu = {
+        platform: 'xiaohongshu',
+        credentials: {},
+    };
+    // Reddit (no credentials needed for public content)
+    config.platforms.reddit = {
+        platform: 'reddit',
+        credentials: {},
+    };
+    // YouTube
+    if (process.env.YOUTUBE_API_KEY) {
+        config.platforms.youtube = {
+            platform: 'youtube',
+            credentials: { apiKey: process.env.YOUTUBE_API_KEY },
+        };
+    }
+    // Moltbook
+    if (process.env.MOLTBOOK_API_KEY) {
+        config.platforms.moltbook = {
+            platform: 'moltbook',
+            credentials: { apiKey: process.env.MOLTBOOK_API_KEY },
+        };
+    }
+    return config;
+}
+export function createService() {
+    return new UnifiedSocialMediaService(createServiceConfig());
+}

package/dist/insights/services/UnifiedSocialMediaService.d.ts

@@ -0,0 +1,94 @@
+/**
+ * Unified Social Media Service
+ * Coordinates all platform adapters and provides a single interface.
+ * Each platform has one adapter — no visual vs legacy distinction.
+ */
+import { PlatformType, PlatformConfig, Post, Comment } from '../core/interfaces/SocialMediaPlatform.js';
+export interface UnifiedServiceConfig {
+    platforms: {
+        tiktok?: PlatformConfig;
+        twitter?: PlatformConfig;
+        reddit?: PlatformConfig;
+        instagram?: PlatformConfig;
+        youtube?: PlatformConfig;
+        moltbook?: PlatformConfig;
+        xiaohongshu?: PlatformConfig;
+    };
+    globalOptions?: {
+        timeout?: number;
+        retries?: number;
+        fallbackStrategy?: 'fail' | 'continue' | 'mock';
+    };
+}
+export declare class UnifiedSocialMediaService {
+    private adapters;
+    private config;
+    private isInitialized;
+    constructor(config: UnifiedServiceConfig);
+    /**
+     * Initialize all configured platform adapters.
+     * One adapter per platform — flat, no tiers.
+     */
+    initialize(): Promise<{
+        [key in PlatformType]?: boolean;
+    }>;
+    /**
+     * Get trending content from a specific platform
+     */
+    getTrendingContent(platform: PlatformType, limit?: number): Promise<Post[]>;
+    /**
+     * Get trending content from all available platforms
+     */
+    getAllTrendingContent(limit?: number): Promise<{
+        [key in PlatformType]?: Post[];
+    }>;
+    /**
+     * Get user content from a specific platform
+     */
+    getUserContent(platform: PlatformType, userId: string, limit?: number): Promise<Post[]>;
+    /**
+     * Search content on a specific platform
+     */
+    searchContent(platform: PlatformType, query: string, limit?: number): Promise<Post[]>;
+    /**
+     * Search content across all available platforms
+     */
+    searchAllPlatforms(query: string, limit?: number): Promise<{
+        [key in PlatformType]?: Post[];
+    }>;
+    /**
+     * Get comments for content on a specific platform
+     */
+    getContentComments(platform: PlatformType, contentId: string, limit?: number): Promise<Comment[]>;
+    /**
+     * Get the list of initialized platform types.
+     * Used by HealthMonitor to know which platforms to probe.
+     */
+    getInitializedPlatforms(): PlatformType[];
+    /**
+     * Get available platforms and their capabilities
+     */
+    getAvailablePlatforms(): {
+        [key in PlatformType]?: any;
+    };
+    /**
+     * Get combined trending content with platform attribution
+     */
+    getCombinedTrendingContent(limit?: number): Promise<Post[]>;
+    /**
+     * Search across all platforms and return combined results
+     */
+    getCombinedSearchResults(query: string, limit?: number): Promise<Post[]>;
+    /**
+     * Platform health check
+     */
+    healthCheck(): Promise<{
+        [key in PlatformType]?: 'healthy' | 'degraded' | 'down';
+    }>;
+    /**
+     * Cleanup all adapters and browser pool
+     */
+    cleanup(): Promise<void>;
+    private getAdapter;
+    private calculateRelevance;
+}

package/dist/insights/services/UnifiedSocialMediaService.js

@@ -0,0 +1,259 @@
+/**
+ * Unified Social Media Service
+ * Coordinates all platform adapters and provides a single interface.
+ * Each platform has one adapter — no visual vs legacy distinction.
+ */
+import { SocialMediaError } from '../core/interfaces/SocialMediaPlatform.js';
+import { TwitterAdapter } from '../platforms/twitter/TwitterAdapter.js';
+import { RedditAdapter } from '../platforms/reddit/RedditAdapter.js';
+import { YouTubeAdapter } from '../platforms/youtube/YouTubeAdapter.js';
+import { MoltbookAdapter } from '../platforms/moltbook/MoltbookAdapter.js';
+import { TikTokAdapter } from '../platforms/tiktok/TikTokAdapter.js';
+import { InstagramAdapter } from '../platforms/instagram/InstagramAdapter.js';
+import { XiaohongshuAdapter } from '../platforms/xiaohongshu/XiaohongshuAdapter.js';
+import { getBrowserPool } from '../browser/BrowserPool.js';
+export class UnifiedSocialMediaService {
+    adapters = new Map();
+    config;
+    isInitialized = false;
+    constructor(config) {
+        this.config = config;
+    }
+    /**
+     * Initialize all configured platform adapters.
+     * One adapter per platform — flat, no tiers.
+     */
+    async initialize() {
+        const results = {};
+        const adapterMap = [
+            { platform: 'twitter', create: () => new TwitterAdapter(this.config.platforms.twitter) },
+            { platform: 'tiktok', create: () => new TikTokAdapter(this.config.platforms.tiktok) },
+            { platform: 'instagram', create: () => new InstagramAdapter(this.config.platforms.instagram) },
+            { platform: 'xiaohongshu', create: () => new XiaohongshuAdapter(this.config.platforms.xiaohongshu) },
+            { platform: 'reddit', create: () => new RedditAdapter(this.config.platforms.reddit) },
+            { platform: 'youtube', create: () => new YouTubeAdapter(this.config.platforms.youtube) },
+            { platform: 'moltbook', create: () => new MoltbookAdapter(this.config.platforms.moltbook) },
+        ];
+        for (const { platform, create } of adapterMap) {
+            if (!this.config.platforms[platform])
+                continue;
+            try {
+                const adapter = create();
+                const success = await adapter.initialize();
+                if (success) {
+                    this.adapters.set(platform, adapter);
+                }
+                results[platform] = success;
+            }
+            catch (error) {
+                console.error(`Failed to initialize ${platform} adapter:`, error);
+                results[platform] = false;
+            }
+        }
+        this.isInitialized = true;
+        console.log(`Unified Service initialized with ${this.adapters.size} platforms:`, Array.from(this.adapters.keys()));
+        return results;
+    }
+    /**
+     * Get trending content from a specific platform
+     */
+    async getTrendingContent(platform, limit) {
+        const adapter = this.getAdapter(platform);
+        return await adapter.getTrendingContent(limit);
+    }
+    /**
+     * Get trending content from all available platforms
+     */
+    async getAllTrendingContent(limit) {
+        const results = {};
+        const limitPerPlatform = limit ? Math.ceil(limit / this.adapters.size) : 10;
+        const promises = Array.from(this.adapters.entries()).map(async ([platform, adapter]) => {
+            try {
+                const posts = await adapter.getTrendingContent(limitPerPlatform);
+                results[platform] = posts;
+            }
+            catch (error) {
+                console.error(`Failed to get trending content from ${platform}:`, error);
+                if (this.config.globalOptions?.fallbackStrategy === 'continue') {
+                    results[platform] = [];
+                }
+            }
+        });
+        await Promise.allSettled(promises);
+        return results;
+    }
+    /**
+     * Get user content from a specific platform
+     */
+    async getUserContent(platform, userId, limit) {
+        const adapter = this.getAdapter(platform);
+        return await adapter.getUserContent(userId, limit);
+    }
+    /**
+     * Search content on a specific platform
+     */
+    async searchContent(platform, query, limit) {
+        const adapter = this.getAdapter(platform);
+        return await adapter.searchContent(query, limit);
+    }
+    /**
+     * Search content across all available platforms
+     */
+    async searchAllPlatforms(query, limit) {
+        const results = {};
+        const limitPerPlatform = limit ? Math.ceil(limit / this.adapters.size) : 10;
+        const promises = Array.from(this.adapters.entries()).map(async ([platform, adapter]) => {
+            try {
+                const posts = await adapter.searchContent(query, limitPerPlatform);
+                results[platform] = posts;
+            }
+            catch (error) {
+                console.error(`Failed to search ${platform} for "${query}":`, error);
+                if (this.config.globalOptions?.fallbackStrategy === 'continue') {
+                    results[platform] = [];
+                }
+            }
+        });
+        await Promise.allSettled(promises);
+        return results;
+    }
+    /**
+     * Get comments for content on a specific platform
+     */
+    async getContentComments(platform, contentId, limit) {
+        const adapter = this.getAdapter(platform);
+        return await adapter.getContentComments(contentId, limit);
+    }
+    /**
+     * Get the list of initialized platform types.
+     * Used by HealthMonitor to know which platforms to probe.
+     */
+    getInitializedPlatforms() {
+        return Array.from(this.adapters.keys());
+    }
+    /**
+     * Get available platforms and their capabilities
+     */
+    getAvailablePlatforms() {
+        const platforms = {};
+        for (const [platform, adapter] of this.adapters) {
+            platforms[platform] = {
+                name: platform,
+                capabilities: adapter.getSupportedFeatures(),
+                initialized: true
+            };
+        }
+        return platforms;
+    }
+    /**
+     * Get combined trending content with platform attribution
+     */
+    async getCombinedTrendingContent(limit = 30) {
+        const allTrending = await this.getAllTrendingContent(limit);
+        const combinedPosts = [];
+        for (const [, posts] of Object.entries(allTrending)) {
+            if (posts && Array.isArray(posts)) {
+                combinedPosts.push(...posts);
+            }
+        }
+        combinedPosts.sort((a, b) => {
+            const aEngagement = (a.engagement.likes || 0) + (a.engagement.comments || 0);
+            const bEngagement = (b.engagement.likes || 0) + (b.engagement.comments || 0);
+            if (aEngagement !== bEngagement) {
+                return bEngagement - aEngagement;
+            }
+            return new Date(b.timestamp).getTime() - new Date(a.timestamp).getTime();
+        });
+        return combinedPosts.slice(0, limit);
+    }
+    /**
+     * Search across all platforms and return combined results
+     */
+    async getCombinedSearchResults(query, limit = 30) {
+        const allResults = await this.searchAllPlatforms(query, limit);
+        const combinedPosts = [];
+        for (const [, posts] of Object.entries(allResults)) {
+            if (posts && Array.isArray(posts)) {
+                combinedPosts.push(...posts);
+            }
+        }
+        combinedPosts.sort((a, b) => {
+            const aRelevance = this.calculateRelevance(a.content, query);
+            const bRelevance = this.calculateRelevance(b.content, query);
+            if (aRelevance !== bRelevance) {
+                return bRelevance - aRelevance;
+            }
+            const aEngagement = (a.engagement.likes || 0) + (a.engagement.comments || 0);
+            const bEngagement = (b.engagement.likes || 0) + (b.engagement.comments || 0);
+            return bEngagement - aEngagement;
+        });
+        return combinedPosts.slice(0, limit);
+    }
+    /**
+     * Platform health check
+     */
+    async healthCheck() {
+        const health = {};
+        const promises = Array.from(this.adapters.entries()).map(async ([platform, adapter]) => {
+            try {
+                await adapter.getTrendingContent(1);
+                health[platform] = 'healthy';
+            }
+            catch (error) {
+                console.warn(`Health check failed for ${platform}:`, error);
+                health[platform] = 'down';
+            }
+        });
+        await Promise.allSettled(promises);
+        return health;
+    }
+    /**
+     * Cleanup all adapters and browser pool
+     */
+    async cleanup() {
+        const cleanupPromises = Array.from(this.adapters.values()).map(adapter => adapter.cleanup().catch(error => console.warn('Cleanup error:', error)));
+        await Promise.allSettled(cleanupPromises);
+        // Cleanup browser pool if any browser-based adapters were used
+        const browserPlatforms = ['tiktok', 'instagram', 'xiaohongshu'];
+        const hasBrowserAdapters = browserPlatforms.some(p => this.adapters.has(p));
+        if (hasBrowserAdapters) {
+            try {
+                const pool = getBrowserPool();
+                await pool.cleanup();
+            }
+            catch (error) {
+                console.warn('Browser pool cleanup error:', error);
+            }
+        }
+        this.adapters.clear();
+        this.isInitialized = false;
+        console.log('Unified Social Media Service cleaned up');
+    }
+    getAdapter(platform) {
+        if (!this.isInitialized) {
+            throw new SocialMediaError('Service not initialized', 'NOT_INITIALIZED', 'tiktok');
+        }
+        const adapter = this.adapters.get(platform);
+        if (!adapter) {
+            throw new SocialMediaError(`Platform ${platform} not available`, 'PLATFORM_NOT_AVAILABLE', platform);
+        }
+        return adapter;
+    }
+    calculateRelevance(content, query) {
+        if (!content || !query)
+            return 0;
+        const contentLower = content.toLowerCase();
+        const queryLower = query.toLowerCase();
+        const queryWords = queryLower.split(/\s+/);
+        let score = 0;
+        if (contentLower.includes(queryLower)) {
+            score += 10;
+        }
+        for (const word of queryWords) {
+            if (contentLower.includes(word)) {
+                score += 2;
+            }
+        }
+        return score;
+    }
+}