@crowdlisten/harness 1.0.0

package/dist/insights/platforms/twitter/TwitterAdapter.js

@@ -0,0 +1,211 @@
+/**
+ * Twitter/X Platform Adapter
+ * Uses @the-convocation/twitter-scraper for HTTP-only access to Twitter's
+ * internal API via cookie auth. No browser needed.
+ *
+ * Auth: TWITTER_USERNAME + TWITTER_PASSWORD env vars, or stored cookies.
+ */
+import { Scraper, SearchMode } from '@the-convocation/twitter-scraper';
+import { BaseAdapter } from '../../core/base/BaseAdapter.js';
+export class TwitterAdapter extends BaseAdapter {
+    scraper;
+    constructor(config) {
+        super(config);
+        this.scraper = new Scraper();
+        this.maxRequestsPerWindow = 20;
+    }
+    async initialize() {
+        try {
+            const username = process.env.TWITTER_USERNAME;
+            const password = process.env.TWITTER_PASSWORD;
+            if (username && password) {
+                await this.scraper.login(username, password);
+            }
+            const loggedIn = await this.scraper.isLoggedIn();
+            if (!loggedIn) {
+                this.log('Twitter scraper not logged in — some features may be limited', 'warn');
+                // Still usable for public content without login
+            }
+            this.isInitialized = true;
+            this.log(`Twitter adapter initialized (logged in: ${loggedIn})`);
+            return true;
+        }
+        catch (error) {
+            this.log(`Failed to initialize Twitter adapter: ${error.message}`, 'error');
+            // Initialize anyway — scraper can still fetch some public content
+            this.isInitialized = true;
+            return true;
+        }
+    }
+    async searchContent(query, limit = 10) {
+        this.ensureInitialized();
+        this.validateLimit(limit);
+        if (!query || query.trim().length === 0) {
+            throw new Error('Search query cannot be empty');
+        }
+        await this.enforceRateLimit();
+        const posts = [];
+        const tweets = this.scraper.searchTweets(query, limit, SearchMode.Latest);
+        for await (const tweet of tweets) {
+            if (posts.length >= limit)
+                break;
+            const post = this.normalizeTweet(tweet);
+            if (post)
+                posts.push(post);
+        }
+        this.log(`Found ${posts.length} tweets for query: ${query}`);
+        return posts;
+    }
+    async getTrendingContent(limit = 10) {
+        this.ensureInitialized();
+        this.validateLimit(limit);
+        await this.enforceRateLimit();
+        try {
+            const trends = await this.scraper.getTrends();
+            if (!trends || trends.length === 0) {
+                return [];
+            }
+            // Search for the top trend to get actual posts
+            const topTrend = trends[0];
+            return this.searchContent(topTrend, limit);
+        }
+        catch (error) {
+            this.handleError(error, 'getTrendingContent');
+        }
+    }
+    async getUserContent(userId, limit = 10) {
+        this.ensureInitialized();
+        this.validateUserId(userId);
+        this.validateLimit(limit);
+        await this.enforceRateLimit();
+        const username = userId.replace(/^@/, '');
+        const posts = [];
+        const tweets = this.scraper.getTweets(username, limit);
+        for await (const tweet of tweets) {
+            if (posts.length >= limit)
+                break;
+            const post = this.normalizeTweet(tweet);
+            if (post)
+                posts.push(post);
+        }
+        this.log(`Retrieved ${posts.length} tweets from user @${username}`);
+        return posts;
+    }
+    async getContentComments(contentId, limit = 20) {
+        this.ensureInitialized();
+        this.validateContentId(contentId);
+        this.validateLimit(limit);
+        await this.enforceRateLimit();
+        // Extract tweet ID from URL if needed
+        let tweetId = contentId;
+        const idMatch = contentId.match(/\/status\/(\d+)/);
+        if (idMatch)
+            tweetId = idMatch[1];
+        try {
+            const tweet = await this.scraper.getTweet(tweetId);
+            if (!tweet) {
+                this.log(`Tweet ${tweetId} not found`, 'warn');
+                return [];
+            }
+            // Get replies by searching for replies to the tweet
+            const comments = [];
+            const username = tweet.username || '';
+            const replyQuery = `to:${username} conversation_id:${tweetId}`;
+            const replies = this.scraper.searchTweets(replyQuery, limit, SearchMode.Latest);
+            for await (const reply of replies) {
+                if (comments.length >= limit)
+                    break;
+                const comment = this.normalizeTweetAsComment(reply);
+                if (comment)
+                    comments.push(comment);
+            }
+            this.log(`Retrieved ${comments.length} replies for tweet ${tweetId}`);
+            return comments;
+        }
+        catch (error) {
+            this.handleError(error, `getContentComments(${tweetId})`);
+        }
+    }
+    // ── Normalization ──────────────────────────────────────────────────────
+    normalizeTweet(tweet) {
+        try {
+            const id = tweet.id || '';
+            if (!id)
+                return null;
+            return {
+                id,
+                platform: 'twitter',
+                author: {
+                    id: tweet.userId || tweet.username || '',
+                    username: tweet.username || '',
+                    displayName: tweet.name || tweet.username || '',
+                    followerCount: tweet.followersCount,
+                    verified: tweet.isVerified || tweet.isBlueVerified,
+                    profileImageUrl: tweet.profileImageUrl,
+                },
+                content: tweet.text || '',
+                mediaUrl: tweet.photos?.[0]?.url || tweet.videos?.[0]?.preview || '',
+                engagement: {
+                    likes: tweet.likes || 0,
+                    comments: tweet.replies || 0,
+                    shares: tweet.retweets || 0,
+                    views: tweet.views || 0,
+                },
+                timestamp: tweet.timeParsed ? new Date(tweet.timeParsed) : new Date(),
+                url: tweet.permanentUrl || `https://x.com/${tweet.username || 'user'}/status/${id}`,
+                hashtags: tweet.hashtags || [],
+            };
+        }
+        catch {
+            return null;
+        }
+    }
+    normalizeTweetAsComment(tweet) {
+        try {
+            const id = tweet.id || '';
+            if (!id)
+                return null;
+            return {
+                id,
+                author: {
+                    id: tweet.userId || tweet.username || '',
+                    username: tweet.username || '',
+                    displayName: tweet.name || tweet.username || '',
+                    verified: tweet.isVerified || tweet.isBlueVerified,
+                },
+                text: tweet.text || '',
+                timestamp: tweet.timeParsed ? new Date(tweet.timeParsed) : new Date(),
+                likes: tweet.likes || 0,
+                engagement: {
+                    shares: tweet.retweets || 0,
+                    views: tweet.views || 0,
+                },
+            };
+        }
+        catch {
+            return null;
+        }
+    }
+    // ── Platform identity ──────────────────────────────────────────────────
+    getPlatformName() {
+        return 'twitter';
+    }
+    getSupportedFeatures() {
+        return {
+            supportsTrending: true,
+            supportsUserContent: true,
+            supportsSearch: true,
+            supportsComments: true,
+            supportsAnalysis: true,
+        };
+    }
+    async cleanup() {
+        try {
+            await this.scraper.logout();
+        }
+        catch {
+            // Ignore logout errors
+        }
+        await super.cleanup();
+    }
+}

package/dist/insights/platforms/xiaohongshu/XiaohongshuAdapter.d.ts

@@ -0,0 +1,35 @@
+/**
+ * Xiaohongshu (RedNote) Platform Adapter — flat browser adapter using API interception.
+ *
+ * Uses BrowserPool + RequestInterceptor to capture XHS's internal API responses.
+ * Preserves mobile viewport, zh-CN locale, and conservative anti-detection delays.
+ * No tiers, no fallback chains.
+ *
+ * API targets:
+ *   - /api/sns/web/v1/search/notes — search
+ *   - /api/sns/web/v2/comment/page — comments
+ *   - /api/sns/web/v1/feed — explore/trending
+ *   - /api/sns/web/v1/user_posted — user posts
+ */
+import { BaseAdapter } from '../../core/base/BaseAdapter.js';
+import { Post, Comment, PlatformCapabilities, PlatformType, PlatformConfig } from '../../core/interfaces/SocialMediaPlatform.js';
+export declare class XiaohongshuAdapter extends BaseAdapter {
+    constructor(config: PlatformConfig);
+    initialize(): Promise<boolean>;
+    searchContent(query: string, limit?: number): Promise<Post[]>;
+    getTrendingContent(limit?: number): Promise<Post[]>;
+    getUserContent(userId: string, limit?: number): Promise<Post[]>;
+    getContentComments(contentId: string, limit?: number): Promise<Comment[]>;
+    private interceptPosts;
+    /**
+     * XHS needs longer waits and slower scrolling due to anti-bot measures.
+     */
+    private waitAndScrollSlow;
+    private humanDelay;
+    private structurePosts;
+    private structureComments;
+    private normalizeNote;
+    private normalizeXHSComment;
+    getPlatformName(): PlatformType;
+    getSupportedFeatures(): PlatformCapabilities;
+}

package/dist/insights/platforms/xiaohongshu/XiaohongshuAdapter.js

@@ -0,0 +1,258 @@
+/**
+ * Xiaohongshu (RedNote) Platform Adapter — flat browser adapter using API interception.
+ *
+ * Uses BrowserPool + RequestInterceptor to capture XHS's internal API responses.
+ * Preserves mobile viewport, zh-CN locale, and conservative anti-detection delays.
+ * No tiers, no fallback chains.
+ *
+ * API targets:
+ *   - /api/sns/web/v1/search/notes — search
+ *   - /api/sns/web/v2/comment/page — comments
+ *   - /api/sns/web/v1/feed — explore/trending
+ *   - /api/sns/web/v1/user_posted — user posts
+ */
+import { BaseAdapter } from '../../core/base/BaseAdapter.js';
+import { getBrowserPool } from '../../browser/BrowserPool.js';
+import { RequestInterceptor } from '../../browser/RequestInterceptor.js';
+const API_PATTERNS = [
+    '/api/sns/web/v1/search/notes',
+    '/api/sns/web/v2/comment/page',
+    '/api/sns/web/v1/feed',
+    '/api/sns/web/v1/user_posted',
+    '/api/sns/web/v1/note/',
+    '/api/sns/web/v2/note/',
+];
+export class XiaohongshuAdapter extends BaseAdapter {
+    constructor(config) {
+        super(config);
+        // Browser platforms risk IP blocks above ~1/min sustained; 3/min allows interactive bursts
+        this.maxRequestsPerWindow = 3;
+    }
+    async initialize() {
+        this.isInitialized = true;
+        this.log('Xiaohongshu adapter initialized (API interception mode, mobile profile)');
+        return true;
+    }
+    async searchContent(query, limit = 10) {
+        this.ensureInitialized();
+        await this.enforceRateLimit();
+        const url = `https://www.xiaohongshu.com/search_result?keyword=${encodeURIComponent(query)}&type=1`;
+        return this.interceptPosts(url, limit);
+    }
+    async getTrendingContent(limit = 10) {
+        this.ensureInitialized();
+        await this.enforceRateLimit();
+        return this.interceptPosts('https://www.xiaohongshu.com/explore', limit);
+    }
+    async getUserContent(userId, limit = 10) {
+        this.ensureInitialized();
+        this.validateUserId(userId);
+        await this.enforceRateLimit();
+        return this.interceptPosts(`https://www.xiaohongshu.com/user/profile/${userId}`, limit);
+    }
+    async getContentComments(contentId, limit = 20) {
+        this.ensureInitialized();
+        this.validateContentId(contentId);
+        await this.enforceRateLimit();
+        const url = contentId.includes('xiaohongshu.com/')
+            ? contentId
+            : `https://www.xiaohongshu.com/explore/${contentId}`;
+        const pool = getBrowserPool();
+        const page = await pool.acquire('xiaohongshu');
+        const interceptor = new RequestInterceptor();
+        try {
+            await interceptor.setup(page, API_PATTERNS);
+            await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 20000 });
+            await this.waitAndScrollSlow(page);
+            const apiData = interceptor.getAllData();
+            if (apiData.length === 0)
+                return [];
+            return this.structureComments(apiData).slice(0, limit);
+        }
+        catch (error) {
+            this.handleError(error, `getContentComments(${contentId})`);
+        }
+        finally {
+            interceptor.stop();
+            await pool.release(page);
+        }
+    }
+    // ── Interception pipeline ──────────────────────────────────────────────
+    async interceptPosts(url, limit) {
+        const pool = getBrowserPool();
+        const page = await pool.acquire('xiaohongshu');
+        const interceptor = new RequestInterceptor();
+        try {
+            await interceptor.setup(page, API_PATTERNS);
+            await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 20000 });
+            await this.waitAndScrollSlow(page);
+            const apiData = interceptor.getAllData();
+            if (apiData.length === 0)
+                return [];
+            return this.structurePosts(apiData).slice(0, limit);
+        }
+        catch (error) {
+            this.handleError(error, `interceptPosts(${url})`);
+        }
+        finally {
+            interceptor.stop();
+            await pool.release(page);
+        }
+    }
+    /**
+     * XHS needs longer waits and slower scrolling due to anti-bot measures.
+     */
+    async waitAndScrollSlow(page) {
+        // Human-like initial wait
+        await this.humanDelay(page, 2000, 4000);
+        try {
+            await page.waitForLoadState('networkidle', { timeout: 20000 });
+        }
+        catch {
+            // Non-fatal
+        }
+        // Slow scrolling with random delays (70% viewport height)
+        for (let i = 0; i < 3; i++) {
+            await page.evaluate(() => window.scrollBy(0, window.innerHeight * 0.7));
+            await this.humanDelay(page, 2000, 5000);
+        }
+    }
+    async humanDelay(page, minMs, maxMs) {
+        const delay = minMs + Math.random() * (maxMs - minMs);
+        await page.waitForTimeout(delay);
+    }
+    // ── Data normalization (preserved from VisualXiaohongshuAdapter) ───────
+    structurePosts(interceptedData) {
+        const posts = [];
+        const seenIds = new Set();
+        for (const data of interceptedData) {
+            try {
+                // Search results shape
+                const items = data?.data?.items || data?.data?.notes || [];
+                if (Array.isArray(items)) {
+                    for (const item of items) {
+                        const noteCard = item.note_card || item;
+                        const post = this.normalizeNote(noteCard, item.id || noteCard.note_id);
+                        if (post && !seenIds.has(post.id)) {
+                            seenIds.add(post.id);
+                            posts.push(post);
+                        }
+                    }
+                }
+                // Single note detail
+                if (data?.data?.note_id || data?.data?.id) {
+                    const post = this.normalizeNote(data.data, data.data.note_id || data.data.id);
+                    if (post && !seenIds.has(post.id)) {
+                        seenIds.add(post.id);
+                        posts.push(post);
+                    }
+                }
+            }
+            catch {
+                // Skip malformed response
+            }
+        }
+        return posts;
+    }
+    structureComments(interceptedData) {
+        const comments = [];
+        const seenIds = new Set();
+        for (const data of interceptedData) {
+            try {
+                const commentList = data?.data?.comments || [];
+                if (Array.isArray(commentList)) {
+                    for (const item of commentList) {
+                        const comment = this.normalizeXHSComment(item);
+                        if (comment && !seenIds.has(comment.id)) {
+                            seenIds.add(comment.id);
+                            comments.push(comment);
+                        }
+                    }
+                }
+            }
+            catch {
+                // Skip malformed response
+            }
+        }
+        return comments;
+    }
+    normalizeNote(note, noteId) {
+        try {
+            const id = noteId || note.note_id || note.id || '';
+            if (!id)
+                return null;
+            const user = note.user || note.author || {};
+            const interactInfo = note.interact_info || {};
+            return {
+                id: String(id),
+                platform: 'xiaohongshu',
+                author: {
+                    id: user.user_id || user.uid || '',
+                    username: user.nickname || user.nick_name || '',
+                    displayName: user.nickname || user.nick_name || '',
+                    profileImageUrl: user.avatar || user.images,
+                },
+                content: note.title || note.desc || note.display_title || '',
+                mediaUrl: note.cover?.url || note.cover?.url_default || note.image_list?.[0]?.url || '',
+                engagement: {
+                    likes: interactInfo.liked_count || note.liked_count || 0,
+                    comments: interactInfo.comment_count || note.comment_count || 0,
+                    shares: interactInfo.share_count || note.share_count || 0,
+                    views: interactInfo.view_count || 0,
+                },
+                timestamp: note.time
+                    ? new Date(note.time)
+                    : note.create_time
+                        ? new Date(note.create_time * 1000)
+                        : new Date(),
+                url: `https://www.xiaohongshu.com/explore/${id}`,
+                hashtags: (note.tag_list || []).map((t) => t.name || t),
+            };
+        }
+        catch {
+            return null;
+        }
+    }
+    normalizeXHSComment(item) {
+        try {
+            const id = item.id || item.comment_id || '';
+            if (!id)
+                return null;
+            const user = item.user_info || item.user || {};
+            const subComments = (item.sub_comments || [])
+                .map((s) => this.normalizeXHSComment(s))
+                .filter(Boolean);
+            return {
+                id: String(id),
+                author: {
+                    id: user.user_id || '',
+                    username: user.nickname || '',
+                    displayName: user.nickname || '',
+                    profileImageUrl: user.image || user.avatar,
+                },
+                text: item.content || '',
+                timestamp: item.create_time
+                    ? new Date(item.create_time * 1000)
+                    : new Date(),
+                likes: item.like_count || 0,
+                replies: subComments.length > 0 ? subComments : undefined,
+            };
+        }
+        catch {
+            return null;
+        }
+    }
+    // ── Platform identity ──────────────────────────────────────────────────
+    getPlatformName() {
+        return 'xiaohongshu';
+    }
+    getSupportedFeatures() {
+        return {
+            supportsTrending: true,
+            supportsUserContent: true,
+            supportsSearch: true,
+            supportsComments: true,
+            supportsAnalysis: true,
+        };
+    }
+}

package/dist/insights/platforms/youtube/YouTubeAdapter.d.ts

@@ -0,0 +1,22 @@
+/**
+ * YouTube Platform Adapter
+ * Uses YouTube Data API v3 for video search, trending, comments, and channel content
+ */
+import { BaseAdapter } from '../../core/base/BaseAdapter.js';
+import { Post, Comment, PlatformCapabilities, PlatformType, PlatformConfig } from '../../core/interfaces/SocialMediaPlatform.js';
+export declare class YouTubeAdapter extends BaseAdapter {
+    private client;
+    private apiKey;
+    constructor(config: PlatformConfig);
+    initialize(): Promise<boolean>;
+    getTrendingContent(limit?: number): Promise<Post[]>;
+    getUserContent(userId: string, limit?: number): Promise<Post[]>;
+    searchContent(query: string, limit?: number): Promise<Post[]>;
+    getContentComments(contentId: string, limit?: number): Promise<Comment[]>;
+    getPlatformName(): PlatformType;
+    getSupportedFeatures(): PlatformCapabilities;
+    protected isRateLimitError(error: any): boolean;
+    protected isAuthError(error: any): boolean;
+    protected isNotFoundError(error: any): boolean;
+    cleanup(): Promise<void>;
+}