@crowdlisten/harness 1.0.0

package/dist/insights/platforms/reddit/RedditAdapter.js

@@ -0,0 +1,212 @@
+/**
+ * Simple Reddit Platform Adapter
+ * Uses Reddit's public JSON API for basic functionality
+ */
+import { BaseAdapter } from '../../core/base/BaseAdapter.js';
+import { DataNormalizer } from '../../core/utils/DataNormalizer.js';
+import { NotFoundError } from '../../core/interfaces/SocialMediaPlatform.js';
+import axios from 'axios';
+export class RedditAdapter extends BaseAdapter {
+    client = null;
+    constructor(config) {
+        super(config);
+        this.maxRequestsPerWindow = 60;
+    }
+    async initialize() {
+        try {
+            this.client = axios.create({
+                baseURL: 'https://www.reddit.com',
+                headers: {
+                    'User-Agent': 'crowdlisten-mcp/1.0.0'
+                },
+                timeout: 10000
+            });
+            this.isInitialized = true;
+            this.log('Reddit adapter initialized successfully (HTTP access)', 'info');
+            return true;
+        }
+        catch (error) {
+            this.log('Failed to initialize Reddit adapter', 'error');
+            this.isInitialized = false;
+            return false;
+        }
+    }
+    async getTrendingContent(limit = 10) {
+        this.ensureInitialized();
+        this.validateLimit(limit);
+        try {
+            await this.enforceRateLimit();
+            const response = await this.client.get('/r/popular.json', {
+                params: { limit }
+            });
+            const posts = [];
+            const items = response.data?.data?.children || [];
+            for (const item of items.slice(0, limit)) {
+                const postData = item.data;
+                const post = DataNormalizer.normalizePost({
+                    id: postData.id,
+                    title: postData.title,
+                    selftext: postData.selftext || '',
+                    author: postData.author,
+                    score: postData.score,
+                    num_comments: postData.num_comments,
+                    created_utc: postData.created_utc,
+                    permalink: postData.permalink,
+                    url: postData.url || `https://reddit.com${postData.permalink}`,
+                    subreddit: postData.subreddit
+                }, 'reddit');
+                posts.push(post);
+            }
+            this.log(`Retrieved ${posts.length} trending Reddit posts`, 'info');
+            return posts;
+        }
+        catch (error) {
+            this.handleError(error, 'getTrendingContent');
+        }
+    }
+    async getUserContent(userId, limit = 10) {
+        this.ensureInitialized();
+        this.validateUserId(userId);
+        this.validateLimit(limit);
+        try {
+            await this.enforceRateLimit();
+            const response = await this.client.get(`/user/${userId}.json`, {
+                params: { limit }
+            });
+            const posts = [];
+            const items = response.data?.data?.children || [];
+            for (const item of items.slice(0, limit)) {
+                const postData = item.data;
+                if (postData.title) { // Only posts, not comments
+                    const post = DataNormalizer.normalizePost({
+                        id: postData.id,
+                        title: postData.title,
+                        selftext: postData.selftext || '',
+                        author: postData.author || userId,
+                        score: postData.score,
+                        num_comments: postData.num_comments,
+                        created_utc: postData.created_utc,
+                        permalink: postData.permalink,
+                        url: postData.url || `https://reddit.com${postData.permalink}`,
+                        subreddit: postData.subreddit
+                    }, 'reddit');
+                    posts.push(post);
+                }
+            }
+            this.log(`Retrieved ${posts.length} posts from Reddit user ${userId}`, 'info');
+            return posts;
+        }
+        catch (error) {
+            if (error.response?.status === 404) {
+                throw new NotFoundError('reddit', `User ${userId}`, error);
+            }
+            this.handleError(error, 'getUserContent');
+        }
+    }
+    async searchContent(query, limit = 10) {
+        this.ensureInitialized();
+        this.validateLimit(limit);
+        if (!query || query.trim().length === 0) {
+            throw new Error('Search query cannot be empty');
+        }
+        try {
+            await this.enforceRateLimit();
+            const response = await this.client.get('/search.json', {
+                params: {
+                    q: query.trim(),
+                    limit,
+                    sort: 'relevance',
+                    type: 'link'
+                }
+            });
+            const posts = [];
+            const items = response.data?.data?.children || [];
+            for (const item of items.slice(0, limit)) {
+                const postData = item.data;
+                const post = DataNormalizer.normalizePost({
+                    id: postData.id,
+                    title: postData.title,
+                    selftext: postData.selftext || '',
+                    author: postData.author,
+                    score: postData.score,
+                    num_comments: postData.num_comments,
+                    created_utc: postData.created_utc,
+                    permalink: postData.permalink,
+                    url: postData.url || `https://reddit.com${postData.permalink}`,
+                    subreddit: postData.subreddit
+                }, 'reddit');
+                posts.push(post);
+            }
+            this.log(`Found ${posts.length} Reddit posts for query: ${query}`, 'info');
+            return posts;
+        }
+        catch (error) {
+            this.handleError(error, 'searchContent');
+        }
+    }
+    async getContentComments(contentId, limit = 20) {
+        this.ensureInitialized();
+        this.validateContentId(contentId);
+        this.validateLimit(limit);
+        try {
+            await this.enforceRateLimit();
+            // Reddit /comments/{postId}.json returns [postListing, commentListing]
+            const response = await this.client.get(`/comments/${contentId}.json`, {
+                params: {
+                    limit,
+                    depth: 5,
+                    sort: 'top'
+                }
+            });
+            const commentListing = response.data?.[1];
+            const children = commentListing?.data?.children || [];
+            const comments = [];
+            for (const child of children) {
+                // Skip "more" stubs (load-more placeholders)
+                if (child.kind !== 't1' || !child.data)
+                    continue;
+                comments.push(DataNormalizer.normalizeComment(child.data, 'reddit'));
+            }
+            this.log(`Retrieved ${comments.length} comments for Reddit post ${contentId}`);
+            return comments.slice(0, limit);
+        }
+        catch (error) {
+            if (error.response?.status === 404) {
+                throw new NotFoundError('reddit', `Post ${contentId}`, error);
+            }
+            this.handleError(error, 'getContentComments');
+        }
+    }
+    getPlatformName() {
+        return 'reddit';
+    }
+    getSupportedFeatures() {
+        return {
+            supportsTrending: true,
+            supportsUserContent: true,
+            supportsSearch: true,
+            supportsComments: true,
+            supportsAnalysis: true
+        };
+    }
+    isRateLimitError(error) {
+        return error.response?.status === 429 ||
+            error.message?.includes('rate limit');
+    }
+    isAuthError(error) {
+        return error.response?.status === 401 ||
+            error.response?.status === 403;
+    }
+    isNotFoundError(error) {
+        return error.response?.status === 404;
+    }
+    async cleanup() {
+        try {
+            this.client = null;
+            await super.cleanup();
+        }
+        catch (error) {
+            this.log('Error during Reddit cleanup', 'warn');
+        }
+    }
+}

package/dist/insights/platforms/tiktok/TikTokAdapter.d.ts

@@ -0,0 +1,34 @@
+/**
+ * TikTok Platform Adapter — flat browser adapter using API interception.
+ *
+ * Uses BrowserPool + RequestInterceptor to capture TikTok's internal API
+ * responses. No tiers, no fallback chains.
+ *
+ * API targets:
+ *   - /api/search/item/ — search results
+ *   - /api/comment/list/ — comment threads
+ *   - /api/recommend/item_list/ — trending/FYP
+ *   - /api/post/item_list/ — user posts
+ */
+import { BaseAdapter } from '../../core/base/BaseAdapter.js';
+import { Post, Comment, PlatformCapabilities, PlatformType, PlatformConfig } from '../../core/interfaces/SocialMediaPlatform.js';
+export declare class TikTokAdapter extends BaseAdapter {
+    constructor(config: PlatformConfig);
+    initialize(): Promise<boolean>;
+    searchContent(query: string, limit?: number): Promise<Post[]>;
+    getTrendingContent(limit?: number): Promise<Post[]>;
+    getUserContent(userId: string, limit?: number): Promise<Post[]>;
+    getContentComments(contentId: string, limit?: number): Promise<Comment[]>;
+    private interceptPosts;
+    /**
+     * TikTok uses IntersectionObserver for lazy loading — stub it before navigation.
+     */
+    private setupPage;
+    private waitAndScroll;
+    private structurePosts;
+    private structureComments;
+    private normalizeItem;
+    private normalizeComment;
+    getPlatformName(): PlatformType;
+    getSupportedFeatures(): PlatformCapabilities;
+}

package/dist/insights/platforms/tiktok/TikTokAdapter.js

@@ -0,0 +1,269 @@
+/**
+ * TikTok Platform Adapter — flat browser adapter using API interception.
+ *
+ * Uses BrowserPool + RequestInterceptor to capture TikTok's internal API
+ * responses. No tiers, no fallback chains.
+ *
+ * API targets:
+ *   - /api/search/item/ — search results
+ *   - /api/comment/list/ — comment threads
+ *   - /api/recommend/item_list/ — trending/FYP
+ *   - /api/post/item_list/ — user posts
+ */
+import { BaseAdapter } from '../../core/base/BaseAdapter.js';
+import { getBrowserPool } from '../../browser/BrowserPool.js';
+import { RequestInterceptor } from '../../browser/RequestInterceptor.js';
+const API_PATTERNS = [
+    '/api/search/item/',
+    '/api/search/general/',
+    '/api/comment/list/',
+    '/api/recommend/item_list/',
+    '/api/post/item_list/',
+    '/v1/search/',
+];
+export class TikTokAdapter extends BaseAdapter {
+    constructor(config) {
+        super(config);
+        // Browser platforms risk IP blocks above ~2/min sustained; 5/min allows interactive bursts
+        this.maxRequestsPerWindow = 5;
+    }
+    async initialize() {
+        this.isInitialized = true;
+        this.log('TikTok adapter initialized (API interception mode)');
+        return true;
+    }
+    async searchContent(query, limit = 10) {
+        this.ensureInitialized();
+        await this.enforceRateLimit();
+        const url = `https://www.tiktok.com/search?q=${encodeURIComponent(query)}`;
+        return this.interceptPosts(url, limit);
+    }
+    async getTrendingContent(limit = 10) {
+        this.ensureInitialized();
+        await this.enforceRateLimit();
+        return this.interceptPosts('https://www.tiktok.com/explore', limit);
+    }
+    async getUserContent(userId, limit = 10) {
+        this.ensureInitialized();
+        this.validateUserId(userId);
+        await this.enforceRateLimit();
+        const username = userId.startsWith('@') ? userId : `@${userId}`;
+        return this.interceptPosts(`https://www.tiktok.com/${username}`, limit);
+    }
+    async getContentComments(contentId, limit = 20) {
+        this.ensureInitialized();
+        this.validateContentId(contentId);
+        await this.enforceRateLimit();
+        const url = contentId.includes('tiktok.com/')
+            ? contentId
+            : `https://www.tiktok.com/video/${contentId}`;
+        const pool = getBrowserPool();
+        const page = await pool.acquire('tiktok');
+        const interceptor = new RequestInterceptor();
+        try {
+            await this.setupPage(page);
+            await interceptor.setup(page, API_PATTERNS);
+            await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 15000 });
+            await this.waitAndScroll(page);
+            const apiData = interceptor.getAllData();
+            if (apiData.length === 0)
+                return [];
+            return this.structureComments(apiData).slice(0, limit);
+        }
+        catch (error) {
+            this.handleError(error, `getContentComments(${contentId})`);
+        }
+        finally {
+            interceptor.stop();
+            await pool.release(page);
+        }
+    }
+    // ── Interception pipeline ──────────────────────────────────────────────
+    async interceptPosts(url, limit) {
+        const pool = getBrowserPool();
+        const page = await pool.acquire('tiktok');
+        const interceptor = new RequestInterceptor();
+        try {
+            await this.setupPage(page);
+            await interceptor.setup(page, API_PATTERNS);
+            await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 15000 });
+            await this.waitAndScroll(page);
+            const apiData = interceptor.getAllData();
+            if (apiData.length === 0)
+                return [];
+            return this.structurePosts(apiData).slice(0, limit);
+        }
+        catch (error) {
+            this.handleError(error, `interceptPosts(${url})`);
+        }
+        finally {
+            interceptor.stop();
+            await pool.release(page);
+        }
+    }
+    /**
+     * TikTok uses IntersectionObserver for lazy loading — stub it before navigation.
+     */
+    async setupPage(page) {
+        await page.addInitScript(() => {
+            if (typeof globalThis.IntersectionObserver === 'undefined') {
+                globalThis.IntersectionObserver = class {
+                    _cb;
+                    constructor(cb) { this._cb = cb; }
+                    observe(target) {
+                        setTimeout(() => this._cb([{
+                                isIntersecting: true,
+                                intersectionRatio: 1,
+                                target,
+                            }]), 50);
+                    }
+                    unobserve() { }
+                    disconnect() { }
+                };
+            }
+        });
+    }
+    async waitAndScroll(page) {
+        try {
+            await page.waitForLoadState('networkidle', { timeout: 15000 });
+        }
+        catch {
+            // Non-fatal
+        }
+        for (let i = 0; i < 5; i++) {
+            await page.evaluate(() => window.scrollBy(0, window.innerHeight));
+            await page.waitForTimeout(2000);
+        }
+    }
+    // ── Data normalization (preserved from VisualTikTokAdapter) ────────────
+    structurePosts(interceptedData) {
+        const posts = [];
+        const seenIds = new Set();
+        for (const data of interceptedData) {
+            try {
+                const items = data?.data || data?.item_list || data?.itemList || [];
+                if (Array.isArray(items)) {
+                    for (const item of items) {
+                        const post = this.normalizeItem(item);
+                        if (post && !seenIds.has(post.id)) {
+                            seenIds.add(post.id);
+                            posts.push(post);
+                        }
+                    }
+                }
+                const searchItems = data?.data?.item_list || data?.data?.items || [];
+                if (Array.isArray(searchItems) && searchItems !== items) {
+                    for (const item of searchItems) {
+                        const post = this.normalizeItem(item);
+                        if (post && !seenIds.has(post.id)) {
+                            seenIds.add(post.id);
+                            posts.push(post);
+                        }
+                    }
+                }
+            }
+            catch {
+                // Skip malformed response
+            }
+        }
+        return posts;
+    }
+    structureComments(interceptedData) {
+        const comments = [];
+        const seenIds = new Set();
+        for (const data of interceptedData) {
+            try {
+                const commentList = data?.comments || data?.data?.comments || [];
+                if (Array.isArray(commentList)) {
+                    for (const item of commentList) {
+                        const comment = this.normalizeComment(item);
+                        if (comment && !seenIds.has(comment.id)) {
+                            seenIds.add(comment.id);
+                            comments.push(comment);
+                        }
+                    }
+                }
+            }
+            catch {
+                // Skip malformed response
+            }
+        }
+        return comments;
+    }
+    normalizeItem(item) {
+        try {
+            const id = item.id || item.aweme_id || item.video?.id || '';
+            if (!id)
+                return null;
+            const author = item.author || {};
+            const stats = item.stats || item.statistics || {};
+            const desc = item.desc || item.title || item.video?.title || '';
+            const createTime = item.createTime || item.create_time;
+            return {
+                id: String(id),
+                platform: 'tiktok',
+                author: {
+                    id: author.id || author.uid || '',
+                    username: author.uniqueId || author.unique_id || author.nickname || '',
+                    displayName: author.nickname || '',
+                    followerCount: author.followerCount || author.follower_count,
+                    verified: author.verified,
+                    profileImageUrl: author.avatarThumb || author.avatar_thumb,
+                },
+                content: desc,
+                mediaUrl: item.video?.cover || item.video?.dynamicCover || '',
+                engagement: {
+                    likes: stats.diggCount || stats.digg_count || stats.likeCount || 0,
+                    comments: stats.commentCount || stats.comment_count || 0,
+                    shares: stats.shareCount || stats.share_count || 0,
+                    views: stats.playCount || stats.play_count || 0,
+                },
+                timestamp: createTime ? new Date(createTime * 1000) : new Date(),
+                url: `https://www.tiktok.com/@${author.uniqueId || author.unique_id || 'user'}/video/${id}`,
+                hashtags: (item.textExtra || [])
+                    .filter((t) => t.hashtagName || t.hashtag_name)
+                    .map((t) => t.hashtagName || t.hashtag_name),
+            };
+        }
+        catch {
+            return null;
+        }
+    }
+    normalizeComment(item) {
+        try {
+            const id = item.cid || item.id || '';
+            if (!id)
+                return null;
+            const user = item.user || {};
+            const createTime = item.create_time || item.createTime;
+            return {
+                id: String(id),
+                author: {
+                    id: user.uid || user.id || '',
+                    username: user.unique_id || user.uniqueId || user.nickname || '',
+                    displayName: user.nickname || '',
+                },
+                text: item.text || '',
+                timestamp: createTime ? new Date(createTime * 1000) : new Date(),
+                likes: item.digg_count || item.diggCount || item.likes || 0,
+                replies: (item.reply_comment || []).map((r) => this.normalizeComment(r)).filter(Boolean),
+            };
+        }
+        catch {
+            return null;
+        }
+    }
+    // ── Platform identity ──────────────────────────────────────────────────
+    getPlatformName() {
+        return 'tiktok';
+    }
+    getSupportedFeatures() {
+        return {
+            supportsTrending: true,
+            supportsUserContent: true,
+            supportsSearch: true,
+            supportsComments: true,
+            supportsAnalysis: true,
+        };
+    }
+}

package/dist/insights/platforms/twitter/TwitterAdapter.d.ts

@@ -0,0 +1,23 @@
+/**
+ * Twitter/X Platform Adapter
+ * Uses @the-convocation/twitter-scraper for HTTP-only access to Twitter's
+ * internal API via cookie auth. No browser needed.
+ *
+ * Auth: TWITTER_USERNAME + TWITTER_PASSWORD env vars, or stored cookies.
+ */
+import { BaseAdapter } from '../../core/base/BaseAdapter.js';
+import { Post, Comment, PlatformCapabilities, PlatformType, PlatformConfig } from '../../core/interfaces/SocialMediaPlatform.js';
+export declare class TwitterAdapter extends BaseAdapter {
+    private scraper;
+    constructor(config: PlatformConfig);
+    initialize(): Promise<boolean>;
+    searchContent(query: string, limit?: number): Promise<Post[]>;
+    getTrendingContent(limit?: number): Promise<Post[]>;
+    getUserContent(userId: string, limit?: number): Promise<Post[]>;
+    getContentComments(contentId: string, limit?: number): Promise<Comment[]>;
+    private normalizeTweet;
+    private normalizeTweetAsComment;
+    getPlatformName(): PlatformType;
+    getSupportedFeatures(): PlatformCapabilities;
+    cleanup(): Promise<void>;
+}