@crowdlisten/harness 1.0.0

package/dist/insights/platforms/instagram/InstagramAdapter.js

@@ -0,0 +1,342 @@
+/**
+ * Instagram Platform Adapter — flat browser adapter using API interception.
+ *
+ * Uses BrowserPool + RequestInterceptor to capture Instagram's internal API
+ * responses (GraphQL + v1 API). No tiers, no fallback chains.
+ *
+ * API targets:
+ *   - /graphql/query/ — posts, user content
+ *   - /api/v1/tags/ — hashtag search
+ *   - /api/v1/feed/ — explore/user feed
+ *   - /api/v1/media/ — media endpoints
+ */
+import { BaseAdapter } from '../../core/base/BaseAdapter.js';
+import { getBrowserPool } from '../../browser/BrowserPool.js';
+import { RequestInterceptor } from '../../browser/RequestInterceptor.js';
+const API_PATTERNS = [
+    '/graphql/query/',
+    '/api/v1/tags/',
+    '/api/v1/feed/',
+    '/api/v1/media/',
+    '/api/v1/web/search/',
+    '/web/search/topsearch/',
+];
+export class InstagramAdapter extends BaseAdapter {
+    constructor(config) {
+        super(config);
+        // Browser platforms risk IP blocks above ~2/min sustained; 5/min allows interactive bursts
+        this.maxRequestsPerWindow = 5;
+    }
+    async initialize() {
+        this.isInitialized = true;
+        this.log('Instagram adapter initialized (API interception mode)');
+        return true;
+    }
+    async searchContent(query, limit = 10) {
+        this.ensureInitialized();
+        await this.enforceRateLimit();
+        const tag = query.replace(/^#/, '');
+        const url = `https://www.instagram.com/explore/tags/${encodeURIComponent(tag)}/`;
+        return this.interceptPosts(url, limit);
+    }
+    async getTrendingContent(limit = 10) {
+        this.ensureInitialized();
+        await this.enforceRateLimit();
+        return this.interceptPosts('https://www.instagram.com/explore/', limit);
+    }
+    async getUserContent(userId, limit = 10) {
+        this.ensureInitialized();
+        this.validateUserId(userId);
+        await this.enforceRateLimit();
+        const username = userId.startsWith('@') ? userId.slice(1) : userId;
+        return this.interceptPosts(`https://www.instagram.com/${username}/`, limit);
+    }
+    async getContentComments(contentId, limit = 20) {
+        this.ensureInitialized();
+        this.validateContentId(contentId);
+        await this.enforceRateLimit();
+        const url = contentId.includes('instagram.com/')
+            ? contentId
+            : `https://www.instagram.com/p/${contentId}/`;
+        const pool = getBrowserPool();
+        const page = await pool.acquire('instagram');
+        const interceptor = new RequestInterceptor();
+        try {
+            await interceptor.setup(page, API_PATTERNS);
+            await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 15000 });
+            await this.waitAndScroll(page);
+            const apiData = interceptor.getAllData();
+            if (apiData.length === 0)
+                return [];
+            return this.structureComments(apiData).slice(0, limit);
+        }
+        catch (error) {
+            this.handleError(error, `getContentComments(${contentId})`);
+        }
+        finally {
+            interceptor.stop();
+            await pool.release(page);
+        }
+    }
+    // ── Interception pipeline ──────────────────────────────────────────────
+    async interceptPosts(url, limit) {
+        const pool = getBrowserPool();
+        const page = await pool.acquire('instagram');
+        const interceptor = new RequestInterceptor();
+        try {
+            await interceptor.setup(page, API_PATTERNS);
+            await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 15000 });
+            await this.waitAndScroll(page);
+            const apiData = interceptor.getAllData();
+            if (apiData.length === 0)
+                return [];
+            return this.structurePosts(apiData).slice(0, limit);
+        }
+        catch (error) {
+            this.handleError(error, `interceptPosts(${url})`);
+        }
+        finally {
+            interceptor.stop();
+            await pool.release(page);
+        }
+    }
+    async waitAndScroll(page) {
+        try {
+            await page.waitForLoadState('networkidle', { timeout: 15000 });
+        }
+        catch {
+            // Non-fatal
+        }
+        for (let i = 0; i < 4; i++) {
+            await page.evaluate(() => window.scrollBy(0, window.innerHeight));
+            await page.waitForTimeout(2000);
+        }
+    }
+    // ── Data normalization (preserved from VisualInstagramAdapter) ─────────
+    structurePosts(interceptedData) {
+        const posts = [];
+        const seenIds = new Set();
+        for (const data of interceptedData) {
+            try {
+                // GraphQL query response shapes
+                const edges = this.extractMediaEdges(data);
+                for (const edge of edges) {
+                    const node = edge.node || edge;
+                    const post = this.normalizeMediaNode(node);
+                    if (post && !seenIds.has(post.id)) {
+                        seenIds.add(post.id);
+                        posts.push(post);
+                    }
+                }
+                // Tag search shape
+                const tagMedia = data?.data?.hashtag?.edge_hashtag_to_media?.edges
+                    || data?.data?.hashtag?.edge_hashtag_to_top_posts?.edges
+                    || [];
+                for (const edge of tagMedia) {
+                    const post = this.normalizeMediaNode(edge.node || edge);
+                    if (post && !seenIds.has(post.id)) {
+                        seenIds.add(post.id);
+                        posts.push(post);
+                    }
+                }
+                // v1 API shape (items array)
+                const items = data?.items || data?.ranked_items || data?.media || [];
+                if (Array.isArray(items)) {
+                    for (const item of items) {
+                        const post = this.normalizeV1Item(item);
+                        if (post && !seenIds.has(post.id)) {
+                            seenIds.add(post.id);
+                            posts.push(post);
+                        }
+                    }
+                }
+            }
+            catch {
+                // Skip malformed response
+            }
+        }
+        return posts;
+    }
+    structureComments(interceptedData) {
+        const comments = [];
+        const seenIds = new Set();
+        for (const data of interceptedData) {
+            try {
+                // GraphQL comments shape
+                const commentEdges = data?.data?.shortcode_media?.edge_media_to_parent_comment?.edges
+                    || data?.data?.shortcode_media?.edge_media_to_comment?.edges
+                    || [];
+                for (const edge of commentEdges) {
+                    const comment = this.normalizeCommentNode(edge.node || edge);
+                    if (comment && !seenIds.has(comment.id)) {
+                        seenIds.add(comment.id);
+                        comments.push(comment);
+                    }
+                }
+                // v1 API comments shape
+                const commentItems = data?.comments || [];
+                if (Array.isArray(commentItems)) {
+                    for (const item of commentItems) {
+                        const comment = this.normalizeV1Comment(item);
+                        if (comment && !seenIds.has(comment.id)) {
+                            seenIds.add(comment.id);
+                            comments.push(comment);
+                        }
+                    }
+                }
+            }
+            catch {
+                // Skip malformed response
+            }
+        }
+        return comments;
+    }
+    extractMediaEdges(data) {
+        const userEdges = data?.data?.user?.edge_owner_to_timeline_media?.edges;
+        if (userEdges)
+            return userEdges;
+        const exploreEdges = data?.data?.user?.edge_web_feed_timeline?.edges;
+        if (exploreEdges)
+            return exploreEdges;
+        const discoverEdges = data?.data?.web_discover_media?.edges;
+        if (discoverEdges)
+            return discoverEdges;
+        return [];
+    }
+    normalizeMediaNode(node) {
+        try {
+            const id = node.id || node.pk || '';
+            if (!id)
+                return null;
+            const owner = node.owner || {};
+            const caption = node.edge_media_to_caption?.edges?.[0]?.node?.text
+                || node.caption?.text
+                || '';
+            return {
+                id: String(id),
+                platform: 'instagram',
+                author: {
+                    id: owner.id || '',
+                    username: owner.username || '',
+                    displayName: owner.full_name || owner.username || '',
+                    profileImageUrl: owner.profile_pic_url,
+                },
+                content: caption,
+                mediaUrl: node.display_url || node.thumbnail_src || '',
+                engagement: {
+                    likes: node.edge_media_preview_like?.count || node.like_count || 0,
+                    comments: node.edge_media_to_comment?.count || node.comment_count || 0,
+                    views: node.video_view_count || 0,
+                },
+                timestamp: node.taken_at_timestamp
+                    ? new Date(node.taken_at_timestamp * 1000)
+                    : new Date(),
+                url: `https://www.instagram.com/p/${node.shortcode || id}/`,
+                hashtags: this.extractHashtags(caption),
+            };
+        }
+        catch {
+            return null;
+        }
+    }
+    normalizeV1Item(item) {
+        try {
+            const id = item.pk || item.id || '';
+            if (!id)
+                return null;
+            const user = item.user || {};
+            const caption = item.caption?.text || '';
+            return {
+                id: String(id),
+                platform: 'instagram',
+                author: {
+                    id: user.pk || user.id || '',
+                    username: user.username || '',
+                    displayName: user.full_name || '',
+                    profileImageUrl: user.profile_pic_url,
+                    verified: user.is_verified,
+                },
+                content: caption,
+                mediaUrl: item.image_versions2?.candidates?.[0]?.url || item.thumbnail_url || '',
+                engagement: {
+                    likes: item.like_count || 0,
+                    comments: item.comment_count || 0,
+                    views: item.view_count || item.play_count || 0,
+                },
+                timestamp: item.taken_at ? new Date(item.taken_at * 1000) : new Date(),
+                url: `https://www.instagram.com/p/${item.code || id}/`,
+                hashtags: this.extractHashtags(caption),
+            };
+        }
+        catch {
+            return null;
+        }
+    }
+    normalizeCommentNode(node) {
+        try {
+            const id = node.id || node.pk || '';
+            if (!id)
+                return null;
+            const user = node.owner || {};
+            const replies = (node.edge_threaded_comments?.edges || [])
+                .map((e) => this.normalizeCommentNode(e.node || e))
+                .filter(Boolean);
+            return {
+                id: String(id),
+                author: {
+                    id: user.id || '',
+                    username: user.username || '',
+                    displayName: user.username || '',
+                    profileImageUrl: user.profile_pic_url,
+                },
+                text: node.text || '',
+                timestamp: node.created_at
+                    ? new Date(node.created_at * 1000)
+                    : new Date(),
+                likes: node.edge_liked_by?.count || 0,
+                replies: replies.length > 0 ? replies : undefined,
+            };
+        }
+        catch {
+            return null;
+        }
+    }
+    normalizeV1Comment(item) {
+        try {
+            const id = item.pk || item.id || '';
+            if (!id)
+                return null;
+            const user = item.user || {};
+            return {
+                id: String(id),
+                author: {
+                    id: user.pk || user.id || '',
+                    username: user.username || '',
+                    displayName: user.full_name || '',
+                },
+                text: item.text || '',
+                timestamp: item.created_at ? new Date(item.created_at * 1000) : new Date(),
+                likes: item.comment_like_count || 0,
+            };
+        }
+        catch {
+            return null;
+        }
+    }
+    extractHashtags(text) {
+        return (text.match(/#\w+/g) || []).map(h => h.slice(1));
+    }
+    // ── Platform identity ──────────────────────────────────────────────────
+    getPlatformName() {
+        return 'instagram';
+    }
+    getSupportedFeatures() {
+        return {
+            supportsTrending: true,
+            supportsUserContent: true,
+            supportsSearch: true,
+            supportsComments: true,
+            supportsAnalysis: true,
+        };
+    }
+}

package/dist/insights/platforms/moltbook/MoltbookAdapter.d.ts

@@ -0,0 +1,31 @@
+/**
+ * Moltbook Platform Adapter
+ * Uses Moltbook REST API for content retrieval and comment extraction.
+ *
+ * API Endpoints:
+ * - GET /api/v1/search?q=QUERY&type=all&limit=N
+ * - GET /api/v1/posts/{ID}/comments?sort=best
+ * - GET /api/v1/posts?sort=hot (trending)
+ *
+ * Auth: Bearer token via MOLTBOOK_API_KEY
+ * Rate limit: 60 reads/min
+ */
+import { BaseAdapter } from '../../core/base/BaseAdapter.js';
+import { Post, Comment, PlatformCapabilities, PlatformType, PlatformConfig } from '../../core/interfaces/SocialMediaPlatform.js';
+export declare class MoltbookAdapter extends BaseAdapter {
+    private client;
+    constructor(config: PlatformConfig);
+    initialize(): Promise<boolean>;
+    getTrendingContent(limit?: number): Promise<Post[]>;
+    getUserContent(userId: string, limit?: number): Promise<Post[]>;
+    searchContent(query: string, limit?: number): Promise<Post[]>;
+    getContentComments(contentId: string, limit?: number): Promise<Comment[]>;
+    getPlatformName(): PlatformType;
+    getSupportedFeatures(): PlatformCapabilities;
+    protected isRateLimitError(error: any): boolean;
+    protected isAuthError(error: any): boolean;
+    protected isNotFoundError(error: any): boolean;
+    cleanup(): Promise<void>;
+    private normalizePost;
+    private normalizeComment;
+}

package/dist/insights/platforms/moltbook/MoltbookAdapter.js

@@ -0,0 +1,227 @@
+/**
+ * Moltbook Platform Adapter
+ * Uses Moltbook REST API for content retrieval and comment extraction.
+ *
+ * API Endpoints:
+ * - GET /api/v1/search?q=QUERY&type=all&limit=N
+ * - GET /api/v1/posts/{ID}/comments?sort=best
+ * - GET /api/v1/posts?sort=hot (trending)
+ *
+ * Auth: Bearer token via MOLTBOOK_API_KEY
+ * Rate limit: 60 reads/min
+ */
+import { BaseAdapter } from '../../core/base/BaseAdapter.js';
+import { NotFoundError } from '../../core/interfaces/SocialMediaPlatform.js';
+import axios from 'axios';
+export class MoltbookAdapter extends BaseAdapter {
+    client = null;
+    constructor(config) {
+        super(config);
+        this.maxRequestsPerWindow = 60;
+    }
+    async initialize() {
+        try {
+            const apiKey = this.config.credentials?.apiKey || '';
+            const headers = {
+                'User-Agent': 'crowdlisten-mcp/1.0.0',
+                'Accept': 'application/json',
+            };
+            if (apiKey) {
+                headers['Authorization'] = `Bearer ${apiKey}`;
+            }
+            this.client = axios.create({
+                baseURL: 'https://moltbook.com/api/v1',
+                headers,
+                timeout: 10000,
+            });
+            this.isInitialized = true;
+            this.log('Moltbook adapter initialized successfully', 'info');
+            return true;
+        }
+        catch (error) {
+            this.log('Failed to initialize Moltbook adapter', 'error');
+            this.isInitialized = false;
+            return false;
+        }
+    }
+    async getTrendingContent(limit = 10) {
+        this.ensureInitialized();
+        this.validateLimit(limit);
+        try {
+            await this.enforceRateLimit();
+            const response = await this.client.get('/posts', {
+                params: { sort: 'hot', limit },
+            });
+            const items = response.data?.posts || response.data?.data || response.data?.results || [];
+            const posts = [];
+            for (const item of (Array.isArray(items) ? items : []).slice(0, limit)) {
+                posts.push(this.normalizePost(item));
+            }
+            this.log(`Retrieved ${posts.length} trending Moltbook posts`, 'info');
+            return posts;
+        }
+        catch (error) {
+            this.handleError(error, 'getTrendingContent');
+        }
+    }
+    async getUserContent(userId, limit = 10) {
+        this.ensureInitialized();
+        this.validateUserId(userId);
+        this.validateLimit(limit);
+        try {
+            await this.enforceRateLimit();
+            const response = await this.client.get(`/users/${userId}/posts`, {
+                params: { limit },
+            });
+            const items = response.data?.posts || response.data?.data || [];
+            const posts = [];
+            for (const item of (Array.isArray(items) ? items : []).slice(0, limit)) {
+                posts.push(this.normalizePost(item));
+            }
+            this.log(`Retrieved ${posts.length} posts from Moltbook user ${userId}`, 'info');
+            return posts;
+        }
+        catch (error) {
+            if (error.response?.status === 404) {
+                throw new NotFoundError('moltbook', `User ${userId}`, error);
+            }
+            this.handleError(error, 'getUserContent');
+        }
+    }
+    async searchContent(query, limit = 10) {
+        this.ensureInitialized();
+        this.validateLimit(limit);
+        if (!query || query.trim().length === 0) {
+            throw new Error('Search query cannot be empty');
+        }
+        try {
+            await this.enforceRateLimit();
+            const response = await this.client.get('/search', {
+                params: { q: query.trim(), type: 'all', limit },
+            });
+            const items = response.data?.posts || response.data?.results || response.data?.data || [];
+            const posts = [];
+            for (const item of (Array.isArray(items) ? items : []).slice(0, limit)) {
+                posts.push(this.normalizePost(item));
+            }
+            this.log(`Found ${posts.length} Moltbook posts for query: ${query}`, 'info');
+            return posts;
+        }
+        catch (error) {
+            this.handleError(error, 'searchContent');
+        }
+    }
+    async getContentComments(contentId, limit = 20) {
+        this.ensureInitialized();
+        this.validateContentId(contentId);
+        this.validateLimit(limit);
+        try {
+            await this.enforceRateLimit();
+            const response = await this.client.get(`/posts/${contentId}/comments`, {
+                params: { sort: 'best', limit },
+            });
+            const items = response.data?.comments || response.data?.data || response.data?.results || [];
+            const comments = [];
+            const flatten = (list, depth = 0) => {
+                for (const item of list) {
+                    if (comments.length >= limit)
+                        return;
+                    comments.push(this.normalizeComment(item));
+                    const replies = item.replies || item.children || [];
+                    if (replies.length > 0 && depth < 3) {
+                        flatten(replies, depth + 1);
+                    }
+                }
+            };
+            flatten(Array.isArray(items) ? items : []);
+            this.log(`Retrieved ${comments.length} comments for Moltbook post ${contentId}`);
+            return comments.slice(0, limit);
+        }
+        catch (error) {
+            if (error.response?.status === 404) {
+                throw new NotFoundError('moltbook', `Post ${contentId}`, error);
+            }
+            this.handleError(error, 'getContentComments');
+        }
+    }
+    getPlatformName() {
+        return 'moltbook';
+    }
+    getSupportedFeatures() {
+        return {
+            supportsTrending: true,
+            supportsUserContent: true,
+            supportsSearch: true,
+            supportsComments: true,
+            supportsAnalysis: true,
+        };
+    }
+    isRateLimitError(error) {
+        return error.response?.status === 429 || error.message?.includes('rate limit');
+    }
+    isAuthError(error) {
+        return error.response?.status === 401 || error.response?.status === 403;
+    }
+    isNotFoundError(error) {
+        return error.response?.status === 404;
+    }
+    async cleanup() {
+        try {
+            this.client = null;
+            await super.cleanup();
+        }
+        catch (error) {
+            this.log('Error during Moltbook cleanup', 'warn');
+        }
+    }
+    // --- Private helpers ---
+    normalizePost(raw) {
+        const author = raw.author || {};
+        const community = raw.community?.name || raw.community_name || '';
+        const postId = raw.id || '';
+        const user = {
+            id: author.id || author.username || '',
+            username: author.username || author.name || '',
+            displayName: author.display_name || author.username || '',
+            followerCount: author.follower_count,
+            verified: author.verified || false,
+        };
+        return {
+            id: String(postId),
+            platform: 'moltbook',
+            author: user,
+            content: raw.body || raw.content || raw.title || '',
+            mediaUrl: raw.media_url || raw.image_url,
+            engagement: {
+                likes: raw.score || raw.upvotes || 0,
+                comments: raw.comment_count || raw.num_comments || 0,
+                shares: raw.share_count || 0,
+                views: raw.view_count || 0,
+            },
+            timestamp: raw.created_at ? new Date(raw.created_at) : new Date(),
+            url: raw.url || (community ? `https://moltbook.com/m/${community}/posts/${postId}` : ''),
+            hashtags: raw.tags || raw.hashtags || [],
+        };
+    }
+    normalizeComment(raw) {
+        const author = raw.author || {};
+        const user = {
+            id: author.id || author.username || '',
+            username: author.username || author.name || 'anonymous',
+            displayName: author.display_name || author.username,
+        };
+        return {
+            id: String(raw.id || ''),
+            author: user,
+            text: raw.body || raw.content || raw.text || '',
+            timestamp: raw.created_at ? new Date(raw.created_at) : new Date(),
+            likes: raw.score || raw.upvotes || 0,
+            replies: raw.replies ? raw.replies.map((r) => this.normalizeComment(r)) : undefined,
+            engagement: {
+                upvotes: raw.upvotes || raw.score || 0,
+                downvotes: raw.downvotes || 0,
+                score: raw.score || 0,
+            },
+        };
+    }
+}

package/dist/insights/platforms/reddit/RedditAdapter.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Simple Reddit Platform Adapter
+ * Uses Reddit's public JSON API for basic functionality
+ */
+import { BaseAdapter } from '../../core/base/BaseAdapter.js';
+import { Post, Comment, PlatformCapabilities, PlatformType, PlatformConfig } from '../../core/interfaces/SocialMediaPlatform.js';
+export declare class RedditAdapter extends BaseAdapter {
+    private client;
+    constructor(config: PlatformConfig);
+    initialize(): Promise<boolean>;
+    getTrendingContent(limit?: number): Promise<Post[]>;
+    getUserContent(userId: string, limit?: number): Promise<Post[]>;
+    searchContent(query: string, limit?: number): Promise<Post[]>;
+    getContentComments(contentId: string, limit?: number): Promise<Comment[]>;
+    getPlatformName(): PlatformType;
+    getSupportedFeatures(): PlatformCapabilities;
+    protected isRateLimitError(error: any): boolean;
+    protected isAuthError(error: any): boolean;
+    protected isNotFoundError(error: any): boolean;
+    cleanup(): Promise<void>;
+}