@crowdlisten/harness 1.0.0

package/dist/insights/core/utils/DataNormalizer.js

@@ -0,0 +1,349 @@
+/**
+ * Data normalization utilities for converting platform-specific data
+ * into standardized formats
+ */
+/**
+ * Normalize user data from different platforms
+ */
+export class DataNormalizer {
+    /**
+     * Normalize a post from any platform to standard format
+     */
+    static normalizePost(rawData, platform, baseUrl = '') {
+        switch (platform) {
+            case 'tiktok':
+                return this.normalizeTikTokPost(rawData);
+            case 'twitter':
+                return this.normalizeTwitterPost(rawData);
+            case 'reddit':
+                return this.normalizeRedditPost(rawData);
+            case 'instagram':
+                return this.normalizeInstagramPost(rawData);
+            case 'youtube':
+                return this.normalizeYouTubePost(rawData);
+            default:
+                throw new Error(`Unsupported platform: ${platform}`);
+        }
+    }
+    /**
+     * Normalize user data from any platform
+     */
+    static normalizeUser(rawData, platform) {
+        switch (platform) {
+            case 'tiktok':
+                return this.normalizeTikTokUser(rawData);
+            case 'twitter':
+                return this.normalizeTwitterUser(rawData);
+            case 'reddit':
+                return this.normalizeRedditUser(rawData);
+            case 'instagram':
+                return this.normalizeInstagramUser(rawData);
+            case 'youtube':
+                return this.normalizeYouTubeUser(rawData);
+            default:
+                throw new Error(`Unsupported platform: ${platform}`);
+        }
+    }
+    /**
+     * Normalize comment data from any platform
+     */
+    static normalizeComment(rawData, platform) {
+        switch (platform) {
+            case 'tiktok':
+                return this.normalizeTikTokComment(rawData);
+            case 'twitter':
+                return this.normalizeTwitterComment(rawData);
+            case 'reddit':
+                return this.normalizeRedditComment(rawData);
+            case 'instagram':
+                return this.normalizeInstagramComment(rawData);
+            case 'youtube':
+                return this.normalizeYouTubeComment(rawData);
+            default:
+                throw new Error(`Unsupported platform: ${platform}`);
+        }
+    }
+    // TikTok normalization methods
+    static normalizeTikTokPost(data) {
+        return {
+            id: data.id || data.aweme_id || '',
+            platform: 'tiktok',
+            author: this.normalizeTikTokUser(data.author || {}),
+            content: data.desc || data.description || '',
+            mediaUrl: data.video?.play_addr?.url_list?.[0] || '',
+            engagement: {
+                likes: data.stats?.diggCount || data.statistics?.digg_count || 0,
+                comments: data.stats?.commentCount || data.statistics?.comment_count || 0,
+                shares: data.stats?.shareCount || data.statistics?.share_count || 0,
+                views: data.stats?.playCount || data.statistics?.play_count || 0
+            },
+            timestamp: new Date((data.createTime || data.create_time || Date.now() / 1000) * 1000),
+            url: data.webVideoUrl || `https://www.tiktok.com/@${data.author?.uniqueId}/video/${data.id}`,
+            hashtags: this.extractHashtags(data.desc || data.description || '')
+        };
+    }
+    static normalizeTikTokUser(data) {
+        return {
+            id: data.id || data.uid || '',
+            username: data.uniqueId || data.unique_id || '',
+            displayName: data.nickname || data.nick_name || '',
+            followerCount: data.followerCount || data.follower_count || 0,
+            verified: data.verified || false,
+            profileImageUrl: data.avatarLarger || data.avatar_larger || '',
+            bio: data.signature || ''
+        };
+    }
+    static normalizeTikTokComment(data) {
+        return {
+            id: data.cid || data.comment_id || '',
+            author: this.normalizeTikTokUser(data.user || {}),
+            text: data.text || '',
+            timestamp: new Date((data.create_time || Date.now() / 1000) * 1000),
+            likes: data.digg_count || 0,
+            replies: data.reply_comment?.map((reply) => this.normalizeTikTokComment(reply)) || []
+        };
+    }
+    // Twitter normalization methods (agent-twitter-client Tweet shape)
+    static normalizeTwitterPost(data) {
+        return {
+            id: data.id || '',
+            platform: 'twitter',
+            author: this.normalizeTwitterUser(data),
+            content: data.text || '',
+            mediaUrl: data.photos?.[0]?.url || data.videos?.[0]?.preview || '',
+            engagement: {
+                likes: data.likes || 0,
+                comments: data.replies || 0,
+                shares: data.retweets || 0,
+                views: data.views || 0
+            },
+            timestamp: data.timeParsed ? new Date(data.timeParsed) :
+                data.timestamp ? new Date(data.timestamp * 1000) : new Date(),
+            url: data.permanentUrl || `https://twitter.com/${data.username || 'user'}/status/${data.id}`,
+            hashtags: data.hashtags || this.extractHashtags(data.text || '')
+        };
+    }
+    static normalizeTwitterUser(data) {
+        return {
+            id: data.userId || data.id || '',
+            username: data.username || '',
+            displayName: data.name || '',
+            followerCount: 0,
+            verified: false,
+            profileImageUrl: '',
+            bio: ''
+        };
+    }
+    static normalizeTwitterComment(data) {
+        // Twitter replies normalized as comments
+        return {
+            id: data.id || '',
+            author: this.normalizeTwitterUser(data),
+            text: data.text || '',
+            timestamp: data.timeParsed ? new Date(data.timeParsed) :
+                data.timestamp ? new Date(data.timestamp * 1000) : new Date(),
+            likes: data.likes || 0,
+            replies: []
+        };
+    }
+    // Reddit normalization methods
+    static normalizeRedditPost(data) {
+        return {
+            id: data.id || '',
+            platform: 'reddit',
+            author: this.normalizeRedditUser(data.author_display_name || data.author || ''),
+            content: data.selftext || data.title || '',
+            mediaUrl: data.url_overridden_by_dest || data.url || '',
+            engagement: {
+                likes: data.score || data.ups || 0,
+                comments: data.num_comments || data.comment_count || 0,
+                shares: 0, // Reddit doesn't have shares
+                views: 0 // Reddit doesn't track views publicly
+            },
+            timestamp: new Date((data.created_utc || data.created || Date.now() / 1000) * 1000),
+            url: `https://reddit.com${data.permalink}`,
+            hashtags: []
+        };
+    }
+    static normalizeRedditUser(data) {
+        if (typeof data === 'string') {
+            return {
+                id: data,
+                username: data,
+                displayName: data
+            };
+        }
+        return {
+            id: data.id || data.name || '',
+            username: data.name || data.display_name || '',
+            displayName: data.display_name || data.name || '',
+            followerCount: 0, // Reddit doesn't expose follower counts
+            verified: data.is_gold || false,
+            profileImageUrl: data.icon_img || '',
+            bio: data.public_description || ''
+        };
+    }
+    static normalizeRedditComment(data) {
+        // Reddit nests replies as a Listing: replies.data.children[]
+        // replies can also be an empty string when there are no replies
+        const replyChildren = data.replies?.data?.children || [];
+        const replies = replyChildren
+            .filter((child) => child.kind === 't1' && child.data)
+            .map((child) => this.normalizeRedditComment(child.data));
+        return {
+            id: data.id || '',
+            author: this.normalizeRedditUser(data.author_display_name || data.author || ''),
+            text: data.body || '',
+            timestamp: new Date((data.created_utc || Date.now() / 1000) * 1000),
+            likes: data.score || data.ups || 0,
+            replies
+        };
+    }
+    // Instagram normalization methods
+    static normalizeInstagramPost(data) {
+        return {
+            id: data.id || data.pk || '',
+            platform: 'instagram',
+            author: this.normalizeInstagramUser(data.user || data.owner || {}),
+            content: data.caption?.text || data.edge_media_to_caption?.edges?.[0]?.node?.text || '',
+            mediaUrl: data.image_versions2?.candidates?.[0]?.url || data.display_url || '',
+            engagement: {
+                likes: data.like_count || data.edge_media_preview_like?.count || 0,
+                comments: data.comment_count || data.edge_media_to_comment?.count || 0,
+                shares: 0, // Instagram doesn't expose shares
+                views: data.view_count || data.video_view_count || 0
+            },
+            timestamp: new Date((data.taken_at || data.taken_at_timestamp || Date.now() / 1000) * 1000),
+            url: `https://www.instagram.com/p/${data.code || data.shortcode}/`,
+            hashtags: this.extractHashtags(data.caption?.text || '')
+        };
+    }
+    static normalizeInstagramUser(data) {
+        return {
+            id: data.pk || data.id || '',
+            username: data.username || '',
+            displayName: data.full_name || data.username || '',
+            followerCount: data.follower_count || data.edge_followed_by?.count || 0,
+            verified: data.is_verified || false,
+            profileImageUrl: data.profile_pic_url || data.profile_pic_url_hd || '',
+            bio: data.biography || data.bio || ''
+        };
+    }
+    static normalizeInstagramComment(data) {
+        return {
+            id: data.pk || data.id || '',
+            author: this.normalizeInstagramUser(data.user || {}),
+            text: data.text || '',
+            timestamp: new Date((data.created_at || data.created_at_utc || Date.now() / 1000) * 1000),
+            likes: data.comment_like_count || 0,
+            replies: data.child_comment_count ? [] : [] // Instagram API complex for replies
+        };
+    }
+    // YouTube normalization methods
+    static normalizeYouTubePost(data) {
+        const snippet = data.snippet || {};
+        const stats = data.statistics || {};
+        const videoId = data.id?.videoId || data.id || '';
+        return {
+            id: videoId,
+            platform: 'youtube',
+            author: this.normalizeYouTubeUser(snippet),
+            content: snippet.title || '',
+            mediaUrl: snippet.thumbnails?.high?.url || snippet.thumbnails?.default?.url || '',
+            engagement: {
+                likes: parseInt(stats.likeCount || '0', 10),
+                comments: parseInt(stats.commentCount || '0', 10),
+                shares: 0,
+                views: parseInt(stats.viewCount || '0', 10)
+            },
+            timestamp: new Date(snippet.publishedAt || Date.now()),
+            url: `https://www.youtube.com/watch?v=${videoId}`,
+            hashtags: this.extractHashtags(snippet.title || '')
+        };
+    }
+    static normalizeYouTubeUser(data) {
+        return {
+            id: data.channelId || '',
+            username: data.channelTitle || '',
+            displayName: data.channelTitle || '',
+            followerCount: 0,
+            verified: false,
+            profileImageUrl: '',
+            bio: ''
+        };
+    }
+    static normalizeYouTubeComment(data) {
+        const topLevel = data.snippet?.topLevelComment?.snippet || {};
+        const replies = (data.replies?.comments || []).map((reply) => {
+            const rs = reply.snippet || {};
+            return {
+                id: reply.id || '',
+                author: {
+                    id: rs.authorChannelId?.value || '',
+                    username: rs.authorDisplayName || '',
+                    displayName: rs.authorDisplayName || ''
+                },
+                text: rs.textDisplay || rs.textOriginal || '',
+                timestamp: new Date(rs.publishedAt || Date.now()),
+                likes: rs.likeCount || 0,
+                replies: []
+            };
+        });
+        return {
+            id: data.id || '',
+            author: {
+                id: topLevel.authorChannelId?.value || '',
+                username: topLevel.authorDisplayName || '',
+                displayName: topLevel.authorDisplayName || ''
+            },
+            text: topLevel.textDisplay || topLevel.textOriginal || '',
+            timestamp: new Date(topLevel.publishedAt || Date.now()),
+            likes: topLevel.likeCount || 0,
+            replies
+        };
+    }
+    /**
+     * Extract hashtags from text
+     */
+    static extractHashtags(text) {
+        const hashtagRegex = /#[\w\u00c0-\u024f\u1e00-\u1eff]+/gi;
+        const matches = text.match(hashtagRegex);
+        return matches ? matches.map(tag => tag.toLowerCase()) : [];
+    }
+    /**
+     * Calculate engagement rate
+     */
+    static calculateEngagementRate(likes, comments, shares = 0, followerCount) {
+        if (followerCount === 0)
+            return 0;
+        return ((likes + comments + shares) / followerCount) * 100;
+    }
+    /**
+     * Sanitize and validate text content
+     */
+    static sanitizeText(text) {
+        if (!text)
+            return '';
+        return text
+            .replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, '') // Remove control characters
+            .trim()
+            .substring(0, 2000); // Limit length
+    }
+    /**
+     * Normalize timestamp from various formats
+     */
+    static normalizeTimestamp(timestamp) {
+        if (!timestamp)
+            return new Date();
+        if (timestamp instanceof Date)
+            return timestamp;
+        if (typeof timestamp === 'string') {
+            return new Date(timestamp);
+        }
+        if (typeof timestamp === 'number') {
+            // Handle both seconds and milliseconds
+            return new Date(timestamp < 1e12 ? timestamp * 1000 : timestamp);
+        }
+        return new Date();
+    }
+}

package/dist/insights/core/utils/InstagramUrlUtils.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Shared Instagram URL utilities for detection, redirect resolution, and shortcode extraction.
+ */
+export declare class InstagramUrlUtils {
+    private static readonly INSTAGRAM_HOST_PATTERN;
+    static isInstagramUrl(input: string): boolean;
+    static isReelUrl(input: string): boolean;
+    static resolveUrl(input: string): Promise<string>;
+    static extractShortcode(input: string): string | null;
+    private static tryParseUrl;
+}

package/dist/insights/core/utils/InstagramUrlUtils.js

@@ -0,0 +1,60 @@
+import axios from 'axios';
+/**
+ * Shared Instagram URL utilities for detection, redirect resolution, and shortcode extraction.
+ */
+export class InstagramUrlUtils {
+    static INSTAGRAM_HOST_PATTERN = /(^|\.)instagram\.com$/i;
+    static isInstagramUrl(input) {
+        const parsed = this.tryParseUrl(input);
+        if (!parsed)
+            return false;
+        return this.INSTAGRAM_HOST_PATTERN.test(parsed.hostname);
+    }
+    static isReelUrl(input) {
+        const parsed = this.tryParseUrl(input);
+        if (!parsed)
+            return false;
+        return /\/(reel|reels)\//i.test(parsed.pathname);
+    }
+    static async resolveUrl(input) {
+        const parsed = this.tryParseUrl(input);
+        if (!parsed) {
+            return input;
+        }
+        try {
+            const response = await axios.get(parsed.toString(), {
+                maxRedirects: 5,
+                timeout: 10000,
+                validateStatus: (status) => status >= 200 && status < 400,
+            });
+            const finalUrl = response.request?.res?.responseUrl;
+            return typeof finalUrl === 'string' && finalUrl.length > 0 ? finalUrl : parsed.toString();
+        }
+        catch {
+            return parsed.toString();
+        }
+    }
+    static extractShortcode(input) {
+        const parsed = this.tryParseUrl(input);
+        if (!parsed) {
+            return null;
+        }
+        // Match /reel/{shortcode}/ or /p/{shortcode}/ or /reels/{shortcode}/
+        const match = parsed.pathname.match(/\/(reel|reels|p)\/([A-Za-z0-9_-]+)/);
+        if (match?.[2]) {
+            return match[2];
+        }
+        return null;
+    }
+    static tryParseUrl(input) {
+        if (!input || input.trim().length === 0) {
+            return null;
+        }
+        try {
+            return new URL(input.trim());
+        }
+        catch {
+            return null;
+        }
+    }
+}

package/dist/insights/core/utils/TikTokUrlUtils.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Shared TikTok URL utilities for detection, redirect resolution, and video ID extraction.
+ */
+export declare class TikTokUrlUtils {
+    private static readonly TIKTOK_HOST_PATTERN;
+    static isTikTokUrl(input: string): boolean;
+    static resolveUrl(input: string): Promise<string>;
+    static extractVideoId(input: string): string | null;
+    private static tryParseUrl;
+}

package/dist/insights/core/utils/TikTokUrlUtils.js

@@ -0,0 +1,57 @@
+import axios from 'axios';
+/**
+ * Shared TikTok URL utilities for detection, redirect resolution, and video ID extraction.
+ */
+export class TikTokUrlUtils {
+    static TIKTOK_HOST_PATTERN = /(^|\.)tiktok\.com$/i;
+    static isTikTokUrl(input) {
+        const parsed = this.tryParseUrl(input);
+        if (!parsed)
+            return false;
+        return this.TIKTOK_HOST_PATTERN.test(parsed.hostname);
+    }
+    static async resolveUrl(input) {
+        const parsed = this.tryParseUrl(input);
+        if (!parsed) {
+            return input;
+        }
+        try {
+            const response = await axios.get(parsed.toString(), {
+                maxRedirects: 5,
+                timeout: 10000,
+                validateStatus: (status) => status >= 200 && status < 400,
+            });
+            const finalUrl = response.request?.res?.responseUrl;
+            return typeof finalUrl === 'string' && finalUrl.length > 0 ? finalUrl : parsed.toString();
+        }
+        catch {
+            return parsed.toString();
+        }
+    }
+    static extractVideoId(input) {
+        const parsed = this.tryParseUrl(input);
+        if (!parsed) {
+            return null;
+        }
+        const pathMatch = parsed.pathname.match(/\/video\/(\d+)/);
+        if (pathMatch?.[1]) {
+            return pathMatch[1];
+        }
+        const itemId = parsed.searchParams.get('item_id') || parsed.searchParams.get('aweme_id');
+        if (itemId && /^\d+$/.test(itemId)) {
+            return itemId;
+        }
+        return null;
+    }
+    static tryParseUrl(input) {
+        if (!input || input.trim().length === 0) {
+            return null;
+        }
+        try {
+            return new URL(input.trim());
+        }
+        catch {
+            return null;
+        }
+    }
+}

package/dist/insights/handlers.d.ts

@@ -0,0 +1,157 @@
+/**
+ * CrowdListen Shared Handlers
+ * Pure functions that return plain objects — used by CLI and MCP server.
+ *
+ * Retrieval handlers (free, local): search, comments, trending, user content, vision
+ * Analysis handlers (paid, API): analyze, cluster, enrich, deep_analyze, insights, research
+ */
+import { UnifiedSocialMediaService } from './services/UnifiedSocialMediaService.js';
+import { HealthMonitor } from './core/health/HealthMonitor.js';
+export interface SearchArgs {
+    platform: string;
+    query: string;
+    limit?: number;
+    useVision?: boolean;
+}
+export interface CommentsArgs {
+    platform: string;
+    contentId: string;
+    limit?: number;
+    useVision?: boolean;
+}
+export interface AnalyzeArgs {
+    platform: string;
+    contentId: string;
+    analysisDepth?: 'surface' | 'standard' | 'deep' | 'comprehensive';
+}
+export interface ClusterArgs {
+    platform: string;
+    contentId: string;
+    clusterCount?: number;
+    includeExamples?: boolean;
+    weightByEngagement?: boolean;
+}
+export interface EnrichArgs {
+    platform: string;
+    contentId: string;
+    question?: string;
+}
+export interface TrendingArgs {
+    platform: string;
+    limit?: number;
+}
+export interface UserContentArgs {
+    platform: string;
+    userId: string;
+    limit?: number;
+}
+export interface ExtractUrlArgs {
+    url: string;
+    mode?: 'posts' | 'comments' | 'raw';
+    limit?: number;
+}
+export declare function extractWithVision(args: ExtractUrlArgs): Promise<{
+    raw?: string | undefined;
+    count?: number | undefined;
+    comments?: import("./core/interfaces/SocialMediaPlatform.js").Comment[] | undefined;
+    posts?: import("./core/interfaces/SocialMediaPlatform.js").Post[] | undefined;
+    url: string;
+    mode: "posts" | "comments" | "raw";
+    provider: string;
+    extractionMethod: string;
+}>;
+export declare function getTrendingContent(service: UnifiedSocialMediaService, args: TrendingArgs): Promise<{
+    platform: string;
+    count: number;
+    posts: import("./core/interfaces/SocialMediaPlatform.js").Post[];
+}>;
+export declare function getUserContent(service: UnifiedSocialMediaService, args: UserContentArgs): Promise<{
+    platform: string;
+    userId: string;
+    count: number;
+    posts: import("./core/interfaces/SocialMediaPlatform.js").Post[];
+}>;
+export declare function searchContent(service: UnifiedSocialMediaService, args: SearchArgs): Promise<{
+    raw?: string | undefined;
+    count?: number | undefined;
+    comments?: import("./core/interfaces/SocialMediaPlatform.js").Comment[] | undefined;
+    posts?: import("./core/interfaces/SocialMediaPlatform.js").Post[] | undefined;
+    url: string;
+    mode: "posts" | "comments" | "raw";
+    provider: string;
+    extractionMethod: string;
+} | {
+    platform: string;
+    query: string;
+    count: number;
+    posts: import("./core/interfaces/SocialMediaPlatform.js").Post[];
+}>;
+export declare function getContentComments(service: UnifiedSocialMediaService, args: CommentsArgs): Promise<{
+    raw?: string | undefined;
+    count?: number | undefined;
+    comments?: import("./core/interfaces/SocialMediaPlatform.js").Comment[] | undefined;
+    posts?: import("./core/interfaces/SocialMediaPlatform.js").Post[] | undefined;
+    url: string;
+    mode: "posts" | "comments" | "raw";
+    provider: string;
+    extractionMethod: string;
+} | {
+    platform: string;
+    contentId: string;
+    count: number;
+    comments: import("./core/interfaces/SocialMediaPlatform.js").Comment[];
+}>;
+export declare function getPlatformStatus(service: UnifiedSocialMediaService): {
+    availablePlatforms: {
+        tiktok?: any;
+        twitter?: any;
+        reddit?: any;
+        instagram?: any;
+        youtube?: any;
+        moltbook?: any;
+        xiaohongshu?: any;
+    };
+    totalPlatforms: number;
+};
+export declare function healthCheck(service: UnifiedSocialMediaService, monitor?: HealthMonitor): Promise<{
+    overall: import("./index.js").HealthStatus;
+    healthStatus: Record<string, unknown>;
+    source: string;
+    lastFullCheck: string | null;
+    timestamp: string;
+} | {
+    healthStatus: {
+        tiktok?: "healthy" | "degraded" | "down" | undefined;
+        twitter?: "healthy" | "degraded" | "down" | undefined;
+        reddit?: "healthy" | "degraded" | "down" | undefined;
+        instagram?: "healthy" | "degraded" | "down" | undefined;
+        youtube?: "healthy" | "degraded" | "down" | undefined;
+        moltbook?: "healthy" | "degraded" | "down" | undefined;
+        xiaohongshu?: "healthy" | "degraded" | "down" | undefined;
+    };
+    timestamp: string;
+    overall?: undefined;
+    source?: undefined;
+    lastFullCheck?: undefined;
+}>;
+export declare function analyzeContent(service: UnifiedSocialMediaService, args: AnalyzeArgs): Promise<any>;
+export declare function clusterOpinions(service: UnifiedSocialMediaService, args: ClusterArgs): Promise<any>;
+export declare function enrichContent(service: UnifiedSocialMediaService, args: EnrichArgs): Promise<any>;
+export interface DeepAnalyzeArgs {
+    platform: string;
+    contentId: string;
+    analysisDepth: 'deep' | 'comprehensive';
+}
+export interface InsightsArgs {
+    platform: string;
+    contentId: string;
+    categories?: string[];
+}
+export interface ResearchArgs {
+    query: string;
+    platforms?: string[];
+    depth?: 'quick' | 'standard' | 'deep';
+}
+export declare function deepAnalyze(args: DeepAnalyzeArgs): Promise<any>;
+export declare function extractInsights(args: InsightsArgs): Promise<any>;
+export declare function researchSynthesis(args: ResearchArgs): Promise<any>;