@crowdlisten/harness 1.0.0

package/dist/insights/core/base/BaseAdapter.js

@@ -0,0 +1,123 @@
+/**
+ * Base adapter class providing common functionality for all social media platforms
+ * Implements shared features like rate limiting, error handling, and logging
+ *
+ * Analysis (clustering, enrichment) has been moved to the CrowdListen API.
+ * This class now only handles data retrieval.
+ */
+import { SocialMediaError, RateLimitError } from '../interfaces/SocialMediaPlatform.js';
+export class BaseAdapter {
+    config;
+    isInitialized = false;
+    lastRequestTime = 0;
+    requestCount = 0;
+    rateLimitWindow = 60000; // 1 minute window
+    maxRequestsPerWindow = 30; // Max 30 requests per minute
+    constructor(config) {
+        this.config = config;
+    }
+    async enforceRateLimit() {
+        const now = Date.now();
+        const timeSinceLastRequest = now - this.lastRequestTime;
+        // Reset request count if window has passed
+        if (timeSinceLastRequest > this.rateLimitWindow) {
+            this.requestCount = 0;
+        }
+        // If we've hit the rate limit, wait
+        if (this.requestCount >= this.maxRequestsPerWindow) {
+            const waitTime = this.rateLimitWindow - timeSinceLastRequest;
+            if (waitTime > 0) {
+                this.log(`Rate limit reached. Waiting ${waitTime}ms...`);
+                await this.sleep(waitTime);
+                this.requestCount = 0;
+            }
+        }
+        this.requestCount++;
+        this.lastRequestTime = Date.now();
+    }
+    async sleep(ms) {
+        return new Promise(resolve => setTimeout(resolve, ms));
+    }
+    handleError(error, operation) {
+        this.log(`Error in ${operation}: ${error.message || error}`, 'error');
+        if (this.isRateLimitError(error)) {
+            throw new RateLimitError(this.getPlatformName(), error);
+        }
+        if (this.isAuthError(error)) {
+            throw new SocialMediaError(`Authentication failed during ${operation}`, 'authentication_error', this.getPlatformName(), error.statusCode);
+        }
+        if (this.isNotFoundError(error)) {
+            throw new SocialMediaError(`Resource not found during ${operation}`, 'not_found', this.getPlatformName(), error.statusCode);
+        }
+        throw new SocialMediaError(`Operation ${operation} failed: ${error.message || error}`, 'unknown_error', this.getPlatformName(), error.statusCode);
+    }
+    isRateLimitError(error) {
+        return error.statusCode === 429 || error.code === 'rate_limit_exceeded';
+    }
+    isAuthError(error) {
+        return error.statusCode === 401 || error.statusCode === 403;
+    }
+    isNotFoundError(error) {
+        return error.statusCode === 404;
+    }
+    log(message, level = 'info') {
+        const timestamp = new Date().toISOString();
+        const platformName = this.getPlatformName();
+        const prefix = `[${timestamp}] [${platformName.toUpperCase()}]`;
+        switch (level) {
+            case 'error':
+                console.error(`${prefix} ERROR: ${message}`);
+                break;
+            case 'warn':
+                console.warn(`${prefix} WARN: ${message}`);
+                break;
+            default:
+                console.log(`${prefix} INFO: ${message}`);
+        }
+    }
+    validateUserId(userId) {
+        if (!userId || userId.trim().length === 0) {
+            throw new SocialMediaError('User ID cannot be empty', 'validation_error', this.getPlatformName());
+        }
+    }
+    validateContentId(contentId) {
+        if (!contentId || contentId.trim().length === 0) {
+            throw new SocialMediaError('Content ID cannot be empty', 'validation_error', this.getPlatformName());
+        }
+    }
+    validateLimit(limit) {
+        if (limit < 1 || limit > 1000) {
+            throw new SocialMediaError('Limit must be between 1 and 1000', 'validation_error', this.getPlatformName());
+        }
+    }
+    async analyzeContent(contentId, enableClustering = true) {
+        this.validateContentId(contentId);
+        try {
+            await this.enforceRateLimit();
+            const comments = await this.getContentComments(contentId, 200);
+            // Return basic post metadata only — analysis is handled by the CrowdListen API
+            const analysis = {
+                postId: contentId,
+                platform: this.getPlatformName(),
+                sentiment: 'neutral',
+                themes: ['general'],
+                summary: `Retrieved ${comments.length} comments for ${contentId}. Use analyze_content with CROWDLISTEN_API_KEY for full analysis.`,
+                commentCount: comments.length,
+                topComments: comments.slice(0, 5),
+            };
+            return analysis;
+        }
+        catch (error) {
+            this.handleError(error, 'analyzeContent');
+        }
+    }
+    ensureInitialized() {
+        if (!this.isInitialized) {
+            throw new SocialMediaError('Adapter not initialized. Call initialize() first.', 'initialization_error', this.getPlatformName());
+        }
+    }
+    async cleanup() {
+        this.isInitialized = false;
+        this.log('Adapter cleaned up');
+    }
+}

package/dist/insights/core/health/HealthMonitor.d.ts

@@ -0,0 +1,75 @@
+/**
+ * Proactive Health Monitor
+ *
+ * Periodically checks platform health in the background, tracks response
+ * times, consecutive failures, and degradation state. The MCP health_check
+ * tool reads cached state from here instead of doing a live probe every
+ * time, making it fast and non-blocking.
+ *
+ * Three-state model: healthy -> degraded -> down
+ *   - healthy:  last check succeeded with results
+ *   - degraded: last check returned 0 results, or 1-2 consecutive failures
+ *   - down:     3+ consecutive failures
+ */
+import { PlatformType } from '../interfaces/SocialMediaPlatform.js';
+export type HealthStatus = 'healthy' | 'degraded' | 'down';
+export interface PlatformHealthState {
+    platform: PlatformType;
+    status: HealthStatus;
+    lastChecked: Date;
+    lastHealthy: Date | null;
+    consecutiveFailures: number;
+    responseTimeMs: number | null;
+    error?: string;
+}
+export interface HealthSummary {
+    overall: HealthStatus;
+    platforms: Record<string, PlatformHealthState>;
+    lastFullCheck: Date | null;
+}
+export type HealthCheckFn = (platform: PlatformType) => Promise<{
+    success: boolean;
+    resultCount: number;
+}>;
+export declare class HealthMonitor {
+    private checkFn;
+    private platforms;
+    private intervalMs;
+    private states;
+    private intervalId;
+    private startupTimeoutId;
+    private lastFullCheck;
+    constructor(checkFn: HealthCheckFn, platforms: PlatformType[], intervalMs?: number);
+    /**
+     * Start background health checks.
+     * The first check runs after a short delay so it does not block MCP
+     * server startup.
+     */
+    start(): void;
+    /**
+     * Stop background health checks and clean up timers.
+     */
+    stop(): void;
+    /**
+     * Check all platforms in parallel. One failing platform does not block
+     * or affect the others.
+     */
+    checkAll(): Promise<HealthSummary>;
+    /**
+     * Check a single platform and update its state.
+     */
+    checkPlatform(platform: PlatformType): Promise<PlatformHealthState>;
+    /**
+     * Get a snapshot summary of all platform health states.
+     */
+    getSummary(): HealthSummary;
+    /**
+     * Get health state for a single platform. Returns undefined if the
+     * platform is not being monitored.
+     */
+    getState(platform: PlatformType): PlatformHealthState | undefined;
+    /**
+     * Whether the monitor has recent data (checked within the given ms).
+     */
+    hasRecentData(withinMs?: number): boolean;
+}

package/dist/insights/core/health/HealthMonitor.js

@@ -0,0 +1,171 @@
+/**
+ * Proactive Health Monitor
+ *
+ * Periodically checks platform health in the background, tracks response
+ * times, consecutive failures, and degradation state. The MCP health_check
+ * tool reads cached state from here instead of doing a live probe every
+ * time, making it fast and non-blocking.
+ *
+ * Three-state model: healthy -> degraded -> down
+ *   - healthy:  last check succeeded with results
+ *   - degraded: last check returned 0 results, or 1-2 consecutive failures
+ *   - down:     3+ consecutive failures
+ */
+/**
+ * Default check interval: 5 minutes.
+ * Default per-platform timeout: 15 seconds.
+ */
+const DEFAULT_INTERVAL_MS = 5 * 60 * 1000;
+const CHECK_TIMEOUT_MS = 15_000;
+const STARTUP_DELAY_MS = 10_000;
+const CONSECUTIVE_FAILURE_THRESHOLD = 3;
+export class HealthMonitor {
+    checkFn;
+    platforms;
+    intervalMs;
+    states = new Map();
+    intervalId = null;
+    startupTimeoutId = null;
+    lastFullCheck = null;
+    constructor(checkFn, platforms, intervalMs = DEFAULT_INTERVAL_MS) {
+        this.checkFn = checkFn;
+        this.platforms = platforms;
+        this.intervalMs = intervalMs;
+        for (const p of platforms) {
+            this.states.set(p, {
+                platform: p,
+                status: 'healthy', // optimistic start
+                lastChecked: new Date(0),
+                lastHealthy: null,
+                consecutiveFailures: 0,
+                responseTimeMs: null,
+            });
+        }
+    }
+    /**
+     * Start background health checks.
+     * The first check runs after a short delay so it does not block MCP
+     * server startup.
+     */
+    start() {
+        if (this.intervalId)
+            return;
+        this.startupTimeoutId = setTimeout(() => {
+            this.checkAll().catch((err) => console.error('[HealthMonitor] Initial check error:', err));
+        }, STARTUP_DELAY_MS);
+        this.intervalId = setInterval(() => {
+            this.checkAll().catch((err) => console.error('[HealthMonitor] Periodic check error:', err));
+        }, this.intervalMs);
+        console.error(`[HealthMonitor] Started — checking ${this.platforms.length} platforms every ${Math.round(this.intervalMs / 1000)}s`);
+    }
+    /**
+     * Stop background health checks and clean up timers.
+     */
+    stop() {
+        if (this.startupTimeoutId) {
+            clearTimeout(this.startupTimeoutId);
+            this.startupTimeoutId = null;
+        }
+        if (this.intervalId) {
+            clearInterval(this.intervalId);
+            this.intervalId = null;
+        }
+        console.error('[HealthMonitor] Stopped');
+    }
+    /**
+     * Check all platforms in parallel. One failing platform does not block
+     * or affect the others.
+     */
+    async checkAll() {
+        await Promise.allSettled(this.platforms.map((p) => this.checkPlatform(p)));
+        this.lastFullCheck = new Date();
+        return this.getSummary();
+    }
+    /**
+     * Check a single platform and update its state.
+     */
+    async checkPlatform(platform) {
+        const state = this.states.get(platform);
+        if (!state) {
+            throw new Error(`Unknown platform: ${platform}`);
+        }
+        const start = Date.now();
+        try {
+            const result = await Promise.race([
+                this.checkFn(platform),
+                new Promise((_, reject) => setTimeout(() => reject(new Error('Health check timeout')), CHECK_TIMEOUT_MS)),
+            ]);
+            const elapsed = Date.now() - start;
+            if (result.success && result.resultCount > 0) {
+                state.status = 'healthy';
+                state.lastHealthy = new Date();
+                state.consecutiveFailures = 0;
+                state.error = undefined;
+            }
+            else if (result.success && result.resultCount === 0) {
+                // Responded but returned nothing — might be a transient issue
+                state.consecutiveFailures++;
+                state.status =
+                    state.consecutiveFailures >= CONSECUTIVE_FAILURE_THRESHOLD
+                        ? 'down'
+                        : 'degraded';
+            }
+            else {
+                state.consecutiveFailures++;
+                state.status =
+                    state.consecutiveFailures >= CONSECUTIVE_FAILURE_THRESHOLD
+                        ? 'down'
+                        : 'degraded';
+            }
+            state.responseTimeMs = elapsed;
+        }
+        catch (error) {
+            state.consecutiveFailures++;
+            state.status =
+                state.consecutiveFailures >= CONSECUTIVE_FAILURE_THRESHOLD
+                    ? 'down'
+                    : 'degraded';
+            state.error =
+                error instanceof Error ? error.message : String(error);
+            state.responseTimeMs = Date.now() - start;
+        }
+        state.lastChecked = new Date();
+        return { ...state };
+    }
+    /**
+     * Get a snapshot summary of all platform health states.
+     */
+    getSummary() {
+        const platforms = {};
+        let hasDown = false;
+        let hasDegraded = false;
+        for (const [key, state] of this.states) {
+            platforms[key] = { ...state };
+            if (state.status === 'down')
+                hasDown = true;
+            if (state.status === 'degraded')
+                hasDegraded = true;
+        }
+        return {
+            overall: hasDown ? 'down' : hasDegraded ? 'degraded' : 'healthy',
+            platforms,
+            lastFullCheck: this.lastFullCheck,
+        };
+    }
+    /**
+     * Get health state for a single platform. Returns undefined if the
+     * platform is not being monitored.
+     */
+    getState(platform) {
+        const state = this.states.get(platform);
+        return state ? { ...state } : undefined;
+    }
+    /**
+     * Whether the monitor has recent data (checked within the given ms).
+     */
+    hasRecentData(withinMs = DEFAULT_INTERVAL_MS) {
+        if (!this.lastFullCheck)
+            return false;
+        return Date.now() - this.lastFullCheck.getTime() < withinMs;
+    }
+}

package/dist/insights/core/interfaces/SocialMediaPlatform.d.ts

@@ -0,0 +1,125 @@
+/**
+ * Core interface that all social media platform adapters must implement
+ * Provides a unified API for content retrieval across different platforms
+ *
+ * Analysis types (EnrichedComment, OpinionUnit, etc.) have been moved to
+ * the CrowdListen API. This file now only contains retrieval types.
+ */
+export interface Post {
+    id: string;
+    platform: PlatformType;
+    author: User;
+    content: string;
+    mediaUrl?: string;
+    engagement: EngagementMetrics;
+    timestamp: Date;
+    url: string;
+    hashtags?: string[];
+}
+export interface User {
+    id: string;
+    username: string;
+    displayName?: string;
+    followerCount?: number;
+    verified?: boolean;
+    profileImageUrl?: string;
+    bio?: string;
+}
+export interface EngagementMetrics {
+    likes: number;
+    comments: number;
+    shares?: number;
+    views?: number;
+    engagementRate?: number;
+}
+export interface Comment {
+    id: string;
+    author: User;
+    text: string;
+    timestamp: Date;
+    likes: number;
+    replies?: Comment[];
+    engagement?: {
+        upvotes?: number;
+        downvotes?: number;
+        shares?: number;
+        views?: number;
+        score?: number;
+    };
+}
+export interface ContentAnalysis {
+    postId: string;
+    platform: PlatformType;
+    sentiment?: 'positive' | 'negative' | 'neutral';
+    themes?: string[];
+    summary?: string;
+    commentCount: number;
+    topComments: Comment[];
+    analysisMetadata?: Record<string, unknown>;
+}
+export interface CommentCluster {
+    id: number;
+    theme: string;
+    sentiment: 'positive' | 'negative' | 'neutral' | 'mixed';
+    comments: Comment[];
+    summary: string;
+    size: number;
+}
+export interface TrendingHashtag {
+    hashtag: string;
+    postCount: number;
+    engagementScore: number;
+}
+export type PlatformType = 'tiktok' | 'twitter' | 'reddit' | 'instagram' | 'youtube' | 'moltbook' | 'xiaohongshu';
+export interface PlatformCapabilities {
+    supportsTrending: boolean;
+    supportsUserContent: boolean;
+    supportsSearch: boolean;
+    supportsComments: boolean;
+    supportsAnalysis: boolean;
+}
+/**
+ * Main interface that all platform adapters must implement
+ */
+export interface SocialMediaPlatform {
+    getTrendingContent(limit?: number): Promise<Post[]>;
+    getUserContent(userId: string, limit?: number): Promise<Post[]>;
+    searchContent(query: string, limit?: number): Promise<Post[]>;
+    getContentComments(contentId: string, limit?: number): Promise<Comment[]>;
+    analyzeContent(contentId: string, enableClustering?: boolean): Promise<ContentAnalysis>;
+    getPlatformName(): PlatformType;
+    getSupportedFeatures(): PlatformCapabilities;
+    initialize(): Promise<boolean>;
+    cleanup(): Promise<void>;
+}
+/**
+ * Configuration interface for platform adapters
+ */
+export interface PlatformConfig {
+    platform: PlatformType;
+    credentials?: Record<string, string>;
+    options?: {
+        rateLimit?: number;
+        timeout?: number;
+        retries?: number;
+    };
+}
+/**
+ * Error types for unified error handling
+ */
+export declare class SocialMediaError extends Error {
+    code: string;
+    platform: PlatformType;
+    statusCode?: number | undefined;
+    originalError?: Error | undefined;
+    constructor(message: string, code: string, platform: PlatformType, statusCode?: number | undefined, originalError?: Error | undefined);
+}
+export declare class AuthenticationError extends SocialMediaError {
+    constructor(platform: PlatformType, originalError?: Error);
+}
+export declare class RateLimitError extends SocialMediaError {
+    constructor(platform: PlatformType, originalError?: Error);
+}
+export declare class NotFoundError extends SocialMediaError {
+    constructor(platform: PlatformType, resource: string, originalError?: Error);
+}

package/dist/insights/core/interfaces/SocialMediaPlatform.js

@@ -0,0 +1,42 @@
+/**
+ * Core interface that all social media platform adapters must implement
+ * Provides a unified API for content retrieval across different platforms
+ *
+ * Analysis types (EnrichedComment, OpinionUnit, etc.) have been moved to
+ * the CrowdListen API. This file now only contains retrieval types.
+ */
+/**
+ * Error types for unified error handling
+ */
+export class SocialMediaError extends Error {
+    code;
+    platform;
+    statusCode;
+    originalError;
+    constructor(message, code, platform, statusCode, originalError) {
+        super(message);
+        this.code = code;
+        this.platform = platform;
+        this.statusCode = statusCode;
+        this.originalError = originalError;
+        this.name = 'SocialMediaError';
+    }
+}
+export class AuthenticationError extends SocialMediaError {
+    constructor(platform, originalError) {
+        super(`Authentication failed for ${platform}`, 'AUTH_ERROR', platform, undefined, originalError);
+        this.name = 'AuthenticationError';
+    }
+}
+export class RateLimitError extends SocialMediaError {
+    constructor(platform, originalError) {
+        super(`Rate limit exceeded for ${platform}`, 'RATE_LIMIT', platform, undefined, originalError);
+        this.name = 'RateLimitError';
+    }
+}
+export class NotFoundError extends SocialMediaError {
+    constructor(platform, resource, originalError) {
+        super(`${resource} not found on ${platform}`, 'NOT_FOUND', platform, undefined, originalError);
+        this.name = 'NotFoundError';
+    }
+}

package/dist/insights/core/utils/DataNormalizer.d.ts

@@ -0,0 +1,53 @@
+/**
+ * Data normalization utilities for converting platform-specific data
+ * into standardized formats
+ */
+import { Post, User, Comment, PlatformType } from '../interfaces/SocialMediaPlatform.js';
+/**
+ * Normalize user data from different platforms
+ */
+export declare class DataNormalizer {
+    /**
+     * Normalize a post from any platform to standard format
+     */
+    static normalizePost(rawData: any, platform: PlatformType, baseUrl?: string): Post;
+    /**
+     * Normalize user data from any platform
+     */
+    static normalizeUser(rawData: any, platform: PlatformType): User;
+    /**
+     * Normalize comment data from any platform
+     */
+    static normalizeComment(rawData: any, platform: PlatformType): Comment;
+    private static normalizeTikTokPost;
+    private static normalizeTikTokUser;
+    private static normalizeTikTokComment;
+    private static normalizeTwitterPost;
+    private static normalizeTwitterUser;
+    private static normalizeTwitterComment;
+    private static normalizeRedditPost;
+    private static normalizeRedditUser;
+    private static normalizeRedditComment;
+    private static normalizeInstagramPost;
+    private static normalizeInstagramUser;
+    private static normalizeInstagramComment;
+    private static normalizeYouTubePost;
+    private static normalizeYouTubeUser;
+    private static normalizeYouTubeComment;
+    /**
+     * Extract hashtags from text
+     */
+    private static extractHashtags;
+    /**
+     * Calculate engagement rate
+     */
+    static calculateEngagementRate(likes: number, comments: number, shares: number | undefined, followerCount: number): number;
+    /**
+     * Sanitize and validate text content
+     */
+    static sanitizeText(text: string): string;
+    /**
+     * Normalize timestamp from various formats
+     */
+    static normalizeTimestamp(timestamp: any): Date;
+}