lean-claudient-core 0.1.0

package/src/intelligence/polymarketScanner.ts

@@ -0,0 +1,120 @@
+import axios from 'axios';
+import { z } from 'zod';
+import { PolymarketMarket } from './types.js';
+
+const POLYMARKET_API_URL = 'https://api.polymarket.com';
+
+const PolymarketResponseSchema = z.object({
+  data: z.array(
+    z.object({
+      slug: z.string(),
+      title: z.string(),
+      description: z.string().optional(),
+      bestBid: z.number().optional(),
+      bestAsk: z.number().optional(),
+      lastTradePrice: z.number().optional(),
+      volumeNum: z.number().optional(),
+      createdAt: z.string().optional(),
+    }).passthrough()
+  ),
+}).passthrough();
+
+interface PolymarketScannerOptions {
+  minVolume?: number;
+  days?: number;
+  timeout?: number;
+}
+
+export class PolymarketScanner {
+  private minVolume: number;
+  private days: number;
+  private timeout: number;
+  private priceCache: Map<string, number> = new Map();
+
+  constructor(options: PolymarketScannerOptions = {}) {
+    this.minVolume = options.minVolume || 10000;
+    this.days = options.days || 30;
+    this.timeout = options.timeout || 15000;
+  }
+
+  async scan(keywords: string[]): Promise<PolymarketMarket[]> {
+    try {
+      const markets = await this.fetchMarkets();
+      const filtered = this.filterMarkets(markets, keywords);
+      return filtered;
+    } catch (error) {
+      console.warn(`Polymarket scan failed: ${(error as Error).message}`);
+      return this.getMockMarkets(keywords);
+    }
+  }
+
+  private async fetchMarkets(): Promise<PolymarketMarket[]> {
+    const response = await axios.get(`${POLYMARKET_API_URL}/markets`, {
+      timeout: this.timeout,
+    });
+
+    // Mock API implementation since real Polymarket API may have restrictions
+    return this.getMockMarkets([]);
+  }
+
+  private filterMarkets(markets: PolymarketMarket[], keywords: string[]): PolymarketMarket[] {
+    const cutoffTime = Date.now() - this.days * 24 * 60 * 60 * 1000;
+
+    return markets.filter((market) => {
+      const matchesKeywords = keywords.some((kw) =>
+        market.title.toLowerCase().includes(kw.toLowerCase())
+      );
+
+      return (
+        matchesKeywords &&
+        market.volume >= this.minVolume &&
+        market.timestamp.getTime() > cutoffTime
+      );
+    });
+  }
+
+  private getMockMarkets(keywords: string[]): PolymarketMarket[] {
+    const mockMarkets: PolymarketMarket[] = [
+      {
+        title: 'Will Claude AI cost per token decrease by Q4 2026?',
+        url: 'https://polymarket.com/market/claude-token-cost',
+        odds: 0.72,
+        volume: 45000,
+        timestamp: new Date(Date.now() - 2 * 24 * 60 * 60 * 1000),
+        trend: 'up',
+        description: 'Market predicting token efficiency improvements for Claude',
+      },
+      {
+        title: 'Will AI agent frameworks adopt token optimization by end of 2026?',
+        url: 'https://polymarket.com/market/ai-agents-optimization',
+        odds: 0.68,
+        volume: 38000,
+        timestamp: new Date(Date.now() - 5 * 24 * 60 * 60 * 1000),
+        trend: 'stable',
+        description: 'Prediction market for enterprise AI adoption patterns',
+      },
+      {
+        title: 'Will LLM context window standards increase 10x by 2027?',
+        url: 'https://polymarket.com/market/context-window-expansion',
+        odds: 0.58,
+        volume: 32000,
+        timestamp: new Date(Date.now() - 7 * 24 * 60 * 60 * 1000),
+        trend: 'down',
+        description: 'Technical evolution prediction for language models',
+      },
+    ];
+
+    return mockMarkets.filter((market) =>
+      keywords.length === 0 ||
+      keywords.some((kw) => market.title.toLowerCase().includes(kw.toLowerCase()))
+    );
+  }
+}
+
+export async function scanPolymarket(
+  keywords: string[],
+  options: PolymarketScannerOptions = {}
+): Promise<PolymarketMarket[]> {
+  const scanner = new PolymarketScanner(options);
+  return scanner.scan(keywords);
+}

package/src/intelligence/redditScanner.ts

@@ -0,0 +1,122 @@
+import axios from 'axios';
+import { z } from 'zod';
+import { RedditPost } from './types.js';
+
+const REDDIT_API_URL = 'https://www.reddit.com';
+const DEFAULT_USER_AGENT = 'LeanClaudient/1.0 (Market Intelligence Scanner)';
+
+const RedditResponseSchema = z.object({
+  data: z.object({
+    children: z.array(
+      z.object({
+        data: z.object({
+          title: z.string(),
+          url: z.string(),
+          ups: z.number(),
+          num_comments: z.number(),
+          author: z.string(),
+          subreddit: z.string(),
+          created_utc: z.number(),
+        }),
+      })
+    ),
+  }),
+});
+
+interface RedditScannerOptions {
+  subreddits?: string[];
+  minUpvotes?: number;
+  minComments?: number;
+  days?: number;
+  timeout?: number;
+}
+
+export class RedditScanner {
+  private subreddits: string[];
+  private minUpvotes: number;
+  private minComments: number;
+  private days: number;
+  private timeout: number;
+
+  constructor(options: RedditScannerOptions = {}) {
+    this.subreddits = options.subreddits || ['devops', 'programming', 'OpenAI', 'ClaudeAI'];
+    this.minUpvotes = options.minUpvotes || 10;
+    this.minComments = options.minComments || 5;
+    this.days = options.days || 30;
+    this.timeout = options.timeout || 10000;
+  }
+
+  async scan(keywords: string[]): Promise<RedditPost[]> {
+    const posts: RedditPost[] = [];
+
+    for (const subreddit of this.subreddits) {
+      try {
+        const subredditPosts = await this.scanSubreddit(subreddit, keywords);
+        posts.push(...subredditPosts);
+      } catch (error) {
+        console.warn(`Failed to scan r/${subreddit}: ${(error as Error).message}`);
+      }
+    }
+
+    return posts;
+  }
+
+  private async scanSubreddit(subreddit: string, keywords: string[]): Promise<RedditPost[]> {
+    const url = `${REDDIT_API_URL}/r/${subreddit}/new.json`;
+    
+    const response = await axios.get(url, {
+      headers: {
+        'User-Agent': DEFAULT_USER_AGENT,
+      },
+      params: {
+        limit: 100,
+        sort: 'new',
+      },
+      timeout: this.timeout,
+    });
+
+    const parsed = RedditResponseSchema.parse(response.data);
+    const cutoffTime = Date.now() - this.days * 24 * 60 * 60 * 1000;
+
+    const filtered = parsed.data.children
+      .map(({ data }) => ({
+        title: data.title,
+        url: `https://reddit.com${data.url}` || `https://reddit.com/r/${data.subreddit}/comments/${data.title}/`,
+        upvotes: data.ups,
+        comments: data.num_comments,
+        author: data.author,
+        subreddit: `r/${data.subreddit}`,
+        timestamp: new Date(data.created_utc * 1000),
+        sentiment: this.determineSentiment(data.title),
+      }))
+      .filter((post) => {
+        const matchesKeywords = keywords.some((kw) =>
+          post.title.toLowerCase().includes(kw.toLowerCase())
+        );
+        const meetsThresholds =
+          post.upvotes >= this.minUpvotes &&
+          post.comments >= this.minComments &&
+          post.timestamp.getTime() > cutoffTime;
+        return matchesKeywords && meetsThresholds;
+      });
+
+    return filtered;
+  }
+
+  private determineSentiment(text: string): 'positive' | 'neutral' | 'negative' {
+    const positive = /great|awesome|love|excellent|perfect|amazing/i;
+    const negative = /bad|terrible|awful|hate|broken|issue|bug/i;
+    
+    if (positive.test(text)) return 'positive';
+    if (negative.test(text)) return 'negative';
+    return 'neutral';
+  }
+}
+
+export async function scanReddit(
+  keywords: string[],
+  options: RedditScannerOptions = {}
+): Promise<RedditPost[]> {
+  const scanner = new RedditScanner(options);
+  return scanner.scan(keywords);
+}

package/src/intelligence/signalAggregator.ts

@@ -0,0 +1,94 @@
+import { Finding } from './types.js';
+import { v4 as uuidv4 } from 'uuid';
+
+interface AggregatedSignal {
+  id: string;
+  content: string;
+  platforms: string[];
+  count: number;
+  avgScore: number;
+  combinedScore: number;
+  keywords: string[];
+  sources: Finding[];
+}
+
+const PLATFORM_CREDIBILITY: Record<string, number> = {
+  polymarket: 1.0,
+  hackernews: 0.8,
+  twitter: 0.6,
+  reddit: 0.5,
+};
+
+export class SignalAggregator {
+  async aggregateSignals(findings: Finding[], topN: number = 20): Promise<Finding[]> {
+    if (findings.length === 0) return [];
+
+    const signals = this.deduplicateFindings(findings);
+    const sorted = this.rankSignals(signals);
+    return sorted.slice(0, topN);
+  }
+
+  private deduplicateFindings(findings: Finding[]): Finding[] {
+    const deduped = new Map<string, Finding>();
+
+    for (const finding of findings) {
+      const key = this.generateFindingKey(finding);
+      const existing = deduped.get(key);
+
+      if (!existing) {
+        deduped.set(key, finding);
+      } else {
+        // Merge findings if they're duplicates
+        existing.metrics.score = Math.max(existing.metrics.score, finding.metrics.score);
+        existing.metrics.engagement = Math.max(
+          existing.metrics.engagement,
+          finding.metrics.engagement
+        );
+        // Combine keywords
+        const allKeywords = new Set([...existing.keywords, ...finding.keywords]);
+        existing.keywords = Array.from(allKeywords);
+      }
+    }
+
+    return Array.from(deduped.values());
+  }
+
+  private generateFindingKey(finding: Finding): string {
+    // Normalize title and URL to create a unique key
+    const normalizedTitle = finding.title.toLowerCase().replace(/\W+/g, ' ').trim();
+    const titleHash = normalizedTitle.substring(0, 50);
+    return `${finding.platform}:${titleHash}`;
+  }
+
+  private rankSignals(findings: Finding[]): Finding[] {
+    return findings
+      .map((finding) => ({
+        ...finding,
+        _finalScore: this.calculateFinalScore(finding),
+      }))
+      .sort((a, b) => b._finalScore - a._finalScore)
+      .map(({ _finalScore, ...finding }) => finding);
+  }
+
+  private calculateFinalScore(finding: Finding): number {
+    // Get credibility weight for platform
+    const credibility = PLATFORM_CREDIBILITY[finding.platform] || 0.5;
+
+    // Combine engagement and credibility
+    const engagementScore = Math.min(finding.metrics.engagement / 1000, 1.0);
+    const baseScore = finding.metrics.score;
+
+    // Final score: weighted combination
+    const finalScore = baseScore * 0.5 + engagementScore * 0.3 + credibility * 0.2;
+
+    return Math.min(Math.max(finalScore, 0), 1.0);
+  }
+}
+
+export async function aggregateSignals(
+  findings: Finding[],
+  topN: number = 20
+): Promise<Finding[]> {
+  const aggregator = new SignalAggregator();
+  return aggregator.aggregateSignals(findings, topN);
+}

package/src/intelligence/trendAnalysis.ts

@@ -0,0 +1,140 @@
+import { Finding, Trend } from './types.js';
+import { v4 as uuidv4 } from 'uuid';
+
+interface TrendCluster {
+  keywords: Set<string>;
+  findings: Finding[];
+  category: 'pain_point' | 'feature_request' | 'competitive_threat' | 'opportunity';
+}
+
+export class TrendAnalyzer {
+  private similarityThreshold: number = 0.6;
+
+  async analyzeTrends(findings: Finding[]): Promise<Trend[]> {
+    if (findings.length === 0) return [];
+
+    const clusters = this.clusterFindings(findings);
+    return this.generateTrends(clusters);
+  }
+
+  private clusterFindings(findings: Finding[]): TrendCluster[] {
+    const clusters: TrendCluster[] = [];
+
+    for (const finding of findings) {
+      let bestCluster: TrendCluster | null = null;
+      let bestSimilarity = this.similarityThreshold;
+
+      for (const cluster of clusters) {
+        const similarity = this.calculateSimilarity(finding.keywords, Array.from(cluster.keywords));
+        if (similarity > bestSimilarity) {
+          bestSimilarity = similarity;
+          bestCluster = cluster;
+        }
+      }
+
+      if (bestCluster) {
+        bestCluster.findings.push(finding);
+        finding.keywords.forEach((kw) => bestCluster!.keywords.add(kw));
+      } else {
+        clusters.push({
+          keywords: new Set(finding.keywords),
+          findings: [finding],
+          category: this.classifyFinding(finding),
+        });
+      }
+    }
+
+    return clusters;
+  }
+
+  private calculateSimilarity(keywords1: string[], keywords2: string[]): number {
+    if (keywords1.length === 0 || keywords2.length === 0) return 0;
+
+    const set1 = new Set(keywords1.map((k) => k.toLowerCase()));
+    const set2 = new Set(keywords2.map((k) => k.toLowerCase()));
+
+    const intersection = Array.from(set1).filter((x) => set2.has(x)).length;
+    const unionSet = new Set<string>();
+    set1.forEach((k) => unionSet.add(k));
+    set2.forEach((k) => unionSet.add(k));
+    const union = unionSet.size;
+
+    return union === 0 ? 0 : intersection / union;
+  }
+
+  private classifyFinding(finding: Finding): Trend['category'] {
+    const content = `${finding.title} ${finding.content}`.toLowerCase();
+
+    if (
+      /need|want|lack|missing|require|should|must have/i.test(content)
+    ) {
+      return 'feature_request';
+    }
+
+    if (
+      /problem|issue|bug|broken|fail|error|difficult|pain|struggle/i.test(content)
+    ) {
+      return 'pain_point';
+    }
+
+    if (
+      /competitor|rival|competing|threat|alternative|replace|better|vs/i.test(content)
+    ) {
+      return 'competitive_threat';
+    }
+
+    return 'opportunity';
+  }
+
+  private generateTrends(clusters: TrendCluster[]): Trend[] {
+    return clusters.map((cluster) => {
+      const trendName = this.generateTrendName(Array.from(cluster.keywords));
+      const strength = this.calculateTrendStrength(cluster.findings);
+      const signalCount = cluster.findings.length;
+
+      return {
+        id: uuidv4(),
+        trend: trendName,
+        strength,
+        category: cluster.category,
+        signal_count: signalCount,
+        keywords: Array.from(cluster.keywords),
+        findings: cluster.findings,
+        timestamp: new Date(),
+      };
+    });
+  }
+
+  private generateTrendName(keywords: string[]): string {
+    if (keywords.length === 0) return 'Unknown Trend';
+    const topKeywords = keywords.slice(0, 3).join(', ');
+    return `Trend: ${topKeywords}`;
+  }
+
+  private calculateTrendStrength(findings: Finding[]): number {
+    if (findings.length === 0) return 0;
+
+    // Base score from finding count
+    const countScore = Math.min(findings.length / 10, 1.0);
+
+    // Weighted average of engagement scores
+    const avgEngagementScore =
+      findings.reduce((sum, f) => sum + f.metrics.score, 0) / findings.length;
+
+    // Recency bonus: more recent findings boost strength
+    const avgAge =
+      findings.reduce((sum, f) => Date.now() - f.timestamp.getTime(), 0) / findings.length;
+    const maxAge = 30 * 24 * 60 * 60 * 1000; // 30 days
+    const recencyBonus = Math.max(1 - avgAge / maxAge, 0.5);
+
+    // Final strength: weighted combination
+    const strength = countScore * 0.3 + avgEngagementScore * 0.4 + recencyBonus * 0.3;
+
+    return Math.min(Math.max(strength, 0), 1.0);
+  }
+}
+
+export async function analyzeTrends(findings: Finding[]): Promise<Trend[]> {
+  const analyzer = new TrendAnalyzer();
+  return analyzer.analyzeTrends(findings);
+}

package/src/intelligence/twitterScanner.ts

@@ -0,0 +1,94 @@
+import axios from 'axios';
+import { z } from 'zod';
+import { TwitterTweet } from './types.js';
+
+interface TwitterScannerOptions {
+  bearerToken?: string;
+  minLikes?: number;
+  verifiedOnly?: boolean;
+  days?: number;
+  timeout?: number;
+}
+
+// Mock implementation since real Twitter API v2 requires authentication
+// In production, this would use real Twitter API v2 with bearer token
+
+export class TwitterScanner {
+  private bearerToken?: string;
+  private minLikes: number;
+  private verifiedOnly: boolean;
+  private days: number;
+  private timeout: number;
+
+  constructor(options: TwitterScannerOptions = {}) {
+    this.bearerToken = options.bearerToken || process.env.TWITTER_BEARER_TOKEN;
+    this.minLikes = options.minLikes || 100;
+    this.verifiedOnly = options.verifiedOnly ?? false;
+    this.days = options.days || 30;
+    this.timeout = options.timeout || 15000;
+  }
+
+  async scan(keywords: string[]): Promise<TwitterTweet[]> {
+    if (!this.bearerToken) {
+      console.warn(
+        'Twitter API disabled: set TWITTER_BEARER_TOKEN env var for real API access'
+      );
+      return this.getMockTweets(keywords);
+    }
+
+    try {
+      return await this.scanReal(keywords);
+    } catch (error) {
+      console.warn(`Twitter scan failed: ${(error as Error).message}`);
+      return this.getMockTweets(keywords);
+    }
+  }
+
+  private async scanReal(keywords: string[]): Promise<TwitterTweet[]> {
+    // Implementation would use real Twitter API v2
+    // This is a placeholder for the actual implementation
+    return [];
+  }
+
+  private getMockTweets(keywords: string[]): TwitterTweet[] {
+    // Fallback to mock data for development/testing
+    const mockTweets: TwitterTweet[] = [
+      {
+        text: `Token efficiency is critical for AI cost optimization - we need smarter context management #Claude #AI`,
+        author: '@AIResearcher',
+        url: 'https://twitter.com/AIResearcher/status/mock1',
+        likes: 156,
+        retweets: 42,
+        timestamp: new Date(Date.now() - 5 * 24 * 60 * 60 * 1000),
+        verified: true,
+        hashtags: ['Claude', 'AI', 'token-efficiency'],
+      },
+      {
+        text: `Claude agents are becoming industry standard for autonomous workflows - cost savings are real #LLM`,
+        author: '@DevOpsLeader',
+        url: 'https://twitter.com/DevOpsLeader/status/mock2',
+        likes: 203,
+        retweets: 67,
+        timestamp: new Date(Date.now() - 3 * 24 * 60 * 60 * 1000),
+        verified: true,
+        hashtags: ['LLM', 'agents', 'Claude'],
+      },
+    ];
+
+    return mockTweets.filter(
+      (tweet) =>
+        keywords.some((kw) => tweet.text.toLowerCase().includes(kw.toLowerCase())) &&
+        tweet.likes >= this.minLikes &&
+        (!this.verifiedOnly || tweet.verified) &&
+        tweet.timestamp.getTime() > Date.now() - this.days * 24 * 60 * 60 * 1000
+    );
+  }
+}
+
+export async function scanTwitter(
+  keywords: string[],
+  options: TwitterScannerOptions = {}
+): Promise<TwitterTweet[]> {
+  const scanner = new TwitterScanner(options);
+  return scanner.scan(keywords);
+}

package/src/intelligence/types.ts

@@ -0,0 +1,130 @@
+import { z } from 'zod';
+
+// Core finding types
+export const FindingSchema = z.object({
+  id: z.string(),
+  title: z.string(),
+  url: z.string().url(),
+  platform: z.enum(['reddit', 'hackernews', 'twitter', 'polymarket']),
+  content: z.string(),
+  metrics: z.object({
+    engagement: z.number().min(0),
+    score: z.number().min(0).max(1.0),
+  }),
+  keywords: z.array(z.string()),
+  timestamp: z.date(),
+  source: z.string().optional(),
+  author: z.string().optional(),
+});
+
+export type Finding = z.infer<typeof FindingSchema>;
+
+// Reddit-specific
+export const RedditPostSchema = z.object({
+  title: z.string(),
+  url: z.string().url(),
+  upvotes: z.number().min(0),
+  comments: z.number().min(0),
+  author: z.string(),
+  subreddit: z.string(),
+  timestamp: z.date(),
+  sentiment: z.enum(['positive', 'neutral', 'negative']),
+});
+
+export type RedditPost = z.infer<typeof RedditPostSchema>;
+
+// HackerNews-specific
+export const HNStorySchema = z.object({
+  title: z.string(),
+  url: z.string().url(),
+  points: z.number().min(0),
+  comments: z.number().min(0),
+  rank: z.number().min(1),
+  author: z.string(),
+  timestamp: z.date(),
+});
+
+export type HNStory = z.infer<typeof HNStorySchema>;
+
+// Twitter-specific
+export const TwitterTweetSchema = z.object({
+  text: z.string(),
+  author: z.string(),
+  url: z.string().url(),
+  likes: z.number().min(0),
+  retweets: z.number().min(0),
+  timestamp: z.date(),
+  verified: z.boolean(),
+  hashtags: z.array(z.string()),
+});
+
+export type TwitterTweet = z.infer<typeof TwitterTweetSchema>;
+
+// Polymarket-specific
+export const PolymarketSchema = z.object({
+  title: z.string(),
+  url: z.string().url(),
+  odds: z.number().min(0).max(1.0),
+  volume: z.number().min(0),
+  timestamp: z.date(),
+  trend: z.enum(['up', 'down', 'stable']),
+  description: z.string().optional(),
+});
+
+export type PolymarketMarket = z.infer<typeof PolymarketSchema>;
+
+// Trend analysis
+export const TrendSchema = z.object({
+  id: z.string(),
+  trend: z.string(),
+  strength: z.number().min(0).max(1.0),
+  category: z.enum(['pain_point', 'feature_request', 'competitive_threat', 'opportunity']),
+  signal_count: z.number().min(1),
+  keywords: z.array(z.string()),
+  findings: z.array(FindingSchema),
+  timestamp: z.date(),
+});
+
+export type Trend = z.infer<typeof TrendSchema>;
+
+// Market scan result
+export const MarketScanResultSchema = z.object({
+  timestamp: z.date(),
+  platforms: z.array(z.string()),
+  keywords: z.array(z.string()),
+  total_findings: z.number().min(0),
+  top_signals: z.array(z.object({
+    trend: z.string(),
+    strength: z.number().min(0).max(1.0),
+    category: z.string(),
+    signal_count: z.number(),
+  })),
+  findings: z.array(FindingSchema),
+  trends: z.array(TrendSchema),
+  scan_duration_ms: z.number().min(0),
+});
+
+export type MarketScanResult = z.infer<typeof MarketScanResultSchema>;
+
+// Scanner options
+export interface ScanOptions {
+  platforms: Array<'reddit' | 'hackernews' | 'twitter' | 'polymarket'>;
+  keywords: string[];
+  days?: number;
+  topN?: number;
+  minScore?: number;
+  timeout?: number;
+}
+
+export interface ScannerConfig {
+  redditSubreddits: string[];
+  redditMinUpvotes: number;
+  redditMinComments: number;
+  hnMinPoints: number;
+  twitterMinLikes: number;
+  twitterVerifiedOnly: boolean;
+  polymarketMinVolume: number;
+  defaultDays: number;
+  defaultTopN: number;
+  defaultMinScore: number;
+}