opencandle 0.3.0 → 0.5.0

package/src/sentiment/adapters/twitter.ts

@@ -0,0 +1,36 @@
+import { randomUUID } from "node:crypto";
+import type { SentinelRecord, SentimentAdapter } from "../types.js";
+import type { TwitterSentimentResult } from "../../types/sentiment.js";
+
+export class TwitterAdapter implements SentimentAdapter {
+  readonly source = "twitter" as const;
+
+  mapToRecords(result: TwitterSentimentResult, query: string): SentinelRecord[] {
+    const fetchedAt = result.fetchedAt;
+    return result.tweets.map((tweet) => ({
+      id: randomUUID(),
+      source: this.source,
+      sourceId: tweet.id,
+      query,
+      title: null,
+      text: tweet.text,
+      author: tweet.author,
+      url: tweet.url,
+      publishedAt: tweet.created,
+      fetchedAt,
+      engagement: {
+        score: tweet.likes,
+        replies: tweet.replies,
+        shares: tweet.retweets,
+        views: tweet.views,
+      },
+      sentiment: { score: 0, confidence: 0, method: "keyword" as const, tickers: [] },
+      metadata: {},
+    }));
+  }
+
+  async fetch(_query: string, _options?: { hours?: number }): Promise<SentinelRecord[]> {
+    // Actual fetching is done by the pipeline via the provider
+    throw new Error("Use pipeline.run() instead of adapter.fetch() directly");
+  }
+}

package/src/sentiment/adapters/web.ts

@@ -0,0 +1,44 @@
+import { randomUUID } from "node:crypto";
+import type { SentinelRecord, SentimentAdapter } from "../types.js";
+import type { WebSearchEnvelope } from "../../types/sentiment.js";
+
+export class WebAdapter implements SentimentAdapter {
+  readonly source = "web" as const;
+
+  mapToRecords(envelope: WebSearchEnvelope, query: string): SentinelRecord[] {
+    const fetchedAt = envelope.fetchedAt;
+    return envelope.results.map((result) => ({
+      id: randomUUID(),
+      source: this.source,
+      sourceId: canonicalizeUrl(result.url),
+      query,
+      title: result.title,
+      text: result.snippet,
+      author: result.source,
+      url: result.url,
+      publishedAt: result.published,
+      fetchedAt,
+      engagement: {
+        score: 0,
+        replies: null,
+        shares: null,
+        views: null,
+      },
+      sentiment: { score: 0, confidence: 0, method: "keyword" as const, tickers: [] },
+      metadata: { category: result.category, provider: envelope.provider },
+    }));
+  }
+
+  async fetch(_query: string, _options?: { hours?: number }): Promise<SentinelRecord[]> {
+    throw new Error("Use pipeline.run() instead of adapter.fetch() directly");
+  }
+}
+
+function canonicalizeUrl(url: string): string {
+  try {
+    const parsed = new URL(url);
+    return parsed.origin + parsed.pathname;
+  } catch {
+    return url;
+  }
+}

package/src/sentiment/index.ts

@@ -0,0 +1,58 @@
+export type {
+  SentinelRecord,
+  SentinelEngagement,
+  SentinelSentiment,
+  SentimentAdapter,
+  ScorerOptions,
+  TrendBucket,
+  TrendResult,
+  DivergenceResult,
+  SentimentSummary,
+  SentimentSource,
+} from "./types.js";
+
+export { isSentinelRecord, SENTIMENT_SOURCES } from "./types.js";
+export { SentimentStore } from "./store.js";
+export { scoreRecords, keywordScore } from "./scorer.js";
+export { SentimentPipeline } from "./pipeline.js";
+export { renderSparkline, computeTrend, computeDivergence } from "./trends.js";
+export { BULLISH_TERMS, BEARISH_TERMS } from "./keywords.js";
+export { TwitterAdapter } from "./adapters/twitter.js";
+export { RedditAdapter } from "./adapters/reddit.js";
+export { WebAdapter } from "./adapters/web.js";
+
+import { SentimentStore } from "./store.js";
+import { SentimentPipeline } from "./pipeline.js";
+import { getConfig } from "../config.js";
+import { resolveOpenCandlePath } from "../infra/opencandle-paths.js";
+
+let _pipeline: SentimentPipeline | null = null;
+let _store: SentimentStore | null = null;
+
+export function getSentimentStore(): SentimentStore {
+  if (!_store) {
+    const dbPath = resolveOpenCandlePath("sentinel.db");
+    _store = new SentimentStore(dbPath);
+    const config = getConfig();
+    _store.prune(config.sentiment?.retentionDays ?? 30);
+  }
+  return _store;
+}
+
+export function getSentimentPipeline(): SentimentPipeline {
+  if (!_pipeline) {
+    const store = getSentimentStore();
+    const config = getConfig();
+    _pipeline = new SentimentPipeline(store, config.sentiment!);
+  }
+  return _pipeline;
+}
+
+/** For testing: reset singletons */
+export function _resetSentimentSingletons(): void {
+  if (_store) {
+    try { _store.close(); } catch { /* ignore */ }
+  }
+  _pipeline = null;
+  _store = null;
+}

package/src/sentiment/keywords.ts

@@ -0,0 +1,9 @@
+export const BULLISH_TERMS = [
+  "moon", "buy", "undervalued", "breakout", "calls", "bullish",
+  "rocket", "diamond hands", "accumulate", "dip buy", "long", "rip", "squeeze",
+] as const;
+
+export const BEARISH_TERMS = [
+  "crash", "overvalued", "sell", "puts", "bearish", "bubble",
+  "dump", "short", "bagholding", "exit", "drill", "tank", "rug",
+] as const;

package/src/sentiment/pipeline.ts

@@ -0,0 +1,68 @@
+import type { SentinelRecord, SentimentSummary, TrendResult, DivergenceResult, SentimentSource } from "./types.js";
+import type { SentimentConfig } from "../config.js";
+import { scoreRecords } from "./scorer.js";
+import { SentimentStore } from "./store.js";
+import { computeTrend, computeDivergence, type SourceStats } from "./trends.js";
+
+export class SentimentPipeline {
+  constructor(
+    private store: SentimentStore,
+    private config: SentimentConfig,
+  ) {}
+
+  async processRecords(records: SentinelRecord[], query: string): Promise<SentimentSummary> {
+    const warnings: string[] = [];
+
+    if (records.length === 0) {
+      return { fresh: [], trend: null, divergence: null, warnings };
+    }
+
+    // Check if store had prior data before this fetch
+    const priorSeries = this.store.getTimeSeries(query, { days: 30, bucketHours: 24 });
+    const hadPriorData = priorSeries.length >= 2;
+
+    // Score all records
+    const scored = scoreRecords(records);
+
+    // Insert into store
+    this.store.insert(scored);
+
+    // Compute trend from historical data (only if we had prior data)
+    let trend: TrendResult[] | null = null;
+    if (hadPriorData) {
+      const series = this.store.getTimeSeries(query, { days: 7, bucketHours: 24 });
+      if (series.length >= 2) {
+        trend = [computeTrend(series, "aggregate")];
+      }
+    }
+
+    // Compute divergence from fresh records
+    let divergence: DivergenceResult | null = null;
+    const sourceGroups = groupBySource(scored);
+    const sourceStats: { twitter?: SourceStats; reddit?: SourceStats; web?: SourceStats; finnhub?: SourceStats } = {};
+
+    for (const [source, recs] of Object.entries(sourceGroups)) {
+      // Exclude comments from divergence calculation
+      const postLevel = recs.filter((r) => !r.metadata.isComment);
+      if (postLevel.length > 0) {
+        const avg = postLevel.reduce((sum, r) => sum + r.sentiment.score, 0) / postLevel.length;
+        sourceStats[source as SentimentSource] = { avg, count: postLevel.length };
+      }
+    }
+
+    if (Object.keys(sourceStats).length >= 2) {
+      divergence = computeDivergence(sourceStats, this.config.divergenceThreshold);
+    }
+
+    return { fresh: scored, trend, divergence, warnings };
+  }
+}
+
+function groupBySource(records: SentinelRecord[]): Record<string, SentinelRecord[]> {
+  const groups: Record<string, SentinelRecord[]> = {};
+  for (const r of records) {
+    if (!groups[r.source]) groups[r.source] = [];
+    groups[r.source].push(r);
+  }
+  return groups;
+}

package/src/sentiment/scorer.ts

@@ -0,0 +1,78 @@
+import { BULLISH_TERMS, BEARISH_TERMS } from "./keywords.js";
+import type { SentinelRecord } from "./types.js";
+
+const TICKER_REGEX = /\$([A-Z]{1,5})\b/g;
+
+interface ScoreResult {
+  score: number;
+  confidence: number;
+  tickers: string[];
+}
+
+export function keywordScore(record: SentinelRecord): ScoreResult {
+  const lower = record.text.toLowerCase();
+  const engagement = 1 + (record.engagement.score ?? 0);
+
+  let bullishWeight = 0;
+  let bearishWeight = 0;
+  let bullishCount = 0;
+  let bearishCount = 0;
+
+  for (const term of BULLISH_TERMS) {
+    if (lower.includes(term)) {
+      bullishCount++;
+      bullishWeight += engagement;
+    }
+  }
+
+  for (const term of BEARISH_TERMS) {
+    if (lower.includes(term)) {
+      bearishCount++;
+      bearishWeight += engagement;
+    }
+  }
+
+  const totalWeight = bullishWeight + bearishWeight;
+  const score = totalWeight === 0 ? 0 : (bullishWeight - bearishWeight) / totalWeight;
+
+  const totalMatches = bullishCount + bearishCount;
+  let confidence = 0;
+  if (totalMatches > 0) {
+    // Base confidence from keyword matches
+    confidence = Math.min(totalMatches / 5, 1) * 0.6;
+    // Boost for longer text
+    const textLength = record.text.length;
+    confidence += Math.min(textLength / 500, 1) * 0.3;
+    // Multiple matches boost
+    if (totalMatches >= 3) confidence += 0.1;
+    // Twitter penalty
+    if (record.source === "twitter") confidence -= 0.1;
+    // Clamp
+    confidence = Math.max(0, Math.min(1, confidence));
+  }
+
+  // Extract tickers
+  const tickers: string[] = [];
+  for (const match of record.text.matchAll(TICKER_REGEX)) {
+    if (!tickers.includes(match[1])) {
+      tickers.push(match[1]);
+    }
+  }
+
+  return { score, confidence, tickers };
+}
+
+export function scoreRecords(records: SentinelRecord[]): SentinelRecord[] {
+  return records.map((record) => {
+    const result = keywordScore(record);
+    return {
+      ...record,
+      sentiment: {
+        score: result.score,
+        confidence: result.confidence,
+        method: "keyword" as const,
+        tickers: result.tickers,
+      },
+    };
+  });
+}

package/src/sentiment/store.ts

@@ -0,0 +1,260 @@
+import Database from "better-sqlite3";
+import { mkdirSync } from "node:fs";
+import { dirname } from "node:path";
+import type { SentinelRecord, SentimentSource, TrendBucket } from "./types.js";
+
+const SCHEMA_VERSION = 1;
+
+const CREATE_SCHEMA = `
+  CREATE TABLE IF NOT EXISTS schema_version (
+    version INTEGER NOT NULL
+  );
+
+  CREATE TABLE IF NOT EXISTS sentinel_records (
+    id TEXT NOT NULL,
+    source TEXT NOT NULL,
+    source_id TEXT NOT NULL,
+    query TEXT NOT NULL,
+    title TEXT,
+    text TEXT NOT NULL,
+    author TEXT,
+    url TEXT NOT NULL,
+    published_at TEXT,
+    fetched_at TEXT NOT NULL,
+    engagement_json TEXT NOT NULL,
+    sentiment_score REAL NOT NULL,
+    sentiment_confidence REAL NOT NULL,
+    sentiment_method TEXT NOT NULL,
+    tickers_json TEXT NOT NULL,
+    metadata_json TEXT NOT NULL,
+    UNIQUE(source, source_id, fetched_at)
+  );
+
+  CREATE INDEX IF NOT EXISTS idx_sentinel_source ON sentinel_records(source);
+  CREATE INDEX IF NOT EXISTS idx_sentinel_fetched_at ON sentinel_records(fetched_at);
+  CREATE INDEX IF NOT EXISTS idx_sentinel_query ON sentinel_records(query);
+
+  CREATE VIRTUAL TABLE IF NOT EXISTS sentinel_fts USING fts5(
+    text, title, author, query, source,
+    content=sentinel_records,
+    content_rowid=rowid
+  );
+`;
+
+const TRIGGERS = `
+  CREATE TRIGGER IF NOT EXISTS sentinel_ai AFTER INSERT ON sentinel_records BEGIN
+    INSERT INTO sentinel_fts(rowid, text, title, author, query, source)
+    VALUES (new.rowid, new.text, new.title, new.author, new.query, new.source);
+  END;
+
+  CREATE TRIGGER IF NOT EXISTS sentinel_ad AFTER DELETE ON sentinel_records BEGIN
+    INSERT INTO sentinel_fts(sentinel_fts, rowid, text, title, author, query, source)
+    VALUES ('delete', old.rowid, old.text, old.title, old.author, old.query, old.source);
+  END;
+`;
+
+export interface SearchOptions {
+  source?: SentimentSource;
+  since?: string;
+  until?: string;
+}
+
+export interface TimeSeriesOptions {
+  days: number;
+  bucketHours: number;
+}
+
+export class SentimentStore {
+  private db: Database.Database;
+
+  constructor(pathOrMemory: string) {
+    if (pathOrMemory !== ":memory:") {
+      mkdirSync(dirname(pathOrMemory), { recursive: true });
+    }
+    this.db = new Database(pathOrMemory);
+    if (pathOrMemory !== ":memory:") {
+      this.db.pragma("journal_mode = WAL");
+    }
+    this.initSchema();
+  }
+
+  private initSchema(): void {
+    this.db.exec(CREATE_SCHEMA);
+    this.db.exec(TRIGGERS);
+
+    const row = this.db.prepare("SELECT version FROM schema_version").get() as
+      | { version: number }
+      | undefined;
+    if (!row) {
+      this.db.prepare("INSERT INTO schema_version (version) VALUES (?)").run(SCHEMA_VERSION);
+    }
+  }
+
+  getSchemaVersion(): number {
+    const row = this.db.prepare("SELECT version FROM schema_version").get() as { version: number };
+    return row.version;
+  }
+
+  insert(records: SentinelRecord[]): void {
+    const stmt = this.db.prepare(`
+      INSERT OR IGNORE INTO sentinel_records
+        (id, source, source_id, query, title, text, author, url,
+         published_at, fetched_at, engagement_json,
+         sentiment_score, sentiment_confidence, sentiment_method,
+         tickers_json, metadata_json)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `);
+
+    const tx = this.db.transaction((recs: SentinelRecord[]) => {
+      for (const r of recs) {
+        stmt.run(
+          r.id,
+          r.source,
+          r.sourceId,
+          r.query,
+          r.title,
+          r.text,
+          r.author,
+          r.url,
+          r.publishedAt,
+          r.fetchedAt,
+          JSON.stringify(r.engagement),
+          r.sentiment.score,
+          r.sentiment.confidence,
+          r.sentiment.method,
+          JSON.stringify(r.sentiment.tickers),
+          JSON.stringify(r.metadata),
+        );
+      }
+    });
+
+    tx(records);
+  }
+
+  search(query: string, options?: SearchOptions): SentinelRecord[] {
+    let sql = `
+      SELECT sr.* FROM sentinel_records sr
+      JOIN sentinel_fts fts ON sr.rowid = fts.rowid
+      WHERE sentinel_fts MATCH ?
+    `;
+    const params: unknown[] = [query];
+
+    if (options?.source) {
+      sql += " AND sr.source = ?";
+      params.push(options.source);
+    }
+    if (options?.since) {
+      sql += " AND sr.fetched_at >= ?";
+      params.push(options.since);
+    }
+    if (options?.until) {
+      sql += " AND sr.fetched_at <= ?";
+      params.push(options.until);
+    }
+
+    sql += " ORDER BY bm25(sentinel_fts) LIMIT 100";
+
+    const rows = this.db.prepare(sql).all(...params) as RawRow[];
+    return rows.map(rowToRecord);
+  }
+
+  getByTicker(ticker: string, options?: { since?: string }): SentinelRecord[] {
+    let sql = `SELECT * FROM sentinel_records WHERE tickers_json LIKE ?`;
+    const pattern = `%"${ticker}"%`;
+    const params: unknown[] = [pattern];
+
+    if (options?.since) {
+      sql += " AND fetched_at >= ?";
+      params.push(options.since);
+    }
+
+    sql += " ORDER BY fetched_at DESC LIMIT 100";
+    const rows = this.db.prepare(sql).all(...params) as RawRow[];
+    return rows.map(rowToRecord);
+  }
+
+  getTimeSeries(query: string, options: TimeSeriesOptions): TrendBucket[] {
+    const since = new Date();
+    since.setDate(since.getDate() - options.days);
+    const sinceStr = since.toISOString();
+    const bucketSeconds = options.bucketHours * 3600;
+
+    const sql = `
+      SELECT
+        (CAST(strftime('%s', fetched_at) AS INTEGER) / ?) * ? AS bucket_ts,
+        AVG(sentiment_score) AS avg_score,
+        COUNT(*) AS cnt
+      FROM sentinel_records
+      WHERE query = ? AND fetched_at >= ?
+      GROUP BY bucket_ts
+      ORDER BY bucket_ts
+    `;
+
+    const rows = this.db.prepare(sql).all(bucketSeconds, bucketSeconds, query, sinceStr) as Array<{
+      bucket_ts: number;
+      avg_score: number;
+      cnt: number;
+    }>;
+
+    return rows.map((r) => ({
+      timestamp: new Date(r.bucket_ts * 1000).toISOString(),
+      avgScore: r.avg_score,
+      count: r.cnt,
+    }));
+  }
+
+  prune(retentionDays: number): number {
+    const cutoff = new Date();
+    cutoff.setDate(cutoff.getDate() - retentionDays);
+    const result = this.db
+      .prepare("DELETE FROM sentinel_records WHERE fetched_at < ?")
+      .run(cutoff.toISOString());
+    return result.changes;
+  }
+
+  close(): void {
+    this.db.close();
+  }
+}
+
+interface RawRow {
+  id: string;
+  source: string;
+  source_id: string;
+  query: string;
+  title: string | null;
+  text: string;
+  author: string | null;
+  url: string;
+  published_at: string | null;
+  fetched_at: string;
+  engagement_json: string;
+  sentiment_score: number;
+  sentiment_confidence: number;
+  sentiment_method: string;
+  tickers_json: string;
+  metadata_json: string;
+}
+
+function rowToRecord(row: RawRow): SentinelRecord {
+  return {
+    id: row.id,
+    source: row.source as SentinelRecord["source"],
+    sourceId: row.source_id,
+    query: row.query,
+    title: row.title,
+    text: row.text,
+    author: row.author,
+    url: row.url,
+    publishedAt: row.published_at,
+    fetchedAt: row.fetched_at,
+    engagement: JSON.parse(row.engagement_json),
+    sentiment: {
+      score: row.sentiment_score,
+      confidence: row.sentiment_confidence,
+      method: row.sentiment_method as "keyword",
+      tickers: JSON.parse(row.tickers_json),
+    },
+    metadata: JSON.parse(row.metadata_json),
+  };
+}

package/src/sentiment/trends.ts

@@ -0,0 +1,90 @@
+import type { TrendBucket, TrendResult, DivergenceResult, SentimentSource } from "./types.js";
+
+const SPARKLINE_CHARS = "▁▂▃▄▅▆▇█";
+
+export function renderSparkline(values: number[]): string {
+  if (values.length === 0) return "";
+
+  const min = Math.min(...values);
+  const max = Math.max(...values);
+  const range = max - min;
+
+  return values
+    .map((v) => {
+      if (range === 0) return SPARKLINE_CHARS[3]; // middle block for flat
+      const idx = Math.round(((v - min) / range) * 7);
+      return SPARKLINE_CHARS[idx];
+    })
+    .join("");
+}
+
+export function computeTrend(
+  buckets: TrendBucket[],
+  source: SentimentSource | "aggregate",
+): TrendResult {
+  const scores = buckets.map((b) => b.avgScore);
+  const totalCount = buckets.reduce((sum, b) => sum + b.count, 0);
+  const avgScore = totalCount === 0 ? 0 : buckets.reduce((sum, b) => sum + b.avgScore * b.count, 0) / totalCount;
+
+  const sparkline = renderSparkline(scores);
+
+  // Delta: last value minus first value
+  const delta = scores.length >= 2 ? scores[scores.length - 1] - scores[0] : 0;
+
+  let direction: "rising" | "falling" | "stable";
+  if (delta > 0.1) direction = "rising";
+  else if (delta < -0.1) direction = "falling";
+  else direction = "stable";
+
+  return { source, sparkline, avgScore, count: totalCount, direction, delta };
+}
+
+export interface SourceStats {
+  avg: number;
+  count: number;
+}
+
+export function computeDivergence(
+  sources: { twitter?: SourceStats; reddit?: SourceStats; web?: SourceStats; finnhub?: SourceStats },
+  threshold: number,
+): DivergenceResult {
+  const retailSources: SourceStats[] = [];
+  if (sources.twitter && sources.twitter.count >= 5) retailSources.push(sources.twitter);
+  if (sources.reddit && sources.reddit.count >= 5) retailSources.push(sources.reddit);
+
+  const newsSources: SourceStats[] = [];
+  if (sources.web && sources.web.count >= 5) newsSources.push(sources.web);
+  if (sources.finnhub && sources.finnhub.count >= 5) newsSources.push(sources.finnhub);
+
+  if (retailSources.length === 0 || newsSources.length === 0) {
+    return {
+      detected: false,
+      retailAvg: null,
+      newsAvg: null,
+      gap: null,
+      message: "Insufficient data for divergence analysis",
+    };
+  }
+
+  const retailAvg = retailSources.reduce((sum, s) => sum + s.avg, 0) / retailSources.length;
+  const newsAvg = newsSources.reduce((sum, s) => sum + s.avg, 0) / newsSources.length;
+  const gap = Math.abs(retailAvg - newsAvg);
+
+  if (gap > threshold) {
+    return {
+      detected: true,
+      retailAvg,
+      newsAvg,
+      gap,
+      message: `⚠ DIVERGENCE: Retail sentiment (${retailAvg >= 0 ? "+" : ""}${retailAvg.toFixed(2)}) vs news sentiment (${newsAvg >= 0 ? "+" : ""}${newsAvg.toFixed(2)}) — gap of ${gap.toFixed(2)}.`,
+    };
+  }
+
+  return {
+    detected: false,
+    retailAvg,
+    newsAvg,
+    gap,
+    message: "Sources broadly aligned",
+  };
+}