opencandle 0.2.0 → 0.4.0

package/src/runtime/workflow-events.ts

@@ -0,0 +1,75 @@
+import type Database from "better-sqlite3";
+
+/** All workflow event types. */
+export type WorkflowEventType =
+  | "workflow_started"
+  | "slot_resolved"
+  | "clarification_asked"
+  | "clarification_answered"
+  | "step_started"
+  | "step_completed"
+  | "step_failed"
+  | "step_skipped"
+  | "tool_called"
+  | "tool_failed"
+  | "validation_passed"
+  | "validation_failed"
+  | "workflow_completed"
+  | "workflow_cancelled";
+
+/** A persisted workflow event row. */
+export interface WorkflowEvent {
+  id: number;
+  runId: string;
+  stepIndex: number;
+  eventType: WorkflowEventType;
+  payloadJson: string | null;
+  timestamp: string;
+}
+
+/** Append-only workflow event logger backed by SQLite. */
+export class WorkflowEventLogger {
+  constructor(private readonly db: Database.Database) {}
+
+  /** Append a workflow event. */
+  log(
+    runId: string,
+    stepIndex: number,
+    eventType: WorkflowEventType,
+    payload?: Record<string, unknown>,
+  ): void {
+    const now = new Date().toISOString();
+    const payloadJson = payload ? JSON.stringify(payload) : null;
+    this.db
+      .prepare(
+        `INSERT INTO workflow_events (run_id, step_index, event_type, payload_json, timestamp)
+         VALUES (?, ?, ?, ?, ?)`,
+      )
+      .run(runId, stepIndex, eventType, payloadJson, now);
+  }
+
+  /** Query all events for a given run ID, ordered by timestamp. */
+  getEventsByRunId(runId: string): WorkflowEvent[] {
+    const rows = this.db
+      .prepare(
+        "SELECT id, run_id, step_index, event_type, payload_json, timestamp FROM workflow_events WHERE run_id = ? ORDER BY id",
+      )
+      .all(runId) as Array<{
+        id: number;
+        run_id: string;
+        step_index: number;
+        event_type: string;
+        payload_json: string | null;
+        timestamp: string;
+      }>;
+
+    return rows.map((r) => ({
+      id: r.id,
+      runId: r.run_id,
+      stepIndex: r.step_index,
+      eventType: r.event_type as WorkflowEventType,
+      payloadJson: r.payload_json,
+      timestamp: r.timestamp,
+    }));
+  }
+}

package/src/runtime/workflow-runner.ts

@@ -0,0 +1,188 @@
+import type { WorkflowRun, StepOutput, WorkflowStep } from "./workflow-types.js";
+import {
+  createWorkflowRun,
+  transitionStepStatus,
+} from "./workflow-types.js";
+import type { WorkflowEventLogger } from "./workflow-events.js";
+import type { ProviderTracker } from "./provider-tracker.js";
+import type { EvidenceRecord } from "./evidence.js";
+
+/** Function that executes a single workflow step. */
+export type StepExecutor = (
+  step: WorkflowStep,
+  stepIndex: number,
+  priorEvidence: EvidenceRecord[],
+  context: StepExecutionContext,
+) => Promise<StepOutput>;
+
+/** Context passed to step executors. */
+export interface StepExecutionContext {
+  runId: string;
+  providerTracker: ProviderTracker;
+}
+
+/** Options for creating a WorkflowRunner. */
+export interface WorkflowRunnerOptions {
+  eventLogger?: WorkflowEventLogger;
+  providerTracker?: ProviderTracker;
+}
+
+let runCounter = 0;
+
+function generateRunId(): string {
+  runCounter += 1;
+  return `run_${Date.now()}_${runCounter}`;
+}
+
+/**
+ * Typed workflow execution engine with run IDs, step definitions,
+ * state transitions, cancellation, and event logging.
+ */
+export class WorkflowRunner {
+  private readonly eventLogger?: WorkflowEventLogger;
+  private readonly providerTracker?: ProviderTracker;
+  private activeRun: WorkflowRun | null = null;
+
+  constructor(options: WorkflowRunnerOptions = {}) {
+    this.eventLogger = options.eventLogger;
+    this.providerTracker = options.providerTracker;
+  }
+
+  /** Get the currently active run, if any. */
+  getActiveRun(): WorkflowRun | null {
+    return this.activeRun;
+  }
+
+  /**
+   * Start a new workflow run. If a run is already active, it is cancelled first.
+   */
+  async start(
+    workflowType: string,
+    stepDefinitions: Omit<WorkflowStep, "status">[],
+    executor: StepExecutor,
+  ): Promise<WorkflowRun> {
+    // Cancel any active run
+    if (this.activeRun && this.activeRun.status === "running") {
+      this.cancel();
+    }
+
+    const runId = generateRunId();
+    const run = createWorkflowRun(runId, workflowType, stepDefinitions);
+    this.activeRun = run;
+    run.status = "running";
+
+    this.providerTracker?.resetAll();
+
+    this.logEvent(runId, 0, "workflow_started", {
+      workflowType,
+      stepCount: stepDefinitions.length,
+    });
+
+    // Execute steps
+    await this.executeSteps(run, executor);
+
+    return run;
+  }
+
+  /** Cancel the active run. */
+  cancel(): void {
+    const run = this.activeRun;
+    if (!run || run.status !== "running") return;
+
+    for (let i = run.currentStepIndex; i < run.steps.length; i++) {
+      const step = run.steps[i];
+      if (step.status === "pending" || step.status === "running") {
+        step.status = transitionStepStatus(step.status, "skipped");
+        this.logEvent(run.runId, i, "step_skipped", {
+          stepType: step.stepType,
+          reason: "cancelled",
+        });
+      }
+    }
+
+    run.status = "cancelled";
+    this.logEvent(run.runId, run.currentStepIndex, "workflow_cancelled", {
+      cancelledAtStep: run.currentStepIndex,
+    });
+  }
+
+  private async executeSteps(
+    run: WorkflowRun,
+    executor: StepExecutor,
+  ): Promise<void> {
+    for (let i = 0; i < run.steps.length; i++) {
+      // Check if run was cancelled externally
+      if (run.status !== "running") return;
+
+      const step = run.steps[i];
+      run.currentStepIndex = i;
+
+      // Collect all prior evidence
+      const priorEvidence: EvidenceRecord[] = [];
+      for (const [, output] of run.stepOutputs) {
+        priorEvidence.push(...output.evidence);
+      }
+
+      // Transition to running
+      step.status = transitionStepStatus(step.status, "running");
+      this.logEvent(run.runId, i, "step_started", { stepType: step.stepType });
+
+      try {
+        const context: StepExecutionContext = {
+          runId: run.runId,
+          providerTracker: this.providerTracker!,
+        };
+
+        const output = await executor(step, i, priorEvidence, context);
+
+        // If run was cancelled during execution, stop without further transitions
+        if (run.status !== "running") return;
+
+        step.status = transitionStepStatus(step.status, "completed");
+        run.stepOutputs.set(i, output);
+
+        this.logEvent(run.runId, i, "step_completed", {
+          stepType: step.stepType,
+          evidenceCount: output.evidence.length,
+        });
+      } catch (error) {
+        // If run was cancelled during execution, stop without further transitions
+        if (run.status !== "running") return;
+
+        const message = error instanceof Error ? error.message : "unknown_error";
+
+        if (step.skippable) {
+          step.status = transitionStepStatus(step.status, "skipped");
+          this.logEvent(run.runId, i, "step_skipped", {
+            stepType: step.stepType,
+            reason: message,
+          });
+        } else {
+          step.status = transitionStepStatus(step.status, "failed");
+          this.logEvent(run.runId, i, "step_failed", {
+            stepType: step.stepType,
+            error: message,
+          });
+          run.status = "failed";
+          return;
+        }
+      }
+    }
+
+    if (run.status === "running") {
+      run.status = "completed";
+      this.logEvent(run.runId, run.steps.length - 1, "workflow_completed", {
+        workflowType: run.workflowType,
+      });
+    }
+  }
+
+  private logEvent(
+    runId: string,
+    stepIndex: number,
+    eventType: string,
+    payload: Record<string, unknown>,
+  ): void {
+    this.eventLogger?.log(runId, stepIndex, eventType as any, payload);
+  }
+}

package/src/runtime/workflow-types.ts

@@ -0,0 +1,102 @@
+import type { EvidenceRecord } from "./evidence.js";
+
+/** Status of a single workflow step. */
+export type StepStatus = "pending" | "running" | "completed" | "failed" | "skipped";
+
+/** Overall status of a workflow run. */
+export type RunStatus = "pending" | "running" | "completed" | "failed" | "cancelled";
+
+/** Valid step status transitions. */
+const VALID_STEP_TRANSITIONS: Record<StepStatus, StepStatus[]> = {
+  pending: ["running", "skipped"],
+  running: ["completed", "failed", "skipped"],
+  completed: [],
+  failed: [],
+  skipped: [],
+};
+
+/** Check whether a step status transition is valid. */
+export function isValidStepTransition(from: StepStatus, to: StepStatus): boolean {
+  return VALID_STEP_TRANSITIONS[from].includes(to);
+}
+
+/** Transition a step status, throwing on invalid transitions. */
+export function transitionStepStatus(from: StepStatus, to: StepStatus): StepStatus {
+  if (!isValidStepTransition(from, to)) {
+    throw new Error(`Invalid step transition: ${from} → ${to}`);
+  }
+  return to;
+}
+
+/** Definition of a single workflow step. */
+export interface WorkflowStep {
+  stepType: string;
+  description: string;
+  requiredInputs: string[];
+  expectedOutputs: string[];
+  skippable: boolean;
+  status: StepStatus;
+}
+
+/** Output produced by a completed workflow step. */
+export interface StepOutput {
+  stepIndex: number;
+  stepType: string;
+  evidence: EvidenceRecord[];
+  rawText?: string;
+}
+
+/** Analyst signal direction. */
+export type AnalystSignal = "BUY" | "HOLD" | "SELL";
+
+/** Structured output from a single analyst role. */
+export interface AnalystOutput {
+  role: string;
+  signal: AnalystSignal;
+  conviction: number;
+  thesis: string;
+  evidence: EvidenceRecord[];
+  rawText?: string;
+}
+
+/** Debate side in bull/bear adversarial debate. */
+export type DebateSide = "bull" | "bear";
+
+/** Structured output from a debate step (eval/test only — not used in live path). */
+export interface DebateOutput {
+  side: DebateSide;
+  thesis: string;
+  keyRisk: string;
+  concessions: string[];
+  remainingConviction: number;
+  evidence: EvidenceRecord[];
+  rawText: string;
+}
+
+/** A complete workflow run definition and state. */
+export interface WorkflowRun {
+  runId: string;
+  workflowType: string;
+  steps: WorkflowStep[];
+  currentStepIndex: number;
+  status: RunStatus;
+  stepOutputs: Map<number, StepOutput>;
+  createdAt: string;
+}
+
+/** Create a new workflow run with all steps in pending state. */
+export function createWorkflowRun(
+  runId: string,
+  workflowType: string,
+  stepDefinitions: Omit<WorkflowStep, "status">[],
+): WorkflowRun {
+  return {
+    runId,
+    workflowType,
+    steps: stepDefinitions.map((def) => ({ ...def, status: "pending" as const })),
+    currentStepIndex: 0,
+    status: "pending",
+    stepOutputs: new Map(),
+    createdAt: new Date().toISOString(),
+  };
+}

package/src/sentiment/adapters/finnhub.ts

@@ -0,0 +1,44 @@
+import { randomUUID } from "node:crypto";
+import type { SentinelRecord } from "../types.js";
+import type { FinnhubArticle } from "../../providers/finnhub.js";
+import { extractEntities } from "../../routing/entity-extractor.js";
+
+const MAX_TICKERS = 3;
+
+export class FinnhubAdapter {
+  readonly source = "finnhub" as const;
+
+  mapToRecords(articles: FinnhubArticle[], query: string): SentinelRecord[] {
+    const fetchedAt = new Date().toISOString();
+    return articles.map((article) => ({
+      id: randomUUID(),
+      source: this.source,
+      sourceId: String(article.id),
+      query,
+      title: article.headline,
+      text: article.summary,
+      author: article.source,
+      url: article.url,
+      publishedAt: new Date(article.datetime * 1000).toISOString(),
+      fetchedAt,
+      engagement: {
+        score: 0,
+        replies: null,
+        shares: null,
+        views: null,
+      },
+      sentiment: {
+        score: 0,
+        confidence: 0,
+        method: "keyword" as const,
+        tickers: article.related ? article.related.split(",").map((t) => t.trim()).filter(Boolean) : [],
+      },
+      metadata: { category: article.category },
+    }));
+  }
+}
+
+export function extractTickersFromQuery(query: string): string[] {
+  const entities = extractEntities(query);
+  return entities.symbols.slice(0, MAX_TICKERS);
+}

package/src/sentiment/adapters/reddit.ts

@@ -0,0 +1,65 @@
+import { randomUUID } from "node:crypto";
+import type { SentinelRecord, SentimentAdapter } from "../types.js";
+import type { RedditSentimentResult } from "../../types/sentiment.js";
+import type { RedditComment } from "../../providers/reddit.js";
+
+export class RedditAdapter implements SentimentAdapter {
+  readonly source = "reddit" as const;
+
+  mapPostsToRecords(result: RedditSentimentResult, query: string): SentinelRecord[] {
+    const fetchedAt = result.fetchedAt;
+    return result.posts.map((post) => ({
+      id: randomUUID(),
+      source: this.source,
+      sourceId: post.id,
+      query,
+      title: post.title,
+      text: post.selftext ? `${post.title}\n${post.selftext}` : post.title,
+      author: post.author,
+      url: post.url,
+      publishedAt: post.created,
+      fetchedAt,
+      engagement: {
+        score: post.score,
+        replies: post.comments,
+        shares: null,
+        views: null,
+      },
+      sentiment: { score: 0, confidence: 0, method: "keyword" as const, tickers: [] },
+      metadata: { subreddit: result.subreddit },
+    }));
+  }
+
+  mapCommentsToRecords(
+    comments: RedditComment[],
+    parentId: string,
+    subreddit: string,
+    query: string,
+  ): SentinelRecord[] {
+    const fetchedAt = new Date().toISOString();
+    return comments.map((comment) => ({
+      id: randomUUID(),
+      source: this.source,
+      sourceId: comment.id,
+      query,
+      title: null,
+      text: comment.body,
+      author: comment.author,
+      url: comment.permalink,
+      publishedAt: null,
+      fetchedAt,
+      engagement: {
+        score: comment.score,
+        replies: null,
+        shares: null,
+        views: null,
+      },
+      sentiment: { score: 0, confidence: 0, method: "keyword" as const, tickers: [] },
+      metadata: { isComment: true, parentId, subreddit },
+    }));
+  }
+
+  async fetch(_query: string, _options?: { hours?: number }): Promise<SentinelRecord[]> {
+    throw new Error("Use pipeline.run() instead of adapter.fetch() directly");
+  }
+}

package/src/sentiment/adapters/twitter.ts

@@ -0,0 +1,36 @@
+import { randomUUID } from "node:crypto";
+import type { SentinelRecord, SentimentAdapter } from "../types.js";
+import type { TwitterSentimentResult } from "../../types/sentiment.js";
+
+export class TwitterAdapter implements SentimentAdapter {
+  readonly source = "twitter" as const;
+
+  mapToRecords(result: TwitterSentimentResult, query: string): SentinelRecord[] {
+    const fetchedAt = result.fetchedAt;
+    return result.tweets.map((tweet) => ({
+      id: randomUUID(),
+      source: this.source,
+      sourceId: tweet.id,
+      query,
+      title: null,
+      text: tweet.text,
+      author: tweet.author,
+      url: tweet.url,
+      publishedAt: tweet.created,
+      fetchedAt,
+      engagement: {
+        score: tweet.likes,
+        replies: tweet.replies,
+        shares: tweet.retweets,
+        views: tweet.views,
+      },
+      sentiment: { score: 0, confidence: 0, method: "keyword" as const, tickers: [] },
+      metadata: {},
+    }));
+  }
+
+  async fetch(_query: string, _options?: { hours?: number }): Promise<SentinelRecord[]> {
+    // Actual fetching is done by the pipeline via the provider
+    throw new Error("Use pipeline.run() instead of adapter.fetch() directly");
+  }
+}

package/src/sentiment/adapters/web.ts

@@ -0,0 +1,44 @@
+import { randomUUID } from "node:crypto";
+import type { SentinelRecord, SentimentAdapter } from "../types.js";
+import type { WebSearchEnvelope } from "../../types/sentiment.js";
+
+export class WebAdapter implements SentimentAdapter {
+  readonly source = "web" as const;
+
+  mapToRecords(envelope: WebSearchEnvelope, query: string): SentinelRecord[] {
+    const fetchedAt = envelope.fetchedAt;
+    return envelope.results.map((result) => ({
+      id: randomUUID(),
+      source: this.source,
+      sourceId: canonicalizeUrl(result.url),
+      query,
+      title: result.title,
+      text: result.snippet,
+      author: result.source,
+      url: result.url,
+      publishedAt: result.published,
+      fetchedAt,
+      engagement: {
+        score: 0,
+        replies: null,
+        shares: null,
+        views: null,
+      },
+      sentiment: { score: 0, confidence: 0, method: "keyword" as const, tickers: [] },
+      metadata: { category: result.category, provider: envelope.provider },
+    }));
+  }
+
+  async fetch(_query: string, _options?: { hours?: number }): Promise<SentinelRecord[]> {
+    throw new Error("Use pipeline.run() instead of adapter.fetch() directly");
+  }
+}
+
+function canonicalizeUrl(url: string): string {
+  try {
+    const parsed = new URL(url);
+    return parsed.origin + parsed.pathname;
+  } catch {
+    return url;
+  }
+}

package/src/sentiment/index.ts

@@ -0,0 +1,58 @@
+export type {
+  SentinelRecord,
+  SentinelEngagement,
+  SentinelSentiment,
+  SentimentAdapter,
+  ScorerOptions,
+  TrendBucket,
+  TrendResult,
+  DivergenceResult,
+  SentimentSummary,
+  SentimentSource,
+} from "./types.js";
+
+export { isSentinelRecord, SENTIMENT_SOURCES } from "./types.js";
+export { SentimentStore } from "./store.js";
+export { scoreRecords, keywordScore } from "./scorer.js";
+export { SentimentPipeline } from "./pipeline.js";
+export { renderSparkline, computeTrend, computeDivergence } from "./trends.js";
+export { BULLISH_TERMS, BEARISH_TERMS } from "./keywords.js";
+export { TwitterAdapter } from "./adapters/twitter.js";
+export { RedditAdapter } from "./adapters/reddit.js";
+export { WebAdapter } from "./adapters/web.js";
+
+import { SentimentStore } from "./store.js";
+import { SentimentPipeline } from "./pipeline.js";
+import { getConfig } from "../config.js";
+import { resolveOpenCandlePath } from "../infra/opencandle-paths.js";
+
+let _pipeline: SentimentPipeline | null = null;
+let _store: SentimentStore | null = null;
+
+export function getSentimentStore(): SentimentStore {
+  if (!_store) {
+    const dbPath = resolveOpenCandlePath("sentinel.db");
+    _store = new SentimentStore(dbPath);
+    const config = getConfig();
+    _store.prune(config.sentiment?.retentionDays ?? 30);
+  }
+  return _store;
+}
+
+export function getSentimentPipeline(): SentimentPipeline {
+  if (!_pipeline) {
+    const store = getSentimentStore();
+    const config = getConfig();
+    _pipeline = new SentimentPipeline(store, config.sentiment!);
+  }
+  return _pipeline;
+}
+
+/** For testing: reset singletons */
+export function _resetSentimentSingletons(): void {
+  if (_store) {
+    try { _store.close(); } catch { /* ignore */ }
+  }
+  _pipeline = null;
+  _store = null;
+}

package/src/sentiment/keywords.ts

@@ -0,0 +1,9 @@
+export const BULLISH_TERMS = [
+  "moon", "buy", "undervalued", "breakout", "calls", "bullish",
+  "rocket", "diamond hands", "accumulate", "dip buy", "long", "rip", "squeeze",
+] as const;
+
+export const BEARISH_TERMS = [
+  "crash", "overvalued", "sell", "puts", "bearish", "bubble",
+  "dump", "short", "bagholding", "exit", "drill", "tank", "rug",
+] as const;

package/src/sentiment/pipeline.ts

@@ -0,0 +1,68 @@
+import type { SentinelRecord, SentimentSummary, TrendResult, DivergenceResult, SentimentSource } from "./types.js";
+import type { SentimentConfig } from "../config.js";
+import { scoreRecords } from "./scorer.js";
+import { SentimentStore } from "./store.js";
+import { computeTrend, computeDivergence, type SourceStats } from "./trends.js";
+
+export class SentimentPipeline {
+  constructor(
+    private store: SentimentStore,
+    private config: SentimentConfig,
+  ) {}
+
+  async processRecords(records: SentinelRecord[], query: string): Promise<SentimentSummary> {
+    const warnings: string[] = [];
+
+    if (records.length === 0) {
+      return { fresh: [], trend: null, divergence: null, warnings };
+    }
+
+    // Check if store had prior data before this fetch
+    const priorSeries = this.store.getTimeSeries(query, { days: 30, bucketHours: 24 });
+    const hadPriorData = priorSeries.length >= 2;
+
+    // Score all records
+    const scored = scoreRecords(records);
+
+    // Insert into store
+    this.store.insert(scored);
+
+    // Compute trend from historical data (only if we had prior data)
+    let trend: TrendResult[] | null = null;
+    if (hadPriorData) {
+      const series = this.store.getTimeSeries(query, { days: 7, bucketHours: 24 });
+      if (series.length >= 2) {
+        trend = [computeTrend(series, "aggregate")];
+      }
+    }
+
+    // Compute divergence from fresh records
+    let divergence: DivergenceResult | null = null;
+    const sourceGroups = groupBySource(scored);
+    const sourceStats: { twitter?: SourceStats; reddit?: SourceStats; web?: SourceStats; finnhub?: SourceStats } = {};
+
+    for (const [source, recs] of Object.entries(sourceGroups)) {
+      // Exclude comments from divergence calculation
+      const postLevel = recs.filter((r) => !r.metadata.isComment);
+      if (postLevel.length > 0) {
+        const avg = postLevel.reduce((sum, r) => sum + r.sentiment.score, 0) / postLevel.length;
+        sourceStats[source as SentimentSource] = { avg, count: postLevel.length };
+      }
+    }
+
+    if (Object.keys(sourceStats).length >= 2) {
+      divergence = computeDivergence(sourceStats, this.config.divergenceThreshold);
+    }
+
+    return { fresh: scored, trend, divergence, warnings };
+  }
+}
+
+function groupBySource(records: SentinelRecord[]): Record<string, SentinelRecord[]> {
+  const groups: Record<string, SentinelRecord[]> = {};
+  for (const r of records) {
+    if (!groups[r.source]) groups[r.source] = [];
+    groups[r.source].push(r);
+  }
+  return groups;
+}