opencandle 0.3.0 → 0.5.0

package/src/infra/rate-limiter.ts

@@ -0,0 +1,73 @@
+interface BucketConfig {
+  maxTokens: number;
+  refillRate: number; // tokens per second
+}
+
+interface Bucket {
+  tokens: number;
+  lastRefill: number;
+  config: BucketConfig;
+}
+
+export const ALPHA_VANTAGE_RATE_LIMIT = {
+  maxTokens: 5,
+  refillRate: 0.083,
+} as const;
+
+export class RateLimiter {
+  private buckets = new Map<string, Bucket>();
+
+  configure(provider: string, maxTokens: number, refillRate: number): void {
+    this.buckets.set(provider, {
+      tokens: maxTokens,
+      lastRefill: Date.now(),
+      config: { maxTokens, refillRate },
+    });
+  }
+
+  async acquire(provider: string): Promise<void> {
+    const bucket = this.buckets.get(provider);
+    if (!bucket) return; // No limit configured
+
+    this.refill(bucket);
+
+    if (bucket.tokens >= 1) {
+      bucket.tokens -= 1;
+      return;
+    }
+
+    // Wait until a token is available
+    const waitMs = ((1 - bucket.tokens) / bucket.config.refillRate) * 1000;
+    await new Promise((resolve) => setTimeout(resolve, Math.ceil(waitMs)));
+    this.refill(bucket);
+    bucket.tokens -= 1;
+  }
+
+  private refill(bucket: Bucket): void {
+    const now = Date.now();
+    const elapsed = (now - bucket.lastRefill) / 1000;
+    bucket.tokens = Math.min(
+      bucket.config.maxTokens,
+      bucket.tokens + elapsed * bucket.config.refillRate,
+    );
+    bucket.lastRefill = now;
+  }
+}
+
+// Shared instance with default provider limits
+export const rateLimiter = new RateLimiter();
+rateLimiter.configure("yahoo", 5, 5);           // 5 req/s
+rateLimiter.configure("coingecko", 10, 0.167);  // 10 req/min
+rateLimiter.configure(
+  "alphavantage",
+  ALPHA_VANTAGE_RATE_LIMIT.maxTokens,
+  ALPHA_VANTAGE_RATE_LIMIT.refillRate,
+); // 5 req/min (free tier)
+rateLimiter.configure("fred", 120, 2);           // 120 req/min
+rateLimiter.configure("twitter", 5, 0.167);      // 5 req, ~10 req/min
+rateLimiter.configure("reddit", 5, 0.167);          // 5 req, ~10 req/min
+rateLimiter.configure("reddit_comments", 10, 0.5); // 10 req, ~30 req/min
+rateLimiter.configure("ddg", 3, 0.1);             // 3 req, ~6 req/min
+rateLimiter.configure("brave_search", 5, 0.083);  // 5 req, ~5 req/min
+rateLimiter.configure("exa", 5, 0.1);              // 5 req, ~6 req/min
+rateLimiter.configure("finnhub", 60, 1);            // 60 req/min (free tier)

package/src/memory/index.ts

@@ -0,0 +1,10 @@
+export { initDatabase, initDefaultDatabase, getTableNames, getSchemaVersion } from "./sqlite.js";
+export { MemoryStorage } from "./storage.js";
+export type { WorkflowPreferences } from "./storage.js";
+export { buildMemoryContext } from "./retrieval.js";
+export { extractPreferences } from "./preference-extractor.js";
+export { MemoryManager } from "./manager.js";
+export { getAllDefaults, getDefaults, setDefault, clearDefault } from "./tool-defaults.js";
+export type { ToolDefaults } from "./tool-defaults.js";
+export type { MemoryCategory, MemoryEntry } from "./types.js";
+export { isStale, STALENESS_THRESHOLDS, NEVER_TRUST_FROM_MEMORY } from "./types.js";

package/src/memory/manager.ts

@@ -0,0 +1,192 @@
+import type { MemoryStorage } from "./storage.js";
+import type { MemoryCategory, MemoryEntry } from "./types.js";
+import {
+  KEY_TO_CATEGORY,
+  WORKFLOW_RELEVANT_CATEGORIES,
+  NEVER_TRUST_FROM_MEMORY,
+  isStale,
+} from "./types.js";
+
+export interface FilteredMemoryEntry {
+  entry: MemoryEntry;
+  reason: "suppressed_by_user_slot" | "never_trust" | "stale" | "irrelevant_category";
+}
+
+export interface MemoryRetrievalResult {
+  entries: MemoryEntry[];
+  filtered: FilteredMemoryEntry[];
+}
+
+/** Slot name → preference key(s) mapping for suppression. */
+const SLOT_TO_PREF_KEYS: Record<string, string[]> = {
+  riskProfile: ["risk_profile"],
+  assetScope: ["asset_scope"],
+  timeHorizon: ["time_horizon"],
+  dteTarget: ["dte_target"],
+  moneynessPreference: ["moneyness_preference"],
+  liquidityMinimum: ["liquidity_minimum", "options_liquidity"],
+};
+
+const MAX_WORKFLOW_HISTORY_PER_TYPE = 3;
+const MAX_PREFERENCE_LINES = 15;
+
+/**
+ * Selective, typed memory retrieval with staleness rules
+ * and override suppression.
+ */
+export class MemoryManager {
+  constructor(private readonly storage: MemoryStorage) {}
+
+  /**
+   * Retrieve memory entries relevant to the given workflow type,
+   * filtering by category, staleness, and overrides.
+   */
+  retrieve(
+    workflowType: string,
+    overriddenSlots?: string[],
+    now: Date = new Date(),
+  ): MemoryEntry[] {
+    return this.retrieveDetailed(workflowType, overriddenSlots, now).entries;
+  }
+
+  retrieveDetailed(
+    workflowType: string,
+    overriddenSlots?: string[],
+    now: Date = new Date(),
+  ): MemoryRetrievalResult {
+    const relevantCategories = WORKFLOW_RELEVANT_CATEGORIES[workflowType] ??
+      WORKFLOW_RELEVANT_CATEGORIES["unclassified"];
+
+    // Build set of preference keys to suppress
+    const suppressedKeys = new Set<string>();
+    if (overriddenSlots) {
+      for (const slot of overriddenSlots) {
+        const keys = SLOT_TO_PREF_KEYS[slot];
+        if (keys) keys.forEach((k) => suppressedKeys.add(k));
+      }
+    }
+
+    const entries: MemoryEntry[] = [];
+    const filtered: FilteredMemoryEntry[] = [];
+
+    // Preferences as investor_profile entries
+    if (relevantCategories.includes("investor_profile")) {
+      const prefs = this.storage.getPreferencesByNamespace("global");
+      for (const pref of prefs) {
+        const key = String(pref.key);
+        const category = KEY_TO_CATEGORY[key] ?? "investor_profile";
+        const entry: MemoryEntry = {
+          key,
+          value: tryParseValue(String(pref.value_json ?? "")),
+          category,
+          recordedAt: String(pref.updated_at ?? pref.created_at ?? now.toISOString()),
+          confidence: pref.confidence != null ? String(pref.confidence) : undefined,
+          source: pref.source != null ? String(pref.source) : undefined,
+        };
+
+        if (suppressedKeys.has(key)) {
+          filtered.push({ entry, reason: "suppressed_by_user_slot" });
+          continue;
+        }
+        if (NEVER_TRUST_FROM_MEMORY.has(key)) {
+          filtered.push({ entry, reason: "never_trust" });
+          continue;
+        }
+        if (!relevantCategories.includes(category)) {
+          filtered.push({ entry, reason: "irrelevant_category" });
+          continue;
+        }
+        if (isStale(entry, now)) {
+          filtered.push({ entry, reason: "stale" });
+          continue;
+        }
+        entries.push(entry);
+      }
+    }
+
+    // Workflow history
+    if (relevantCategories.includes("workflow_history")) {
+      const runs = this.storage.getRecentWorkflowRuns(MAX_WORKFLOW_HISTORY_PER_TYPE * 4);
+      const countsByType = new Map<string, number>();
+
+      for (const run of runs) {
+        const wfType = String(run.workflow_type);
+        const count = countsByType.get(wfType) ?? 0;
+        if (count >= MAX_WORKFLOW_HISTORY_PER_TYPE) continue;
+        countsByType.set(wfType, count + 1);
+
+        const recordedAt = String(run.created_at ?? now.toISOString());
+        const entry: MemoryEntry = {
+          key: `workflow_run_${run.id}`,
+          value: run.output_summary
+            ? `${wfType}: ${run.output_summary}`
+            : wfType,
+          category: "workflow_history",
+          recordedAt,
+        };
+
+        if (isStale(entry, now)) {
+          filtered.push({ entry, reason: "stale" });
+          continue;
+        }
+        entries.push(entry);
+      }
+    }
+
+    return {
+      entries: entries.slice(0, MAX_PREFERENCE_LINES + MAX_WORKFLOW_HISTORY_PER_TYPE * 4),
+      filtered,
+    };
+  }
+
+  /**
+   * Build compact text context from retrieved memory entries.
+   */
+  buildContext(
+    workflowType: string,
+    overriddenSlots?: string[],
+    now: Date = new Date(),
+  ): string {
+    const entries = this.retrieve(workflowType, overriddenSlots, now);
+    if (entries.length === 0) return "";
+
+    const sections: string[] = [];
+
+    // Group by category
+    const byCategory = new Map<MemoryCategory, MemoryEntry[]>();
+    for (const entry of entries) {
+      const list = byCategory.get(entry.category) ?? [];
+      list.push(entry);
+      byCategory.set(entry.category, list);
+    }
+
+    const profileEntries = byCategory.get("investor_profile");
+    if (profileEntries && profileEntries.length > 0) {
+      const lines = profileEntries.map((e) => `- ${e.key}: ${e.value}`);
+      sections.push("User Preferences:\n" + lines.join("\n"));
+    }
+
+    const historyEntries = byCategory.get("workflow_history");
+    if (historyEntries && historyEntries.length > 0) {
+      const lines = historyEntries.map((e) => `- ${e.value} (${e.recordedAt})`);
+      sections.push("Recent Workflows:\n" + lines.join("\n"));
+    }
+
+    const feedbackEntries = byCategory.get("interaction_feedback");
+    if (feedbackEntries && feedbackEntries.length > 0) {
+      const lines = feedbackEntries.map((e) => `- ${e.key}: ${e.value}`);
+      sections.push("Feedback:\n" + lines.join("\n"));
+    }
+
+    return sections.join("\n\n");
+  }
+}
+
+function tryParseValue(json: string): string {
+  try {
+    const parsed = JSON.parse(json);
+    return typeof parsed === "string" ? parsed : JSON.stringify(parsed);
+  } catch {
+    return json;
+  }
+}

package/src/memory/preference-extractor.ts

@@ -0,0 +1,106 @@
+export interface ExtractedPreference {
+  key: string;
+  value: string;
+  confidence: "high" | "medium" | "low";
+}
+
+interface PatternRule {
+  pattern: RegExp;
+  key: string;
+  value: string;
+  confidence: "high" | "medium" | "low";
+}
+
+const PREFERENCE_PATTERNS: PatternRule[] = [
+  // Asset scope
+  {
+    pattern: /\bprefer\s+etfs?\b/i,
+    key: "asset_scope",
+    value: "etf_focused",
+    confidence: "high",
+  },
+  {
+    pattern: /\betf[\s-]*(?:heavy|focused|only)\b/i,
+    key: "asset_scope",
+    value: "etf_focused",
+    confidence: "high",
+  },
+
+  // Risk profile
+  {
+    pattern: /\b(?:i'?m|i\s+am)\s+conservative\b/i,
+    key: "risk_profile",
+    value: "conservative",
+    confidence: "high",
+  },
+  {
+    pattern: /\brisk\s*averse\b/i,
+    key: "risk_profile",
+    value: "conservative",
+    confidence: "high",
+  },
+  {
+    pattern: /\b(?:i'?m|i\s+am)\s+aggressive\b/i,
+    key: "risk_profile",
+    value: "aggressive",
+    confidence: "high",
+  },
+  {
+    pattern: /\baggressive\s+growth\b/i,
+    key: "risk_profile",
+    value: "aggressive",
+    confidence: "high",
+  },
+  {
+    pattern: /\b(?:i'?m|i\s+am)\s+(?:moderate|balanced)\b/i,
+    key: "risk_profile",
+    value: "balanced",
+    confidence: "high",
+  },
+
+  // Time horizon
+  {
+    pattern: /\b(?:12\s*month|one\s*year|1\s*year)\s*horizon/i,
+    key: "time_horizon",
+    value: "1y_plus",
+    confidence: "high",
+  },
+  {
+    pattern: /\blong[\s-]*term\s+(?:invest|hold|horizon)/i,
+    key: "time_horizon",
+    value: "long",
+    confidence: "medium",
+  },
+  {
+    pattern: /\bshort[\s-]*term\s+(?:trad|invest|horizon)/i,
+    key: "time_horizon",
+    value: "short",
+    confidence: "medium",
+  },
+
+  // Options liquidity
+  {
+    pattern: /\b(?:only|prefer)\s+(?:trade\s+)?liquid\s+options?\b/i,
+    key: "options_liquidity",
+    value: "high",
+    confidence: "high",
+  },
+];
+
+export function extractPreferences(input: string): ExtractedPreference[] {
+  const preferences: ExtractedPreference[] = [];
+  const seenKeys = new Set<string>();
+
+  for (const rule of PREFERENCE_PATTERNS) {
+    if (rule.pattern.test(input) && !seenKeys.has(rule.key)) {
+      preferences.push({
+        key: rule.key,
+        value: rule.value,
+        confidence: rule.confidence,
+      });
+      seenKeys.add(rule.key);
+    }
+  }
+
+  return preferences;
+}

package/src/memory/retrieval.ts

@@ -0,0 +1,70 @@
+import type { MemoryStorage } from "./storage.js";
+
+const MAX_PREFERENCE_LINES = 15;
+const MAX_WORKFLOW_SUMMARY_LINES = 12;
+
+/** Slot name → preference key(s) mapping for suppression. */
+const SLOT_TO_PREF_KEYS: Record<string, string[]> = {
+  riskProfile: ["risk_profile"],
+  assetScope: ["asset_scope"],
+  timeHorizon: ["time_horizon"],
+  dteTarget: ["dte_target"],
+  moneynessPreference: ["moneyness_preference"],
+  liquidityMinimum: ["liquidity_minimum", "options_liquidity"],
+};
+
+/**
+ * Build compact memory context for agent injection.
+ * @param overriddenSlots Slot names whose values were overridden by current-turn user input.
+ *   The corresponding preference keys will be excluded from memory context to avoid
+ *   conflicting provenance signals.
+ */
+export function buildMemoryContext(
+  storage: MemoryStorage,
+  overriddenSlots?: string[],
+): string {
+  const sections: string[] = [];
+
+  // Build set of preference keys to suppress
+  const suppressedKeys = new Set<string>();
+  if (overriddenSlots) {
+    for (const slot of overriddenSlots) {
+      const keys = SLOT_TO_PREF_KEYS[slot];
+      if (keys) keys.forEach((k) => suppressedKeys.add(k));
+    }
+  }
+
+  // Preferences
+  const prefs = storage.getPreferencesByNamespace("global");
+  if (prefs.length > 0) {
+    const filtered = prefs.filter((p) => !suppressedKeys.has(String(p.key)));
+    if (filtered.length > 0) {
+      const lines = filtered.slice(0, MAX_PREFERENCE_LINES).map((p) => {
+        const value = tryParseJson(p.value_json as string);
+        return `- ${p.key}: ${value}`;
+      });
+      sections.push("User Preferences:\n" + lines.join("\n"));
+    }
+  }
+
+  // Recent workflow runs
+  const runs = storage.getRecentWorkflowRuns(3);
+  if (runs.length > 0) {
+    const lines = runs.slice(0, MAX_WORKFLOW_SUMMARY_LINES).map((r) => {
+      const summary = r.output_summary ? ` — ${r.output_summary}` : "";
+      return `- ${r.workflow_type} (${r.created_at})${summary}`;
+    });
+    sections.push("Recent Workflows:\n" + lines.join("\n"));
+  }
+
+  return sections.join("\n\n");
+}
+
+function tryParseJson(json: string): string {
+  try {
+    const parsed = JSON.parse(json);
+    return typeof parsed === "string" ? parsed : JSON.stringify(parsed);
+  } catch {
+    return json;
+  }
+}

package/src/memory/sqlite.ts

@@ -0,0 +1,172 @@
+import { mkdirSync } from "node:fs";
+import { dirname } from "node:path";
+import Database from "better-sqlite3";
+import { getStateDbPath } from "../infra/opencandle-paths.js";
+
+const CURRENT_SCHEMA_VERSION = 4;
+
+const CURRENT_SCHEMA = `
+  CREATE TABLE IF NOT EXISTS schema_version (
+    version INTEGER NOT NULL
+  );
+
+  CREATE TABLE IF NOT EXISTS user_preferences (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    namespace TEXT NOT NULL DEFAULT 'global',
+    key TEXT NOT NULL,
+    value_json TEXT NOT NULL,
+    confidence TEXT DEFAULT 'medium',
+    source TEXT DEFAULT 'explicit',
+    created_at TEXT NOT NULL,
+    updated_at TEXT NOT NULL,
+    UNIQUE(namespace, key)
+  );
+
+  CREATE TABLE IF NOT EXISTS workflow_runs (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    session_id TEXT NOT NULL,
+    workflow_type TEXT NOT NULL,
+    input_slots_json TEXT,
+    resolved_slots_json TEXT,
+    defaults_used_json TEXT,
+    output_summary TEXT,
+    created_at TEXT NOT NULL,
+    turn_type TEXT NOT NULL DEFAULT 'workflow'
+  );
+
+  CREATE TABLE IF NOT EXISTS recommendations (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    workflow_run_id INTEGER NOT NULL,
+    recommendation_type TEXT NOT NULL,
+    symbol TEXT,
+    payload_json TEXT,
+    created_at TEXT NOT NULL,
+    FOREIGN KEY (workflow_run_id) REFERENCES workflow_runs(id)
+  );
+
+  CREATE TABLE IF NOT EXISTS workflow_events (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    run_id TEXT NOT NULL,
+    step_index INTEGER NOT NULL,
+    event_type TEXT NOT NULL,
+    payload_json TEXT,
+    timestamp TEXT NOT NULL
+  );
+
+  CREATE INDEX IF NOT EXISTS idx_workflow_events_run_id ON workflow_events(run_id);
+
+  CREATE TABLE IF NOT EXISTS tool_defaults (
+    tool_name TEXT NOT NULL,
+    param_path TEXT NOT NULL,
+    value_json TEXT NOT NULL,
+    set_at TEXT NOT NULL,
+    PRIMARY KEY (tool_name, param_path)
+  );
+`;
+
+export function initDatabase(path: string): Database.Database {
+  if (path !== ":memory:") {
+    mkdirSync(dirname(path), { recursive: true });
+  }
+  const db = new Database(path);
+  db.pragma("journal_mode = WAL");
+  db.pragma("foreign_keys = ON");
+  ensureCurrentSchema(db);
+
+  return db;
+}
+
+export function initDefaultDatabase(): Database.Database {
+  return initDatabase(getStateDbPath());
+}
+
+function ensureCurrentSchema(db: Database.Database): void {
+  const currentVersion = readSchemaVersion(db);
+
+  if (currentVersion === CURRENT_SCHEMA_VERSION) {
+    // Up to date — still run CREATE TABLE IF NOT EXISTS for any missing auxiliary
+    // tables (e.g. workflow_events added out-of-band).
+    db.exec(CURRENT_SCHEMA);
+    return;
+  }
+
+  if (currentVersion === 3) {
+    migrateV3ToV4(db);
+    return;
+  }
+
+  // Additive v2 → v3 → v4 migration without dropping data.
+  if (currentVersion === 2) {
+    migrateV2ToV3(db);
+    migrateV3ToV4(db);
+    return;
+  }
+
+  // Any other mismatch (null first-run, or a foreign schema): reset.
+  resetSchema(db);
+}
+
+function migrateV2ToV3(db: Database.Database): void {
+  const cols = (db.pragma("table_info(workflow_runs)") as Array<{ name: string }>).map(
+    (c) => c.name,
+  );
+
+  if (!cols.includes("turn_type")) {
+    db.exec(
+      `ALTER TABLE workflow_runs ADD COLUMN turn_type TEXT NOT NULL DEFAULT 'workflow'`,
+    );
+  }
+
+  // Ensure any tables or indexes added between versions are present.
+  db.exec(CURRENT_SCHEMA);
+
+  db.prepare("DELETE FROM schema_version").run();
+  db.prepare("INSERT INTO schema_version (version) VALUES (?)").run(3);
+}
+
+function migrateV3ToV4(db: Database.Database): void {
+  db.exec(CURRENT_SCHEMA);
+
+  db.prepare("DELETE FROM schema_version").run();
+  db.prepare("INSERT INTO schema_version (version) VALUES (?)").run(CURRENT_SCHEMA_VERSION);
+}
+
+function readSchemaVersion(db: Database.Database): number | null {
+  const table = db
+    .prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'schema_version'")
+    .get() as { name: string } | undefined;
+  if (!table) {
+    return null;
+  }
+
+  const row = db.prepare("SELECT version FROM schema_version LIMIT 1").get() as
+    | { version: number }
+    | undefined;
+  return row?.version ?? null;
+}
+
+function resetSchema(db: Database.Database): void {
+  db.exec(`
+    DROP TABLE IF EXISTS recommendations;
+    DROP TABLE IF EXISTS workflow_runs;
+    DROP TABLE IF EXISTS user_preferences;
+    DROP TABLE IF EXISTS tool_defaults;
+    DROP TABLE IF EXISTS schema_version;
+  `);
+  db.exec(CURRENT_SCHEMA);
+  db.prepare("INSERT INTO schema_version (version) VALUES (?)").run(CURRENT_SCHEMA_VERSION);
+}
+
+export function getTableNames(db: Database.Database): string[] {
+  const rows = db
+    .prepare("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name")
+    .all() as Array<{ name: string }>;
+  return rows.map((r) => r.name);
+}
+
+export function getSchemaVersion(db: Database.Database): number {
+  const row = db.prepare("SELECT version FROM schema_version LIMIT 1").get() as
+    | { version: number }
+    | undefined;
+  return row?.version ?? 0;
+}