opencandle 0.2.0 → 0.4.0

package/src/infra/browser.ts

@@ -0,0 +1,111 @@
+/**
+ * Shared stealth browser infrastructure using Camoufox (anti-detection Firefox).
+ * Provides a singleton browser instance that any tool can use for scraping.
+ *
+ * Usage:
+ *   import { StealthBrowser } from "../infra/browser.js";
+ *   const data = await StealthBrowser.fetchJson<MyType>(url);
+ *   const result = await StealthBrowser.evaluate(url, () => document.title);
+ */
+import "./node-version.js";
+import { Camoufox } from "camoufox-js";
+import type { Browser, Page } from "playwright-core";
+
+let browser: Browser | null = null;
+let page: Page | null = null;
+let launching: Promise<void> | null = null;
+let pageQueue: Promise<void> = Promise.resolve();
+
+async function ensureBrowser(): Promise<Page> {
+  if (page && browser?.isConnected()) return page;
+
+  // Prevent concurrent launches
+  if (launching) {
+    await launching;
+    if (page && browser?.isConnected()) return page;
+  }
+
+  launching = (async () => {
+    const b = await Camoufox({ headless: true });
+    browser = b;
+    page = await b.newPage();
+  })();
+
+  await launching;
+  launching = null;
+  return page!;
+}
+
+async function withPage<T>(fn: (p: Page) => Promise<T>): Promise<T> {
+  let resolve!: () => void;
+  const next = new Promise<void>((r) => { resolve = r; });
+  const prev = pageQueue;
+  pageQueue = next;
+  await prev;
+  try {
+    const p = await ensureBrowser();
+    return await fn(p);
+  } finally {
+    resolve();
+  }
+}
+
+export const StealthBrowser = {
+  /**
+   * Navigate to a URL, run a JS function in the page context, and return the result.
+   */
+  async evaluate<T>(url: string, fn: () => T | Promise<T>): Promise<T> {
+    return withPage(async (p) => {
+      await p.goto(url, { waitUntil: "domcontentloaded", timeout: 15000 });
+      return p.evaluate(fn);
+    });
+  },
+
+  /**
+   * Fetch JSON from a URL using the browser's session (cookies, TLS fingerprint).
+   * Useful for APIs that block Node.js fetch but allow real browsers.
+   */
+  async fetchJson<T>(url: string, options?: { cookies?: string }): Promise<T> {
+    return withPage(async (p) => {
+      const result = await p.evaluate(async (fetchUrl: string) => {
+        const res = await fetch(fetchUrl, { credentials: "include" });
+        if (!res.ok) throw new Error(`HTTP ${res.status}`);
+        return res.json();
+      }, url);
+      return result as T;
+    });
+  },
+
+  /**
+   * Run a custom async function in the browser page context.
+   * The page must already be on a relevant domain for cookies to work.
+   */
+  async run<T>(fn: (page: Page) => Promise<T>): Promise<T> {
+    return withPage(async (p) => fn(p));
+  },
+
+  /**
+   * Navigate to a URL and establish session cookies for that domain.
+   */
+  async initSession(url: string): Promise<void> {
+    return withPage(async (p) => {
+      await p.goto(url, { waitUntil: "domcontentloaded", timeout: 15000 });
+    });
+  },
+
+  /**
+   * Close the browser. It will be re-launched on next use.
+   */
+  async close(): Promise<void> {
+    if (browser) {
+      await browser.close().catch(() => {});
+      browser = null;
+      page = null;
+    }
+  },
+};
+
+// Clean up on process exit
+process.on("exit", () => {
+  browser?.close().catch(() => {});
+});

package/src/infra/cache.ts

@@ -0,0 +1,103 @@
+interface CacheEntry<T> {
+  value: T;
+  expiresAt: number;
+  cachedAt: number;
+}
+
+export interface StaleResult<T> {
+  value: T;
+  stale: true;
+  cachedAt: number;
+}
+
+export class Cache {
+  private store = new Map<string, CacheEntry<unknown>>();
+  private lastStaleHit = false;
+  private lastStaleCachedAt = 0;
+
+  get<T>(key: string): T | undefined {
+    const entry = this.store.get(key);
+    if (!entry) return undefined;
+    if (Date.now() > entry.expiresAt) return undefined;
+    return entry.value as T;
+  }
+
+  /**
+   * Return an expired entry if it exists and is within the stale limit.
+   * Unlike get(), this does not require the entry to be within its TTL.
+   * Entries beyond the stale limit are deleted.
+   */
+  getStale<T>(key: string, staleLimitMs: number): StaleResult<T> | undefined {
+    const entry = this.store.get(key);
+    if (!entry) return undefined;
+
+    if (Date.now() > entry.cachedAt + staleLimitMs) {
+      this.store.delete(key);
+      return undefined;
+    }
+
+    this.lastStaleHit = true;
+    this.lastStaleCachedAt = entry.cachedAt;
+    return { value: entry.value as T, stale: true, cachedAt: entry.cachedAt };
+  }
+
+  /**
+   * Consume the stale flag set by the last getStale() hit.
+   * Returns { stale: true, cachedAt } if the last getStale() found data,
+   * then resets the flag. Used by wrapProvider to propagate stale metadata.
+   */
+  consumeStaleFlag(): { stale: boolean; cachedAt: number } {
+    const result = { stale: this.lastStaleHit, cachedAt: this.lastStaleCachedAt };
+    this.lastStaleHit = false;
+    this.lastStaleCachedAt = 0;
+    return result;
+  }
+
+  set<T>(key: string, value: T, ttlMs: number): void {
+    this.store.set(key, { value, expiresAt: Date.now() + ttlMs, cachedAt: Date.now() });
+  }
+
+  invalidate(pattern: string): void {
+    for (const key of this.store.keys()) {
+      if (key.includes(pattern)) {
+        this.store.delete(key);
+      }
+    }
+  }
+
+  clear(): void {
+    this.store.clear();
+  }
+
+  get size(): number {
+    return this.store.size;
+  }
+}
+
+// Shared cache instance
+export const cache = new Cache();
+
+// Default TTLs
+export const TTL = {
+  QUOTE: 60_000,         // 1 minute
+  HISTORY: 3_600_000,    // 1 hour
+  FUNDAMENTALS: 86_400_000, // 24 hours
+  MACRO: 3_600_000,      // 1 hour
+  SENTIMENT: 300_000,    // 5 minutes
+  OPTIONS_CHAIN: 120_000, // 2 minutes
+  CRUMB: 900_000,        // 15 minutes
+  WEB_SEARCH: 300_000,   // 5 minutes
+  FINNHUB_NEWS: 300_000, // 5 minutes
+} as const;
+
+// Stale limits — how long past TTL expiry a cached value is still useful as fallback
+export const STALE_LIMIT = {
+  QUOTE: 15 * 60_000,             // 15 minutes
+  HISTORY: 24 * 3_600_000,        // 24 hours
+  FUNDAMENTALS: 7 * 86_400_000,   // 7 days
+  MACRO: 24 * 3_600_000,          // 24 hours
+  SENTIMENT: 3_600_000,           // 1 hour
+  OPTIONS_CHAIN: 30 * 60_000,     // 30 minutes
+  WEB_SEARCH: 3_600_000,          // 1 hour
+  FINNHUB_NEWS: 3_600_000,        // 1 hour
+} as const;

package/src/infra/http-client.ts

@@ -0,0 +1,68 @@
+export interface HttpClientOptions {
+  timeoutMs?: number;
+  maxRetries?: number;
+  retryDelayMs?: number;
+  headers?: Record<string, string>;
+}
+
+const DEFAULT_OPTIONS: Required<HttpClientOptions> = {
+  timeoutMs: 10_000,
+  maxRetries: 2,
+  retryDelayMs: 1_000,
+  headers: {},
+};
+
+export class HttpError extends Error {
+  constructor(
+    public readonly status: number,
+    public readonly statusText: string,
+    public readonly body: string,
+  ) {
+    super(`HTTP ${status} ${statusText}`);
+    this.name = "HttpError";
+  }
+}
+
+export async function httpGet<T>(
+  url: string,
+  options: HttpClientOptions = {},
+): Promise<T> {
+  const opts = { ...DEFAULT_OPTIONS, ...options };
+  let lastError: Error | undefined;
+
+  for (let attempt = 0; attempt <= opts.maxRetries; attempt++) {
+    if (attempt > 0) {
+      await sleep(opts.retryDelayMs * attempt);
+    }
+
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), opts.timeoutMs);
+
+    try {
+      const response = await fetch(url, {
+        signal: controller.signal,
+        headers: opts.headers,
+      });
+
+      if (!response.ok) {
+        const body = await response.text().catch(() => "");
+        throw new HttpError(response.status, response.statusText, body);
+      }
+
+      return (await response.json()) as T;
+    } catch (error) {
+      lastError = error as Error;
+      if (error instanceof HttpError && error.status >= 400 && error.status < 500) {
+        throw error; // Don't retry client errors
+      }
+    } finally {
+      clearTimeout(timeout);
+    }
+  }
+
+  throw lastError!;
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}

package/src/infra/index.ts

@@ -0,0 +1,18 @@
+export { Cache, cache, TTL } from "./cache.js";
+export { RateLimiter, rateLimiter } from "./rate-limiter.js";
+export { httpGet, HttpError, type HttpClientOptions } from "./http-client.js";
+export { StealthBrowser } from "./browser.js";
+export {
+  getOpenCandleHomeDir,
+  ensureOpenCandleHomeDir,
+  resolveOpenCandlePath,
+  ensureParentDir,
+  getWatchlistPath,
+  getPortfolioPath,
+  getPredictionsPath,
+  getConfigPath,
+  getOnboardingPath,
+  getStateDbPath,
+  getLogsDir,
+  getBrowserProfileDir,
+} from "./opencandle-paths.js";

package/src/infra/native-dependencies.ts

@@ -0,0 +1,12 @@
+export function getNativeDependencyErrorMessage(error: unknown, dependencyName: string): string | null {
+  const message = error instanceof Error ? error.message : String(error);
+  if (
+    !message.includes("NODE_MODULE_VERSION") &&
+    !message.includes("was compiled against a different Node.js version")
+  ) {
+    return null;
+  }
+
+  return `${dependencyName} native binding was built for a different Node ABI than the active Node ${process.versions.node}. ` +
+    `Run \`npm rebuild ${dependencyName}\` or reinstall dependencies under the active Node with \`npm install\`.`;
+}

package/src/infra/node-version.ts

@@ -0,0 +1,24 @@
+const SUPPORTED_NODE_RANGE = "20.19+, 22.12+, or 24.x-26.x";
+
+function isSupportedNodeVersion(version: string): boolean {
+  const [majorRaw, minorRaw] = version.split(".");
+  const major = Number(majorRaw);
+  const minor = Number(minorRaw);
+
+  if (major === 20) return minor >= 19;
+  if (major === 22) return minor >= 12;
+  return major >= 24 && major < 27;
+}
+
+export function getUnsupportedNodeVersionMessage(version: string = process.versions.node): string | null {
+  if (isSupportedNodeVersion(version)) return null;
+
+  return `OpenCandle supports Node ${SUPPORTED_NODE_RANGE}. Current Node is ${version}. Use Node ${SUPPORTED_NODE_RANGE}; the repo default is Node 22.22.0 via \`nvm use\`. After switching Node versions, reinstall dependencies under the active Node with \`npm install\` or rebuild native modules with \`npm rebuild better-sqlite3\`.`;
+}
+
+export function assertSupportedNodeVersion(version?: string): void {
+  const message = getUnsupportedNodeVersionMessage(version);
+  if (message) throw new Error(message);
+}
+
+assertSupportedNodeVersion();

package/src/infra/open-url.ts

@@ -0,0 +1,28 @@
+import { execFile } from "node:child_process";
+
+export function openInBrowser(url: string): Promise<void> {
+  return new Promise<void>((resolve, reject) => {
+    let command: string;
+    let args: string[];
+
+    if (process.platform === "darwin") {
+      command = "open";
+      args = [url];
+    } else if (process.platform === "win32") {
+      command = "cmd";
+      args = ["/c", "start", "", url];
+    } else {
+      command = "xdg-open";
+      args = [url];
+    }
+
+    const child = execFile(command, args, { timeout: 5000 }, (error) => {
+      if (error) {
+        reject(error);
+      } else {
+        resolve();
+      }
+    });
+    child.unref();
+  });
+}

package/src/infra/opencandle-paths.ts

@@ -0,0 +1,64 @@
+import { existsSync, mkdirSync } from "node:fs";
+import { homedir } from "node:os";
+import { dirname, join, resolve } from "node:path";
+
+const OPENCANDLE_HOME_ENV = "OPENCANDLE_HOME";
+
+function ensureDir(path: string): void {
+  if (!existsSync(path)) {
+    mkdirSync(path, { recursive: true });
+  }
+}
+
+export function getOpenCandleHomeDir(): string {
+  const override = process.env[OPENCANDLE_HOME_ENV];
+  return override ? resolve(override) : join(homedir(), ".opencandle");
+}
+
+export function ensureOpenCandleHomeDir(): string {
+  const home = getOpenCandleHomeDir();
+  ensureDir(home);
+  return home;
+}
+
+export function resolveOpenCandlePath(...segments: string[]): string {
+  return join(getOpenCandleHomeDir(), ...segments);
+}
+
+export function ensureParentDir(path: string): string {
+  const parent = dirname(path);
+  ensureDir(parent);
+  return parent;
+}
+
+export function getWatchlistPath(): string {
+  return resolveOpenCandlePath("watchlist.json");
+}
+
+export function getPortfolioPath(): string {
+  return resolveOpenCandlePath("portfolio.json");
+}
+
+export function getPredictionsPath(): string {
+  return resolveOpenCandlePath("predictions.json");
+}
+
+export function getConfigPath(): string {
+  return resolveOpenCandlePath("config.json");
+}
+
+export function getOnboardingPath(): string {
+  return resolveOpenCandlePath("onboarding.json");
+}
+
+export function getStateDbPath(): string {
+  return resolveOpenCandlePath("state.db");
+}
+
+export function getLogsDir(): string {
+  return resolveOpenCandlePath("logs");
+}
+
+export function getBrowserProfileDir(): string {
+  return resolveOpenCandlePath("browser-profile");
+}

package/src/infra/rate-limiter.ts

@@ -0,0 +1,64 @@
+interface BucketConfig {
+  maxTokens: number;
+  refillRate: number; // tokens per second
+}
+
+interface Bucket {
+  tokens: number;
+  lastRefill: number;
+  config: BucketConfig;
+}
+
+export class RateLimiter {
+  private buckets = new Map<string, Bucket>();
+
+  configure(provider: string, maxTokens: number, refillRate: number): void {
+    this.buckets.set(provider, {
+      tokens: maxTokens,
+      lastRefill: Date.now(),
+      config: { maxTokens, refillRate },
+    });
+  }
+
+  async acquire(provider: string): Promise<void> {
+    const bucket = this.buckets.get(provider);
+    if (!bucket) return; // No limit configured
+
+    this.refill(bucket);
+
+    if (bucket.tokens >= 1) {
+      bucket.tokens -= 1;
+      return;
+    }
+
+    // Wait until a token is available
+    const waitMs = ((1 - bucket.tokens) / bucket.config.refillRate) * 1000;
+    await new Promise((resolve) => setTimeout(resolve, Math.ceil(waitMs)));
+    this.refill(bucket);
+    bucket.tokens -= 1;
+  }
+
+  private refill(bucket: Bucket): void {
+    const now = Date.now();
+    const elapsed = (now - bucket.lastRefill) / 1000;
+    bucket.tokens = Math.min(
+      bucket.config.maxTokens,
+      bucket.tokens + elapsed * bucket.config.refillRate,
+    );
+    bucket.lastRefill = now;
+  }
+}
+
+// Shared instance with default provider limits
+export const rateLimiter = new RateLimiter();
+rateLimiter.configure("yahoo", 5, 5);           // 5 req/s
+rateLimiter.configure("coingecko", 10, 0.167);  // 10 req/min
+rateLimiter.configure("alphavantage", 5, 0.083); // 5 req/min (free tier)
+rateLimiter.configure("fred", 120, 2);           // 120 req/min
+rateLimiter.configure("twitter", 5, 0.167);      // 5 req, ~10 req/min
+rateLimiter.configure("reddit", 5, 0.167);          // 5 req, ~10 req/min
+rateLimiter.configure("reddit_comments", 10, 0.5); // 10 req, ~30 req/min
+rateLimiter.configure("ddg", 3, 0.1);             // 3 req, ~6 req/min
+rateLimiter.configure("brave_search", 5, 0.083);  // 5 req, ~5 req/min
+rateLimiter.configure("exa", 5, 0.1);              // 5 req, ~6 req/min
+rateLimiter.configure("finnhub", 60, 1);            // 60 req/min (free tier)

package/src/memory/index.ts

@@ -0,0 +1,10 @@
+export { initDatabase, initDefaultDatabase, getTableNames, getSchemaVersion } from "./sqlite.js";
+export { MemoryStorage } from "./storage.js";
+export type { WorkflowPreferences } from "./storage.js";
+export { buildMemoryContext } from "./retrieval.js";
+export { extractPreferences } from "./preference-extractor.js";
+export { MemoryManager } from "./manager.js";
+export { getAllDefaults, getDefaults, setDefault, clearDefault } from "./tool-defaults.js";
+export type { ToolDefaults } from "./tool-defaults.js";
+export type { MemoryCategory, MemoryEntry } from "./types.js";
+export { isStale, STALENESS_THRESHOLDS, NEVER_TRUST_FROM_MEMORY } from "./types.js";

package/src/memory/manager.ts

@@ -0,0 +1,159 @@
+import type { MemoryStorage } from "./storage.js";
+import type { MemoryCategory, MemoryEntry } from "./types.js";
+import {
+  KEY_TO_CATEGORY,
+  WORKFLOW_RELEVANT_CATEGORIES,
+  NEVER_TRUST_FROM_MEMORY,
+  isStale,
+} from "./types.js";
+
+/** Slot name → preference key(s) mapping for suppression. */
+const SLOT_TO_PREF_KEYS: Record<string, string[]> = {
+  riskProfile: ["risk_profile"],
+  assetScope: ["asset_scope"],
+  timeHorizon: ["time_horizon"],
+  dteTarget: ["dte_target"],
+  moneynessPreference: ["moneyness_preference"],
+  liquidityMinimum: ["liquidity_minimum", "options_liquidity"],
+};
+
+const MAX_WORKFLOW_HISTORY_PER_TYPE = 3;
+const MAX_PREFERENCE_LINES = 15;
+
+/**
+ * Selective, typed memory retrieval with staleness rules
+ * and override suppression.
+ */
+export class MemoryManager {
+  constructor(private readonly storage: MemoryStorage) {}
+
+  /**
+   * Retrieve memory entries relevant to the given workflow type,
+   * filtering by category, staleness, and overrides.
+   */
+  retrieve(
+    workflowType: string,
+    overriddenSlots?: string[],
+    now: Date = new Date(),
+  ): MemoryEntry[] {
+    const relevantCategories = WORKFLOW_RELEVANT_CATEGORIES[workflowType] ??
+      WORKFLOW_RELEVANT_CATEGORIES["unclassified"];
+
+    // Build set of preference keys to suppress
+    const suppressedKeys = new Set<string>();
+    if (overriddenSlots) {
+      for (const slot of overriddenSlots) {
+        const keys = SLOT_TO_PREF_KEYS[slot];
+        if (keys) keys.forEach((k) => suppressedKeys.add(k));
+      }
+    }
+
+    const entries: MemoryEntry[] = [];
+
+    // Preferences as investor_profile entries
+    if (relevantCategories.includes("investor_profile")) {
+      const prefs = this.storage.getPreferencesByNamespace("global");
+      for (const pref of prefs) {
+        const key = String(pref.key);
+        if (suppressedKeys.has(key)) continue;
+        if (NEVER_TRUST_FROM_MEMORY.has(key)) continue;
+
+        const category = KEY_TO_CATEGORY[key] ?? "investor_profile";
+        if (!relevantCategories.includes(category)) continue;
+
+        const entry: MemoryEntry = {
+          key,
+          value: tryParseValue(String(pref.value_json ?? "")),
+          category,
+          recordedAt: String(pref.updated_at ?? pref.created_at ?? now.toISOString()),
+          confidence: pref.confidence != null ? String(pref.confidence) : undefined,
+          source: pref.source != null ? String(pref.source) : undefined,
+        };
+
+        if (!isStale(entry, now)) {
+          entries.push(entry);
+        }
+      }
+    }
+
+    // Workflow history
+    if (relevantCategories.includes("workflow_history")) {
+      const runs = this.storage.getRecentWorkflowRuns(MAX_WORKFLOW_HISTORY_PER_TYPE * 4);
+      const countsByType = new Map<string, number>();
+
+      for (const run of runs) {
+        const wfType = String(run.workflow_type);
+        const count = countsByType.get(wfType) ?? 0;
+        if (count >= MAX_WORKFLOW_HISTORY_PER_TYPE) continue;
+        countsByType.set(wfType, count + 1);
+
+        const recordedAt = String(run.created_at ?? now.toISOString());
+        const entry: MemoryEntry = {
+          key: `workflow_run_${run.id}`,
+          value: run.output_summary
+            ? `${wfType}: ${run.output_summary}`
+            : wfType,
+          category: "workflow_history",
+          recordedAt,
+        };
+
+        if (!isStale(entry, now)) {
+          entries.push(entry);
+        }
+      }
+    }
+
+    return entries.slice(0, MAX_PREFERENCE_LINES + MAX_WORKFLOW_HISTORY_PER_TYPE * 4);
+  }
+
+  /**
+   * Build compact text context from retrieved memory entries.
+   */
+  buildContext(
+    workflowType: string,
+    overriddenSlots?: string[],
+    now: Date = new Date(),
+  ): string {
+    const entries = this.retrieve(workflowType, overriddenSlots, now);
+    if (entries.length === 0) return "";
+
+    const sections: string[] = [];
+
+    // Group by category
+    const byCategory = new Map<MemoryCategory, MemoryEntry[]>();
+    for (const entry of entries) {
+      const list = byCategory.get(entry.category) ?? [];
+      list.push(entry);
+      byCategory.set(entry.category, list);
+    }
+
+    const profileEntries = byCategory.get("investor_profile");
+    if (profileEntries && profileEntries.length > 0) {
+      const lines = profileEntries.map((e) => `- ${e.key}: ${e.value}`);
+      sections.push("User Preferences:\n" + lines.join("\n"));
+    }
+
+    const historyEntries = byCategory.get("workflow_history");
+    if (historyEntries && historyEntries.length > 0) {
+      const lines = historyEntries.map((e) => `- ${e.value} (${e.recordedAt})`);
+      sections.push("Recent Workflows:\n" + lines.join("\n"));
+    }
+
+    const feedbackEntries = byCategory.get("interaction_feedback");
+    if (feedbackEntries && feedbackEntries.length > 0) {
+      const lines = feedbackEntries.map((e) => `- ${e.key}: ${e.value}`);
+      sections.push("Feedback:\n" + lines.join("\n"));
+    }
+
+    return sections.join("\n\n");
+  }
+}
+
+function tryParseValue(json: string): string {
+  try {
+    const parsed = JSON.parse(json);
+    return typeof parsed === "string" ? parsed : JSON.stringify(parsed);
+  } catch {
+    return json;
+  }
+}