kalshi-trading-bot-cli 2.1.0

package/src/tools/fetch/web-fetch.ts

@@ -0,0 +1,419 @@
+/**
+ * web_fetch tool — lightweight one-shot page reader with caching.
+ *
+ * Core extraction logic ported from OpenClaw's src/agents/tools/web-fetch.ts (MIT license).
+ * Adapted for the app's LangChain DynamicStructuredTool + Zod framework.
+ *
+ * Differences from OpenClaw:
+ * - fetchWithSsrFGuard replaced with plain fetch + manual redirect handling
+ * - Firecrawl fallback removed (falls back to htmlToMarkdown instead)
+ * - Config resolution replaced with hardcoded defaults
+ * - Tool wrapper uses LangChain DynamicStructuredTool + Zod (not AnyAgentTool + TypeBox)
+ */
+import { DynamicStructuredTool } from '@langchain/core/tools';
+import { z } from 'zod';
+import { formatToolResult } from '../types.js';
+import { wrapExternalContent, wrapWebContent } from './external-content.js';
+import {
+  extractReadableContent,
+  htmlToMarkdown,
+  markdownToText,
+  truncateText,
+  type ExtractMode,
+} from './web-fetch-utils.js';
+import {
+  type CacheEntry,
+  DEFAULT_CACHE_TTL_MINUTES,
+  DEFAULT_TIMEOUT_SECONDS,
+  normalizeCacheKey,
+  readCache,
+  readResponseText,
+  resolveCacheTtlMs,
+  resolveTimeoutSeconds,
+  withTimeout,
+  writeCache,
+} from './cache.js';
+
+/**
+ * Rich description for the web_fetch tool.
+ * Used in the system prompt to guide the LLM on when and how to use this tool.
+ */
+export const WEB_FETCH_DESCRIPTION = `
+Fetch and extract readable content from a URL (HTML -> markdown/text). Returns the page content directly in a single call.
+
+## This is the DEFAULT tool for reading web pages
+
+Use web_fetch as your FIRST choice whenever you need to read the content of a web page. It is faster and simpler than the browser tool.
+
+## When to Use
+
+- Reading earnings reports, press releases, or investor relations pages
+- Reading articles from news sites (CNBC, Bloomberg, Reuters, etc.)
+- Accessing any URL discovered via web_search
+- Reading documentation, blog posts, or any static web content
+- When you need the full text content of a known URL
+
+## When NOT to Use
+
+- Interactive pages that require JavaScript rendering, clicking, or form filling (use browser instead)
+- Structured financial data like prices, metrics, or estimates (use financial_search instead)
+- SEC filings content (use read_filings instead)
+- When you need to navigate through multiple pages by clicking links (use browser instead)
+
+## Schema
+
+- **url** (required): The HTTP or HTTPS URL to fetch
+- **extractMode** (optional): "markdown" (default) or "text" - controls output format
+- **maxChars** (optional): Maximum characters to return (default 20,000)
+
+## Returns
+
+Returns the page content directly as markdown or text. No multi-step workflow needed - one call gets you the full content.
+
+Response includes: url, finalUrl, title, text, extractMode, extractor, truncated, tookMs
+
+## Usage Notes
+
+- Returns content in a single call - no need for navigate/snapshot/read steps
+- Results are cached for 15 minutes - repeated fetches of the same URL are instant
+- Handles redirects automatically (up to 3 hops)
+- Extracts readable content using Mozilla Readability (same as Firefox Reader View)
+- Falls back to raw HTML-to-markdown conversion if Readability extraction fails
+- Works with HTML pages, JSON responses, and plain text
+`.trim();
+
+// ============================================================================
+// Constants (identical to OpenClaw)
+// ============================================================================
+
+const DEFAULT_FETCH_MAX_CHARS = 20_000;
+const DEFAULT_FETCH_MAX_REDIRECTS = 3;
+const DEFAULT_ERROR_MAX_CHARS = 4_000;
+const DEFAULT_FETCH_USER_AGENT =
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_7_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36";
+
+// ============================================================================
+// Cache (identical to OpenClaw)
+// ============================================================================
+
+const FETCH_CACHE = new Map<string, CacheEntry<Record<string, unknown>>>();
+
+// ============================================================================
+// Content wrapping (identical to OpenClaw)
+// ============================================================================
+
+const WEB_FETCH_WRAPPER_WITH_WARNING_OVERHEAD = wrapWebContent("", "web_fetch").length;
+const WEB_FETCH_WRAPPER_NO_WARNING_OVERHEAD = wrapExternalContent("", {
+  source: "web_fetch",
+  includeWarning: false,
+}).length;
+
+function wrapWebFetchContent(
+  value: string,
+  maxChars: number,
+): {
+  text: string;
+  truncated: boolean;
+  rawLength: number;
+  wrappedLength: number;
+} {
+  if (maxChars <= 0) {
+    return { text: "", truncated: true, rawLength: 0, wrappedLength: 0 };
+  }
+  const includeWarning = maxChars >= WEB_FETCH_WRAPPER_WITH_WARNING_OVERHEAD;
+  const wrapperOverhead = includeWarning
+    ? WEB_FETCH_WRAPPER_WITH_WARNING_OVERHEAD
+    : WEB_FETCH_WRAPPER_NO_WARNING_OVERHEAD;
+  if (wrapperOverhead > maxChars) {
+    const minimal = includeWarning
+      ? wrapWebContent("", "web_fetch")
+      : wrapExternalContent("", { source: "web_fetch", includeWarning: false });
+    const truncatedWrapper = truncateText(minimal, maxChars);
+    return {
+      text: truncatedWrapper.text,
+      truncated: true,
+      rawLength: 0,
+      wrappedLength: truncatedWrapper.text.length,
+    };
+  }
+  const maxInner = Math.max(0, maxChars - wrapperOverhead);
+  let truncated = truncateText(value, maxInner);
+  let wrappedText = includeWarning
+    ? wrapWebContent(truncated.text, "web_fetch")
+    : wrapExternalContent(truncated.text, { source: "web_fetch", includeWarning: false });
+
+  if (wrappedText.length > maxChars) {
+    const excess = wrappedText.length - maxChars;
+    const adjustedMaxInner = Math.max(0, maxInner - excess);
+    truncated = truncateText(value, adjustedMaxInner);
+    wrappedText = includeWarning
+      ? wrapWebContent(truncated.text, "web_fetch")
+      : wrapExternalContent(truncated.text, { source: "web_fetch", includeWarning: false });
+  }
+
+  return {
+    text: wrappedText,
+    truncated: truncated.truncated,
+    rawLength: truncated.text.length,
+    wrappedLength: wrappedText.length,
+  };
+}
+
+function wrapWebFetchField(value: string | undefined): string | undefined {
+  if (!value) {
+    return value;
+  }
+  return wrapExternalContent(value, { source: "web_fetch", includeWarning: false });
+}
+
+// ============================================================================
+// Helpers (identical to OpenClaw)
+// ============================================================================
+
+function normalizeContentType(value: string | null | undefined): string | undefined {
+  if (!value) {
+    return undefined;
+  }
+  const [raw] = value.split(";");
+  const trimmed = raw?.trim();
+  return trimmed || undefined;
+}
+
+function looksLikeHtml(value: string): boolean {
+  const trimmed = value.trimStart();
+  if (!trimmed) {
+    return false;
+  }
+  const head = trimmed.slice(0, 256).toLowerCase();
+  return head.startsWith("<!doctype html") || head.startsWith("<html");
+}
+
+function formatWebFetchErrorDetail(params: {
+  detail: string;
+  contentType?: string | null;
+  maxChars: number;
+}): string {
+  const { detail, contentType, maxChars } = params;
+  if (!detail) {
+    return "";
+  }
+  let text = detail;
+  const contentTypeLower = contentType?.toLowerCase();
+  if (contentTypeLower?.includes("text/html") || looksLikeHtml(detail)) {
+    const rendered = htmlToMarkdown(detail);
+    const withTitle = rendered.title ? `${rendered.title}\n${rendered.text}` : rendered.text;
+    text = markdownToText(withTitle);
+  }
+  const truncatedResult = truncateText(text.trim(), maxChars);
+  return truncatedResult.text;
+}
+
+function resolveMaxChars(value: unknown, fallback: number, cap: number): number {
+  const parsed = typeof value === "number" && Number.isFinite(value) ? value : fallback;
+  const clamped = Math.max(100, Math.floor(parsed));
+  return Math.min(clamped, cap);
+}
+
+// ============================================================================
+// HTTP fetch with manual redirect handling (replaces OpenClaw's fetchWithSsrFGuard)
+// ============================================================================
+
+function isRedirectStatus(status: number): boolean {
+  return status === 301 || status === 302 || status === 303 || status === 307 || status === 308;
+}
+
+async function fetchWithRedirects(params: {
+  url: string;
+  maxRedirects: number;
+  timeoutMs: number;
+  headers: Record<string, string>;
+}): Promise<{ response: Response; finalUrl: string }> {
+  const signal = withTimeout(undefined, params.timeoutMs);
+  const visited = new Set<string>();
+  let currentUrl = params.url;
+  let redirectCount = 0;
+
+  while (true) {
+    let parsedUrl: URL;
+    try {
+      parsedUrl = new URL(currentUrl);
+    } catch {
+      throw new Error("[Web Fetch] Invalid URL: must be http or https");
+    }
+    if (!["http:", "https:"].includes(parsedUrl.protocol)) {
+      throw new Error("[Web Fetch] Invalid URL: must be http or https");
+    }
+
+    const response = await fetch(parsedUrl.toString(), {
+      redirect: "manual",
+      headers: params.headers,
+      signal,
+    });
+
+    if (isRedirectStatus(response.status)) {
+      const location = response.headers.get("location");
+      if (!location) {
+        throw new Error(`[Web Fetch] Redirect missing location header (${response.status})`);
+      }
+      redirectCount += 1;
+      if (redirectCount > params.maxRedirects) {
+        throw new Error(`[Web Fetch] Too many redirects (limit: ${params.maxRedirects})`);
+      }
+      const nextUrl = new URL(location, parsedUrl).toString();
+      if (visited.has(nextUrl)) {
+        throw new Error("[Web Fetch] Redirect loop detected");
+      }
+      visited.add(nextUrl);
+      currentUrl = nextUrl;
+      continue;
+    }
+
+    return { response, finalUrl: currentUrl };
+  }
+}
+
+// ============================================================================
+// Core fetch logic (ported from OpenClaw's runWebFetch, Firecrawl branches removed)
+// ============================================================================
+
+async function runWebFetch(params: {
+  url: string;
+  extractMode: ExtractMode;
+  maxChars: number;
+  maxRedirects: number;
+  timeoutSeconds: number;
+  cacheTtlMs: number;
+  userAgent: string;
+}): Promise<Record<string, unknown>> {
+  const cacheKey = normalizeCacheKey(
+    `fetch:${params.url}:${params.extractMode}:${params.maxChars}`,
+  );
+  const cached = readCache(FETCH_CACHE, cacheKey);
+  if (cached) {
+    return { ...cached.value, cached: true };
+  }
+
+  let parsedUrl: URL;
+  try {
+    parsedUrl = new URL(params.url);
+  } catch {
+    throw new Error("[Web Fetch] Invalid URL: must be http or https");
+  }
+  if (!["http:", "https:"].includes(parsedUrl.protocol)) {
+    throw new Error("[Web Fetch] Invalid URL: must be http or https");
+  }
+
+  const start = Date.now();
+  const { response: res, finalUrl } = await fetchWithRedirects({
+    url: params.url,
+    maxRedirects: params.maxRedirects,
+    timeoutMs: params.timeoutSeconds * 1000,
+    headers: {
+      Accept: "*/*",
+      "User-Agent": params.userAgent,
+      "Accept-Language": "en-US,en;q=0.9",
+    },
+  });
+
+  if (!res.ok) {
+    const rawDetail = await readResponseText(res);
+    const detail = formatWebFetchErrorDetail({
+      detail: rawDetail,
+      contentType: res.headers.get("content-type"),
+      maxChars: DEFAULT_ERROR_MAX_CHARS,
+    });
+    const wrappedDetail = wrapWebFetchContent(detail || res.statusText, DEFAULT_ERROR_MAX_CHARS);
+    throw new Error(`[Web Fetch] failed (${res.status}): ${wrappedDetail.text}`);
+  }
+
+  const contentType = res.headers.get("content-type") ?? "application/octet-stream";
+  const normalizedContentType = normalizeContentType(contentType) ?? "application/octet-stream";
+  const body = await readResponseText(res);
+
+  let title: string | undefined;
+  let extractor = "raw";
+  let text = body;
+  if (contentType.includes("text/html")) {
+    const readable = await extractReadableContent({
+      html: body,
+      url: finalUrl,
+      extractMode: params.extractMode,
+    });
+    if (readable?.text) {
+      text = readable.text;
+      title = readable.title;
+      extractor = "readability";
+    } else {
+      // Fallback to htmlToMarkdown (OpenClaw falls to Firecrawl here)
+      const rendered = htmlToMarkdown(body);
+      text = params.extractMode === "text" ? markdownToText(rendered.text) : rendered.text;
+      title = rendered.title;
+      extractor = "htmlToMarkdown";
+    }
+  } else if (contentType.includes("application/json")) {
+    try {
+      text = JSON.stringify(JSON.parse(body), null, 2);
+      extractor = "json";
+    } catch {
+      text = body;
+      extractor = "raw";
+    }
+  }
+
+  const wrapped = wrapWebFetchContent(text, params.maxChars);
+  const wrappedTitle = title ? wrapWebFetchField(title) : undefined;
+  const payload = {
+    url: params.url,
+    finalUrl,
+    status: res.status,
+    contentType: normalizedContentType,
+    title: wrappedTitle,
+    extractMode: params.extractMode,
+    extractor,
+    truncated: wrapped.truncated,
+    length: wrapped.wrappedLength,
+    rawLength: wrapped.rawLength,
+    wrappedLength: wrapped.wrappedLength,
+    fetchedAt: new Date().toISOString(),
+    tookMs: Date.now() - start,
+    text: wrapped.text,
+  };
+  writeCache(FETCH_CACHE, cacheKey, payload, params.cacheTtlMs);
+  return payload;
+}
+
+// ============================================================================
+// Tool definition (adapted for the app's LangChain + Zod framework)
+// ============================================================================
+
+export const webFetchTool = new DynamicStructuredTool({
+  name: 'web_fetch',
+  description:
+    'Fetch and extract readable content from a URL (HTML → markdown/text). Use for lightweight page access without browser automation.',
+  schema: z.object({
+    url: z.string().describe('HTTP or HTTPS URL to fetch.'),
+    extractMode: z
+      .enum(['markdown', 'text'])
+      .optional()
+      .describe('Extraction mode ("markdown" or "text"). Defaults to "markdown".'),
+    maxChars: z
+      .number()
+      .min(100)
+      .optional()
+      .describe('Maximum characters to return (truncates when exceeded).'),
+  }),
+  func: async (input) => {
+    const extractMode: ExtractMode = input.extractMode === 'text' ? 'text' : 'markdown';
+    const maxChars = resolveMaxChars(input.maxChars, DEFAULT_FETCH_MAX_CHARS, DEFAULT_FETCH_MAX_CHARS);
+    const result = await runWebFetch({
+      url: input.url,
+      extractMode,
+      maxChars,
+      maxRedirects: DEFAULT_FETCH_MAX_REDIRECTS,
+      timeoutSeconds: resolveTimeoutSeconds(undefined, DEFAULT_TIMEOUT_SECONDS),
+      cacheTtlMs: resolveCacheTtlMs(undefined, DEFAULT_CACHE_TTL_MINUTES),
+      userAgent: DEFAULT_FETCH_USER_AGENT,
+    });
+    return formatToolResult(result, [input.url]);
+  },
+});

package/src/tools/index.ts

@@ -0,0 +1,10 @@
+// Tool registry - the primary way to access tools and their descriptions
+export { getToolRegistry, getTools, buildToolDescriptions } from './registry.js';
+export type { RegisteredTool } from './registry.js';
+
+// Kalshi tools
+export * from './kalshi/index.js';
+
+// Search
+export { tavilySearch } from './search/index.js';
+export { WEB_SEARCH_DESCRIPTION } from './search/index.js';

package/src/tools/kalshi/api.ts

@@ -0,0 +1,251 @@
+import { createSign, constants } from 'node:crypto';
+import { readFileSync } from 'node:fs';
+import { logger } from '../../utils/logger.js';
+import { auditTrail } from '../../audit/index.js';
+import { dlqWriter } from './dlq.js';
+import type { KalshiMarket } from './types.js';
+
+const PROD_BASE_URL = 'https://api.elections.kalshi.com/trade-api/v2';
+const DEMO_BASE_URL = 'https://demo-api.kalshi.co/trade-api/v2';
+
+function getBaseUrl(): string {
+  return process.env.KALSHI_USE_DEMO === 'true' ? DEMO_BASE_URL : PROD_BASE_URL;
+}
+
+function getPrivateKey(): string {
+  if (process.env.KALSHI_PRIVATE_KEY) {
+    return process.env.KALSHI_PRIVATE_KEY;
+  }
+  if (process.env.KALSHI_PRIVATE_KEY_FILE) {
+    return readFileSync(process.env.KALSHI_PRIVATE_KEY_FILE, 'utf-8');
+  }
+  throw new Error('Kalshi private key not configured. Set KALSHI_PRIVATE_KEY or KALSHI_PRIVATE_KEY_FILE.');
+}
+
+function getApiKey(): string {
+  const key = process.env.KALSHI_API_KEY;
+  if (!key) throw new Error('KALSHI_API_KEY not set');
+  return key;
+}
+
+function buildSignature(method: string, path: string): { timestamp: string; signature: string } {
+  const timestamp = Date.now().toString();
+  const message = timestamp + method.toUpperCase() + path;
+
+  const privateKey = getPrivateKey();
+  const sign = createSign('SHA256');
+  sign.update(message);
+  sign.end();
+
+  const signature = sign.sign(
+    {
+      key: privateKey,
+      padding: constants.RSA_PKCS1_PSS_PADDING,
+      saltLength: 32,
+    },
+    'base64'
+  );
+
+  return { timestamp, signature };
+}
+
+// --- Error class ---
+
+export class KalshiApiError extends Error {
+  constructor(
+    public readonly statusCode: number,
+    public readonly statusText: string,
+    public readonly body: string
+  ) {
+    super(`Kalshi API error: ${statusCode} ${statusText}${body ? ` — ${body}` : ''}`);
+    this.name = 'KalshiApiError';
+  }
+}
+
+// --- Dollar conversion utilities ---
+
+export function toDollarString(cents: number): string {
+  return (cents / 100).toFixed(2);
+}
+
+export function fromDollarString(dollar: string): number {
+  return Math.round(parseFloat(dollar) * 100);
+}
+
+export function supportsFractional(market: KalshiMarket): boolean {
+  return market.supports_fractional === true || market.tick_size < 1;
+}
+
+// --- Retry logic ---
+
+interface RetryContext {
+  method: string;
+  path: string;
+  body?: Record<string, unknown>;
+}
+
+const MAX_RETRIES = 5;
+const BASE_DELAY_MS = 1000;
+const MAX_DELAY_MS = 120_000;
+const JITTER_FACTOR = 0.2;
+
+function isRetryable(error: unknown): boolean {
+  if (!(error instanceof KalshiApiError)) return false;
+  if (error.statusCode === 429) return true;
+  if (error.statusCode >= 500) return true;
+  return false;
+}
+
+function computeDelay(attempt: number): number {
+  const base = Math.min(BASE_DELAY_MS * Math.pow(2, attempt), MAX_DELAY_MS);
+  const jitter = base * JITTER_FACTOR * (2 * Math.random() - 1);
+  return Math.max(0, base + jitter);
+}
+
+async function withRetry<T>(fn: () => Promise<T>, context: RetryContext): Promise<T> {
+  for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
+    try {
+      return await fn();
+    } catch (error) {
+      if (!isRetryable(error) || attempt === MAX_RETRIES) {
+        if (attempt > 0 && error instanceof KalshiApiError) {
+          // Exhausted retries — write to DLQ
+          dlqWriter.append({
+            method: context.method,
+            path: context.path,
+            body: context.body,
+            error: error.message,
+            attempts: attempt + 1,
+          });
+          auditTrail.log({
+            type: 'DLQ_ENTRY',
+            method: context.method,
+            path: context.path,
+            error: error.message,
+            attempts: attempt + 1,
+          });
+        }
+        throw error;
+      }
+
+      const apiError = error as KalshiApiError;
+      const delay = computeDelay(attempt);
+
+      auditTrail.log({
+        type: 'API_RETRY',
+        method: context.method,
+        path: context.path,
+        attempt: attempt + 1,
+        max_retries: MAX_RETRIES,
+        status_code: apiError.statusCode,
+        delay_ms: Math.round(delay),
+      });
+
+      logger.warn(
+        `[Kalshi API] ${apiError.statusCode} on ${context.method} ${context.path}, retrying in ${Math.round(delay)}ms (attempt ${attempt + 1}/${MAX_RETRIES})`
+      );
+      await new Promise((resolve) => setTimeout(resolve, delay));
+    }
+  }
+  throw new Error('Max retries exceeded');
+}
+
+// --- Public API ---
+
+export interface KalshiApiResponse {
+  [key: string]: unknown;
+}
+
+export async function callKalshiApi(
+  method: 'GET' | 'POST' | 'PUT' | 'DELETE',
+  path: string,
+  options?: {
+    params?: Record<string, string | number | boolean | string[] | undefined>;
+    body?: Record<string, unknown>;
+  }
+): Promise<KalshiApiResponse> {
+  return withRetry(
+    async () => {
+      const baseUrl = getBaseUrl();
+
+      // Build URL with query params
+      const url = new URL(`${baseUrl}${path}`);
+      if (options?.params) {
+        for (const [key, value] of Object.entries(options.params)) {
+          if (value !== undefined && value !== null) {
+            if (Array.isArray(value)) {
+              value.forEach((v) => url.searchParams.append(key, String(v)));
+            } else {
+              url.searchParams.append(key, String(value));
+            }
+          }
+        }
+      }
+
+      // Sign the request using only the path (no query string)
+      // Kalshi signature covers: timestamp + method + path (without query params)
+      const pathWithQuery = url.pathname;
+      const { timestamp, signature } = buildSignature(method, pathWithQuery);
+
+      const headers: Record<string, string> = {
+        'KALSHI-ACCESS-KEY': getApiKey(),
+        'KALSHI-ACCESS-SIGNATURE': signature,
+        'KALSHI-ACCESS-TIMESTAMP': timestamp,
+        'Content-Type': 'application/json',
+      };
+
+      const fetchOptions: RequestInit = { method, headers };
+
+      if (options?.body && (method === 'POST' || method === 'PUT' || method === 'DELETE')) {
+        fetchOptions.body = JSON.stringify(options.body);
+      }
+
+      const response = await fetch(url.toString(), fetchOptions);
+
+      if (!response.ok) {
+        const text = await response.text().catch(() => '');
+        throw new KalshiApiError(response.status, response.statusText, text);
+      }
+
+      // 204 No Content — DELETE operations often return this
+      if (response.status === 204) {
+        return {};
+      }
+
+      return response.json();
+    },
+    { method, path, body: options?.body }
+  );
+}
+
+/**
+ * Fetch all pages of a cursor-paginated endpoint.
+ */
+export async function fetchAllPages<T>(
+  path: string,
+  params: Record<string, string | number | boolean | undefined>,
+  dataKey: string,
+  maxPages = 10,
+  onProgress?: (info: { fetchedItems: number; page: number; maxPages: number }) => void
+): Promise<T[]> {
+  const results: T[] = [];
+  let cursor: string | undefined;
+  let page = 0;
+
+  while (page < maxPages) {
+    const response = await callKalshiApi('GET', path, {
+      params: cursor ? { ...params, cursor } : params,
+    });
+
+    const data = response[dataKey] as T[] | undefined;
+    if (!data || data.length === 0) break;
+
+    results.push(...data);
+    cursor = response.cursor as string | undefined;
+    page++;
+    onProgress?.({ fetchedItems: results.length, page, maxPages });
+    if (!cursor) break;
+  }
+
+  return results;
+}