@utopia-ai/cli 0.1.0

package/src/server/routes/probes.ts

@@ -0,0 +1,286 @@
+import { Router, Request, Response } from 'express';
+import { getDb } from '../db/index.js';
+
+const router: Router = Router();
+
+interface ProbeRow {
+  id: string;
+  project_id: string;
+  probe_type: string;
+  timestamp: string;
+  file: string;
+  line: number;
+  function_name: string;
+  data: string;
+  metadata: string;
+}
+
+interface ProbeResponse {
+  id: string;
+  projectId: string;
+  probeType: string;
+  timestamp: string;
+  file: string;
+  line: number;
+  functionName: string;
+  data: Record<string, unknown>;
+  metadata: Record<string, unknown>;
+}
+
+function rowToProbe(row: ProbeRow): ProbeResponse {
+  return {
+    id: row.id,
+    projectId: row.project_id,
+    probeType: row.probe_type,
+    timestamp: row.timestamp,
+    file: row.file,
+    line: row.line,
+    functionName: row.function_name,
+    data: JSON.parse(row.data),
+    metadata: JSON.parse(row.metadata),
+  };
+}
+
+const VALID_PROBE_TYPES = new Set(['error', 'database', 'api', 'infra', 'function']);
+
+function validateProbe(probe: Record<string, unknown>): string | null {
+  if (!probe.id || typeof probe.id !== 'string') return 'Missing or invalid "id"';
+  if (!probe.project_id && !probe.projectId) return 'Missing "project_id"';
+  if (!probe.probe_type && !probe.probeType) return 'Missing "probe_type"';
+  const probeType = (probe.probe_type || probe.probeType) as string;
+  if (!VALID_PROBE_TYPES.has(probeType)) {
+    return `Invalid probe_type "${probeType}". Must be one of: error, database, api, infra, function`;
+  }
+  if (!probe.file || typeof probe.file !== 'string') return 'Missing or invalid "file"';
+  if (probe.line === undefined || probe.line === null || typeof probe.line !== 'number') {
+    return 'Missing or invalid "line"';
+  }
+  return null;
+}
+
+// POST / - Ingest probe data (single or array)
+router.post('/', (req: Request, res: Response) => {
+  const db = getDb();
+  const body = req.body;
+
+  const probes: Record<string, unknown>[] = Array.isArray(body) ? body : [body];
+
+  if (probes.length === 0) {
+    res.status(400).json({ error: 'Request body must contain probe data' });
+    return;
+  }
+
+  const errors: Array<{ index: number; error: string }> = [];
+
+  const insertStmt = db.prepare(`
+    INSERT OR REPLACE INTO probes (id, project_id, probe_type, timestamp, file, line, function_name, data, metadata)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+  `);
+
+  const insertMany = db.transaction((probeList: Record<string, unknown>[]) => {
+    let inserted = 0;
+    for (let i = 0; i < probeList.length; i++) {
+      const probe = probeList[i];
+      const validationError = validateProbe(probe);
+      if (validationError) {
+        errors.push({ index: i, error: validationError });
+        continue;
+      }
+
+      const projectId = (probe.project_id || probe.projectId) as string;
+      const probeType = (probe.probe_type || probe.probeType) as string;
+      const timestamp = (probe.timestamp as string) || new Date().toISOString();
+      const functionName = (probe.function_name || probe.functionName || '') as string;
+
+      insertStmt.run(
+        probe.id,
+        projectId,
+        probeType,
+        timestamp,
+        probe.file,
+        probe.line,
+        functionName,
+        JSON.stringify(probe.data ?? {}),
+        JSON.stringify(probe.metadata ?? {}),
+      );
+      inserted++;
+    }
+    return inserted;
+  });
+
+  const inserted = insertMany(probes);
+
+  if (errors.length > 0 && inserted === 0) {
+    res.status(400).json({ error: 'All probes failed validation', details: errors });
+    return;
+  }
+
+  res.status(201).json({
+    inserted,
+    errors: errors.length > 0 ? errors : undefined,
+  });
+});
+
+// GET /errors/recent - Get recent errors
+// Must be defined BEFORE /:id to avoid matching "errors" as an id
+router.get('/errors/recent', (req: Request, res: Response) => {
+  const db = getDb();
+  const hours = parseInt(req.query.hours as string, 10) || 24;
+  const limit = Math.min(parseInt(req.query.limit as string, 10) || 100, 1000);
+
+  const since = new Date(Date.now() - hours * 60 * 60 * 1000).toISOString();
+
+  const rows = db.prepare(`
+    SELECT * FROM probes
+    WHERE probe_type = 'error' AND timestamp >= ?
+    ORDER BY timestamp DESC
+    LIMIT ?
+  `).all(since, limit) as ProbeRow[];
+
+  res.json({
+    count: rows.length,
+    probes: rows.map(rowToProbe),
+  });
+});
+
+// GET /context - Smart context retrieval
+// Must be defined BEFORE /:id to avoid matching "context" as an id
+router.get('/context', (req: Request, res: Response) => {
+  const db = getDb();
+  const prompt = req.query.prompt as string;
+
+  if (!prompt) {
+    res.status(400).json({ error: 'Missing "prompt" query parameter' });
+    return;
+  }
+
+  const limit = Math.min(parseInt(req.query.limit as string, 10) || 20, 100);
+
+  // Extract keywords: split on whitespace/non-alphanumeric, filter stopwords and short tokens
+  const stopWords = new Set([
+    'the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for',
+    'of', 'with', 'by', 'is', 'it', 'this', 'that', 'are', 'was', 'were',
+    'be', 'been', 'has', 'have', 'had', 'do', 'does', 'did', 'will', 'would',
+    'could', 'should', 'may', 'might', 'can', 'not', 'no', 'what', 'how',
+    'why', 'when', 'where', 'which', 'who', 'whom', 'from', 'into', 'than',
+    'then', 'there', 'here', 'just', 'also', 'very', 'too', 'some', 'any',
+    'all', 'each', 'every', 'both', 'few', 'more', 'most', 'other', 'about',
+  ]);
+
+  const keywords = prompt
+    .toLowerCase()
+    .split(/[^a-z0-9_.-]+/)
+    .filter(w => w.length >= 2 && !stopWords.has(w));
+
+  if (keywords.length === 0) {
+    res.json({ count: 0, probes: [], keywords: [] });
+    return;
+  }
+
+  // Build WHERE clause: match any keyword against file, function_name, or data
+  const conditions = keywords.map(() =>
+    '(LOWER(file) LIKE ? OR LOWER(function_name) LIKE ? OR LOWER(data) LIKE ?)'
+  );
+
+  const whereParams: string[] = [];
+  for (const kw of keywords) {
+    const pattern = `%${kw}%`;
+    whereParams.push(pattern, pattern, pattern);
+  }
+
+  // Build relevance scoring: count how many keyword/column matches
+  const relevanceExpr = keywords.map(() =>
+    '(CASE WHEN LOWER(file) LIKE ? THEN 1 ELSE 0 END + ' +
+    'CASE WHEN LOWER(function_name) LIKE ? THEN 1 ELSE 0 END + ' +
+    'CASE WHEN LOWER(data) LIKE ? THEN 1 ELSE 0 END)'
+  ).join(' + ');
+
+  const relevanceParams: string[] = [];
+  for (const kw of keywords) {
+    const pattern = `%${kw}%`;
+    relevanceParams.push(pattern, pattern, pattern);
+  }
+
+  const sql = `
+    SELECT *, (${relevanceExpr}) as relevance
+    FROM probes
+    WHERE ${conditions.join(' OR ')}
+    ORDER BY relevance DESC, timestamp DESC
+    LIMIT ?
+  `;
+
+  const allParams = [...relevanceParams, ...whereParams, limit];
+  const rows = db.prepare(sql).all(...allParams) as (ProbeRow & { relevance: number })[];
+
+  res.json({
+    count: rows.length,
+    keywords,
+    probes: rows.map(rowToProbe),
+  });
+});
+
+// GET /:id - Get single probe by ID
+router.get('/:id', (req: Request, res: Response) => {
+  const db = getDb();
+  const row = db.prepare('SELECT * FROM probes WHERE id = ?').get(req.params.id) as ProbeRow | undefined;
+
+  if (!row) {
+    res.status(404).json({ error: 'Probe not found' });
+    return;
+  }
+
+  res.json(rowToProbe(row));
+});
+
+// GET / - Query probes with filters
+router.get('/', (req: Request, res: Response) => {
+  const db = getDb();
+
+  const conditions: string[] = [];
+  const params: (string | number)[] = [];
+
+  if (req.query.probe_type) {
+    conditions.push('probe_type = ?');
+    params.push(req.query.probe_type as string);
+  }
+
+  if (req.query.file) {
+    conditions.push('file = ?');
+    params.push(req.query.file as string);
+  }
+
+  if (req.query.function_name) {
+    conditions.push('function_name = ?');
+    params.push(req.query.function_name as string);
+  }
+
+  if (req.query.project_id) {
+    conditions.push('project_id = ?');
+    params.push(req.query.project_id as string);
+  }
+
+  if (req.query.since) {
+    conditions.push('timestamp >= ?');
+    params.push(req.query.since as string);
+  }
+
+  if (req.query.until) {
+    conditions.push('timestamp <= ?');
+    params.push(req.query.until as string);
+  }
+
+  const limit = Math.min(parseInt(req.query.limit as string, 10) || 100, 1000);
+
+  const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '';
+  const sql = `SELECT * FROM probes ${whereClause} ORDER BY timestamp DESC LIMIT ?`;
+  params.push(limit);
+
+  const rows = db.prepare(sql).all(...params) as ProbeRow[];
+
+  res.json({
+    count: rows.length,
+    probes: rows.map(rowToProbe),
+  });
+});
+
+export default router;

package/src/types.ts

@@ -0,0 +1,147 @@
+// Utopia shared types
+
+export type CloudProvider = 'aws' | 'gcp' | 'vercel' | 'azure' | 'other';
+export type DeploymentMethod = 'manual' | 'github-actions' | 'vercel-trigger' | 'other';
+export type Language = 'javascript' | 'typescript' | 'python';
+export type Framework = 'react' | 'nextjs' | 'express' | 'fastapi' | 'flask' | 'django' | 'other';
+
+export interface UtopiaConfig {
+  version: string;
+  projectId: string;
+  cloudProvider: CloudProvider;
+  service: string;
+  deploymentMethod: DeploymentMethod;
+  isStandalone: boolean;
+  dataEndpoint: string;
+  language: Language[];
+  framework: Framework;
+}
+
+export type ProbeType = 'error' | 'database' | 'api' | 'infra' | 'function';
+
+export interface ProbeData {
+  id: string;
+  projectId: string;
+  probeType: ProbeType;
+  timestamp: string;
+  file: string;
+  line: number;
+  functionName: string;
+  data: Record<string, unknown>;
+  metadata: ProbeMetadata;
+}
+
+export interface ProbeMetadata {
+  runtime: 'node' | 'python';
+  environment?: string;
+  hostname?: string;
+  pid?: number;
+  version?: string;
+}
+
+export interface ErrorProbeData extends ProbeData {
+  probeType: 'error';
+  data: {
+    errorType: string;
+    message: string;
+    stack: string;
+    inputData: Record<string, unknown>;
+    codeLine: string;
+  };
+}
+
+export interface DatabaseProbeData extends ProbeData {
+  probeType: 'database';
+  data: {
+    operation: string;
+    query?: string;
+    table?: string;
+    duration: number;
+    rowCount?: number;
+    connectionInfo: {
+      type: string;
+      host?: string;
+      database?: string;
+    };
+    params?: unknown[];
+  };
+}
+
+export interface ApiProbeData extends ProbeData {
+  probeType: 'api';
+  data: {
+    method: string;
+    url: string;
+    statusCode?: number;
+    duration: number;
+    requestHeaders?: Record<string, string>;
+    responseHeaders?: Record<string, string>;
+    requestBody?: unknown;
+    responseBody?: unknown;
+    error?: string;
+  };
+}
+
+export interface InfraProbeData extends ProbeData {
+  probeType: 'infra';
+  data: {
+    provider: CloudProvider;
+    region?: string;
+    serviceType?: string;
+    instanceId?: string;
+    containerInfo?: {
+      containerId?: string;
+      image?: string;
+    };
+    envVars: Record<string, string>;
+    memoryUsage: number;
+    cpuUsage?: number;
+  };
+}
+
+export interface FunctionProbeData extends ProbeData {
+  probeType: 'function';
+  data: {
+    args: unknown[];
+    returnValue?: unknown;
+    duration: number;
+    llmContext?: string; // Utopia mode: LLM-generated context
+    callStack: string[];
+  };
+}
+
+// Impact Graph types
+export interface GraphNode {
+  id: string;
+  type: 'function' | 'service' | 'database' | 'api' | 'file';
+  name: string;
+  file?: string;
+  metadata: Record<string, unknown>;
+}
+
+export interface GraphEdge {
+  source: string;
+  target: string;
+  type: 'calls' | 'queries' | 'serves' | 'depends_on';
+  weight: number;
+  lastSeen: string;
+}
+
+export interface ImpactGraph {
+  nodes: GraphNode[];
+  edges: GraphEdge[];
+}
+
+// MCP tool types
+export interface ContextQuery {
+  prompt: string;
+  file?: string;
+  probeTypes?: ProbeType[];
+  limit?: number;
+}
+
+export interface ContextResult {
+  relevantProbes: ProbeData[];
+  impactedNodes: GraphNode[];
+  summary: string;
+}

package/src/utopia-mode/index.ts

@@ -0,0 +1,206 @@
+/**
+ * Utopia Mode — LLM-Enhanced Probes
+ *
+ * At runtime, probes capture function data and send it (async, non-blocking)
+ * to an LLM that generates rich semantic context about what the code is doing.
+ * This context is then stored in the data service for AI coding agents to query.
+ *
+ * The LLM call happens server-side (in the data service) to avoid adding
+ * the Anthropic SDK as a dependency of the probe runtime.
+ */
+
+import Anthropic from '@anthropic-ai/sdk';
+
+interface LlmContextRequest {
+  file: string;
+  line: number;
+  functionName: string;
+  args: unknown[];
+  returnValue?: unknown;
+  duration: number;
+  probeType: string;
+  additionalContext?: Record<string, unknown>;
+}
+
+interface LlmContextResult {
+  summary: string;
+  behavior: string;
+  dataFlow: string;
+  sideEffects: string[];
+  dependencies: string[];
+  risks: string[];
+}
+
+let client: Anthropic | null = null;
+let processingQueue: LlmContextRequest[] = [];
+let isProcessing = false;
+const BATCH_SIZE = 5;
+const FLUSH_INTERVAL = 10_000; // 10 seconds
+let flushTimer: ReturnType<typeof setInterval> | null = null;
+
+function getClient(): Anthropic | null {
+  if (client) return client;
+  const apiKey = process.env.ANTHROPIC_API_KEY || process.env.UTOPIA_LLM_API_KEY;
+  if (!apiKey) return null;
+  client = new Anthropic({ apiKey });
+  return client;
+}
+
+function startFlushTimer(): void {
+  if (flushTimer) return;
+  flushTimer = setInterval(() => {
+    processQueue().catch(() => {});
+  }, FLUSH_INTERVAL);
+  if (flushTimer.unref) flushTimer.unref();
+}
+
+/**
+ * Queue a function's runtime data for LLM context generation.
+ * Called by the data service when it receives a function probe or llm_context probe.
+ */
+export function queueForLlmContext(request: LlmContextRequest): void {
+  processingQueue.push(request);
+  startFlushTimer();
+
+  if (processingQueue.length >= BATCH_SIZE) {
+    processQueue().catch(() => {});
+  }
+}
+
+/**
+ * Process queued requests — generate LLM context for each and return results.
+ */
+async function processQueue(): Promise<LlmContextResult[]> {
+  if (isProcessing || processingQueue.length === 0) return [];
+  isProcessing = true;
+
+  const batch = processingQueue.splice(0, BATCH_SIZE);
+  const results: LlmContextResult[] = [];
+
+  try {
+    const anthropic = getClient();
+    if (!anthropic) {
+      isProcessing = false;
+      return [];
+    }
+
+    // Process each request concurrently but with a concurrency limit
+    const promises = batch.map(req => generateContext(anthropic, req));
+    const settled = await Promise.allSettled(promises);
+
+    for (const result of settled) {
+      if (result.status === 'fulfilled' && result.value) {
+        results.push(result.value);
+      }
+    }
+  } catch {
+    // Silently fail — never impact the host application
+  } finally {
+    isProcessing = false;
+  }
+
+  return results;
+}
+
+/**
+ * Generate semantic context for a single function invocation using Claude.
+ */
+async function generateContext(
+  anthropic: Anthropic,
+  request: LlmContextRequest
+): Promise<LlmContextResult | null> {
+  try {
+    const prompt = buildPrompt(request);
+
+    const message = await anthropic.messages.create({
+      model: 'claude-haiku-4-5-20251001',
+      max_tokens: 512,
+      messages: [{ role: 'user', content: prompt }],
+      system: `You are analyzing runtime data from a production function invocation. Respond ONLY with valid JSON matching this schema:
+{
+  "summary": "one-line description of what this function call did",
+  "behavior": "description of the function's runtime behavior based on inputs/outputs",
+  "dataFlow": "how data flows through this function (inputs → transformations → outputs)",
+  "sideEffects": ["list of side effects: DB writes, API calls, file I/O, etc."],
+  "dependencies": ["list of external dependencies this function relies on"],
+  "risks": ["potential issues: N+1 queries, slow calls, error-prone patterns"]
+}`,
+    });
+
+    const content = message.content[0];
+    if (content.type !== 'text') return null;
+
+    const parsed = JSON.parse(content.text) as LlmContextResult;
+    return parsed;
+  } catch {
+    return null;
+  }
+}
+
+function buildPrompt(request: LlmContextRequest): string {
+  const argsSummary = truncate(JSON.stringify(request.args), 500);
+  const returnSummary = truncate(JSON.stringify(request.returnValue), 500);
+
+  return `Analyze this production function invocation:
+
+File: ${request.file}
+Function: ${request.functionName}
+Line: ${request.line}
+Duration: ${request.duration}ms
+Arguments: ${argsSummary}
+Return Value: ${returnSummary}
+${request.additionalContext ? `Additional Context: ${JSON.stringify(request.additionalContext)}` : ''}
+
+Based on the function name, file path, arguments, return value, and duration, generate a semantic context summary.`;
+}
+
+function truncate(str: string, maxLen: number): string {
+  if (str.length <= maxLen) return str;
+  return str.substring(0, maxLen) + '...';
+}
+
+/**
+ * Process a probe from the data service and generate LLM context.
+ * Returns the context to be stored alongside the probe.
+ */
+export async function processProbeForLlmContext(probe: {
+  file: string;
+  line: number;
+  function_name: string;
+  probe_type: string;
+  data: string;
+}): Promise<LlmContextResult | null> {
+  const anthropic = getClient();
+  if (!anthropic) return null;
+
+  let parsedData: Record<string, unknown> = {};
+  try {
+    parsedData = JSON.parse(probe.data);
+  } catch {
+    return null;
+  }
+
+  return generateContext(anthropic, {
+    file: probe.file,
+    line: probe.line,
+    functionName: probe.function_name,
+    args: (parsedData.args as unknown[]) || [],
+    returnValue: parsedData.returnValue,
+    duration: (parsedData.duration as number) || 0,
+    probeType: probe.probe_type,
+    additionalContext: parsedData,
+  });
+}
+
+/**
+ * Shutdown: flush remaining queue.
+ */
+export async function shutdown(): Promise<void> {
+  if (flushTimer) {
+    clearInterval(flushTimer);
+    flushTimer = null;
+  }
+  await processQueue();
+}
+
+export type { LlmContextRequest, LlmContextResult };

package/tsconfig.json

@@ -0,0 +1,19 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "ESNext",
+    "moduleResolution": "bundler",
+    "outDir": "./dist",
+    "rootDir": "./src",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "resolveJsonModule": true,
+    "declaration": true,
+    "declarationMap": true,
+    "sourceMap": true
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist"]
+}