@lakitu/sdk 0.1.63 → 0.1.65

package/convex/cloud/workflows/compileSandbox.ts

@@ -1,6 +1,6 @@
 /**
  * Sandbox Compiler
- * 
+ *
  * Manages compilation manifest for sandbox definitions.
  */
 
@@ -27,11 +27,11 @@ export const saveManifest = internalMutation({
       .query("compiledSandbox")
       .withIndex("by_version", (q) => q.eq("version", args.version))
       .collect();
-    
+
     for (const entry of existing) {
       await ctx.db.delete(entry._id);
     }
-    
+
     // Insert new entries
     for (const item of args.manifest) {
       await ctx.db.insert("compiledSandbox", {
@@ -43,7 +43,7 @@ export const saveManifest = internalMutation({
         createdAt: Date.now(),
       });
     }
-    
+
     return { saved: args.manifest.length };
   },
 });
@@ -60,15 +60,15 @@ export const getManifest = query({
         .withIndex("by_version", (q) => q.eq("version", args.version!))
         .collect();
     }
-    
+
     // Get latest version
     const latest = await ctx.db
       .query("compiledSandbox")
       .order("desc")
       .first();
-    
+
     if (!latest) return [];
-    
+
     return await ctx.db
       .query("compiledSandbox")
       .withIndex("by_version", (q) => q.eq("version", latest.version))
@@ -84,7 +84,7 @@ export const getLatestVersion = query({
       .query("compiledSandbox")
       .order("desc")
       .first();
-    
+
     return latest?.version ?? null;
   },
 });
@@ -112,35 +112,7 @@ export const getToolImplementation = query({
       .query("customTools")
       .withIndex("by_toolId", (q) => q.eq("toolId", args.toolId))
       .first();
-    
+
     return tool?.implementation || null;
   },
 });
-
-// ============================================
-// Built-in Metadata Queries (deprecated - metadata now in Lakitu)
-// ============================================
-
-/** Get all built-in tool metadata */
-export const getBuiltInTools = query({
-  args: {},
-  handler: async () => [],
-});
-
-/** Get all built-in skill metadata */
-export const getBuiltInSkills = query({
-  args: {},
-  handler: async () => [],
-});
-
-/** Get all built-in deliverable metadata */
-export const getBuiltInDeliverables = query({
-  args: {},
-  handler: async () => [],
-});
-
-/** Get all agent definitions */
-export const getAgents = query({
-  args: {},
-  handler: async () => [],
-});

package/convex/sandbox/agent/index.ts

@@ -11,7 +11,7 @@
  * 4. Response returns through the chain
  */
 
-import { action, internalAction, query, mutation } from "../_generated/server";
+import { action, query, mutation } from "../_generated/server";
 import { api, internal } from "../_generated/api";
 import { v } from "convex/values";
 import type { ChainOfThoughtStep, StepStatus } from "../../../shared/chain-of-thought";
@@ -68,7 +68,7 @@ interface GatewayConfig {
   jwt: string;
 }
 
-// Module-level gateway config (set by startThread/continueThread)
+// Module-level gateway config (set by startCodeExecThread)
 let gatewayConfig: GatewayConfig | null = null;
 
 // Module-level chain-of-thought steps for real-time UI (in-memory per sandbox session)
@@ -105,7 +105,7 @@ function createToolStep(
   status: StepStatus
 ): Omit<ChainOfThoughtStep, "id" | "timestamp"> {
   const stepType = getStepTypeForTool(toolName);
-  
+
   switch (stepType) {
     case "search": {
       // Extract URLs from search results
@@ -123,14 +123,14 @@ function createToolStep(
         results: urls.length > 0 ? urls : undefined,
       };
     }
-    
+
     case "browser": {
       const action = toolName.replace("browser_", "") as any;
       return {
         type: "browser",
         status,
         action: action === "open" ? "navigate" : action,
-        label: toolName === "browser_open" 
+        label: toolName === "browser_open"
           ? `Navigating to ${(args as any).url}`
           : toolName === "browser_screenshot"
           ? "Taking screenshot"
@@ -139,9 +139,9 @@ function createToolStep(
         screenshot: toolName === "browser_screenshot" ? (result as any)?.screenshot : undefined,
       };
     }
-    
+
     case "file": {
-      const operation = toolName.includes("read") ? "read" 
+      const operation = toolName.includes("read") ? "read"
         : toolName.includes("edit") ? "edit"
         : toolName.includes("pdf") ? "save"
         : "write";
@@ -156,7 +156,7 @@ function createToolStep(
           : `Saving ${path}`,
       };
     }
-    
+
     default:
       return {
         type: "tool",
@@ -306,83 +306,6 @@ function createThreadId(): string {
   return `thread_${Date.now()}_${Math.random().toString(36).slice(2, 9)}`;
 }
 
-// ============================================
-// Legacy Tool Execution Loop (DEPRECATED)
-// ============================================
-
-/**
- * @deprecated Use startCodeExecThread instead. Legacy JSON tool calling is no longer supported.
- */
-async function runAgentLoop(
-  _ctx: any,
-  _systemPrompt: string,
-  _userPrompt: string,
-  _maxSteps: number = 10,
-  _threadId?: string
-): Promise<{ text: string; toolCalls: ToolCall[] }> {
-  throw new Error(
-    "Legacy tool calling mode is deprecated. Use startCodeExecThread instead, " +
-    "which uses the new KSA (Knowledge, Skills, Abilities) architecture with code execution."
-  );
-}
-
-// ============================================
-// Agent Actions
-// ============================================
-
-/**
- * Start a new agent thread
- * @deprecated Use startCodeExecThread instead. Legacy JSON tool calling is no longer supported.
- */
-export const startThread = action({
-  args: {
-    prompt: v.string(),
-    context: v.optional(v.any()),
-  },
-  handler: async (_ctx, _args): Promise<AgentResult> => {
-    throw new Error(
-      "startThread is deprecated. Use startCodeExecThread instead, " +
-      "which uses the new KSA (Knowledge, Skills, Abilities) architecture with code execution."
-    );
-  },
-});
-
-/**
- * Continue an existing thread
- * @deprecated Use startCodeExecThread instead. Legacy JSON tool calling is no longer supported.
- */
-export const continueThread = action({
-  args: {
-    threadId: v.string(),
-    prompt: v.string(),
-  },
-  handler: async (_ctx, _args): Promise<AgentResult> => {
-    throw new Error(
-      "continueThread is deprecated. Use startCodeExecThread instead, " +
-      "which uses the new KSA (Knowledge, Skills, Abilities) architecture with code execution."
-    );
-  },
-});
-
-/**
- * Run agent with timeout for chained execution
- * @deprecated Use startCodeExecThread instead. Legacy JSON tool calling is no longer supported.
- */
-export const runWithTimeout = internalAction({
-  args: {
-    prompt: v.string(),
-    context: v.optional(v.any()),
-    timeoutMs: v.number(),
-    checkpointId: v.optional(v.id("checkpoints")),
-  },
-  handler: async (_ctx, _args) => {
-    throw new Error(
-      "runWithTimeout is deprecated. Use startCodeExecThread instead, " +
-      "which uses the new KSA (Knowledge, Skills, Abilities) architecture with code execution."
-    );
-  },
-});
-
 // ============================================
 // Queries
 // ============================================
@@ -412,7 +335,7 @@ export const getChainOfThoughtSteps = query({
 });
 
 // ============================================
-// Code Execution Mode (NEW ARCHITECTURE)
+// Code Execution Mode
 // ============================================
 
 import { runCodeExecLoop, getSteps } from "./codeExecLoop";
@@ -421,12 +344,10 @@ import { getCodeExecSystemPrompt, generateKSAInstructions } from "../prompts/cod
 /**
  * Start a thread using code execution mode.
  *
- * This is the NEW architecture:
+ * This is the primary agent entry point:
  * - LLM generates TypeScript code
- * - Code imports from skills/ and executes
+ * - Code imports from KSA modules and executes
  * - No JSON tool calls
- *
- * Use this instead of startThread for the new code execution model.
  */
 export const startCodeExecThread = action({
   args: {

package/convex/sandbox/http.ts

@@ -0,0 +1,107 @@
+/**
+ * Sandbox HTTP Endpoints
+ *
+ * Provides observability endpoints for the E2B sandbox.
+ * These run on port 3211 (site port) alongside Convex backend.
+ */
+
+import { httpRouter } from "convex/server";
+import { httpAction } from "./_generated/server";
+
+const http = httpRouter();
+
+/**
+ * Metrics endpoint - returns sandbox health and resource usage.
+ * Used by pool health checks and Claude Code observability.
+ *
+ * GET /metrics
+ *
+ * Response:
+ * {
+ *   uptime: number,           // Process uptime in seconds
+ *   cpu: { usage: number, cores: number },
+ *   memory: { total, free, used, heapUsed, heapTotal },
+ *   timestamp: number
+ * }
+ */
+http.route({
+  path: "/metrics",
+  method: "GET",
+  handler: httpAction(async () => {
+    // Dynamic imports for Node.js APIs (required for Convex bundling)
+    const os = await import("os");
+
+    const metrics = {
+      uptime: process.uptime(),
+      cpu: {
+        usage: os.loadavg()[0], // 1-minute load average
+        cores: os.cpus().length,
+      },
+      memory: {
+        total: os.totalmem(),
+        free: os.freemem(),
+        used: os.totalmem() - os.freemem(),
+        heapUsed: process.memoryUsage().heapUsed,
+        heapTotal: process.memoryUsage().heapTotal,
+      },
+      services: {
+        convex: "running", // We're responding, so Convex is up
+      },
+      timestamp: Date.now(),
+    };
+
+    return new Response(JSON.stringify(metrics), {
+      status: 200,
+      headers: {
+        "Content-Type": "application/json",
+        "Cache-Control": "no-cache",
+      },
+    });
+  }),
+});
+
+/**
+ * Health check endpoint - simple ping for pool management.
+ * Used by E2B pool to verify sandbox is responsive.
+ *
+ * GET /health
+ */
+http.route({
+  path: "/health",
+  method: "GET",
+  handler: httpAction(async () => {
+    return new Response(JSON.stringify({ status: "ok", timestamp: Date.now() }), {
+      status: 200,
+      headers: { "Content-Type": "application/json" },
+    });
+  }),
+});
+
+/**
+ * Version endpoint - returns SDK version for debugging.
+ * Used by pool health checks.
+ *
+ * GET /version
+ */
+http.route({
+  path: "/version",
+  method: "GET",
+  handler: httpAction(async () => {
+    // Read version from package.json at runtime
+    const fs = await import("fs/promises");
+    let version = "unknown";
+    try {
+      const pkg = JSON.parse(await fs.readFile("/home/user/lakitu/package.json", "utf-8"));
+      version = pkg.version || "unknown";
+    } catch {
+      // Fallback if file not found
+    }
+
+    return new Response(JSON.stringify({ version, timestamp: Date.now() }), {
+      status: 200,
+      headers: { "Content-Type": "application/json" },
+    });
+  }),
+});
+
+export default http;

package/convex/sandbox/index.ts

@@ -8,11 +8,10 @@
 // Agent
 // ============================================
 export {
-  startThread,
-  continueThread,
-  runWithTimeout,
+  startCodeExecThread,
   getThreadMessages,
   getStreamDeltas,
+  getChainOfThoughtSteps,
 } from "./agent";
 export * as decisions from "./agent/decisions";
 
@@ -38,12 +37,6 @@ export * as sync from "./planning/sync";
 export * as context from "./context";
 export * as session from "./context/session";
 
-// ============================================
-// Tools (DEPRECATED - Use KSAs instead)
-// ============================================
-// Legacy tool system has been removed. Use the KSA (Knowledge, Skills, Abilities)
-// architecture with code execution mode instead. See packages/lakitu/ksa/
-
 // ============================================
 // Prompts
 // ============================================

package/convex/sandbox/nodeActions/codeExec.ts

@@ -1,11 +1,14 @@
 "use node";
 
 /**
- * Code Execution Action
+ * Code Execution Action (with Streaming Support)
  *
  * Executes TypeScript code generated by the LLM.
  * This is the core of the code execution model - instead of JSON tool calls,
  * the agent writes code that imports from ksa/ and we execute it.
+ *
+ * Streaming: Output chunks are forwarded to cloud via fire-and-forget
+ * for real-time visibility during long-running executions.
  */
 
 import { internalAction } from "../_generated/server";
@@ -15,11 +18,14 @@ import { v } from "convex/values";
 const MAX_TIMEOUT_MS = 120_000; // 2 minutes
 const MAX_OUTPUT_LENGTH = 50_000; // 50KB
 
+// Chunk size for streaming (balance between latency and overhead)
+const STREAM_CHUNK_INTERVAL_MS = 100; // Batch output every 100ms
+
 /**
- * Execute TypeScript code in the sandbox.
+ * Execute TypeScript code in the sandbox with streaming output.
  *
  * The code can import from /home/user/ksa/ to use available capabilities.
- * Output is captured from stdout/stderr.
+ * Output is captured and optionally streamed to the cloud.
  */
 export const execute = internalAction({
   args: {
@@ -33,20 +39,25 @@ export const execute = internalAction({
       CARD_ID: v.optional(v.string()),
       THREAD_ID: v.optional(v.string()),
     })),
+    // Streaming configuration
+    streamConfig: v.optional(v.object({
+      sessionId: v.optional(v.string()),
+      stepId: v.optional(v.string()),
+      cloudUrl: v.optional(v.string()),
+    })),
   },
   handler: async (_ctx, args): Promise<{
     success: boolean;
     output: string;
     error?: string;
     exitCode: number;
+    streamedChunks?: number; // Track how many chunks were streamed
   }> => {
     // Dynamic imports for Node.js modules (required for Convex bundling)
-    const { exec } = await import("child_process");
-    const { promisify } = await import("util");
+    const { spawn } = await import("child_process");
     const fs = await import("fs/promises");
     const crypto = await import("crypto");
 
-    const execAsync = promisify(exec);
     const timeout = Math.min(args.timeoutMs || 60_000, MAX_TIMEOUT_MS);
 
     // Generate unique filename for this execution
@@ -54,32 +65,140 @@ export const execute = internalAction({
     const hash = crypto.createHash("md5").update(args.code).digest("hex").slice(0, 8);
     const filename = `/home/user/agent_exec_${Date.now()}_${hash}.ts`;
 
+    // Streaming state
+    let streamBuffer = "";
+    let streamedChunks = 0;
+    let streamInterval: NodeJS.Timeout | null = null;
+
+    // Fire-and-forget stream chunk to cloud
+    const flushStreamBuffer = () => {
+      if (!streamBuffer || !args.streamConfig?.cloudUrl) return;
+
+      const chunk = streamBuffer;
+      streamBuffer = "";
+      streamedChunks++;
+
+      // Non-blocking POST to cloud
+      fetch(`${args.streamConfig.cloudUrl}/agent/stream`, {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+          Authorization: `Bearer ${args.env?.SANDBOX_JWT || ""}`,
+        },
+        body: JSON.stringify({
+          sessionId: args.streamConfig.sessionId,
+          stepId: args.streamConfig.stepId,
+          chunk,
+          chunkIndex: streamedChunks,
+          timestamp: Date.now(),
+        }),
+      }).catch(() => {
+        // Fire-and-forget - ignore errors
+      });
+    };
+
     try {
       // Write code to temp file
       await fs.writeFile(filename, args.code, "utf-8");
 
-      // Execute with bun (use full path since PATH may not be set in Convex process)
+      // Execute with bun using spawn (for streaming)
       const bunPath = "/home/user/.bun/bin/bun";
-      const { stdout, stderr } = await execAsync(`${bunPath} run ${filename}`, {
-        timeout,
+      const proc = spawn(bunPath, ["run", filename], {
         cwd: "/home/user/workspace",
         env: {
           ...process.env,
-          // Ensure PATH includes bun
           PATH: "/home/user/.bun/bin:/usr/local/bin:/usr/bin:/bin",
           HOME: "/home/user",
-          // Make KSAs available via import path
           NODE_PATH: "/home/user",
-          // Gateway config for KSAs to call cloud services
           ...(args.env?.CONVEX_URL && { CONVEX_URL: args.env.CONVEX_URL }),
           ...(args.env?.GATEWAY_URL && { GATEWAY_URL: args.env.GATEWAY_URL }),
           ...(args.env?.SANDBOX_JWT && { SANDBOX_JWT: args.env.SANDBOX_JWT }),
           ...(args.env?.CARD_ID && { CARD_ID: args.env.CARD_ID }),
           ...(args.env?.THREAD_ID && { THREAD_ID: args.env.THREAD_ID }),
         },
-        maxBuffer: MAX_OUTPUT_LENGTH * 2,
       });
 
+      // Collect output
+      let stdout = "";
+      let stderr = "";
+
+      // Start streaming interval if configured
+      if (args.streamConfig?.cloudUrl) {
+        streamInterval = setInterval(flushStreamBuffer, STREAM_CHUNK_INTERVAL_MS);
+      }
+
+      // Handle stdout
+      proc.stdout?.on("data", (data: Buffer) => {
+        const text = data.toString();
+        stdout += text;
+
+        // Add to stream buffer if streaming enabled
+        if (args.streamConfig?.cloudUrl) {
+          streamBuffer += text;
+        }
+      });
+
+      // Handle stderr
+      proc.stderr?.on("data", (data: Buffer) => {
+        const text = data.toString();
+        stderr += text;
+
+        // Add to stream buffer if streaming enabled
+        if (args.streamConfig?.cloudUrl) {
+          streamBuffer += `[stderr] ${text}`;
+        }
+      });
+
+      // Wait for process to complete with timeout
+      const result = await new Promise<{ success: boolean; exitCode: number; error?: string }>((resolve) => {
+        let resolved = false;
+
+        // Timeout handler
+        const timeoutId = setTimeout(() => {
+          if (!resolved) {
+            resolved = true;
+            proc.kill("SIGTERM");
+            resolve({
+              success: false,
+              exitCode: 124,
+              error: `Execution timed out after ${timeout}ms`,
+            });
+          }
+        }, timeout);
+
+        // Normal completion
+        proc.on("close", (code) => {
+          if (!resolved) {
+            resolved = true;
+            clearTimeout(timeoutId);
+            resolve({
+              success: code === 0,
+              exitCode: code ?? 1,
+              error: code !== 0 ? `Process exited with code ${code}` : undefined,
+            });
+          }
+        });
+
+        // Error handler
+        proc.on("error", (err) => {
+          if (!resolved) {
+            resolved = true;
+            clearTimeout(timeoutId);
+            resolve({
+              success: false,
+              exitCode: 1,
+              error: err.message,
+            });
+          }
+        });
+      });
+
+      // Stop streaming interval and flush remaining buffer
+      if (streamInterval) {
+        clearInterval(streamInterval);
+        flushStreamBuffer();
+      }
+
       // Combine and truncate output
       let output = stdout || "";
       if (stderr) {
@@ -90,34 +209,27 @@ export const execute = internalAction({
       }
 
       return {
-        success: true,
+        success: result.success,
         output: output.trim(),
-        exitCode: 0,
+        error: result.error,
+        exitCode: result.exitCode,
+        streamedChunks: args.streamConfig?.cloudUrl ? streamedChunks : undefined,
       };
     } catch (error: unknown) {
-      const err = error as { message?: string; stdout?: string; stderr?: string; code?: number };
-      const message = err.message || String(error);
-      let output = "";
-
-      // Capture any partial output
-      if (err.stdout) output += err.stdout;
-      if (err.stderr) output += `\n[stderr]\n${err.stderr}`;
-
-      // Check for timeout
-      if (message.includes("TIMEOUT") || message.includes("timed out")) {
-        return {
-          success: false,
-          output: output.trim(),
-          error: `Execution timed out after ${timeout}ms`,
-          exitCode: 124,
-        };
+      // Stop streaming on error
+      if (streamInterval) {
+        clearInterval(streamInterval);
       }
 
+      const err = error as { message?: string };
+      const message = err.message || String(error);
+
       return {
         success: false,
-        output: output.trim(),
+        output: "",
         error: message,
-        exitCode: err.code || 1,
+        exitCode: 1,
+        streamedChunks: args.streamConfig?.cloudUrl ? streamedChunks : undefined,
       };
     } finally {
       // Clean up temp file

package/convex/sandbox/schema.ts

@@ -476,7 +476,7 @@ export default defineSchema({
     .index("by_thread", ["threadId"]),
 
   // ============================================
-  // Sync Queue - Items to sync to cloud
+  // Sync Queue - Items to sync to cloud (legacy)
   // ============================================
 
   syncQueue: defineTable({
@@ -507,4 +507,135 @@ export default defineSchema({
   })
     .index("by_status", ["status"])
     .index("by_type", ["type"]),
+
+  // ============================================
+  // Sync Outbox - Transactional outbox for cloud sync
+  // ============================================
+  // Records all local writes that need to sync to cloud.
+  // Uses idempotency keys for at-least-once delivery without duplicates.
+
+  syncOutbox: defineTable({
+    // Idempotency key - UUID generated at write time, ensures no duplicates
+    idempotencyKey: v.string(),
+
+    // Entity reference
+    entityType: v.string(),  // "brands.products", "brands.assets", "artifacts", etc.
+    entityId: v.string(),    // Local ID of the entity
+
+    // Operation
+    operation: v.union(
+      v.literal("create"),
+      v.literal("update"),
+      v.literal("delete"),
+      v.literal("upsert")
+    ),
+
+    // Payload to send to cloud
+    payload: v.any(),
+
+    // Cloud target (gateway path)
+    cloudPath: v.string(),  // e.g., "features.brands.core.products.saveProducts"
+
+    // Status tracking
+    status: v.union(
+      v.literal("pending"),
+      v.literal("syncing"),
+      v.literal("synced"),
+      v.literal("failed"),
+      v.literal("skipped")
+    ),
+
+    // Retry tracking
+    attempts: v.number(),
+    maxAttempts: v.number(),
+    lastAttemptAt: v.optional(v.number()),
+    lastError: v.optional(v.string()),
+
+    // Result tracking
+    cloudId: v.optional(v.string()),  // ID returned from cloud
+    syncedAt: v.optional(v.number()),
+
+    // Metadata
+    priority: v.number(),  // 0=highest, 10=lowest
+    createdAt: v.number(),
+    sessionId: v.optional(v.string()),
+    threadId: v.optional(v.string()),
+  })
+    .index("by_status", ["status"])
+    .index("by_idempotency", ["idempotencyKey"])
+    .index("by_entity", ["entityType", "entityId"])
+    .index("by_session", ["sessionId", "status"])
+    .index("by_priority", ["status", "priority", "createdAt"]),
+
+  // ============================================
+  // Marketing Copy - Extracted marketing content
+  // ============================================
+
+  discoveredMarketingCopy: defineTable({
+    domain: v.string(),
+    sourceUrl: v.string(),
+    copyType: v.union(
+      v.literal("headline"),
+      v.literal("tagline"),
+      v.literal("value_prop"),
+      v.literal("cta"),
+      v.literal("testimonial"),
+      v.literal("feature_description"),
+      v.literal("other")
+    ),
+    content: v.string(),
+    context: v.optional(v.string()),
+    extractedAt: v.number(),
+    threadId: v.optional(v.string()),
+    syncedToCloud: v.boolean(),
+  })
+    .index("by_domain", ["domain"])
+    .index("by_type", ["copyType"])
+    .index("by_synced", ["syncedToCloud"]),
+
+  // ============================================
+  // Services - Extracted service offerings
+  // ============================================
+
+  discoveredServices: defineTable({
+    domain: v.string(),
+    sourceUrl: v.string(),
+    name: v.string(),
+    serviceType: v.union(
+      v.literal("consulting"),
+      v.literal("implementation"),
+      v.literal("support"),
+      v.literal("training"),
+      v.literal("managed_service"),
+      v.literal("professional_service"),
+      v.literal("other")
+    ),
+    description: v.optional(v.string()),
+    pricing: v.optional(v.string()),
+    extractedAt: v.number(),
+    threadId: v.optional(v.string()),
+    syncedToCloud: v.boolean(),
+  })
+    .index("by_domain", ["domain"])
+    .index("by_type", ["serviceType"])
+    .index("by_synced", ["syncedToCloud"]),
+
+  // ============================================
+  // Reviews - Extracted reviews/testimonials
+  // ============================================
+
+  discoveredReviews: defineTable({
+    domain: v.string(),
+    sourceUrl: v.string(),
+    reviewerName: v.optional(v.string()),
+    rating: v.optional(v.number()),
+    content: v.string(),
+    date: v.optional(v.string()),
+    platform: v.optional(v.string()),
+    extractedAt: v.number(),
+    threadId: v.optional(v.string()),
+    syncedToCloud: v.boolean(),
+  })
+    .index("by_domain", ["domain"])
+    .index("by_synced", ["syncedToCloud"]),
 });

package/convex/sandbox/sync/index.ts

@@ -0,0 +1,237 @@
+/**
+ * Sync Outbox - Local to Cloud Sync
+ *
+ * Transactional outbox pattern for syncing local sandbox data to cloud.
+ * All local writes are recorded atomically, then synced via gateway.
+ */
+
+import { v } from "convex/values";
+import { mutation, query } from "../_generated/server";
+
+// ============================================================================
+// Outbox Operations
+// ============================================================================
+
+/**
+ * Add an item to the sync outbox.
+ * Called atomically with the local write.
+ */
+export const addToOutbox = mutation({
+  args: {
+    idempotencyKey: v.string(),
+    entityType: v.string(),
+    entityId: v.string(),
+    operation: v.union(
+      v.literal("create"),
+      v.literal("update"),
+      v.literal("delete"),
+      v.literal("upsert")
+    ),
+    payload: v.any(),
+    cloudPath: v.string(),
+    priority: v.optional(v.number()),
+    sessionId: v.optional(v.string()),
+    threadId: v.optional(v.string()),
+  },
+  handler: async (ctx, args) => {
+    // Check for existing entry with same idempotency key
+    const existing = await ctx.db
+      .query("syncOutbox")
+      .withIndex("by_idempotency", q => q.eq("idempotencyKey", args.idempotencyKey))
+      .first();
+
+    if (existing) {
+      // Already in outbox, skip
+      return existing._id;
+    }
+
+    return await ctx.db.insert("syncOutbox", {
+      idempotencyKey: args.idempotencyKey,
+      entityType: args.entityType,
+      entityId: args.entityId,
+      operation: args.operation,
+      payload: args.payload,
+      cloudPath: args.cloudPath,
+      status: "pending",
+      attempts: 0,
+      maxAttempts: 3,
+      priority: args.priority ?? 5,
+      createdAt: Date.now(),
+      sessionId: args.sessionId,
+      threadId: args.threadId,
+    });
+  },
+});
+
+/**
+ * Get pending items from outbox, ordered by priority and creation time.
+ */
+export const getPendingItems = query({
+  args: {
+    limit: v.optional(v.number()),
+    sessionId: v.optional(v.string()),
+  },
+  handler: async (ctx, args) => {
+    const limit = args.limit ?? 50;
+
+    if (args.sessionId) {
+      return await ctx.db
+        .query("syncOutbox")
+        .withIndex("by_session", q => q.eq("sessionId", args.sessionId).eq("status", "pending"))
+        .take(limit);
+    }
+
+    return await ctx.db
+      .query("syncOutbox")
+      .withIndex("by_priority", q => q.eq("status", "pending"))
+      .take(limit);
+  },
+});
+
+/**
+ * Mark item as syncing (in progress).
+ */
+export const markSyncing = mutation({
+  args: {
+    id: v.id("syncOutbox"),
+  },
+  handler: async (ctx, args) => {
+    const item = await ctx.db.get(args.id);
+    if (!item) return null;
+
+    await ctx.db.patch(args.id, {
+      status: "syncing",
+      lastAttemptAt: Date.now(),
+      attempts: item.attempts + 1,
+    });
+
+    return item;
+  },
+});
+
+/**
+ * Mark item as synced (success).
+ */
+export const markSynced = mutation({
+  args: {
+    id: v.id("syncOutbox"),
+    cloudId: v.optional(v.string()),
+  },
+  handler: async (ctx, args) => {
+    await ctx.db.patch(args.id, {
+      status: "synced",
+      syncedAt: Date.now(),
+      cloudId: args.cloudId,
+    });
+  },
+});
+
+/**
+ * Mark item as failed.
+ */
+export const markFailed = mutation({
+  args: {
+    id: v.id("syncOutbox"),
+    error: v.string(),
+  },
+  handler: async (ctx, args) => {
+    const item = await ctx.db.get(args.id);
+    if (!item) return;
+
+    const newStatus = item.attempts >= item.maxAttempts ? "failed" : "pending";
+
+    await ctx.db.patch(args.id, {
+      status: newStatus,
+      lastError: args.error,
+      lastAttemptAt: Date.now(),
+    });
+  },
+});
+
+/**
+ * Get outbox statistics.
+ */
+export const getStats = query({
+  args: {
+    sessionId: v.optional(v.string()),
+  },
+  handler: async (ctx, args) => {
+    let items;
+    if (args.sessionId) {
+      items = await ctx.db
+        .query("syncOutbox")
+        .withIndex("by_session", q => q.eq("sessionId", args.sessionId))
+        .collect();
+    } else {
+      items = await ctx.db.query("syncOutbox").collect();
+    }
+
+    return {
+      total: items.length,
+      pending: items.filter(i => i.status === "pending").length,
+      syncing: items.filter(i => i.status === "syncing").length,
+      synced: items.filter(i => i.status === "synced").length,
+      failed: items.filter(i => i.status === "failed").length,
+      byType: items.reduce((acc, i) => {
+        acc[i.entityType] = (acc[i.entityType] || 0) + 1;
+        return acc;
+      }, {} as Record<string, number>),
+    };
+  },
+});
+
+/**
+ * Clear synced items (cleanup).
+ */
+export const clearSynced = mutation({
+  args: {
+    olderThanMs: v.optional(v.number()),
+  },
+  handler: async (ctx, args) => {
+    const cutoff = Date.now() - (args.olderThanMs ?? 24 * 60 * 60 * 1000); // 24h default
+
+    const synced = await ctx.db
+      .query("syncOutbox")
+      .withIndex("by_status", q => q.eq("status", "synced"))
+      .collect();
+
+    let deleted = 0;
+    for (const item of synced) {
+      if (item.syncedAt && item.syncedAt < cutoff) {
+        await ctx.db.delete(item._id);
+        deleted++;
+      }
+    }
+
+    return { deleted };
+  },
+});
+
+/**
+ * Retry failed items.
+ */
+export const retryFailed = mutation({
+  args: {
+    entityType: v.optional(v.string()),
+  },
+  handler: async (ctx, args) => {
+    const failed = await ctx.db
+      .query("syncOutbox")
+      .withIndex("by_status", q => q.eq("status", "failed"))
+      .collect();
+
+    let retried = 0;
+    for (const item of failed) {
+      if (!args.entityType || item.entityType === args.entityType) {
+        await ctx.db.patch(item._id, {
+          status: "pending",
+          attempts: 0,
+          lastError: undefined,
+        });
+        retried++;
+      }
+    }
+
+    return { retried };
+  },
+});

package/ksa/_shared/trace.ts

@@ -0,0 +1,240 @@
+/**
+ * KSA Tracing Module
+ *
+ * Provides tracing utilities for KSA gateway calls.
+ * Enables per-step call tracing for observability into agent execution.
+ *
+ * Usage:
+ * 1. Agent loop calls setStepContext() at start of each step
+ * 2. KSAs use tracedGatewayCall() instead of callGateway()
+ * 3. Agent loop calls flushTraces() after step completion
+ */
+
+import { callGateway, fireAndForget } from "./gateway";
+
+// =============================================================================
+// Types
+// =============================================================================
+
+export interface TraceEntry {
+  traceId: string;
+  stepId: string | null;
+  event: "ksa_call_start" | "ksa_call_end";
+  path: string;
+  timestamp: number;
+  durationMs?: number;
+  success?: boolean;
+  error?: string;
+}
+
+export interface StepContext {
+  stepId: string;
+  sessionId?: string;
+  stepNumber?: number;
+}
+
+// =============================================================================
+// Tracing State (per-sandbox)
+// =============================================================================
+
+let currentStepContext: StepContext | null = null;
+let traceBuffer: TraceEntry[] = [];
+
+/**
+ * Generate a unique trace ID for KSA calls.
+ * Format: ksa-{timestamp}-{random} for easy identification.
+ */
+function generateTraceId(): string {
+  const timestamp = Date.now().toString(36);
+  const random = Math.random().toString(36).substring(2, 6);
+  return `ksa-${timestamp}-${random}`;
+}
+
+// =============================================================================
+// Context Management
+// =============================================================================
+
+/**
+ * Set the current step context for tracing.
+ * Call this at the start of each agent step.
+ *
+ * @param context - Step context with stepId and optional metadata
+ *
+ * @example
+ * setStepContext({ stepId: 'step-001', sessionId: 'sess-123', stepNumber: 1 });
+ */
+export function setStepContext(context: StepContext): void {
+  currentStepContext = context;
+}
+
+/**
+ * Clear the current step context.
+ * Call this after step completion.
+ */
+export function clearStepContext(): void {
+  currentStepContext = null;
+}
+
+/**
+ * Get the current step context (for debugging).
+ */
+export function getStepContext(): StepContext | null {
+  return currentStepContext;
+}
+
+// =============================================================================
+// Traced Gateway Calls
+// =============================================================================
+
+/**
+ * Make a traced gateway call.
+ * Records start/end events with timing for observability.
+ *
+ * @param path - Service path (e.g., 'services.Valyu.internal.search')
+ * @param args - Arguments to pass to the service
+ * @param type - Operation type (query, mutation, action)
+ * @returns Service response data
+ *
+ * @example
+ * const data = await tracedGatewayCall('features.brands.core.crud.get', { id: 'abc123' });
+ */
+export async function tracedGatewayCall<T = unknown>(
+  path: string,
+  args: Record<string, unknown>,
+  type?: "query" | "mutation" | "action"
+): Promise<T> {
+  const traceId = generateTraceId();
+  const startTime = Date.now();
+  const stepId = currentStepContext?.stepId ?? null;
+
+  // Record start event
+  traceBuffer.push({
+    traceId,
+    stepId,
+    event: "ksa_call_start",
+    path,
+    timestamp: startTime,
+  });
+
+  try {
+    const result = await callGateway<T>(path, args, type);
+
+    // Record success event
+    traceBuffer.push({
+      traceId,
+      stepId,
+      event: "ksa_call_end",
+      path,
+      timestamp: Date.now(),
+      durationMs: Date.now() - startTime,
+      success: true,
+    });
+
+    return result;
+  } catch (error: unknown) {
+    const err = error as { message?: string };
+
+    // Record failure event
+    traceBuffer.push({
+      traceId,
+      stepId,
+      event: "ksa_call_end",
+      path,
+      timestamp: Date.now(),
+      durationMs: Date.now() - startTime,
+      success: false,
+      error: err.message || String(error),
+    });
+
+    throw error;
+  }
+}
+
+// =============================================================================
+// Trace Buffer Management
+// =============================================================================
+
+/**
+ * Get all traces in the buffer without clearing.
+ * Useful for debugging.
+ */
+export function getTraces(): TraceEntry[] {
+  return [...traceBuffer];
+}
+
+/**
+ * Flush traces from the buffer and return them.
+ * Call this after each step to collect traces.
+ *
+ * @returns Array of trace entries
+ *
+ * @example
+ * const traces = flushTraces();
+ * console.log(`Step had ${traces.length} KSA calls`);
+ */
+export function flushTraces(): TraceEntry[] {
+  const traces = [...traceBuffer];
+  traceBuffer = [];
+  return traces;
+}
+
+/**
+ * Flush traces to the cloud via fire-and-forget.
+ * Use this for real-time observability without blocking.
+ *
+ * @param cloudPath - Path to send traces (default: 'components.lakitu.workflows.sandboxConvex.appendTraces')
+ *
+ * @example
+ * flushTracesToCloud();
+ */
+export function flushTracesToCloud(
+  cloudPath = "components.lakitu.workflows.sandboxConvex.appendTraces"
+): void {
+  const traces = flushTraces();
+  if (traces.length === 0) return;
+
+  const sessionId = currentStepContext?.sessionId;
+  if (!sessionId) {
+    console.warn("[trace] Cannot flush traces: no sessionId in context");
+    return;
+  }
+
+  fireAndForget(cloudPath, {
+    sessionId,
+    traces,
+  });
+}
+
+/**
+ * Get trace statistics for the current buffer.
+ * Useful for debugging and observability.
+ */
+export function getTraceStats(): {
+  totalCalls: number;
+  successfulCalls: number;
+  failedCalls: number;
+  avgDurationMs: number;
+  callsByPath: Record<string, number>;
+} {
+  const endTraces = traceBuffer.filter((t) => t.event === "ksa_call_end");
+
+  const successfulCalls = endTraces.filter((t) => t.success).length;
+  const failedCalls = endTraces.filter((t) => !t.success).length;
+
+  const durations = endTraces.filter((t) => t.durationMs !== undefined).map((t) => t.durationMs!);
+  const avgDurationMs =
+    durations.length > 0 ? Math.round(durations.reduce((a, b) => a + b, 0) / durations.length) : 0;
+
+  const callsByPath: Record<string, number> = {};
+  for (const trace of endTraces) {
+    callsByPath[trace.path] = (callsByPath[trace.path] || 0) + 1;
+  }
+
+  return {
+    totalCalls: endTraces.length,
+    successfulCalls,
+    failedCalls,
+    avgDurationMs,
+    callsByPath,
+  };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lakitu/sdk",
-  "version": "0.1.63",
+  "version": "0.1.65",
   "description": "Self-hosted AI agent framework for Convex + E2B with code execution",
   "type": "module",
   "main": "./dist/sdk/index.js",
@@ -71,6 +71,10 @@
     "./ksa/browser": {
       "bun": "./ksa/browser.ts",
       "default": "./ksa/browser.ts"
+    },
+    "./ksa/trace": {
+      "bun": "./ksa/_shared/trace.ts",
+      "default": "./ksa/_shared/trace.ts"
     }
   },
   "files": [

package/template/build.ts

@@ -280,7 +280,7 @@ const customTemplate = (baseId: string, buildDir: string, config: TemplateConfig
   if (aptPackages.length > 0 || pipPackages.length > 0 || npmPackages.length > 0) {
     console.log("Installing custom packages from template config...");
     const installCmds: string[] = [];
-    
+
     if (aptPackages.length > 0) {
       console.log(`  APT: ${aptPackages.join(", ")}`);
       installCmds.push(generateAptInstall(aptPackages));
@@ -396,20 +396,27 @@ async function buildCustom(baseId = "lakitu-base") {
     ${LAKITU_DIR}/ ${BUILD_DIR}/lakitu/`.quiet();
   await $`cp ${import.meta.dir}/e2b/start.sh ${BUILD_DIR}/`;
 
-  // Copy project-specific KSAs from convex/lakitu/ksa/ (new location) or lakitu/ (legacy)
-  const NEW_KSA_DIR = `${LAKITU_DIR}/../../convex/lakitu/ksa`;
-  const LEGACY_KSA_DIR = `${LAKITU_DIR}/../../lakitu`;
+  // Copy project-specific KSAs
+  // Supported paths (checked in order):
+  //   1. convex/lakitu/ksa/ - Alternative location (colocated with convex backend)
+  //   2. lakitu/           - Standard location for project.social (27 KSA modules)
+  //
+  // Note: project.social uses lakitu/ at the project root as its standard KSA location.
+  // The convex/lakitu/ksa/ path is supported for projects that prefer colocating KSAs
+  // with their Convex backend code.
+  const ALT_KSA_DIR = `${LAKITU_DIR}/../../convex/lakitu/ksa`;
+  const STANDARD_KSA_DIR = `${LAKITU_DIR}/../../lakitu`;
   await $`mkdir -p ${BUILD_DIR}/project-ksa`.quiet();
-  
-  // Try new location first, fall back to legacy
+
+  // Try alternative location first (convex/lakitu/ksa/), fall back to standard (lakitu/)
   try {
-    await $`test -d ${NEW_KSA_DIR}`.quiet();
-    await $`cp -r ${NEW_KSA_DIR}/* ${BUILD_DIR}/project-ksa/`.quiet();
+    await $`test -d ${ALT_KSA_DIR}`.quiet();
+    await $`cp -r ${ALT_KSA_DIR}/* ${BUILD_DIR}/project-ksa/`.quiet();
     console.log("Copied project KSAs from convex/lakitu/ksa/");
   } catch {
     try {
-      await $`cp -r ${LEGACY_KSA_DIR}/* ${BUILD_DIR}/project-ksa/`.quiet();
-      console.log("Copied project KSAs from lakitu/ (legacy location)");
+      await $`cp -r ${STANDARD_KSA_DIR}/* ${BUILD_DIR}/project-ksa/`.quiet();
+      console.log("Copied project KSAs from lakitu/");
     } catch {
       console.log("No project KSAs found");
     }