@poncho-ai/harness 0.35.0 → 0.36.1

package/src/storage/sqlite-engine.ts

@@ -0,0 +1,99 @@
+// ---------------------------------------------------------------------------
+// SqliteEngine – better-sqlite3 backed storage engine.
+// ---------------------------------------------------------------------------
+
+import { mkdirSync } from "node:fs";
+import { dirname, resolve } from "node:path";
+import type { QueryExecutor, QueryRow } from "./sql-dialect.js";
+import { SqlStorageEngine, sqliteDialect } from "./sql-dialect.js";
+
+type Database = import("better-sqlite3").Database;
+
+const isServerless = (): boolean =>
+  !!(
+    process.env.VERCEL ||
+    process.env.AWS_LAMBDA_FUNCTION_NAME ||
+    process.env.NETLIFY ||
+    process.env.RENDER
+  );
+
+export class SqliteEngine extends SqlStorageEngine {
+  private db!: Database;
+  private readonly dbPath: string;
+  protected readonly executor: QueryExecutor;
+
+  constructor(options: { workingDir: string; agentId: string; dbPath?: string }) {
+    super(sqliteDialect, options.agentId);
+
+    this.dbPath =
+      options.dbPath ??
+      (isServerless()
+        ? resolve("/tmp/.poncho/poncho.db")
+        : resolve(options.workingDir, ".poncho", "poncho.db"));
+
+    this.executor = {
+      run: async (sql: string, params?: unknown[]): Promise<void> => {
+        this.db.prepare(sql).run(...(params ?? []));
+      },
+      get: async <T extends QueryRow = QueryRow>(
+        sql: string,
+        params?: unknown[],
+      ): Promise<T | undefined> => {
+        return this.db.prepare(sql).get(...(params ?? [])) as T | undefined;
+      },
+      all: async <T extends QueryRow = QueryRow>(
+        sql: string,
+        params?: unknown[],
+      ): Promise<T[]> => {
+        return this.db.prepare(sql).all(...(params ?? [])) as T[];
+      },
+      exec: async (sql: string): Promise<void> => {
+        this.db.exec(sql);
+      },
+      transaction: async (fn: () => Promise<void>): Promise<void> => {
+        this.db.exec("BEGIN");
+        try {
+          await fn();
+          this.db.exec("COMMIT");
+        } catch (err) {
+          this.db.exec("ROLLBACK");
+          throw err;
+        }
+      },
+    };
+  }
+
+  protected override async onBeforeInit(): Promise<void> {
+    mkdirSync(dirname(this.dbPath), { recursive: true });
+
+    const BetterSqlite3 = (await import("better-sqlite3")).default;
+    this.db = new BetterSqlite3(this.dbPath);
+
+    this.db.pragma("journal_mode = WAL");
+    this.db.pragma("busy_timeout = 5000");
+
+    if (isServerless()) {
+      console.warn(
+        "[poncho] SQLite storage detected in serverless environment. " +
+          "Data will NOT persist between invocations. " +
+          "Configure `storage.provider: 'postgresql'` for persistent storage.",
+      );
+    }
+  }
+
+  override async initialize(): Promise<void> {
+    await super.initialize();
+    // Patch listAllPaths to use synchronous better-sqlite3 queries
+    this.vfs.listAllPaths = (tenantId: string): string[] => {
+      if (!this.db) return [];
+      const rows = this.db
+        .prepare("SELECT path FROM vfs_entries WHERE agent_id = ? AND tenant_id = ?")
+        .all(this.agentId, tenantId) as Array<{ path: string }>;
+      return rows.map((r) => r.path);
+    };
+  }
+
+  async close(): Promise<void> {
+    this.db?.close();
+  }
+}

package/src/storage/store-adapters.ts

@@ -0,0 +1,100 @@
+// ---------------------------------------------------------------------------
+// Thin adapters that wrap StorageEngine namespaces into the existing
+// ConversationStore / MemoryStore / TodoStore / ReminderStore interfaces.
+// This keeps backward compatibility while routing through the new engine.
+// ---------------------------------------------------------------------------
+
+import type {
+  Conversation,
+  ConversationStore,
+  ConversationSummary,
+  PendingSubagentResult,
+} from "../state.js";
+import type { MainMemory, MemoryStore } from "../memory.js";
+import type { TodoItem, TodoStore } from "../todo-tools.js";
+import type { Reminder, ReminderStore } from "../reminder-store.js";
+import type { StorageEngine } from "./engine.js";
+
+// ---------------------------------------------------------------------------
+// ConversationStore adapter
+// ---------------------------------------------------------------------------
+
+export function createConversationStoreFromEngine(
+  engine: StorageEngine,
+): ConversationStore {
+  return {
+    list: (ownerId?: string, tenantId?: string | null) =>
+      engine.conversations.list(ownerId, tenantId).then((summaries) => {
+        // list() returns full Conversation[] in the old interface.
+        // For backward compat, fetch full conversations.
+        return Promise.all(
+          summaries.map((s) => engine.conversations.get(s.conversationId)),
+        ).then((convs) => convs.filter(Boolean) as Conversation[]);
+      }),
+    listSummaries: (ownerId?: string, tenantId?: string | null) =>
+      engine.conversations.list(ownerId, tenantId),
+    get: (conversationId: string) =>
+      engine.conversations.get(conversationId),
+    create: (ownerId?: string, title?: string, tenantId?: string | null) =>
+      engine.conversations.create(ownerId, title, tenantId),
+    update: (conversation: Conversation) =>
+      engine.conversations.update(conversation),
+    rename: (conversationId: string, title: string) =>
+      engine.conversations.rename(conversationId, title),
+    delete: (conversationId: string) =>
+      engine.conversations.delete(conversationId),
+    appendSubagentResult: (conversationId: string, result: PendingSubagentResult) =>
+      engine.conversations.appendSubagentResult(conversationId, result),
+    clearCallbackLock: (conversationId: string) =>
+      engine.conversations.clearCallbackLock(conversationId),
+  };
+}
+
+// ---------------------------------------------------------------------------
+// MemoryStore adapter
+// ---------------------------------------------------------------------------
+
+export function createMemoryStoreFromEngine(
+  engine: StorageEngine,
+  tenantId?: string | null,
+): MemoryStore {
+  return {
+    getMainMemory: () => engine.memory.get(tenantId),
+    updateMainMemory: (input: { content: string }) =>
+      engine.memory.update(input.content, tenantId),
+  };
+}
+
+// ---------------------------------------------------------------------------
+// TodoStore adapter
+// ---------------------------------------------------------------------------
+
+export function createTodoStoreFromEngine(engine: StorageEngine): TodoStore {
+  return {
+    get: (conversationId: string) => engine.todos.get(conversationId),
+    set: (conversationId: string, todos: TodoItem[]) =>
+      engine.todos.set(conversationId, todos),
+  };
+}
+
+// ---------------------------------------------------------------------------
+// ReminderStore adapter
+// ---------------------------------------------------------------------------
+
+export function createReminderStoreFromEngine(
+  engine: StorageEngine,
+): ReminderStore {
+  return {
+    list: () => engine.reminders.list(),
+    create: (input: {
+      task: string;
+      scheduledAt: number;
+      timezone?: string;
+      conversationId: string;
+      ownerId?: string;
+      tenantId?: string | null;
+    }) => engine.reminders.create(input),
+    cancel: (id: string) => engine.reminders.cancel(id),
+    delete: (id: string) => engine.reminders.delete(id),
+  };
+}

package/src/todo-tools.ts

@@ -1,14 +1,4 @@
-import { mkdir, readFile, rename, writeFile } from "node:fs/promises";
-import { dirname, resolve } from "node:path";
 import { defineTool, type ToolDefinition } from "@poncho-ai/sdk";
-import type { StateConfig } from "./state.js";
-import {
-  ensureAgentIdentity,
-  getAgentStoreDirectory,
-  slugifyStorageComponent,
-  STORAGE_SCHEMA_VERSION,
-} from "./agent-identity.js";
-import { createRawKVStore, type RawKVStore } from "./kv-store.js";
 
 // ---------------------------------------------------------------------------
 // Data model
@@ -37,25 +27,6 @@ export interface TodoStore {
 
 const VALID_STATUSES: TodoStatus[] = ["pending", "in_progress", "completed"];
 const VALID_PRIORITIES: TodoPriority[] = ["high", "medium", "low"];
-const TODOS_DIRECTORY = "todos";
-
-const writeJsonAtomic = async (filePath: string, payload: unknown): Promise<void> => {
-  await mkdir(dirname(filePath), { recursive: true });
-  const tmpPath = `${filePath}.tmp`;
-  await writeFile(tmpPath, JSON.stringify(payload, null, 2), "utf8");
-  await rename(tmpPath, filePath);
-};
-
-const parseTodoList = (raw: unknown): TodoItem[] => {
-  if (!Array.isArray(raw)) return [];
-  return raw.filter(
-    (item): item is TodoItem =>
-      typeof item === "object" &&
-      item !== null &&
-      typeof (item as Record<string, unknown>).id === "string" &&
-      typeof (item as Record<string, unknown>).content === "string",
-  );
-};
 
 const generateId = (): string =>
   (globalThis.crypto?.randomUUID?.() ?? `${Date.now()}-${Math.random()}`).slice(0, 8);
@@ -76,117 +47,11 @@ class InMemoryTodoStore implements TodoStore {
   }
 }
 
-// ---------------------------------------------------------------------------
-// FileTodoStore — one JSON file per conversation
-// ---------------------------------------------------------------------------
-
-class FileTodoStore implements TodoStore {
-  private readonly workingDir: string;
-  private todosDir = "";
-
-  constructor(workingDir: string) {
-    this.workingDir = workingDir;
-  }
-
-  private async ensureTodosDir(): Promise<string> {
-    if (this.todosDir) return this.todosDir;
-    const identity = await ensureAgentIdentity(this.workingDir);
-    this.todosDir = resolve(getAgentStoreDirectory(identity), TODOS_DIRECTORY);
-    await mkdir(this.todosDir, { recursive: true });
-    return this.todosDir;
-  }
-
-  private async filePath(conversationId: string): Promise<string> {
-    const dir = await this.ensureTodosDir();
-    return resolve(dir, `${slugifyStorageComponent(conversationId)}.json`);
-  }
-
-  async get(conversationId: string): Promise<TodoItem[]> {
-    try {
-      const fp = await this.filePath(conversationId);
-      const raw = await readFile(fp, "utf8");
-      return parseTodoList(JSON.parse(raw));
-    } catch {
-      return [];
-    }
-  }
-
-  async set(conversationId: string, todos: TodoItem[]): Promise<void> {
-    const fp = await this.filePath(conversationId);
-    await writeJsonAtomic(fp, todos);
-  }
-}
-
-// ---------------------------------------------------------------------------
-// KVBackedTodoStore — wraps any RawKVStore (Upstash, Redis, DynamoDB)
-// ---------------------------------------------------------------------------
-
-class KVBackedTodoStore implements TodoStore {
-  private readonly kv: RawKVStore;
-  private readonly baseKey: string;
-  private readonly ttl?: number;
-  private readonly memoryFallback = new InMemoryTodoStore();
-
-  constructor(kv: RawKVStore, baseKey: string, ttl?: number) {
-    this.kv = kv;
-    this.baseKey = baseKey;
-    this.ttl = ttl;
-  }
-
-  private keyFor(conversationId: string): string {
-    return `${this.baseKey}:${slugifyStorageComponent(conversationId)}`;
-  }
-
-  async get(conversationId: string): Promise<TodoItem[]> {
-    try {
-      const raw = await this.kv.get(this.keyFor(conversationId));
-      if (!raw) return [];
-      return parseTodoList(JSON.parse(raw));
-    } catch {
-      return this.memoryFallback.get(conversationId);
-    }
-  }
-
-  async set(conversationId: string, todos: TodoItem[]): Promise<void> {
-    try {
-      const serialized = JSON.stringify(todos);
-      const key = this.keyFor(conversationId);
-      if (typeof this.ttl === "number") {
-        await this.kv.setWithTtl(key, serialized, Math.max(1, this.ttl));
-      } else {
-        await this.kv.set(key, serialized);
-      }
-    } catch {
-      await this.memoryFallback.set(conversationId, todos);
-    }
-  }
-}
-
 // ---------------------------------------------------------------------------
 // Factory
 // ---------------------------------------------------------------------------
 
-export const createTodoStore = (
-  agentId: string,
-  config?: StateConfig,
-  options?: { workingDir?: string },
-): TodoStore => {
-  const provider = config?.provider ?? "local";
-  const ttl = config?.ttl;
-  const workingDir = options?.workingDir ?? process.cwd();
-
-  if (provider === "local") {
-    return new FileTodoStore(workingDir);
-  }
-  if (provider === "memory") {
-    return new InMemoryTodoStore();
-  }
-
-  const kv = createRawKVStore(config);
-  if (kv) {
-    const baseKey = `poncho:${STORAGE_SCHEMA_VERSION}:${slugifyStorageComponent(agentId)}:todos`;
-    return new KVBackedTodoStore(kv, baseKey, ttl);
-  }
+export const createTodoStore = (): TodoStore => {
   return new InMemoryTodoStore();
 };
 

package/src/upload-store.ts

@@ -26,6 +26,7 @@ const tryImport = async (mod: string, workingDir?: string): Promise<any> => {
 };
 
 export const PONCHO_UPLOAD_SCHEME = "poncho-upload://";
+export const VFS_SCHEME = "vfs://";
 
 export interface UploadStore {
   put(key: string, data: Buffer, mediaType: string): Promise<string>;

package/src/vfs/bash-manager.ts

@@ -0,0 +1,120 @@
+// ---------------------------------------------------------------------------
+// BashEnvironmentManager – manages per-tenant bash instances.
+// ---------------------------------------------------------------------------
+
+import { Bash } from "just-bash";
+import type {
+  BashOptions,
+  CommandName,
+  NetworkConfig as JustBashNetworkConfig,
+} from "just-bash";
+import type { StorageEngine } from "../storage/engine.js";
+import type { BashConfig, NetworkConfig } from "../config.js";
+import { PonchoFsAdapter } from "./poncho-fs-adapter.js";
+import { createBashFs } from "./create-bash-fs.js";
+import type { PostgresEngine } from "../storage/postgres-engine.js";
+
+/** Convert poncho NetworkConfig → just-bash NetworkConfig. */
+function toJustBashNetwork(cfg: NetworkConfig): JustBashNetworkConfig {
+  return {
+    allowedUrlPrefixes: cfg.allowedUrls,
+    allowedMethods: cfg.allowedMethods,
+    dangerouslyAllowFullInternetAccess: cfg.dangerouslyAllowAll,
+    maxRedirects: cfg.maxRedirects,
+    timeoutMs: cfg.timeoutMs,
+    maxResponseSize: cfg.maxResponseSize,
+    denyPrivateRanges: cfg.denyPrivateRanges,
+  };
+}
+
+/** Build the just-bash BashOptions from poncho BashConfig + NetworkConfig. */
+function toBashOptions(
+  cfg: BashConfig | undefined,
+  network: NetworkConfig | undefined,
+): Partial<BashOptions> {
+  const opts: Partial<BashOptions> = {};
+
+  if (network) {
+    opts.network = toJustBashNetwork(network);
+  }
+
+  if (!cfg) return opts;
+
+  if (cfg.commands) {
+    opts.commands = cfg.commands as CommandName[];
+  }
+
+  if (cfg.executionLimits) {
+    opts.executionLimits = { ...cfg.executionLimits };
+  }
+
+  if (cfg.python) {
+    opts.python = true;
+  }
+
+  if (cfg.javascript) {
+    opts.javascript = true;
+  }
+
+  if (cfg.env) {
+    opts.env = cfg.env;
+  }
+
+  return opts;
+}
+
+export class BashEnvironmentManager {
+  private environments = new Map<string, Bash>();
+  private readonly workingDir: string | null;
+  private readonly bashOptions: Partial<BashOptions>;
+
+  constructor(
+    private engine: StorageEngine,
+    private limits: { maxFileSize: number; maxTotalStorage: number },
+    workingDir: string | null,
+    bashConfig?: BashConfig,
+    network?: NetworkConfig,
+  ) {
+    this.workingDir = workingDir;
+    this.bashOptions = toBashOptions(bashConfig, network);
+  }
+
+  getOrCreate(tenantId: string): Bash {
+    let bash = this.environments.get(tenantId);
+    if (!bash) {
+      const adapter = new PonchoFsAdapter(this.engine, tenantId, this.limits);
+      const fs = createBashFs(adapter, this.workingDir);
+      bash = new Bash({
+        fs,
+        cwd: "/",
+        ...this.bashOptions,
+      });
+      this.environments.set(tenantId, bash);
+    }
+    return bash;
+  }
+
+  getAdapter(tenantId: string): PonchoFsAdapter {
+    return new PonchoFsAdapter(this.engine, tenantId, this.limits);
+  }
+
+  /** Refresh the PostgreSQL path cache before a bash.exec() call. */
+  async refreshPathCache(tenantId: string): Promise<void> {
+    if ("refreshPathCache" in this.engine) {
+      // Not on StorageEngine interface but on PostgresEngine
+    }
+    // Check if the engine is a PostgresEngine with refreshPathCache
+    const pg = this.engine as unknown as PostgresEngine;
+    if (typeof pg.refreshPathCache === "function") {
+      await pg.refreshPathCache(tenantId);
+    }
+  }
+
+  destroy(tenantId: string): void {
+    this.environments.delete(tenantId);
+  }
+
+  destroyAll(): void {
+    this.environments.clear();
+  }
+}

package/src/vfs/bash-tool.ts

@@ -0,0 +1,59 @@
+// ---------------------------------------------------------------------------
+// Bash tool definition – agent-facing bash interpreter tool.
+// ---------------------------------------------------------------------------
+
+import { defineTool, type ToolDefinition } from "@poncho-ai/sdk";
+import type { BashEnvironmentManager } from "./bash-manager.js";
+
+export const createBashTool = (
+  bashManager: BashEnvironmentManager,
+): ToolDefinition => defineTool({
+  name: "bash",
+  description:
+    "Execute a bash command or script in a sandboxed environment. " +
+    "The environment has a persistent virtual filesystem — files written in one call " +
+    "are available in subsequent calls. Supports standard commands: ls, cat, echo, " +
+    "grep, awk, jq, sed, sort, head, tail, wc, find, mkdir, cp, mv, rm, etc. " +
+    "Use this for data processing, file manipulation, and script execution.",
+  inputSchema: {
+    type: "object",
+    properties: {
+      command: {
+        type: "string",
+        description: "The bash command or script to execute",
+      },
+      timeout: {
+        type: "number",
+        description: "Timeout in milliseconds (default: 30000)",
+      },
+    },
+    required: ["command"],
+    additionalProperties: false,
+  },
+  handler: async (input, context) => {
+    const tenantId = context.tenantId ?? "__default__";
+
+    // Refresh PostgreSQL path cache before exec
+    await bashManager.refreshPathCache(tenantId);
+
+    const bash = bashManager.getOrCreate(tenantId);
+    const timeout = typeof input.timeout === "number" ? input.timeout : 30_000;
+
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), timeout);
+
+    try {
+      const result = await bash.exec(input.command as string, {
+        signal: controller.signal,
+      });
+
+      return {
+        stdout: result.stdout,
+        stderr: result.stderr,
+        exitCode: result.exitCode,
+      };
+    } finally {
+      clearTimeout(timer);
+    }
+  },
+});

package/src/vfs/create-bash-fs.ts

@@ -0,0 +1,32 @@
+// ---------------------------------------------------------------------------
+// createBashFs – assembles MountableFs with VFS root + optional /project mount.
+// ---------------------------------------------------------------------------
+
+import type { IFileSystem } from "just-bash";
+import { MountableFs, ReadWriteFs } from "just-bash";
+import type { PonchoFsAdapter } from "./poncho-fs-adapter.js";
+import { ProtectedFs } from "./protected-fs.js";
+
+/**
+ * Create the filesystem tree for the bash environment.
+ *
+ * - Production: VFS only (no project access).
+ * - Development: VFS root + real project files at /project.
+ */
+export function createBashFs(
+  adapter: PonchoFsAdapter,
+  workingDir: string | null,
+): IFileSystem {
+  if (!workingDir) {
+    // Prod: VFS only
+    return adapter;
+  }
+
+  const realFs = new ReadWriteFs({ root: workingDir });
+  const protectedFs = new ProtectedFs(realFs);
+
+  return new MountableFs({
+    base: adapter,
+    mounts: [{ mountPoint: "/project", filesystem: protectedFs }],
+  });
+}

package/src/vfs/edit-file-tool.ts

@@ -0,0 +1,72 @@
+// ---------------------------------------------------------------------------
+// edit_file tool – targeted string replacement in VFS files.
+// ---------------------------------------------------------------------------
+
+import { defineTool, type ToolDefinition } from "@poncho-ai/sdk";
+import type { StorageEngine } from "../storage/engine.js";
+
+export const createEditFileTool = (
+  engine: StorageEngine,
+): ToolDefinition => defineTool({
+  name: "edit_file",
+  description:
+    "Edit a file by replacing an exact string match with new content. " +
+    "The old_str must match exactly one location in the file. " +
+    "Use an empty new_str to delete matched content. " +
+    "Use read_file first to see current content before editing.",
+  inputSchema: {
+    type: "object",
+    properties: {
+      path: {
+        type: "string",
+        description: "Absolute path of the file to edit",
+      },
+      old_str: {
+        type: "string",
+        description:
+          "The exact text to find and replace (must be unique in the file). " +
+          "Include surrounding context if needed to ensure uniqueness.",
+      },
+      new_str: {
+        type: "string",
+        description: "The replacement text (use empty string to delete the matched content)",
+      },
+    },
+    required: ["path", "old_str", "new_str"],
+    additionalProperties: false,
+  },
+  handler: async (input, context) => {
+    const filePath = typeof input.path === "string" ? input.path.trim() : "";
+    const oldStr = typeof input.old_str === "string" ? input.old_str : "";
+    const newStr = typeof input.new_str === "string" ? input.new_str : "";
+
+    if (!filePath) throw new Error("path is required");
+    if (!oldStr) throw new Error("old_str must not be empty");
+
+    const tenantId = context.tenantId ?? "__default__";
+    const stat = await engine.vfs.stat(tenantId, filePath);
+    if (!stat) throw new Error(`File not found: ${filePath}`);
+    if (stat.type === "directory") throw new Error(`${filePath} is a directory`);
+
+    const buf = await engine.vfs.readFile(tenantId, filePath);
+    const content = Buffer.from(buf).toString("utf8");
+
+    const first = content.indexOf(oldStr);
+    if (first === -1) {
+      throw new Error(
+        "old_str not found in file. Make sure it matches exactly, including whitespace and line breaks.",
+      );
+    }
+    const last = content.lastIndexOf(oldStr);
+    if (first !== last) {
+      throw new Error(
+        "old_str appears multiple times in the file. Include more surrounding context to ensure a unique match.",
+      );
+    }
+
+    const updated = content.slice(0, first) + newStr + content.slice(first + oldStr.length);
+    await engine.vfs.writeFile(tenantId, filePath, new TextEncoder().encode(updated));
+
+    return { ok: true, path: filePath };
+  },
+});

package/src/vfs/index.ts

@@ -0,0 +1,5 @@
+export { PonchoFsAdapter } from "./poncho-fs-adapter.js";
+export { ProtectedFs } from "./protected-fs.js";
+export { createBashFs } from "./create-bash-fs.js";
+export { BashEnvironmentManager } from "./bash-manager.js";
+export { createBashTool } from "./bash-tool.js";