@pushpalsdev/cli 1.0.18 → 1.0.20

package/runtime/sandbox/apps/workerpals/src/job_runner.ts

@@ -0,0 +1,194 @@
+#!/usr/bin/env bun
+/**
+ * Docker Job Runner - Standalone job execution daemon inside Docker
+ *
+ * This script runs inside a Docker container and executes a single job.
+ * It's designed to be the entrypoint for sandboxed job execution.
+ *
+ * Usage (inside container):
+ *   bun run job_runner.ts <base64-encoded-job-spec>
+ *
+ * The job spec is base64-encoded JSON: { jobId, taskId, kind, params, workerId }
+ *
+ * Output:
+ *   stderr → JSON log lines: {"stream":"stdout|stderr","line":"..."}
+ *   stdout → Result with sentinel: ___RESULT___ {"ok":true,...,"commit":{...}}
+ */
+
+import { executeJob, shouldCommit, createJobCommit } from "./execute_job.js";
+import { loadPushPalsConfig } from "shared";
+import { writeFileSync } from "fs";
+
+const CONFIG = loadPushPalsConfig();
+
+// ─── Types ──────────────────────────────────────────────────────────────────
+
+interface JobSpec {
+  jobId: string;
+  taskId: string;
+  kind: string;
+  params: Record<string, unknown>;
+  workerId: string;
+}
+
+interface JobResult {
+  ok: boolean;
+  summary: string;
+  stdout?: string;
+  stderr?: string;
+  exitCode?: number;
+  commit?: {
+    branch: string;
+    sha: string;
+  };
+}
+
+// ─── Logging helpers ────────────────────────────────────────────────────────
+
+function log(stream: "stdout" | "stderr", line: string): void {
+  const json = JSON.stringify({ stream, line });
+  // eslint-disable-next-line no-console
+  console.error(json);
+}
+
+// ─── Git credentials setup ──────────────────────────────────────────────────
+
+function setupGitCredentials(): void {
+  const token = CONFIG.gitToken ?? process.env.GIT_TOKEN ?? null;
+  if (!token) return;
+  try {
+    // Use a credential helper script and avoid remote URL rewriting with embedded secrets.
+    // This keeps push auth stable and prevents token leakage in git stderr output.
+    const helperScript = `#!/bin/sh
+echo "username=x-access-token"
+echo "password=${token}"
+`;
+    const helperPath = "/tmp/git-credential-helper";
+    writeFileSync(helperPath, helperScript, { mode: 0o755 });
+
+    // Remove any legacy URL rewrite rules that may have embedded token credentials.
+    const urlRules = Bun.spawnSync(["git", "config", "--global", "--get-regexp", "^url\\..*\\.insteadOf$"], {
+      stdout: "pipe",
+      stderr: "pipe",
+    });
+    if (urlRules.exitCode === 0) {
+      const lines = String(urlRules.stdout ?? "")
+        .split(/\r?\n/)
+        .map((line) => line.trim())
+        .filter(Boolean);
+      for (const line of lines) {
+        const key = line.split(/\s+/, 1)[0] ?? "";
+        if (!key) continue;
+        const lower = key.toLowerCase();
+        if (!lower.startsWith("url.")) continue;
+        if (!lower.endsWith(".insteadof")) continue;
+        if (!lower.includes("oauth2") && !lower.includes("%3a//")) continue;
+        Bun.spawnSync(["git", "config", "--global", "--unset-all", key], {
+          stdout: "pipe",
+          stderr: "pipe",
+        });
+      }
+    }
+
+    Bun.spawnSync(["git", "config", "--global", "credential.helper", helperPath]);
+  } catch (err) {
+    log("stderr", `Failed to setup git credentials: ${err}`);
+  }
+}
+
+// ─── Main ───────────────────────────────────────────────────────────────────
+
+async function main(): Promise<void> {
+  const args = process.argv.slice(2);
+  const base64Spec = args[0];
+
+  if (!base64Spec) {
+    // eslint-disable-next-line no-console
+    console.error("Usage: bun run job_runner.ts <base64-encoded-job-spec>");
+    process.exit(1);
+  }
+
+  // Decode base64 job spec
+  let spec: JobSpec;
+  try {
+    const json = Buffer.from(base64Spec, "base64").toString("utf-8");
+    spec = JSON.parse(json);
+  } catch (err) {
+    // eslint-disable-next-line no-console
+    console.error(`Failed to decode job spec: ${err}`);
+    process.exit(1);
+  }
+  log("stdout", `[JobRunner] Starting job ${spec.jobId} (${spec.kind})`);
+  // Setup git credentials for pushing
+  setupGitCredentials();
+  // Execute inside the mounted job worktree (docker -w), not the baked image copy.
+  const jobRepo = process.cwd();
+  const result = await executeJob(
+    spec.kind,
+    spec.params,
+    jobRepo,
+    (stream, line) => {
+      log(stream, line);
+    },
+    CONFIG,
+  );
+  // Build result object
+  const jobResult: JobResult = {
+    ok: result.ok,
+    summary: result.summary,
+    stdout: result.stdout,
+    stderr: result.stderr,
+    exitCode: result.exitCode,
+  };
+  // Create commit for file-modifying jobs
+  if (result.ok && shouldCommit(spec.kind, CONFIG)) {
+    log("stdout", `[JobRunner] Job modified files, creating commit...`);
+    const commitResult = await createJobCommit(
+      jobRepo,
+      spec.workerId,
+      {
+        id: spec.jobId,
+        taskId: spec.taskId,
+        kind: spec.kind,
+        params: spec.params,
+        context: "docker",
+      },
+      CONFIG,
+    );
+
+    if (commitResult.ok && commitResult.sha && commitResult.branch) {
+      jobResult.commit = {
+        branch: commitResult.branch!,
+        sha: commitResult.sha,
+      };
+      if (commitResult.sha === "no-changes") {
+        log("stdout", `[JobRunner] No changes to commit for ${spec.jobId}`);
+      } else {
+        log("stdout", `[JobRunner] Created commit ${commitResult.sha} on ${commitResult.branch}`);
+      }
+    } else {
+      const commitError =
+        commitResult.error ??
+        `Commit metadata missing for ${spec.kind} (${spec.jobId}) while running in Docker mode`;
+      jobResult.ok = false;
+      jobResult.summary = `Failed to create commit for ${spec.kind}`;
+      jobResult.stderr = [jobResult.stderr, commitError].filter(Boolean).join("\n");
+      jobResult.exitCode = jobResult.exitCode && jobResult.exitCode !== 0 ? jobResult.exitCode : 1;
+      log("stderr", `[JobRunner] Failed to create commit: ${commitError}`);
+    }
+  }
+
+  // Output result with sentinel
+  const resultJson = JSON.stringify(jobResult);
+  // eslint-disable-next-line no-console
+  console.log(`___RESULT___ ${resultJson}`);
+
+  // Exit with appropriate code
+  process.exit(jobResult.exitCode ?? (jobResult.ok ? 0 : 1));
+}
+
+main().catch((err) => {
+  // eslint-disable-next-line no-console
+  console.error(`[JobRunner] Fatal error: ${err}`);
+  process.exit(1);
+});

package/runtime/sandbox/apps/workerpals/src/shell_manager.ts

@@ -0,0 +1,210 @@
+// Persistent ShellManager for multi-worker shell sessions
+// Milestone 1: Correctness, lease management, command framing, audit logging
+
+import { spawn } from "bun";
+import Database from "bun:sqlite";
+import { randomUUID } from "crypto";
+import { ContextManager } from "./context_manager";
+
+const SESSION_TTL_MS = 10 * 60 * 1000; // 10 min
+const LEASE_DURATION_MS = 60 * 1000; // 1 min
+
+export interface ShellCommand {
+  cmdId: string;
+  command: string;
+  status: "pending" | "running" | "completed" | "crashed" | "killed";
+  exitCode?: number;
+  stdout?: string;
+  stderr?: string;
+  startedAt?: number;
+  completedAt?: number;
+}
+
+export class ShellManager {
+  private db: Database;
+  private workerId: string;
+  private sessions = new Map<string, ShellSessionRuntime>();
+  private contextMgr: ContextManager;
+
+  constructor(db: Database, workerId: string) {
+    this.db = db;
+    this.workerId = workerId;
+    this.contextMgr = new ContextManager(db);
+    this.ensureTables();
+  }
+
+  ensureTables() {
+    this.db.run(`CREATE TABLE IF NOT EXISTS shell_sessions (
+      session_id TEXT PRIMARY KEY,
+      worker_id TEXT,
+      lease_expiry INTEGER,
+      last_used_at INTEGER,
+      last_known_cwd TEXT,
+      status TEXT
+    )`);
+    this.db.run(`CREATE TABLE IF NOT EXISTS shell_commands (
+      cmd_id TEXT PRIMARY KEY,
+      session_id TEXT,
+      command TEXT,
+      status TEXT,
+      exit_code INTEGER,
+      stdout TEXT,
+      stderr TEXT,
+      started_at INTEGER,
+      completed_at INTEGER
+    )`);
+  }
+
+  acquireLease(sessionId: string): boolean {
+    const now = Date.now();
+    const expiry = now + LEASE_DURATION_MS;
+    // Try to acquire lease
+    this.db.run(
+      `INSERT OR IGNORE INTO shell_sessions (session_id, worker_id, lease_expiry, last_used_at, status) VALUES (?, ?, ?, ?, ?)`,
+      [sessionId, this.workerId, expiry, now, "active"],
+    );
+    const row = this.db
+      .query(`SELECT worker_id, lease_expiry FROM shell_sessions WHERE session_id = ?`)
+      .get(sessionId) as { worker_id: string; lease_expiry: number } | null;
+    if (row && (row.worker_id === this.workerId || row.lease_expiry < now)) {
+      this.db.run(
+        `UPDATE shell_sessions SET worker_id = ?, lease_expiry = ?, last_used_at = ?, status = ? WHERE session_id = ?`,
+        [this.workerId, expiry, now, "active", sessionId],
+      );
+      return true;
+    }
+    return false;
+  }
+
+  getOrCreate(sessionId: string): ShellSessionRuntime | null {
+    if (!this.acquireLease(sessionId)) return null;
+    let runtime = this.sessions.get(sessionId);
+    if (!runtime) {
+      // Restore last_known_cwd from context
+      const cwd = this.contextMgr.get(sessionId, "last_known_cwd") || process.cwd();
+      runtime = new ShellSessionRuntime(sessionId, this.workerId, this.db, this.contextMgr, cwd);
+      this.sessions.set(sessionId, runtime);
+    }
+    return runtime;
+  }
+
+  cleanupIdleSessions() {
+    const now = Date.now();
+    for (const [sessionId, runtime] of this.sessions) {
+      if (now - runtime.lastUsedAt > SESSION_TTL_MS) {
+        runtime.terminate();
+        this.sessions.delete(sessionId);
+        this.db.run(`UPDATE shell_sessions SET status = ? WHERE session_id = ?`, [
+          "stopped",
+          sessionId,
+        ]);
+      }
+    }
+  }
+
+  startContextRefresh(intervalMs = 10000) {
+    setInterval(() => {
+      for (const sessionId of this.sessions.keys()) {
+        // Refresh context for this session
+        const context = this.contextMgr.getAll(sessionId);
+        // Feed context to agent if handler exists
+        const session = this.sessions.get(sessionId);
+        if (session && typeof session.onContextRefresh === "function") {
+          session.onContextRefresh(context);
+        }
+      }
+    }, intervalMs);
+  }
+}
+
+export class ShellSessionRuntime {
+  private sessionId: string;
+  private workerId: string;
+  private db: Database;
+  private shellProc: ReturnType<typeof spawn> | null = null;
+  private commandQueue: ShellCommand[] = [];
+  public lastUsedAt: number = Date.now();
+  public cwd: string = process.cwd();
+  private running: boolean = false;
+  private contextMgr: ContextManager;
+  public onContextRefresh?: (context: Record<string, string>) => void;
+
+  constructor(
+    sessionId: string,
+    workerId: string,
+    db: Database,
+    contextMgr: ContextManager,
+    cwd: string,
+  ) {
+    this.sessionId = sessionId;
+    this.workerId = workerId;
+    this.db = db;
+    this.contextMgr = contextMgr;
+    this.cwd = cwd;
+    this.spawnShell();
+  }
+
+  spawnShell() {
+    // Spawn shell process (PowerShell or bash)
+    const isWindows = process.platform === "win32";
+    const shell = isWindows ? "powershell.exe" : "bash";
+    this.shellProc = spawn([shell], {
+      cwd: this.cwd,
+      stdin: "pipe",
+      stdout: "pipe",
+      stderr: "pipe",
+    });
+    // Attach parser, etc.
+  }
+
+  terminate() {
+    if (this.shellProc) {
+      this.shellProc.kill();
+      this.shellProc = null;
+    }
+  }
+
+  enqueueCommand(command: string) {
+    const cmdId = randomUUID();
+    const shellCmd: ShellCommand = {
+      cmdId,
+      command,
+      status: "pending",
+      startedAt: Date.now(),
+    };
+    this.commandQueue.push(shellCmd);
+    const startedAt = shellCmd.startedAt ?? Date.now();
+    this.db.run(
+      `INSERT INTO shell_commands (cmd_id, session_id, command, status, started_at) VALUES (?, ?, ?, ?, ?)`,
+      [cmdId, this.sessionId, command, "pending", startedAt],
+    );
+    this.processQueue();
+    return cmdId;
+  }
+
+  async processQueue() {
+    if (this.running || this.commandQueue.length === 0) return;
+    this.running = true;
+    const cmd = this.commandQueue.shift();
+    if (!cmd || !this.shellProc) {
+      this.running = false;
+      return;
+    }
+    // Framing: BEGIN/EXIT/CWD/END
+    const framed = `echo BEGIN ${cmd.cmdId}\n${cmd.command}\necho EXIT ${cmd.cmdId} $?\necho CWD ${cmd.cmdId} $(pwd)\necho END ${cmd.cmdId}`;
+    const stdin = this.shellProc.stdin;
+    if (!stdin || typeof stdin === "number" || typeof stdin.write !== "function") {
+      this.running = false;
+      return;
+    }
+    stdin.write(framed + "\n");
+    // TODO: parse output, update db, handle exit/cwd
+    // TODO: renew lease, update last_used_at
+    // On command completion, update last_known_cwd in context
+    if (this.cwd) {
+      this.contextMgr.set(this.sessionId, "last_known_cwd", this.cwd);
+    }
+    this.running = false;
+    this.processQueue();
+  }
+}

package/runtime/sandbox/apps/workerpals/src/timeout_policy.ts

@@ -0,0 +1,24 @@
+export const DEFAULT_OPENHANDS_TIMEOUT_MS = 1_800_000;
+export const DEFAULT_DOCKER_TIMEOUT_MS = 1_860_000;
+
+export function parseOpenHandsTimeoutMs(raw: string | undefined): number {
+  const parsed = parseInt(raw ?? "", 10);
+  if (!Number.isFinite(parsed) || parsed <= 0) return DEFAULT_OPENHANDS_TIMEOUT_MS;
+  return Math.max(10_000, parsed);
+}
+
+export function parseDockerTimeoutMs(raw: string | undefined): number {
+  const parsed = parseInt(raw ?? "", 10);
+  if (!Number.isFinite(parsed) || parsed <= 0) return DEFAULT_DOCKER_TIMEOUT_MS;
+  return Math.max(10_000, parsed);
+}
+
+export function computeTimeoutWarningWindow(timeoutMs: number): {
+  leadMs: number;
+  delayMs: number;
+} {
+  const normalized = Math.max(10_000, Math.floor(timeoutMs));
+  const leadMs = Math.min(60_000, Math.max(10_000, normalized - 5_000));
+  const delayMs = Math.max(1_000, normalized - leadMs);
+  return { leadMs, delayMs };
+}