@cmetech/otto 1.1.0 → 1.2.4

package/packages/coworker-scratchpad/src/kernel-bindings.ts

@@ -0,0 +1,279 @@
+import pl from 'nodejs-polars';
+import ExcelJS from 'exceljs';
+import lodash from 'lodash';
+import axios from 'axios';
+import { z } from 'zod';
+import * as dateFns from 'date-fns';
+import * as DuckDB from '@duckdb/node-api';
+import type { DuckDBInstance, DuckDBConnection } from '@duckdb/node-api';
+import { SecretScanner, type AuditLog, type AuditRecord } from '@otto/coworker-utils';
+
+/**
+ * The data libraries pre-bound into every scratchpad cell's vm sandbox.
+ * DuckDB is bound as an in-memory-capable lib here; on-disk kernel.db wiring is 1d2.
+ */
+export function buildDataLibBindings(): Record<string, unknown> {
+  return {
+    polars: pl,
+    DuckDB,
+    ExcelJS,
+    dateFns,
+    lodash,
+    zod: z,
+    axios,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Task F: otto.duckdb.registerDf — single-call polars/Arrow/records → DuckDB
+// ---------------------------------------------------------------------------
+
+/** SQL column type string passed straight into CREATE TABLE (e.g. VARCHAR, BIGINT). */
+export type DuckDBColumnType = string;
+
+export interface RegisterDfOptions {
+  /**
+   * Explicit per-column SQL types. When omitted, the schema is inferred from
+   * the first non-null value of each column in the first 10 rows.
+   */
+  schema?: Record<string, DuckDBColumnType> | Array<[string, DuckDBColumnType]>;
+}
+
+interface PolarsDataFrameShape {
+  toRecords: () => Record<string, unknown>[];
+  width: number;
+  height: number;
+}
+
+interface ArrowTableShape {
+  toArray: () => Record<string, unknown>[];
+  numRows: number;
+  schema: unknown;
+}
+
+function isPolarsDataFrame(x: unknown): x is PolarsDataFrameShape {
+  return (
+    !!x &&
+    typeof x === 'object' &&
+    typeof (x as { toRecords?: unknown }).toRecords === 'function' &&
+    typeof (x as { width?: unknown }).width === 'number' &&
+    typeof (x as { height?: unknown }).height === 'number'
+  );
+}
+
+function isArrowTable(x: unknown): x is ArrowTableShape {
+  return (
+    !!x &&
+    typeof x === 'object' &&
+    typeof (x as { toArray?: unknown }).toArray === 'function' &&
+    typeof (x as { numRows?: unknown }).numRows === 'number' &&
+    'schema' in (x as object)
+  );
+}
+
+function coerceToRecords(input: unknown): Record<string, unknown>[] {
+  if (isPolarsDataFrame(input)) return input.toRecords();
+  if (isArrowTable(input)) return input.toArray();
+  if (Array.isArray(input)) return input as Record<string, unknown>[];
+  throw new TypeError(
+    'registerDf: input must be a polars DataFrame, Arrow Table, or array of records',
+  );
+}
+
+function inferSchema(records: Record<string, unknown>[]): Record<string, DuckDBColumnType> {
+  if (records.length === 0) {
+    throw new Error(
+      'registerDf: cannot infer schema from empty input. Provide opts.schema or pass at least one row.',
+    );
+  }
+  const cols = Object.keys(records[0]!);
+  const out: Record<string, DuckDBColumnType> = {};
+  for (const col of cols) {
+    // Null-walk: scan up to the first 10 rows for the first non-null value of this column.
+    // Defaults to VARCHAR for all-null columns (safe — DuckDB will accept NULL into a VARCHAR).
+    out[col] = 'VARCHAR';
+    const walk = Math.min(10, records.length);
+    for (let i = 0; i < walk; i++) {
+      const v = records[i]![col];
+      if (v === null || v === undefined) continue;
+      if (typeof v === 'string') { out[col] = 'VARCHAR'; break; }
+      if (typeof v === 'bigint') { out[col] = 'BIGINT'; break; }
+      if (typeof v === 'number') { out[col] = 'DOUBLE'; break; }
+      if (typeof v === 'boolean') { out[col] = 'BOOLEAN'; break; }
+      if (v instanceof Date) { out[col] = 'TIMESTAMP'; break; }
+      out[col] = 'VARCHAR'; // unknown object → store as VARCHAR
+      break;
+    }
+  }
+  return out;
+}
+
+function normalizeSchema(
+  schema: Record<string, DuckDBColumnType> | Array<[string, DuckDBColumnType]>,
+): Array<[string, DuckDBColumnType]> {
+  if (Array.isArray(schema)) return schema;
+  return Object.entries(schema);
+}
+
+function quoteIdent(name: string): string {
+  // DuckDB identifier quoting: wrap in double quotes, escape embedded double quotes.
+  return `"${name.replace(/"/g, '""')}"`;
+}
+
+async function registerViaAppender(
+  conn: DuckDBConnection,
+  name: string,
+  records: Record<string, unknown>[],
+  schema: Array<[string, DuckDBColumnType]>,
+  sourceHint: 'inferred' | 'explicit',
+): Promise<void> {
+  const ddl = `CREATE TABLE ${quoteIdent(name)} (${schema
+    .map(([c, t]) => `${quoteIdent(c)} ${t}`)
+    .join(', ')})`;
+  await conn.run(ddl);
+  // All-or-nothing semantic: on any failure we drop the table so the caller
+  // can retry the same name without hitting "Table already exists." On the
+  // success path we flip this flag and the finally becomes a no-op.
+  let createdTable = true;
+  const app = await conn.createAppender(name);
+  try {
+    for (let i = 0; i < records.length; i++) {
+      let failingColIdx = -1;
+      try {
+        for (let colIdx = 0; colIdx < schema.length; colIdx++) {
+          failingColIdx = colIdx;
+          const col = schema[colIdx]![0];
+          const v = records[i]![col];
+          if (v === null || v === undefined) {
+            app.appendNull();
+          } else {
+            // appendValue accepts DuckDBValue (null | boolean | number | bigint | string | ...)
+            // and lets the engine coerce based on the declared column type.
+            app.appendValue(v as never);
+          }
+        }
+        // endRow() may itself reject the row on a type mismatch detected at
+        // end-of-row; in that case we can't attribute to a single column so
+        // we fall back to '<column>' literally rather than blame col 0.
+        failingColIdx = -1;
+        app.endRow();
+      } catch (e) {
+        const failingCol =
+          failingColIdx >= 0 ? schema[failingColIdx]![0] : '<column>';
+        const hint =
+          sourceHint === 'inferred'
+            ? ` Pass an explicit schema via the third argument: registerDf(name, df, { schema: { ${failingCol}: 'VARCHAR' } })`
+            : '';
+        throw new Error(
+          `registerDf row ${i}: append failed for column '${failingCol}' (${(e as Error).message}).${hint}`,
+        );
+      }
+    }
+    // Success path: close the appender to commit the rows, then disarm cleanup.
+    app.closeSync();
+    createdTable = false;
+  } finally {
+    if (createdTable) {
+      // Failure path: close the appender (may already be closed; best-effort)
+      // then drop the partially-populated table.
+      try { app.closeSync(); } catch { /* already closed or errored */ }
+      try { await conn.run(`DROP TABLE IF EXISTS ${quoteIdent(name)}`); } catch { /* best-effort */ }
+    }
+  }
+}
+
+/**
+ * Patches a DuckDBInstance with a `registerDf(name, input, opts?)` method
+ * so the LLM can drop a polars DataFrame / Arrow Table / array of records
+ * into DuckDB in one call instead of discovering the appender API.
+ *
+ * The method opens a fresh connection on each call (cheap — DuckDB connections
+ * share the underlying instance/db) and CREATE-TABLE-then-appends.
+ */
+export function attachRegisterDf(instance: DuckDBInstance): void {
+  const patched = instance as unknown as {
+    registerDf: (
+      name: string,
+      input: unknown,
+      opts?: RegisterDfOptions,
+    ) => Promise<void>;
+  };
+  patched.registerDf = async function registerDf(
+    name: string,
+    input: unknown,
+    opts: RegisterDfOptions = {},
+  ): Promise<void> {
+    const conn = await instance.connect();
+    try {
+      const records = coerceToRecords(input);
+      const sourceHint: 'inferred' | 'explicit' = opts.schema ? 'explicit' : 'inferred';
+      const schemaObj = opts.schema ?? inferSchema(records);
+      const schema = normalizeSchema(schemaObj);
+      await registerViaAppender(conn, name, records, schema, sourceHint);
+    } finally {
+      // Release the connection — DuckDBConnection exposes closeSync(): void.
+      // Without this, every registerDf call leaks one connection over the
+      // lifetime of the scratchpad session.
+      conn.closeSync();
+    }
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Phase 2 Task 14: SecretScanner gate on cell-output journal writes.
+// Live TUI output is UPSTREAM of the journal write (the runtime returns the
+// raw stdout, then the manager forks: the same string is returned to the tool
+// for live display AND passed through redactForJournal before archive.append).
+// Only the journal copy is redacted; the TUI copy is untouched.
+// ---------------------------------------------------------------------------
+
+/**
+ * Context passed to `redactForJournal`. The audit instance is the same one
+ * held by the CredentialInjector — supplied by the manager so secret-redaction
+ * audit records land alongside vault inject/inject-skipped records.
+ */
+export interface RedactionContext {
+  audit: AuditLog;
+  sessionId: string;
+  scratchpadName: string;
+  pid: number;
+  cellId: string;
+}
+
+const journalScanner = new SecretScanner();
+
+/**
+ * Scans `raw` for known secret patterns; for each hit, appends a
+ * `producer: 'secret-scanner', action: 'redact'` audit record (severity warn)
+ * with `detail = { cell_id, kind, offset, length }` — NEVER the secret value
+ * or `SecretHit.preview`. Returns the redacted text (`[REDACTED:<kind>]` per
+ * hit). If no hits, returns the input verbatim with no audit emission.
+ *
+ * Backward compatibility: callers that don't have an AuditLog should skip this
+ * helper entirely (pass-through). Audit is required here because the redaction
+ * MUST be observable.
+ */
+export function redactForJournal(raw: string, ctx: RedactionContext): string {
+  const hits = journalScanner.scan(raw);
+  if (hits.length === 0) return raw;
+  const ts = new Date().toISOString();
+  for (const h of hits) {
+    ctx.audit.append({
+      _schema: 1,
+      ts,
+      producer: 'secret-scanner',
+      action: 'redact',
+      severity: 'warn',
+      sessionId: ctx.sessionId,
+      scratchpadName: ctx.scratchpadName,
+      pid: ctx.pid,
+      detail: {
+        cell_id: ctx.cellId,
+        kind: h.kind,
+        offset: h.start,
+        length: h.end - h.start,
+      },
+    } satisfies AuditRecord);
+  }
+  return journalScanner.redact(raw);
+}

package/packages/coworker-scratchpad/src/kernel-entry.test.ts

@@ -0,0 +1,123 @@
+import { describe, it, beforeEach, afterEach } from 'node:test';
+import assert from 'node:assert/strict';
+import { spawn, type ChildProcessWithoutNullStreams } from 'node:child_process';
+import { existsSync } from 'node:fs';
+import { mkdtemp, mkdir, writeFile, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { pathToFileURL } from 'node:url';
+import process from 'node:process';
+import { writeNdjson, readNdjson } from '@otto/coworker-utils';
+import { ChildProcessRuntime } from './child-process-runtime.js';
+import { resolveKernelEntry, kernelExecArgv, filterEnv } from './kernel-spawn.js';
+import type { KernelFrame, ProgressEvent, ResultResponse } from './kernel-protocol.js';
+
+let workspace: string;
+let inputs: string;
+let child: ChildProcessWithoutNullStreams;
+
+function startKernel(ws: string): ChildProcessWithoutNullStreams {
+  return spawn(
+    process.execPath,
+    [...kernelExecArgv(), resolveKernelEntry(), ws],
+    { stdio: ['pipe', 'pipe', 'inherit'], cwd: process.cwd(), env: filterEnv(process.env) },
+  ) as unknown as ChildProcessWithoutNullStreams;
+}
+
+// Drain frames until `count` result frames have arrived; ignore events.
+async function collectResults(c: ChildProcessWithoutNullStreams, count: number): Promise<ResultResponse[]> {
+  const results: ResultResponse[] = [];
+  for await (const raw of readNdjson(c.stdout)) {
+    const frame = raw as KernelFrame;
+    if (frame.type === 'result') {
+      results.push(frame);
+      if (results.length === count) break;
+    }
+  }
+  return results;
+}
+
+describe('kernel-entry (child process)', () => {
+  beforeEach(async () => {
+    workspace = await mkdtemp(join(tmpdir(), 'ke-ws-'));
+    inputs = join(workspace, '.otto', 'inputs');
+    await mkdir(inputs, { recursive: true });
+  });
+
+  afterEach(async () => {
+    child?.kill('SIGKILL');
+    await rm(workspace, { recursive: true, force: true });
+  });
+
+  it('evaluates a cell and returns its value + captured stdout', async () => {
+    child = startKernel(workspace);
+    await writeNdjson(child.stdin, { id: 1, type: 'run', code: "console.log('hi'); return 1 + 1;" });
+    const [res] = await collectResults(child, 1);
+    assert.equal(res.ok, true);
+    assert.equal((res as { value: unknown }).value, 2);
+    assert.equal((res as { stdout: string }).stdout, 'hi');
+  });
+
+  it('persists globalThis across cells', async () => {
+    child = startKernel(workspace);
+    await writeNdjson(child.stdin, { id: 1, type: 'run', code: 'globalThis.counter = 41; return globalThis.counter;' });
+    await writeNdjson(child.stdin, { id: 2, type: 'run', code: 'return globalThis.counter + 1;' });
+    const results = await collectResults(child, 2);
+    assert.equal(results[0].ok, true);
+    assert.equal((results[0] as { value: unknown }).value, 41);
+    assert.equal((results[1] as { value: unknown }).value, 42);
+  });
+
+  it('binds otto.collectors.list() to the workspace inputs dir', async () => {
+    await writeFile(join(inputs, 'cmdb.csv'), 'a,b\n1,2\n');
+    child = startKernel(workspace);
+    await writeNdjson(child.stdin, { id: 1, type: 'run', code: 'return (await otto.collectors.list()).map((r) => r.uri);' });
+    const [res] = await collectResults(child, 1);
+    assert.equal(res.ok, true);
+    assert.deepEqual((res as { value: string[] }).value, [pathToFileURL(join(inputs, 'cmdb.csv')).href]);
+  });
+
+  it('returns ok:false with the error message when a cell throws', async () => {
+    child = startKernel(workspace);
+    await writeNdjson(child.stdin, { id: 1, type: 'run', code: "throw new Error('boom');" });
+    const [res] = await collectResults(child, 1);
+    assert.equal(res.ok, false);
+    assert.match((res as { error: { message: string } }).error.message, /boom/);
+  });
+
+  it('emits a progress event when a cell calls progress()', async () => {
+    child = startKernel(workspace);
+    await writeNdjson(child.stdin, { id: 1, type: 'run', code: "progress('halfway'); return 1;" });
+    const frames: KernelFrame[] = [];
+    for await (const raw of readNdjson(child.stdout)) {
+      const f = raw as KernelFrame;
+      frames.push(f);
+      if (f.type === 'result') break;
+    }
+    const prog = frames.find((f) => f.type === 'event' && f.event === 'progress');
+    assert.ok(prog, 'expected a progress event before the result');
+    assert.equal((prog as ProgressEvent).message, 'halfway');
+  });
+
+  it('snapshot writes namespace.json via tmp + rename (no .tmp leak — 1g3)', async () => {
+    // Strategy: drive the kernel via ChildProcessRuntime, run a cell, snapshot,
+    // confirm namespace.json exists and namespace.json.tmp does NOT.
+    // (kernel-entry's snapshot handler is exercised through a real subprocess
+    // in the existing tests; this just adds the .tmp absence assertion.)
+    const ws = await mkdtemp(join(tmpdir(), 'ke-ws-'));
+    const dir = await mkdtemp(join(tmpdir(), 'ke-dir-'));
+    try {
+      const rt = new ChildProcessRuntime({ workspace: ws, scratchpadDir: dir });
+      await rt.start();
+      await rt.runCell('globalThis.x = 1;');
+      const res = await rt.snapshot();
+      assert.equal(res.ok, true, 'snapshot acked');
+      assert.ok(existsSync(join(dir, 'namespace.json')), 'namespace.json present');
+      assert.equal(existsSync(join(dir, 'namespace.json.tmp')), false, 'no .tmp leak');
+      await rt.dispose();
+    } finally {
+      await rm(ws, { recursive: true, force: true });
+      await rm(dir, { recursive: true, force: true });
+    }
+  });
+});