@cmetech/otto 1.1.0 → 1.2.4

package/packages/coworker-scratchpad/src/file-collector.test.ts

@@ -0,0 +1,109 @@
+import { describe, it, beforeEach, afterEach } from 'node:test';
+import assert from 'node:assert/strict';
+import { mkdtemp, mkdir, writeFile, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { pathToFileURL } from 'node:url';
+import { FileCollector } from './file-collector.js';
+
+let workspace: string;
+let inputs: string;
+
+async function collectRefs(it: AsyncIterable<{ uri: string; kind: string; bytes?: number; metadata: unknown }>) {
+  const out = [];
+  for await (const ref of it) out.push(ref);
+  return out;
+}
+
+describe('FileCollector', () => {
+  beforeEach(async () => {
+    workspace = await mkdtemp(join(tmpdir(), 'fc-ws-'));
+    inputs = join(workspace, '.otto', 'inputs');
+    await mkdir(inputs, { recursive: true });
+  });
+
+  afterEach(async () => {
+    await rm(workspace, { recursive: true, force: true });
+  });
+
+  it('describe() advertises file:// support, the six kinds, watching, no streaming', () => {
+    const fc = new FileCollector({ workspace });
+    const cap = fc.describe();
+    assert.deepEqual(cap.supports_uris, ['file://*']);
+    assert.deepEqual([...cap.supports_kinds].sort(), ['csv', 'json', 'md', 'parquet', 'txt', 'xlsx']);
+    assert.equal(cap.supports_streaming, false);
+    assert.equal(cap.supports_watching, true);
+  });
+
+  it('list() yields refs only for supported files, recursively, skipping unsupported', async () => {
+    await writeFile(join(inputs, 'cmdb.csv'), 'a,b\n1,2\n');
+    await writeFile(join(inputs, 'notes.txt'), 'hi');
+    await writeFile(join(inputs, 'ignore.pdf'), 'x');
+    await mkdir(join(inputs, 'nested'), { recursive: true });
+    await writeFile(join(inputs, 'nested', 'data.json'), '{"k":1}');
+
+    const fc = new FileCollector({ workspace });
+    const refs = await collectRefs(fc.list());
+    const byKind = Object.fromEntries(refs.map((r) => [r.kind, r]));
+
+    assert.deepEqual(Object.keys(byKind).sort(), ['csv', 'json', 'txt']);
+    assert.equal(byKind.csv.collector, 'file');
+    assert.equal(byKind.csv.uri, pathToFileURL(join(inputs, 'cmdb.csv')).href);
+    assert.equal(byKind.csv.bytes, 8);
+    assert.deepEqual(byKind.csv.metadata, {});
+    assert.equal(typeof byKind.csv.modified, 'string');
+  });
+
+  it('list() honors the limit option', async () => {
+    await writeFile(join(inputs, 'a.csv'), 'x');
+    await writeFile(join(inputs, 'b.csv'), 'x');
+    await writeFile(join(inputs, 'c.csv'), 'x');
+    const fc = new FileCollector({ workspace });
+    const refs = await collectRefs(fc.list({ limit: 2 }));
+    assert.equal(refs.length, 2);
+  });
+
+  it('list() yields nothing when the inputs dir does not exist', async () => {
+    await rm(inputs, { recursive: true, force: true });
+    const fc = new FileCollector({ workspace });
+    const refs = await collectRefs(fc.list());
+    assert.deepEqual(refs, []);
+  });
+
+  it('open().load() parses JSON, returns text as string, and binary as Buffer', async () => {
+    await writeFile(join(inputs, 'd.json'), '{"hello":"world"}');
+    await writeFile(join(inputs, 'd.csv'), 'a,b\n1,2\n');
+    await writeFile(join(inputs, 'd.parquet'), Buffer.from([0x50, 0x41, 0x52, 0x31]));
+    const fc = new FileCollector({ workspace });
+
+    const byKind = Object.fromEntries((await collectRefs(fc.list())).map((r) => [r.kind, r]));
+
+    const json = await (await fc.open(byKind.json as never)).load();
+    assert.deepEqual(json, { hello: 'world' });
+
+    const csv = await (await fc.open(byKind.csv as never)).load();
+    assert.equal(csv, 'a,b\n1,2\n');
+
+    const parquet = await (await fc.open(byKind.parquet as never)).load();
+    assert.ok(Buffer.isBuffer(parquet));
+    assert.deepEqual(parquet, Buffer.from([0x50, 0x41, 0x52, 0x31]));
+  });
+
+  it('watch() invokes onChange when the file is modified and unsubscribe stops it', async () => {
+    const file = join(inputs, 'live.csv');
+    await writeFile(file, 'v1');
+    const fc = new FileCollector({ workspace });
+    const ref = (await collectRefs(fc.list()))[0];
+
+    const changed = new Promise<void>((resolve) => {
+      const unsub = fc.watch(ref as never, () => {
+        unsub();
+        resolve();
+      });
+      // Give chokidar a moment to register before mutating.
+      setTimeout(() => void writeFile(file, 'v2-longer'), 300);
+    });
+
+    await changed; // test times out (and fails) if onChange never fires
+  });
+});

package/packages/coworker-scratchpad/src/file-collector.ts

@@ -0,0 +1,114 @@
+import { readdir, readFile, stat } from 'node:fs/promises';
+import type { Dirent } from 'node:fs';
+import { join } from 'node:path';
+import { pathToFileURL, fileURLToPath } from 'node:url';
+import { watch as chokidarWatch } from 'chokidar';
+import type {
+  Collector,
+  CollectorCapabilities,
+  DataSource,
+  DataSourceRef,
+  ListOpts,
+  Unsubscribe,
+} from '@otto/coworker-types';
+import { detectKind, FILE_COLLECTOR_KINDS } from './detect-kind.js';
+
+const TEXT_KINDS = new Set(['csv', 'txt', 'md']);
+
+export interface FileCollectorOptions {
+  /** Absolute path to the workspace root. inputs/ resolves to <workspace>/.otto/inputs. */
+  workspace: string;
+}
+
+export class FileCollector implements Collector {
+  readonly id = 'file';
+  readonly kind = 'file' as const;
+  private readonly inputsDir: string;
+
+  constructor(opts: FileCollectorOptions) {
+    this.inputsDir = join(opts.workspace, '.otto', 'inputs');
+  }
+
+  describe(): CollectorCapabilities {
+    return {
+      supports_uris: ['file://*'],
+      supports_kinds: [...FILE_COLLECTOR_KINDS],
+      supports_streaming: false,
+      supports_watching: true,
+    };
+  }
+
+  async *list(opts?: ListOpts): AsyncIterable<DataSourceRef> {
+    let remaining = opts?.limit ?? Number.POSITIVE_INFINITY;
+    if (remaining <= 0) return;
+    const prefixDir = opts?.prefix ? join(this.inputsDir, opts.prefix) : this.inputsDir;
+    for await (const abs of walk(this.inputsDir)) {
+      if (opts?.prefix && !abs.startsWith(prefixDir)) continue;
+      const kind = detectKind(abs);
+      if (!kind) continue;
+      const st = await stat(abs);
+      yield {
+        collector: this.id,
+        uri: pathToFileURL(abs).href,
+        kind,
+        bytes: st.size,
+        modified: st.mtime.toISOString(),
+        metadata: {},
+      };
+      remaining -= 1;
+      if (remaining <= 0) return;
+    }
+  }
+
+  async open(ref: DataSourceRef): Promise<DataSource> {
+    const abs = fileURLToPath(ref.uri);
+    const kind = ref.kind;
+    return {
+      ref,
+      async load(): Promise<Buffer | string | object> {
+        if (kind === 'json') {
+          return JSON.parse(await readFile(abs, 'utf8')) as object;
+        }
+        if (TEXT_KINDS.has(kind)) {
+          return readFile(abs, 'utf8');
+        }
+        return readFile(abs); // Buffer for xlsx/parquet
+      },
+    };
+  }
+
+  watch(ref: DataSourceRef, onChange: (ref: DataSourceRef) => void): Unsubscribe {
+    const abs = fileURLToPath(ref.uri);
+    const watcher = chokidarWatch(abs, { ignoreInitial: true });
+    const handler = async (): Promise<void> => {
+      try {
+        const st = await stat(abs);
+        onChange({ ...ref, bytes: st.size, modified: st.mtime.toISOString() });
+      } catch {
+        onChange(ref);
+      }
+    };
+    watcher.on('change', () => void handler());
+    watcher.on('add', () => void handler());
+    return () => {
+      void watcher.close();
+    };
+  }
+}
+
+async function* walk(dir: string): AsyncIterable<string> {
+  let entries: Dirent[];
+  try {
+    entries = await readdir(dir, { withFileTypes: true });
+  } catch {
+    return; // inputs/ may not exist yet — yield nothing
+  }
+  for (const entry of entries) {
+    const full = join(dir, entry.name);
+    if (entry.isDirectory()) {
+      yield* walk(full);
+    } else if (entry.isFile()) {
+      yield full;
+    }
+  }
+}

package/packages/coworker-scratchpad/src/index.ts

@@ -0,0 +1,74 @@
+export { detectKind, FILE_COLLECTOR_KINDS } from './detect-kind.js';
+export { FileCollector, type FileCollectorOptions } from './file-collector.js';
+export { DefaultCollectorRegistry, uriMatchesPattern } from './collector-registry.js';
+export {
+  isDataLoadEvent,
+  isProgressEvent,
+  isStartupErrorEvent,
+  isSnapshotResult,
+  type RunRequest,
+  type SnapshotRequest,
+  type KernelRequest,
+  type ResultOk,
+  type ResultErr,
+  type ResultResponse,
+  type SnapshotResultOk,
+  type SnapshotResultErr,
+  type SnapshotResult,
+  type SkippedKey,
+  type RecoveryNote,
+  type StartupErrorEvent,
+  type DataLoadDrawer,
+  type ArtifactCreateDrawer,
+  type ReadyEvent,
+  type DataLoadEvent,
+  type ProgressEvent,
+  type KernelEvent,
+  type KernelFrame,
+} from './kernel-protocol.js';
+export { filterEnv, kernelExecArgv, resolveKernelEntry } from './kernel-spawn.js';
+export {
+  ChildProcessRuntime,
+  type CellResult,
+  type ChildProcessRuntimeOptions,
+} from './child-process-runtime.js';
+export {
+  ScratchpadManager,
+  ForkKernelHangError,
+  type ScratchpadManagerOptions,
+  type AttachOptions,
+  type ScratchpadInfo,
+} from './scratchpad-manager.js';
+export {
+  acquireLock,
+  releaseLock,
+  readLock,
+  isStaleLock,
+  ScratchpadBusyError,
+  type LockInfo,
+  type AcquireOptions,
+} from './scratchpad-lock.js';
+export {
+  CellArchive,
+  CELLS_SCHEMA_VERSION,
+  type CellEntry,
+  type AppendInput,
+} from './cell-archive.js';
+export { buildDataLibBindings } from './kernel-bindings.js';
+export {
+  NAMESPACE_SCHEMA_VERSION,
+  encodeNamespace,
+  decodeNamespace,
+  type NamespaceEnvelope,
+  type EncodeResult,
+  type DecodeResult,
+} from './namespace-codec.js';
+export {
+  projectTree,
+  findLeaves,
+  validateLeafId,
+  formatTreeText,
+  type TreeNode,
+  type CellTree,
+} from './cell-tree.js';
+export { StalenessBanner, type StalenessCheck } from './staleness-banner.js';

package/packages/coworker-scratchpad/src/kernel-bindings.test.ts

@@ -0,0 +1,188 @@
+import { describe, it, beforeEach, afterEach } from 'node:test';
+import assert from 'node:assert/strict';
+import { mkdtemp, mkdir, rm } from 'node:fs/promises';
+import { mkdtempSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { AuditLog, type AuditRecord } from '@otto/coworker-utils';
+import { buildDataLibBindings, redactForJournal } from './kernel-bindings.js';
+import { ChildProcessRuntime } from './child-process-runtime.js';
+
+describe('kernel-bindings', () => {
+  it('exposes all seven pre-bound data libraries', () => {
+    const b = buildDataLibBindings();
+    for (const key of ['polars', 'DuckDB', 'ExcelJS', 'dateFns', 'lodash', 'zod', 'axios']) {
+      assert.ok(key in b, `missing binding: ${key}`);
+      assert.notEqual(b[key], undefined, `binding is undefined: ${key}`);
+    }
+  });
+
+  it('binds usable shapes (polars.DataFrame, zod.string, dateFns.format)', () => {
+    const b = buildDataLibBindings() as Record<string, any>;
+    assert.equal(typeof b.polars.DataFrame, 'function');
+    assert.equal(typeof b.zod.string, 'function');
+    assert.equal(typeof b.dateFns.format, 'function');
+    assert.equal(typeof b.lodash.chunk, 'function');
+    assert.equal(typeof b.axios.get, 'function');
+    assert.equal(typeof b.ExcelJS.Workbook, 'function');
+  });
+});
+
+describe('otto.duckdb.registerDf (Task F)', () => {
+  let ws: string;
+  let sp: string;
+  let rt: ChildProcessRuntime | undefined;
+
+  beforeEach(async () => {
+    ws = await mkdtemp(join(tmpdir(), 'kb-registerdf-ws-'));
+    await mkdir(join(ws, '.otto', 'inputs'), { recursive: true });
+    sp = await mkdtemp(join(tmpdir(), 'kb-registerdf-sp-'));
+    rt = new ChildProcessRuntime({
+      workspace: ws,
+      scratchpadDir: sp,
+      cellTimeoutMs: 30_000,
+      inactivityTimeoutMs: 30_000,
+    });
+    await rt.start();
+  });
+
+  afterEach(async () => {
+    await rt?.dispose();
+    rt = undefined;
+    await rm(ws, { recursive: true, force: true });
+    await rm(sp, { recursive: true, force: true });
+  });
+
+  const runCell = async (code: string): Promise<unknown> => {
+    const { value } = await rt!.runCell(code);
+    return value;
+  };
+
+  it('round-trips an array of records', async () => {
+    const result = (await runCell(`
+      await otto.duckdb.registerDf('rec', [{a: 1, b: 'x'}, {a: 2, b: 'y'}]);
+      const c = await otto.duckdb.connect();
+      return (await c.runAndReadAll('SELECT SUM(a) FROM rec')).getRows();
+    `)) as unknown[][];
+    assert.equal(Number(result[0]![0]), 3);
+  });
+
+  it('round-trips a polars DataFrame', async () => {
+    const result = (await runCell(`
+      const df = polars.DataFrame({ a: [1, 2, 3], b: ['x', 'y', 'z'] });
+      await otto.duckdb.registerDf('pdf', df);
+      const c = await otto.duckdb.connect();
+      return (await c.runAndReadAll('SELECT SUM(a) FROM pdf')).getRows();
+    `)) as unknown[][];
+    assert.equal(Number(result[0]![0]), 6);
+  });
+
+  it('throws TypeError for unsupported input', async () => {
+    await assert.rejects(
+      runCell(`await otto.duckdb.registerDf('bad', 42);`),
+      /must be a polars DataFrame, Arrow Table, or array of records/,
+    );
+  });
+
+  it('opts.schema override skips inference and uses provided types', async () => {
+    const result = (await runCell(`
+      await otto.duckdb.registerDf(
+        'sized',
+        [{n: 1}, {n: 2}, {n: 3}],
+        { schema: { n: 'BIGINT' } }
+      );
+      const c = await otto.duckdb.connect();
+      const desc = await c.runAndReadAll('DESCRIBE sized');
+      return desc.getRows();
+    `)) as unknown[][];
+    assert.equal(result[0]![0], 'n');
+    assert.equal(String(result[0]![1]).toUpperCase(), 'BIGINT');
+  });
+
+  it('null-walk inference picks the first non-null value type', async () => {
+    const result = (await runCell(`
+      const rows = Array(8).fill({ rev: null }).concat([{ rev: 1200 }, { rev: 980 }]);
+      await otto.duckdb.registerDf('rev', rows);
+      const c = await otto.duckdb.connect();
+      const desc = await c.runAndReadAll('DESCRIBE rev');
+      return desc.getRows();
+    `)) as unknown[][];
+    assert.equal(result[0]![0], 'rev');
+    assert.equal(String(result[0]![1]).toUpperCase(), 'DOUBLE');
+  });
+
+  it('partial-failure leaves no table behind (all-or-nothing) and names failing column', async () => {
+    // First call: BIGINT schema with a non-coercible string value in row 11 → append fails mid-batch.
+    // The all-or-nothing semantic should DROP the partially-populated table so the
+    // retry below with the same name + clean data succeeds without "Table already exists."
+    await assert.rejects(
+      runCell(`
+        const rows = [];
+        for (let i = 0; i < 11; i++) rows.push({ n: i });
+        rows.push({ n: 'not-a-number' });
+        await otto.duckdb.registerDf('aon', rows, { schema: { n: 'BIGINT' } });
+      `),
+      /registerDf row 11: append failed for column 'n'/,
+    );
+    const result = (await runCell(`
+      await otto.duckdb.registerDf('aon', [{n: 7}, {n: 8}], { schema: { n: 'BIGINT' } });
+      const c = await otto.duckdb.connect();
+      // Cast to DOUBLE so the value survives JSON serialization across the
+      // child-process boundary (BIGINT sums become BigInt and JSON-stringify-throw).
+      return (await c.runAndReadAll('SELECT CAST(SUM(n) AS DOUBLE) FROM aon')).getRows();
+    `)) as unknown[][];
+    // Sum of clean retry batch — proves both that the first-call table was DROPped
+    // (otherwise CREATE TABLE here would fail) and that the retry persisted rows.
+    assert.equal(Number(result[0]![0]), 15);
+  });
+});
+
+describe('kernel-bindings — secret redaction (Phase 2 Task 14)', () => {
+  it('redacts secret patterns in stdout before journaling and emits one audit record per hit', async () => {
+    const auditPath = join(mkdtempSync(join(tmpdir(), 'kb-audit-')), 'audit.jsonl');
+    const audit = new AuditLog({ path: auditPath });
+    // AKIA + 16 [A-Z0-9] chars — matches the aws_access_key_id pattern.
+    const raw = 'before AKIAABCDEFGHIJKLMNOP after';
+    const result = redactForJournal(raw, {
+      audit,
+      sessionId: 's',
+      scratchpadName: 'sp',
+      pid: 1,
+      cellId: 'c1',
+    });
+    assert.equal(result, 'before [REDACTED:aws_access_key_id] after');
+    const records: AuditRecord[] = [];
+    for await (const r of audit.read({ producer: 'secret-scanner' })) records.push(r);
+    assert.equal(records.length, 1);
+    const rec = records[0]!;
+    assert.equal(rec.action, 'redact');
+    assert.equal(rec.severity, 'warn');
+    assert.equal(rec.scratchpadName, 'sp');
+    assert.equal(rec.sessionId, 's');
+    assert.equal(rec.pid, 1);
+    // Detail must carry kind/offset/length/cell_id and NOT include the secret value or preview.
+    assert.equal(rec.detail.kind, 'aws_access_key_id');
+    assert.equal(rec.detail.cell_id, 'c1');
+    assert.equal(rec.detail.offset, 'before '.length);
+    assert.equal(rec.detail.length, 'AKIAABCDEFGHIJKLMNOP'.length);
+    assert.equal('preview' in rec.detail, false);
+    // Defense-in-depth: the serialized record must not contain the raw secret substring.
+    assert.equal(JSON.stringify(rec).includes('AKIAABCDEFGHIJKLMNOP'), false);
+  });
+
+  it('returns input unchanged when no secrets present and writes no audit record', async () => {
+    const auditPath = join(mkdtempSync(join(tmpdir(), 'kb-audit-clean-')), 'audit.jsonl');
+    const audit = new AuditLog({ path: auditPath });
+    const out = redactForJournal('hello world', {
+      audit,
+      sessionId: 's',
+      scratchpadName: 'sp',
+      pid: 1,
+      cellId: 'c1',
+    });
+    assert.equal(out, 'hello world');
+    const records: AuditRecord[] = [];
+    for await (const r of audit.read({ producer: 'secret-scanner' })) records.push(r);
+    assert.equal(records.length, 0);
+  });
+});