@cmetech/otto 1.1.1 → 1.2.4

package/packages/coworker-scratchpad/dist/kernel-bindings.d.ts

@@ -0,0 +1,49 @@
+import type { DuckDBInstance } from '@duckdb/node-api';
+import { type AuditLog } from '@otto/coworker-utils';
+/**
+ * The data libraries pre-bound into every scratchpad cell's vm sandbox.
+ * DuckDB is bound as an in-memory-capable lib here; on-disk kernel.db wiring is 1d2.
+ */
+export declare function buildDataLibBindings(): Record<string, unknown>;
+/** SQL column type string passed straight into CREATE TABLE (e.g. VARCHAR, BIGINT). */
+export type DuckDBColumnType = string;
+export interface RegisterDfOptions {
+    /**
+     * Explicit per-column SQL types. When omitted, the schema is inferred from
+     * the first non-null value of each column in the first 10 rows.
+     */
+    schema?: Record<string, DuckDBColumnType> | Array<[string, DuckDBColumnType]>;
+}
+/**
+ * Patches a DuckDBInstance with a `registerDf(name, input, opts?)` method
+ * so the LLM can drop a polars DataFrame / Arrow Table / array of records
+ * into DuckDB in one call instead of discovering the appender API.
+ *
+ * The method opens a fresh connection on each call (cheap — DuckDB connections
+ * share the underlying instance/db) and CREATE-TABLE-then-appends.
+ */
+export declare function attachRegisterDf(instance: DuckDBInstance): void;
+/**
+ * Context passed to `redactForJournal`. The audit instance is the same one
+ * held by the CredentialInjector — supplied by the manager so secret-redaction
+ * audit records land alongside vault inject/inject-skipped records.
+ */
+export interface RedactionContext {
+    audit: AuditLog;
+    sessionId: string;
+    scratchpadName: string;
+    pid: number;
+    cellId: string;
+}
+/**
+ * Scans `raw` for known secret patterns; for each hit, appends a
+ * `producer: 'secret-scanner', action: 'redact'` audit record (severity warn)
+ * with `detail = { cell_id, kind, offset, length }` — NEVER the secret value
+ * or `SecretHit.preview`. Returns the redacted text (`[REDACTED:<kind>]` per
+ * hit). If no hits, returns the input verbatim with no audit emission.
+ *
+ * Backward compatibility: callers that don't have an AuditLog should skip this
+ * helper entirely (pass-through). Audit is required here because the redaction
+ * MUST be observable.
+ */
+export declare function redactForJournal(raw: string, ctx: RedactionContext): string;

package/packages/coworker-scratchpad/dist/kernel-bindings.js

@@ -0,0 +1,220 @@
+import pl from 'nodejs-polars';
+import ExcelJS from 'exceljs';
+import lodash from 'lodash';
+import axios from 'axios';
+import { z } from 'zod';
+import * as dateFns from 'date-fns';
+import * as DuckDB from '@duckdb/node-api';
+import { SecretScanner } from '@otto/coworker-utils';
+/**
+ * The data libraries pre-bound into every scratchpad cell's vm sandbox.
+ * DuckDB is bound as an in-memory-capable lib here; on-disk kernel.db wiring is 1d2.
+ */
+export function buildDataLibBindings() {
+    return {
+        polars: pl,
+        DuckDB,
+        ExcelJS,
+        dateFns,
+        lodash,
+        zod: z,
+        axios,
+    };
+}
+function isPolarsDataFrame(x) {
+    return (!!x &&
+        typeof x === 'object' &&
+        typeof x.toRecords === 'function' &&
+        typeof x.width === 'number' &&
+        typeof x.height === 'number');
+}
+function isArrowTable(x) {
+    return (!!x &&
+        typeof x === 'object' &&
+        typeof x.toArray === 'function' &&
+        typeof x.numRows === 'number' &&
+        'schema' in x);
+}
+function coerceToRecords(input) {
+    if (isPolarsDataFrame(input))
+        return input.toRecords();
+    if (isArrowTable(input))
+        return input.toArray();
+    if (Array.isArray(input))
+        return input;
+    throw new TypeError('registerDf: input must be a polars DataFrame, Arrow Table, or array of records');
+}
+function inferSchema(records) {
+    if (records.length === 0) {
+        throw new Error('registerDf: cannot infer schema from empty input. Provide opts.schema or pass at least one row.');
+    }
+    const cols = Object.keys(records[0]);
+    const out = {};
+    for (const col of cols) {
+        // Null-walk: scan up to the first 10 rows for the first non-null value of this column.
+        // Defaults to VARCHAR for all-null columns (safe — DuckDB will accept NULL into a VARCHAR).
+        out[col] = 'VARCHAR';
+        const walk = Math.min(10, records.length);
+        for (let i = 0; i < walk; i++) {
+            const v = records[i][col];
+            if (v === null || v === undefined)
+                continue;
+            if (typeof v === 'string') {
+                out[col] = 'VARCHAR';
+                break;
+            }
+            if (typeof v === 'bigint') {
+                out[col] = 'BIGINT';
+                break;
+            }
+            if (typeof v === 'number') {
+                out[col] = 'DOUBLE';
+                break;
+            }
+            if (typeof v === 'boolean') {
+                out[col] = 'BOOLEAN';
+                break;
+            }
+            if (v instanceof Date) {
+                out[col] = 'TIMESTAMP';
+                break;
+            }
+            out[col] = 'VARCHAR'; // unknown object → store as VARCHAR
+            break;
+        }
+    }
+    return out;
+}
+function normalizeSchema(schema) {
+    if (Array.isArray(schema))
+        return schema;
+    return Object.entries(schema);
+}
+function quoteIdent(name) {
+    // DuckDB identifier quoting: wrap in double quotes, escape embedded double quotes.
+    return `"${name.replace(/"/g, '""')}"`;
+}
+async function registerViaAppender(conn, name, records, schema, sourceHint) {
+    const ddl = `CREATE TABLE ${quoteIdent(name)} (${schema
+        .map(([c, t]) => `${quoteIdent(c)} ${t}`)
+        .join(', ')})`;
+    await conn.run(ddl);
+    // All-or-nothing semantic: on any failure we drop the table so the caller
+    // can retry the same name without hitting "Table already exists." On the
+    // success path we flip this flag and the finally becomes a no-op.
+    let createdTable = true;
+    const app = await conn.createAppender(name);
+    try {
+        for (let i = 0; i < records.length; i++) {
+            let failingColIdx = -1;
+            try {
+                for (let colIdx = 0; colIdx < schema.length; colIdx++) {
+                    failingColIdx = colIdx;
+                    const col = schema[colIdx][0];
+                    const v = records[i][col];
+                    if (v === null || v === undefined) {
+                        app.appendNull();
+                    }
+                    else {
+                        // appendValue accepts DuckDBValue (null | boolean | number | bigint | string | ...)
+                        // and lets the engine coerce based on the declared column type.
+                        app.appendValue(v);
+                    }
+                }
+                // endRow() may itself reject the row on a type mismatch detected at
+                // end-of-row; in that case we can't attribute to a single column so
+                // we fall back to '<column>' literally rather than blame col 0.
+                failingColIdx = -1;
+                app.endRow();
+            }
+            catch (e) {
+                const failingCol = failingColIdx >= 0 ? schema[failingColIdx][0] : '<column>';
+                const hint = sourceHint === 'inferred'
+                    ? ` Pass an explicit schema via the third argument: registerDf(name, df, { schema: { ${failingCol}: 'VARCHAR' } })`
+                    : '';
+                throw new Error(`registerDf row ${i}: append failed for column '${failingCol}' (${e.message}).${hint}`);
+            }
+        }
+        // Success path: close the appender to commit the rows, then disarm cleanup.
+        app.closeSync();
+        createdTable = false;
+    }
+    finally {
+        if (createdTable) {
+            // Failure path: close the appender (may already be closed; best-effort)
+            // then drop the partially-populated table.
+            try {
+                app.closeSync();
+            }
+            catch { /* already closed or errored */ }
+            try {
+                await conn.run(`DROP TABLE IF EXISTS ${quoteIdent(name)}`);
+            }
+            catch { /* best-effort */ }
+        }
+    }
+}
+/**
+ * Patches a DuckDBInstance with a `registerDf(name, input, opts?)` method
+ * so the LLM can drop a polars DataFrame / Arrow Table / array of records
+ * into DuckDB in one call instead of discovering the appender API.
+ *
+ * The method opens a fresh connection on each call (cheap — DuckDB connections
+ * share the underlying instance/db) and CREATE-TABLE-then-appends.
+ */
+export function attachRegisterDf(instance) {
+    const patched = instance;
+    patched.registerDf = async function registerDf(name, input, opts = {}) {
+        const conn = await instance.connect();
+        try {
+            const records = coerceToRecords(input);
+            const sourceHint = opts.schema ? 'explicit' : 'inferred';
+            const schemaObj = opts.schema ?? inferSchema(records);
+            const schema = normalizeSchema(schemaObj);
+            await registerViaAppender(conn, name, records, schema, sourceHint);
+        }
+        finally {
+            // Release the connection — DuckDBConnection exposes closeSync(): void.
+            // Without this, every registerDf call leaks one connection over the
+            // lifetime of the scratchpad session.
+            conn.closeSync();
+        }
+    };
+}
+const journalScanner = new SecretScanner();
+/**
+ * Scans `raw` for known secret patterns; for each hit, appends a
+ * `producer: 'secret-scanner', action: 'redact'` audit record (severity warn)
+ * with `detail = { cell_id, kind, offset, length }` — NEVER the secret value
+ * or `SecretHit.preview`. Returns the redacted text (`[REDACTED:<kind>]` per
+ * hit). If no hits, returns the input verbatim with no audit emission.
+ *
+ * Backward compatibility: callers that don't have an AuditLog should skip this
+ * helper entirely (pass-through). Audit is required here because the redaction
+ * MUST be observable.
+ */
+export function redactForJournal(raw, ctx) {
+    const hits = journalScanner.scan(raw);
+    if (hits.length === 0)
+        return raw;
+    const ts = new Date().toISOString();
+    for (const h of hits) {
+        ctx.audit.append({
+            _schema: 1,
+            ts,
+            producer: 'secret-scanner',
+            action: 'redact',
+            severity: 'warn',
+            sessionId: ctx.sessionId,
+            scratchpadName: ctx.scratchpadName,
+            pid: ctx.pid,
+            detail: {
+                cell_id: ctx.cellId,
+                kind: h.kind,
+                offset: h.start,
+                length: h.end - h.start,
+            },
+        });
+    }
+    return journalScanner.redact(raw);
+}

package/packages/coworker-scratchpad/dist/kernel-entry.d.ts

@@ -0,0 +1 @@
+export {};

package/packages/coworker-scratchpad/dist/kernel-entry.js

@@ -0,0 +1,355 @@
+import process, { argv, stdin, stdout } from 'node:process';
+import vm from 'node:vm';
+import { existsSync, readFileSync, renameSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { DuckDBInstance } from '@duckdb/node-api';
+import { writeNdjson, readNdjson } from '@otto/coworker-utils';
+import { DefaultCollectorRegistry } from './collector-registry.js';
+import { FileCollector } from './file-collector.js';
+import { isKernelRpcResponse } from './kernel-protocol.js';
+import { buildDataLibBindings, attachRegisterDf } from './kernel-bindings.js';
+import { encodeNamespace, decodeNamespace } from './namespace-codec.js';
+const workspace = argv[2] ?? process.cwd();
+const scratchpadDir = argv[3];
+const trace = process.env.OTTO_SCRATCHPAD_IPC_TRACE === '1';
+const KNOWN_BOUND_KEYS = new Set([
+    'otto',
+    'console',
+    'progress',
+    'setTimeout',
+    'clearTimeout',
+    'setInterval',
+    'clearInterval',
+    'polars',
+    'DuckDB',
+    'ExcelJS',
+    'dateFns',
+    'lodash',
+    'zod',
+    'axios',
+    'Date',
+    // Phase 2 Task 13: process is bound so cells can read OTTO_DS_* env vars
+    // injected by CredentialInjector. Filtered from namespace snapshots.
+    'process',
+    'Buffer',
+]);
+const registry = new DefaultCollectorRegistry();
+registry.register(new FileCollector({ workspace }));
+process.on('SIGINT', () => {
+    // Ignored on purpose. A stray cancel between cells must not tear down a healthy
+    // kernel; the parent escalates to SIGTERM/SIGKILL to actually stop a hung kernel.
+});
+function send(frame) {
+    if (trace)
+        process.stderr.write(`[kernel→] ${JSON.stringify(frame)}\n`);
+    void writeNdjson(stdout, frame);
+}
+const pendingRpc = new Map();
+let nextRpcId = 1;
+function newRpcId() {
+    return `art-${process.pid}-${nextRpcId++}`;
+}
+function rpcRequest(payload) {
+    return new Promise((resolve, reject) => {
+        pendingRpc.set(payload.id, {
+            resolve: resolve,
+            reject,
+        });
+        send(payload);
+    });
+}
+// Returns true when the frame was consumed as an RPC response.
+function tryRouteRpcResponse(frame) {
+    if (!isKernelRpcResponse(frame))
+        return false;
+    const p = pendingRpc.get(frame.id);
+    if (!p)
+        return true; // unknown id — silently drop; nothing to do
+    pendingRpc.delete(frame.id);
+    if (frame.ok === false)
+        p.reject(new Error(frame.error));
+    else
+        p.resolve(frame);
+    return true;
+}
+const ottoCollectors = {
+    async list() {
+        const refs = [];
+        for (const collector of registry.list()) {
+            for await (const ref of collector.list())
+                refs.push(ref);
+        }
+        return refs;
+    },
+    async open(uri) {
+        const hit = await registry.resolve(uri);
+        if (!hit)
+            throw new Error(`no collector resolves uri: ${uri}`);
+        const source = await hit.collector.open(hit.ref);
+        return {
+            ref: source.ref,
+            async load() {
+                const value = await source.load();
+                send({
+                    type: 'event',
+                    event: 'data_load',
+                    drawer: {
+                        kind: 'data_load',
+                        collector: source.ref.collector,
+                        uri: source.ref.uri,
+                        bytes: source.ref.bytes ?? null,
+                        rows_loaded: Array.isArray(value) ? value.length : null,
+                        loaded_at: new Date().toISOString(),
+                        schema: null,
+                    },
+                });
+                return value;
+            },
+        };
+    },
+};
+function makeArtifactProxy(args) {
+    return {
+        slug: args.slug,
+        uri: args.uri,
+        primaryPath: args.primaryPath,
+        async update(files) {
+            const req = {
+                type: 'request',
+                request: 'artifact_update',
+                id: newRpcId(),
+                slug: args.slug,
+                files,
+            };
+            const resp = await rpcRequest(req);
+            if (resp.ok === false)
+                throw new Error(`artifact_update failed: ${resp.error}`);
+            return { files_touched: resp.files_touched };
+        },
+    };
+}
+const ottoArtifact = {
+    async create(kind, name) {
+        if (kind !== 'report') {
+            throw new Error(`unsupported artifact kind: ${kind}. v1 ships only 'report'.`);
+        }
+        const req = {
+            type: 'request',
+            request: 'artifact_create',
+            id: newRpcId(),
+            kind,
+            name,
+        };
+        const resp = await rpcRequest(req);
+        if (resp.ok === false)
+            throw new Error(`artifact_create failed: ${resp.error}`);
+        // Broadcast artifact_create event so the manager can call recordArtifact
+        // into memory (Phase 4 Task 12 closure: getMemoryRecorder()?.recordArtifact).
+        // Without this, Layer B is silent for artifacts.
+        send({
+            type: 'event',
+            event: 'artifact_create',
+            drawer: {
+                kind: 'artifact',
+                slug: resp.slug,
+                artifact_kind: kind,
+                uri: resp.uri,
+                primary_path: resp.primary_path,
+                created_at: new Date().toISOString(),
+            },
+        });
+        return makeArtifactProxy({
+            slug: resp.slug,
+            uri: resp.uri,
+            primaryPath: resp.primary_path,
+        });
+    },
+    async spillIfLarge(value, opts) {
+        const threshold = opts?.thresholdBytes ?? 10_240;
+        let serialized;
+        try {
+            serialized = typeof value === 'string' ? value : JSON.stringify(value, null, 2);
+        }
+        catch {
+            serialized = String(value);
+        }
+        if (Buffer.byteLength(serialized, 'utf8') < threshold)
+            return null;
+        const name = opts?.name ??
+            `cell-output-${new Date().toISOString().slice(0, 19).replace(/[T:]/g, '-')}`;
+        const handle = await ottoArtifact.create('report', name);
+        await handle.update([{ path: 'report.md', content: serialized }]);
+        return handle;
+    },
+};
+const otto = { collectors: ottoCollectors, artifact: ottoArtifact };
+const sandbox = {
+    otto,
+    ...buildDataLibBindings(),
+    // Timers are not part of a fresh vm context; bind the host ones so async cells
+    // (await new Promise((r) => setTimeout(r, ...))) and progress() heartbeats work.
+    setTimeout,
+    clearTimeout,
+    setInterval,
+    clearInterval,
+    // Bind the host Date so that v8-deserialized Date objects (from namespace restore)
+    // pass `instanceof Date` checks inside the vm context (cross-realm fix).
+    Date,
+    // Phase 2 Task 13: cells need process.env to read OTTO_DS_* vault-injected
+    // env vars. Bind the host process directly; vm sandbox isolation otherwise
+    // hides Node globals. Buffer is bound alongside since cells frequently
+    // base64-encode credentials (e.g. Basic auth from env vars).
+    process,
+    Buffer,
+};
+const context = vm.createContext(sandbox);
+async function runCell(code) {
+    const logs = [];
+    sandbox.console = {
+        log: (...args) => logs.push(args.map((a) => String(a)).join(' ')),
+        error: (...args) => logs.push(args.map((a) => String(a)).join(' ')),
+    };
+    sandbox.progress = (message) => {
+        send({
+            type: 'event',
+            event: 'progress',
+            message: message === undefined ? undefined : String(message),
+        });
+    };
+    const wrapped = `(async () => {\n${code}\n})()`;
+    const value = await vm.runInContext(wrapped, context, { filename: 'cell.js' });
+    return { value, stdout: logs.join('\n') };
+}
+function toSerializable(value) {
+    try {
+        return JSON.parse(JSON.stringify(value ?? null));
+    }
+    catch {
+        return { valuePreview: String(value) };
+    }
+}
+async function openKernelDb(dir) {
+    try {
+        const instance = await DuckDBInstance.create(join(dir, 'kernel.db'));
+        attachRegisterDf(instance);
+        otto.duckdb = instance;
+    }
+    catch (err) {
+        const e = err;
+        await writeNdjson(stdout, {
+            type: 'event',
+            event: 'startup_error',
+            kind: 'duckdb_open',
+            error: { name: e.name, message: e.message },
+        });
+        process.exit(1);
+    }
+}
+function restoreNamespace(dir) {
+    const path = join(dir, 'namespace.json');
+    if (!existsSync(path))
+        return [{ kind: 'namespace-absent' }];
+    let raw;
+    try {
+        raw = readFileSync(path, 'utf8');
+    }
+    catch (err) {
+        return [{ kind: 'namespace-corrupt', message: err.message }];
+    }
+    try {
+        const { values } = decodeNamespace(raw);
+        for (const key of Object.keys(values)) {
+            if (KNOWN_BOUND_KEYS.has(key))
+                continue; // never let a tampered namespace.json overwrite host bindings
+            sandbox[key] = values[key];
+        }
+        return [];
+    }
+    catch (err) {
+        return [{ kind: 'namespace-corrupt', message: err.message }];
+    }
+}
+function writeNamespaceSnapshot(dir) {
+    // Enumerate live globalThis additions, excluding the known-bound surface.
+    const live = {};
+    for (const key of Object.keys(sandbox)) {
+        if (KNOWN_BOUND_KEYS.has(key))
+            continue;
+        live[key] = sandbox[key];
+    }
+    const { envelope, skipped } = encodeNamespace(live, () => Date.now());
+    const nsPath = join(dir, 'namespace.json');
+    const tmp = `${nsPath}.tmp`;
+    writeFileSync(tmp, JSON.stringify(envelope));
+    renameSync(tmp, nsPath);
+    return { skipped, snapshotted_at: envelope.ts };
+}
+async function main() {
+    const recoveryNotes = [];
+    if (scratchpadDir !== undefined) {
+        await openKernelDb(scratchpadDir);
+        recoveryNotes.push(...restoreNamespace(scratchpadDir));
+    }
+    send({ type: 'event', event: 'ready', recovery_notes: recoveryNotes });
+    try {
+        for await (const raw of readNdjson(stdin)) {
+            if (trace)
+                process.stderr.write(`[kernel←] ${JSON.stringify(raw)}\n`);
+            // Phase 4 Task 9 — route artifact RPC responses to the pending map before
+            // falling through to the existing run/snapshot request dispatch.
+            if (tryRouteRpcResponse(raw))
+                continue;
+            const req = raw;
+            if (req.type === 'snapshot') {
+                if (scratchpadDir === undefined) {
+                    send({ id: req.id, type: 'snapshot_result', ok: true, skipped: [], snapshotted_at: new Date().toISOString() });
+                    continue;
+                }
+                try {
+                    const { skipped, snapshotted_at } = writeNamespaceSnapshot(scratchpadDir);
+                    send({ id: req.id, type: 'snapshot_result', ok: true, skipped, snapshotted_at });
+                }
+                catch (err) {
+                    const e = err;
+                    send({ id: req.id, type: 'snapshot_result', ok: false, error: { name: e.name, message: e.message } });
+                }
+                continue;
+            }
+            if (req.type !== 'run')
+                continue;
+            // Phase 4 Task 10 fix — `runCell` may issue an artifact RPC and await its
+            // response. The response arrives over stdin, which means the main loop
+            // MUST keep iterating to route it via tryRouteRpcResponse. Awaiting
+            // runCell inline would deadlock the kernel against its own RPC. Fire it
+            // off; runCell sends its own result frame. The manager already serializes
+            // cell submissions, so there's no concurrency to worry about.
+            void (async () => {
+                let res;
+                try {
+                    const { value, stdout: out } = await runCell(req.code);
+                    res = { id: req.id, type: 'result', ok: true, value: toSerializable(value), stdout: out };
+                }
+                catch (err) {
+                    const e = err;
+                    res = {
+                        id: req.id,
+                        type: 'result',
+                        ok: false,
+                        error: { name: e.name, message: e.message, stack: e.stack },
+                    };
+                }
+                send(res);
+            })();
+        }
+    }
+    finally {
+        // Phase 4 Task 9 review fix — if stdin closes (manager died / EOF) while
+        // RPCs are still in flight, reject them so awaiting cells fail fast
+        // instead of hanging until SIGKILL.
+        for (const [, p] of pendingRpc) {
+            p.reject(new Error('kernel: stdin closed with pending RPC'));
+        }
+        pendingRpc.clear();
+    }
+}
+void main();