@tiny-fish/cli 0.51.2-next.417 → 0.52.0

package/README.md

@@ -442,6 +442,14 @@ checks npm once a day for a newer CLI. Set `TINYFISH_NO_TELEMETRY` to any value
 TinyFish-owned origins (`*.tinyfish.ai`, `*.tinyfish.io`) or a local development endpoint
 (`localhost`, `127.0.0.1`, `[::1]`).
 
+### Datasets
+
+`tinyfish dataset create "prompt"` plans a schema, creates the dataset and follows the run: rows print as they land (`--pretty` prints them tab-separated in column order, status and log lines go to stderr; JSON mode prints one `{event, data}` line per frame and the final dataset last). `--rows N` sets the target (1-250, default 100); `--async` returns the 202 dataset at once. To review the schema first, `tinyfish dataset plan "prompt" > spec.json`, edit it, then `tinyfish dataset create --from-spec spec.json`.
+
+Manage datasets with `tinyfish dataset list`, `get <id>`, `rows <id> [--limit] [--after] [--all]`, `populate <id> [--rows N] [--reset] [--async]`, `watch <id> <run_id>`, `cancel <id> <run_id>` and `delete <id>`. `rows --all` follows `next_cursor` to the last page. `populate` and `delete` are refused with a 409 while a run is live.
+
+Ctrl+C during `create` or `populate` cancels the run it started; during `watch` it only detaches. A failed or cancelled run exits unsuccessfully with the run's error. A stream that closes while the run is still live is picked up again, up to three times; after that the recovery hint names `watch`.
+
 ## CI/CD
 
 ```yaml

package/dist/commands/dataset.d.ts

@@ -0,0 +1,2 @@
+import { Command } from 'commander';
+export declare function registerDataset(program: Command): void;

package/dist/commands/dataset.js

@@ -0,0 +1,268 @@
+import { readFileSync } from 'node:fs';
+import { withResolvedKey } from '../lib/auth.js';
+import * as api from '../lib/client.js';
+import * as datasets from '../lib/datasets.js';
+import { integerInRange } from '../lib/list-options.js';
+import { ApiError, errLine, handleApiError, out, outLine, parseJson } from '../lib/output.js';
+import { installSignalGuard, onBrokenPipe } from '../lib/signals.js';
+const datasetLine = (d) => `${d.id}  ${d.status}  ${d.row_count}/${d.target_rows}  ${d.name}`;
+function printDataset(d, pretty, columns = false) {
+    if (!pretty)
+        return out(d);
+    outLine(datasetLine(d));
+    if (d.run)
+        outLine(`Run: ${d.run.id}  ${d.run.status}${d.run.error ? `: ${d.run.error}` : ''}`);
+    if (columns) {
+        const names = d.spec.columns.map((c) => `${c.name}(${c.type})${c.is_primary_key ? '*' : ''}`);
+        outLine(`Columns: ${names.join(', ')}`);
+    }
+}
+const cell = (value) => value === null || value === undefined
+    ? ''
+    : typeof value === 'object'
+        ? JSON.stringify(value)
+        : String(value);
+function printRows(page, pretty, header = true) {
+    if (!pretty)
+        return out(page);
+    const names = page.columns.map((c) => c.name);
+    if (header)
+        outLine(names.join('\t'));
+    for (const row of page.rows)
+        outLine(names.map((name) => cell(row.data[name])).join('\t'));
+}
+function renderFrame(frame, columns, pretty) {
+    if (!pretty)
+        return out(frame);
+    const { event, data } = frame;
+    if (event !== 'row')
+        return noteFrame(event, data);
+    const row = datasets.rowFrameSchema.safeParse(data).data;
+    if (!row)
+        return;
+    if (row.deleted)
+        return errLine(`row removed ${row.row_id}`);
+    outLine(columns.map((name) => cell(row.data[name])).join('\t'));
+}
+// Everything but rows goes to stderr, so stdout stays a table.
+function noteFrame(event, data) {
+    if (event === 'status') {
+        const status = datasets.statusFrameSchema.safeParse(data).data;
+        return errLine(`status: ${status?.status ?? '?'}${status?.error ? `: ${status.error}` : ''}`);
+    }
+    if (event === 'log')
+        return errLine(`log: ${datasets.logFrameSchema.safeParse(data).data?.text ?? ''}`);
+    if (event === 'error')
+        return errLine(`error: ${datasets.errorFrameSchema.safeParse(data).data?.message ?? ''}`);
+    if (event !== 'heartbeat' && event !== 'done')
+        errLine(event);
+}
+// True once a status frame says the run is over. A bare `done` is not: a task
+// that does not hold the run answers `running` then `done`, so it reattaches.
+function isTerminal(frame) {
+    const status = datasets.statusFrameSchema.safeParse(frame.data).data;
+    return (frame.event === 'status' && status !== undefined && !datasets.RUN_IN_PROGRESS.has(status.status));
+}
+// The route caps one stream; a run still going is picked up again.
+async function followStream(id, runId, apiKey, opts) {
+    for (let attempt = 0;; attempt++) {
+        let terminal = false;
+        for await (const frame of api.datasetEvents(id, runId, apiKey, opts.signal)) {
+            renderFrame(frame, opts.columns, opts.pretty);
+            if (isTerminal(frame))
+                terminal = true;
+        }
+        if (terminal)
+            return;
+        const live = (await api.getDataset(id, apiKey)).run;
+        if (live?.id !== runId)
+            throw new Error('Run stream ended before completion');
+        // Finished in the gap; the caller prints the dataset.
+        if (!datasets.RUN_IN_PROGRESS.has(live.status))
+            return;
+        if (attempt >= datasets.MAX_REATTACHES)
+            throw new Error('Run stream ended before completion');
+        await datasets.pause(datasets.reattachDelayMs(attempt), opts.signal);
+        if (opts.pretty)
+            errLine('reconnected');
+    }
+}
+async function follow(id, runId, apiKey, opts) {
+    const { started, pretty, timeout = datasets.RUN_TIMEOUT_MS } = opts;
+    // Guard exits 130 once the handler settles.
+    const uninstall = installSignalGuard(() => interrupted(started, id, runId, apiKey));
+    const removePipeHandler = onBrokenPipe(() => interrupted(started, id, runId, apiKey));
+    try {
+        await followStream(id, runId, apiKey, { ...opts, signal: AbortSignal.timeout(timeout) });
+        const result = await api.getDataset(id, apiKey);
+        printDataset(result, pretty);
+        if (result.status === 'failed' || result.status === 'cancelled')
+            throw new Error(`Dataset ${result.status}: ${result.run?.error ?? runId}`);
+    }
+    catch (error) {
+        recoveryHint(error, started, id, runId);
+        throw error;
+    }
+    finally {
+        uninstall();
+        removePipeHandler();
+    }
+}
+function recoveryHint(error, started, id, runId) {
+    // A 4xx before the run existed means nothing to recover.
+    if (error instanceof ApiError && error.status < 500 && !started)
+        return;
+    errLine(`Check tinyfish dataset get ${id} or tinyfish dataset watch ${id} ${runId}.`);
+}
+async function interrupted(started, id, runId, apiKey) {
+    if (!started)
+        return;
+    try {
+        errLine(`Run ${(await api.cancelDatasetRun(id, runId, apiKey)).status}.`);
+    }
+    catch {
+        errLine(`Cancellation unconfirmed. Use tinyfish dataset cancel ${id} ${runId}.`);
+    }
+}
+// The planner's output, bare or as `{spec}`, so `plan > f.json` feeds back in.
+function readSpecFile(path) {
+    let text;
+    try {
+        text = readFileSync(path, 'utf8');
+    }
+    catch {
+        return handleApiError(new Error(`Cannot read ${path}`));
+    }
+    const parsed = parseJson(text);
+    const wrapped = parsed?.spec;
+    return api.parseWithSchema(datasets.specSchema, wrapped ?? parsed, 'Invalid spec file');
+}
+async function printOrFollow(dataset, apiKey, opts) {
+    if (opts.async || !dataset.run)
+        return printDataset(dataset, opts.pretty);
+    errLine(`Dataset ID: ${dataset.id}`);
+    errLine(`Run ID: ${dataset.run.id}`);
+    const columns = dataset.spec.columns.map((c) => c.name);
+    return follow(dataset.id, dataset.run.id, apiKey, {
+        started: true,
+        columns,
+        pretty: opts.pretty,
+    });
+}
+function create(prompt, opts) {
+    const hasPrompt = Boolean(prompt?.trim());
+    if (hasPrompt === Boolean(opts.fromSpec))
+        handleApiError(new Error('prompt or --from-spec required, not both'));
+    return withResolvedKey(async (apiKey) => {
+        const request = api.parseWithSchema(datasets.createRequestSchema, {
+            ...(opts.fromSpec ? { spec: readSpecFile(opts.fromSpec) } : { prompt }),
+            target_rows: opts.rows,
+        }, 'Invalid dataset request');
+        return printOrFollow(await api.createDataset(request, apiKey), apiKey, opts);
+    });
+}
+async function* walkRows(id, opts, apiKey) {
+    let after = opts.after;
+    do {
+        const page = await api.getDatasetRows(id, { ...opts, after }, apiKey);
+        yield page;
+        after = page.next_cursor ?? undefined;
+    } while (after);
+}
+async function rows(id, opts, apiKey) {
+    const query = api.parseWithSchema(datasets.rowsQuerySchema, { limit: opts.limit, after: opts.after }, 'Invalid rows query');
+    if (!opts.all) {
+        const page = await api.getDatasetRows(id, query, apiKey);
+        printRows(page, opts.pretty);
+        if (opts.pretty && page.next_cursor)
+            outLine(`Next: --after ${page.next_cursor}`);
+        return;
+    }
+    const merged = { columns: [], rows: [], truncated: false, next_cursor: null };
+    let first = true;
+    for await (const page of walkRows(id, query, apiKey)) {
+        if (opts.pretty)
+            printRows(page, true, first);
+        merged.columns = page.columns;
+        merged.rows.push(...page.rows);
+        merged.truncated = page.truncated;
+        first = false;
+    }
+    if (!opts.pretty)
+        out(merged);
+}
+async function list(opts, apiKey) {
+    const query = api.parseWithSchema(datasets.listQuerySchema, { limit: opts.limit, cursor: opts.cursor }, 'Invalid list query');
+    const result = await api.listDatasets(query, apiKey);
+    if (!opts.pretty)
+        return out(result);
+    if (result.data.length === 0)
+        outLine('No datasets.');
+    result.data.forEach((item) => outLine(datasetLine(item)));
+    if (result.pagination.next_cursor) {
+        const limit = opts.limit === undefined ? '' : `--limit ${opts.limit} `;
+        outLine(`Next: ${limit}--cursor ${result.pagination.next_cursor}`);
+    }
+}
+async function watch(id, runId, opts, apiKey) {
+    const run = api.parseRunId(runId);
+    const dataset = await api.getDataset(api.parseDatasetId(id), apiKey);
+    const columns = dataset.spec.columns.map((c) => c.name);
+    const follows = { started: false, columns, pretty: opts.pretty };
+    if (opts.timeout !== undefined)
+        follows.timeout = opts.timeout;
+    return follow(dataset.id, run, apiKey, follows);
+}
+export function registerDataset(program) {
+    const group = program
+        .command('dataset')
+        .description('Web-populated datasets')
+        .enablePositionalOptions()
+        .addHelpText('after', '\nCtrl+C cancels a run this command started; watch only detaches.');
+    const command = (name, description) => group.command(name).description(description).option('--pretty', 'Human-readable output');
+    const rowsOption = (cmd) => cmd.option('--rows <n>', `Rows to fill to (1-${datasets.MAX_TARGET_ROWS}; default: 100)`, integerInRange(1, datasets.MAX_TARGET_ROWS));
+    rowsOption(command('create [prompt]', 'Plan a dataset from a prompt or a spec file, then fill it')
+        .option('--from-spec <file>', 'A spec JSON file, as printed by `plan`')
+        .option('--async', 'Submit without waiting for the rows')).action(create);
+    command('plan <prompt>', 'Print the schema the planner designs, to edit and pass to create').action((prompt, opts) => withResolvedKey(async (apiKey) => {
+        const { spec } = await api.planDataset(prompt, apiKey);
+        if (opts.pretty)
+            outLine(JSON.stringify(spec, null, 2));
+        else
+            out(spec);
+    }));
+    command('list', 'List datasets, newest first')
+        .option('--limit <n>', `Maximum datasets (1-${datasets.LIST_LIMIT_MAX}; default: 20)`, integerInRange(1, datasets.LIST_LIMIT_MAX))
+        .option('--cursor <cursor>', 'Pagination cursor from a previous response')
+        .action((opts) => withResolvedKey((apiKey) => list(opts, apiKey)));
+    command('get <id>', 'Get a dataset: status, row count and latest run').action((id, opts) => withResolvedKey(async (apiKey) => printDataset(await api.getDataset(api.parseDatasetId(id), apiKey), opts.pretty, true)));
+    command('rows <id>', 'Print rows; --all walks every page')
+        .option('--limit <n>', `Rows per page (1-${datasets.ROWS_LIMIT_MAX}; default: 500)`, integerInRange(1, datasets.ROWS_LIMIT_MAX))
+        .option('--after <cursor>', 'Row cursor from a previous response')
+        .option('--all', 'Follow next_cursor until the last page')
+        .action((id, opts) => withResolvedKey((apiKey) => rows(api.parseDatasetId(id), opts, apiKey)));
+    rowsOption(command('populate <id>', 'Fill the dataset again; refused while a run is live')
+        .option('--reset', 'Empty the dataset first')
+        .option('--async', 'Submit without waiting for the rows')).action((id, opts) => withResolvedKey(async (apiKey) => {
+        const request = api.parseWithSchema(datasets.populateRequestSchema, { reset: opts.reset, target_rows: opts.rows }, 'Invalid populate request');
+        const dataset = await api.populateDataset(api.parseDatasetId(id), request, apiKey);
+        return printOrFollow(dataset, apiKey, opts);
+    }));
+    command('watch <id> <run_id>', 'Follow a run; Ctrl+C detaches')
+        .option('--timeout <ms>', 'Maximum watch duration in milliseconds', integerInRange(1, 2147483647), datasets.RUN_TIMEOUT_MS)
+        .action((id, runId, opts) => withResolvedKey((apiKey) => watch(id, runId, opts, apiKey)));
+    command('cancel <id> <run_id>', 'Stop a run; the final status arrives on watch').action((id, runId, opts) => withResolvedKey(async (apiKey) => {
+        const result = await api.cancelDatasetRun(api.parseDatasetId(id), api.parseRunId(runId), apiKey);
+        if (opts.pretty)
+            outLine(`${result.run_id}  ${result.status}`);
+        else
+            out(result);
+    }));
+    command('delete <id>', 'Delete a dataset and its rows; refused while a run is live').action((id, opts) => withResolvedKey(async (apiKey) => {
+        const result = await api.deleteDataset(api.parseDatasetId(id), apiKey);
+        if (opts.pretty)
+            outLine(`Dataset ${result.id} deleted`);
+        else
+            out(result);
+    }));
+}

package/dist/lib/client.d.ts

@@ -2,6 +2,7 @@ import { type AgentRunAsyncResponse, type AgentRunResponse, type AgentRunWithStr
 import type { PageParams } from './list-options.js';
 import type { BatchCancelResponse, BatchGetResponse, BatchRunRequest, BatchRunResponse, CancelRunResponse, CliAgentRunParams, CliBrowserSessionCreateParams, CookieRecord, ProfileCreateResponse, ProfileSummary, ProfileUploadResponse, RunStepsResponse, VaultConnectRequest, VaultConnectResponse, VaultConnectionsListResponse, VaultDisconnectResponse, VaultItemsListResponse, VaultItemsSyncResponse } from './types.js';
 import { z } from 'zod';
+import * as datasets from './datasets.js';
 import * as research from './research.js';
 type ConnectMap = Record<string, {
     attempt_id: string;
@@ -211,4 +212,198 @@ export declare function cancelResearch(id: string, apiKey: string): Promise<{
     cancelled: boolean;
 }>;
 export declare function researchStream(input: research.Request | string, apiKey: string, signal: AbortSignal): AsyncGenerator<research.Event>;
+export declare const parseDatasetId: (id: string) => string;
+export declare const parseRunId: (id: string) => string;
+export declare function createDataset(req: datasets.CreateRequest, apiKey: string): Promise<{
+    [x: string]: unknown;
+    id: string;
+    name: string;
+    spec: {
+        [x: string]: unknown;
+        name: string;
+        description: string;
+        columns: {
+            [x: string]: unknown;
+            name: string;
+            type: string;
+            is_primary_key: boolean;
+            label?: string | undefined;
+            retrieval_hint?: string | undefined;
+        }[];
+        keep_when?: {
+            [x: string]: unknown;
+            column: string;
+        } | undefined;
+    };
+    status: "ready" | "empty" | "populating" | "failed" | "cancelled";
+    target_rows: number;
+    row_count: number;
+    run: {
+        [x: string]: unknown;
+        id: string;
+        status: "failed" | "cancelled" | "queued" | "running" | "done";
+        started_at: string;
+        finished_at?: string | undefined;
+        error?: string | undefined;
+    } | null;
+    created_at: string;
+}>;
+export declare function planDataset(prompt: string, apiKey: string): Promise<{
+    [x: string]: unknown;
+    spec: {
+        [x: string]: unknown;
+        name: string;
+        description: string;
+        columns: {
+            [x: string]: unknown;
+            name: string;
+            type: string;
+            is_primary_key: boolean;
+            label?: string | undefined;
+            retrieval_hint?: string | undefined;
+        }[];
+        keep_when?: {
+            [x: string]: unknown;
+            column: string;
+        } | undefined;
+    };
+}>;
+export declare function listDatasets(opts: datasets.ListQuery, apiKey: string): Promise<{
+    [x: string]: unknown;
+    data: {
+        [x: string]: unknown;
+        id: string;
+        name: string;
+        spec: {
+            [x: string]: unknown;
+            name: string;
+            description: string;
+            columns: {
+                [x: string]: unknown;
+                name: string;
+                type: string;
+                is_primary_key: boolean;
+                label?: string | undefined;
+                retrieval_hint?: string | undefined;
+            }[];
+            keep_when?: {
+                [x: string]: unknown;
+                column: string;
+            } | undefined;
+        };
+        status: "ready" | "empty" | "populating" | "failed" | "cancelled";
+        target_rows: number;
+        row_count: number;
+        run: {
+            [x: string]: unknown;
+            id: string;
+            status: "failed" | "cancelled" | "queued" | "running" | "done";
+            started_at: string;
+            finished_at?: string | undefined;
+            error?: string | undefined;
+        } | null;
+        created_at: string;
+    }[];
+    pagination: {
+        [x: string]: unknown;
+        next_cursor: string | null;
+        has_more: boolean;
+    };
+}>;
+export declare function getDataset(id: string, apiKey: string): Promise<{
+    [x: string]: unknown;
+    id: string;
+    name: string;
+    spec: {
+        [x: string]: unknown;
+        name: string;
+        description: string;
+        columns: {
+            [x: string]: unknown;
+            name: string;
+            type: string;
+            is_primary_key: boolean;
+            label?: string | undefined;
+            retrieval_hint?: string | undefined;
+        }[];
+        keep_when?: {
+            [x: string]: unknown;
+            column: string;
+        } | undefined;
+    };
+    status: "ready" | "empty" | "populating" | "failed" | "cancelled";
+    target_rows: number;
+    row_count: number;
+    run: {
+        [x: string]: unknown;
+        id: string;
+        status: "failed" | "cancelled" | "queued" | "running" | "done";
+        started_at: string;
+        finished_at?: string | undefined;
+        error?: string | undefined;
+    } | null;
+    created_at: string;
+}>;
+export declare function getDatasetRows(id: string, opts: datasets.RowsQuery, apiKey: string): Promise<{
+    [x: string]: unknown;
+    columns: {
+        [x: string]: unknown;
+        name: string;
+        label: string;
+        type: string;
+        is_primary_key: boolean;
+    }[];
+    rows: {
+        [x: string]: unknown;
+        _id: string;
+        data: Record<string, unknown>;
+    }[];
+    truncated: boolean;
+    next_cursor: string | null;
+}>;
+export declare function populateDataset(id: string, req: datasets.PopulateRequest, apiKey: string): Promise<{
+    [x: string]: unknown;
+    id: string;
+    name: string;
+    spec: {
+        [x: string]: unknown;
+        name: string;
+        description: string;
+        columns: {
+            [x: string]: unknown;
+            name: string;
+            type: string;
+            is_primary_key: boolean;
+            label?: string | undefined;
+            retrieval_hint?: string | undefined;
+        }[];
+        keep_when?: {
+            [x: string]: unknown;
+            column: string;
+        } | undefined;
+    };
+    status: "ready" | "empty" | "populating" | "failed" | "cancelled";
+    target_rows: number;
+    row_count: number;
+    run: {
+        [x: string]: unknown;
+        id: string;
+        status: "failed" | "cancelled" | "queued" | "running" | "done";
+        started_at: string;
+        finished_at?: string | undefined;
+        error?: string | undefined;
+    } | null;
+    created_at: string;
+}>;
+export declare function cancelDatasetRun(id: string, runId: string, apiKey: string): Promise<{
+    [x: string]: unknown;
+    run_id: string;
+    status: string;
+}>;
+export declare function deleteDataset(id: string, apiKey: string): Promise<{
+    [x: string]: unknown;
+    id: string;
+    deleted: true;
+}>;
+export declare function datasetEvents(id: string, runId: string, apiKey: string, signal: AbortSignal): AsyncGenerator<datasets.Frame>;
 export {};

package/dist/lib/client.js

@@ -5,6 +5,7 @@ import { detectHarness, detectHumanInitiated } from './harness.js';
 import { captureNotice } from './notice.js';
 import { ApiError, parseJson } from './output.js';
 import { z } from 'zod';
+import * as datasets from './datasets.js';
 import * as research from './research.js';
 // Install-level connect context (PF-3169). A known caller is credited only to its own
 // install; under an unknown caller we credit a lone install, else send source only.
@@ -86,7 +87,7 @@ class TinyFishCliClient extends TinyFish {
         return headers;
     }
     // The SDK lacks GET streaming; reuse its authentication headers.
-    async researchEvents(path, signal) {
+    async getEventStream(path, signal) {
         const response = await fetch(`${this.baseURL}${path}`, {
             headers: { ...this._buildHeaders(), Accept: 'text/event-stream' },
             signal,
@@ -95,10 +96,10 @@ class TinyFishCliClient extends TinyFish {
         if (!response.ok)
             throw await apiErrorFrom(response);
         if (!response.body)
-            throw new Error('Research events response has no body');
+            throw new Error('Event stream response has no body');
         return response.body.pipeThrough(new TextDecoderStream());
     }
-    // SDK has no PATCH and its delete() throws on 204.
+    // The SDK's delete() and patch() parse a JSON body; 204 routes come through here.
     async send(method, path, json) {
         const response = await fetch(`${this.baseURL}${path}`, {
             method,
@@ -264,6 +265,50 @@ async function* parseSseStream(stream, onUnparseableLine) {
         reader.releaseLock();
     }
 }
+// Frames named on the event line; `data:` only streams keep using parseSseStream.
+async function* parseSseFrames(stream) {
+    const reader = stream.getReader();
+    let buffer = '';
+    const parseFrame = (raw) => {
+        let event = 'message';
+        const data = [];
+        for (const line of raw.split('\n')) {
+            if (line.startsWith('event:'))
+                event = line.slice('event:'.length).trim();
+            else if (line.startsWith('data:'))
+                data.push(line.slice('data:'.length).trim());
+        }
+        if (data.length === 0)
+            return undefined;
+        try {
+            return { event, data: JSON.parse(data.join('\n')) };
+        }
+        catch {
+            throw new Error('Invalid dataset event JSON');
+        }
+    };
+    try {
+        while (true) {
+            const { done, value } = await reader.read();
+            if (done)
+                break;
+            buffer += value.replaceAll('\r\n', '\n');
+            const frames = buffer.split('\n\n');
+            buffer = frames.pop() ?? '';
+            for (const raw of frames) {
+                const frame = parseFrame(raw);
+                if (frame)
+                    yield frame;
+            }
+        }
+        const last = parseFrame(buffer);
+        if (last)
+            yield last;
+    }
+    finally {
+        reader.releaseLock();
+    }
+}
 export function runSync(req, apiKey) {
     return sdk(apiKey, async (client) => {
         const response = await client.post('/v1/automation/run', {
@@ -643,7 +688,7 @@ export async function* researchStream(input, apiKey, signal) {
     try {
         stream =
             typeof input === 'string'
-                ? await client.researchEvents(`${runPath(input)}/events`, signal)
+                ? await client.getEventStream(`${runPath(input)}/events`, signal)
                 : await client.postStream('/v1/automation/run-research', {
                     json: research.requestSchema.parse(input),
                     signal,
@@ -660,3 +705,58 @@ export async function* researchStream(input, apiKey, signal) {
         await stream?.cancel().catch(() => { });
     }
 }
+// A dataset may exist; never lose its ID.
+function parseDataset(schema, body) {
+    const id = z.looseObject({ id: datasets.idSchema }).safeParse(body).data;
+    const suffix = id ? ` for ${id.id}` : '';
+    return parseWithSchema(schema, body, `Invalid dataset response${suffix}`);
+}
+export const parseDatasetId = (id) => parseWithSchema(datasets.idSchema, id, 'Invalid dataset ID');
+export const parseRunId = (id) => parseWithSchema(datasets.idSchema, id, 'Invalid run ID');
+const datasetPath = (id) => `/v1/datasets/${parseDatasetId(id)}`;
+const datasetRunPath = (id, runId) => `${datasetPath(id)}/runs/${parseRunId(runId)}`;
+export function createDataset(req, apiKey) {
+    return sdk(apiKey, async (client) => parseDataset(datasets.datasetSchema, await client.post('/v1/datasets', { json: datasets.createRequestSchema.parse(req) })), datasets.START_TIMEOUT_MS);
+}
+export function planDataset(prompt, apiKey) {
+    return sdk(apiKey, async (client) => parseDataset(datasets.planResponseSchema, await client.post('/v1/datasets/plan', {
+        json: datasets.planRequestSchema.parse({ prompt }),
+    })), datasets.START_TIMEOUT_MS);
+}
+export function listDatasets(opts, apiKey) {
+    return sdk(apiKey, async (client) => parseDataset(datasets.listSchema, await client.get('/v1/datasets', { params: datasets.listQuerySchema.parse(opts) })), datasets.READ_TIMEOUT_MS);
+}
+export function getDataset(id, apiKey) {
+    return sdk(apiKey, async (client) => parseDataset(datasets.datasetSchema, await client.get(datasetPath(id))), datasets.READ_TIMEOUT_MS);
+}
+export function getDatasetRows(id, opts, apiKey) {
+    return sdk(apiKey, async (client) => parseDataset(datasets.rowsSchema, await client.get(`${datasetPath(id)}/rows`, {
+        params: datasets.rowsQuerySchema.parse(opts),
+    })), datasets.READ_TIMEOUT_MS);
+}
+export function populateDataset(id, req, apiKey) {
+    return sdk(apiKey, async (client) => parseDataset(datasets.datasetSchema, await client.post(`${datasetPath(id)}/populate`, {
+        json: datasets.populateRequestSchema.parse(req),
+    })), datasets.START_TIMEOUT_MS);
+}
+export function cancelDatasetRun(id, runId, apiKey) {
+    return sdk(apiKey, async (client) => parseDataset(datasets.cancelSchema, await client.post(`${datasetRunPath(id, runId)}/cancel`)), datasets.CANCEL_TIMEOUT_MS);
+}
+export function deleteDataset(id, apiKey) {
+    return sdk(apiKey, async (client) => parseDataset(datasets.deleteSchema, await client.delete(datasetPath(id))), datasets.READ_TIMEOUT_MS);
+}
+export async function* datasetEvents(id, runId, apiKey, signal) {
+    let stream;
+    const client = createSdkClient(apiKey, datasets.RUN_TIMEOUT_MS);
+    try {
+        stream = await client.getEventStream(`${datasetRunPath(id, runId)}/events`, signal);
+        for await (const frame of parseSseFrames(stream))
+            yield parseWithSchema(datasets.frameSchema, frame, 'Invalid dataset event');
+    }
+    catch (error) {
+        rethrowSdkError(error, client.baseURL);
+    }
+    finally {
+        await stream?.cancel().catch(() => { });
+    }
+}

package/dist/lib/datasets.d.ts

@@ -0,0 +1,246 @@
+import { z } from 'zod';
+export declare const RUN_TIMEOUT_MS = 1800000;
+export declare const START_TIMEOUT_MS = 90000;
+export declare const READ_TIMEOUT_MS = 15000;
+export declare const CANCEL_TIMEOUT_MS = 5000;
+export declare const MAX_TARGET_ROWS = 250;
+export declare const LIST_LIMIT_MAX = 100;
+export declare const ROWS_LIMIT_MAX = 2000;
+export declare const MAX_REATTACHES = 3;
+export declare const reattachDelayMs: (attempt: number) => number;
+export declare function pause(ms: number, signal: AbortSignal): Promise<void>;
+export declare const idSchema: z.ZodUUID;
+export declare const statusSchema: z.ZodEnum<{
+    ready: "ready";
+    empty: "empty";
+    populating: "populating";
+    failed: "failed";
+    cancelled: "cancelled";
+}>;
+export declare const runStatusSchema: z.ZodEnum<{
+    failed: "failed";
+    cancelled: "cancelled";
+    queued: "queued";
+    running: "running";
+    done: "done";
+}>;
+export declare const RUN_IN_PROGRESS: ReadonlySet<string>;
+export declare const specSchema: z.ZodObject<{
+    name: z.ZodString;
+    description: z.ZodString;
+    columns: z.ZodArray<z.ZodObject<{
+        name: z.ZodString;
+        label: z.ZodOptional<z.ZodString>;
+        type: z.ZodString;
+        is_primary_key: z.ZodBoolean;
+        retrieval_hint: z.ZodOptional<z.ZodString>;
+    }, z.core.$loose>>;
+    keep_when: z.ZodOptional<z.ZodObject<{
+        column: z.ZodString;
+    }, z.core.$loose>>;
+}, z.core.$loose>;
+export type Spec = z.infer<typeof specSchema>;
+export declare const runSchema: z.ZodNullable<z.ZodObject<{
+    id: z.ZodUUID;
+    status: z.ZodEnum<{
+        failed: "failed";
+        cancelled: "cancelled";
+        queued: "queued";
+        running: "running";
+        done: "done";
+    }>;
+    started_at: z.ZodString;
+    finished_at: z.ZodOptional<z.ZodString>;
+    error: z.ZodOptional<z.ZodString>;
+}, z.core.$loose>>;
+export declare const datasetSchema: z.ZodObject<{
+    id: z.ZodUUID;
+    name: z.ZodString;
+    spec: z.ZodObject<{
+        name: z.ZodString;
+        description: z.ZodString;
+        columns: z.ZodArray<z.ZodObject<{
+            name: z.ZodString;
+            label: z.ZodOptional<z.ZodString>;
+            type: z.ZodString;
+            is_primary_key: z.ZodBoolean;
+            retrieval_hint: z.ZodOptional<z.ZodString>;
+        }, z.core.$loose>>;
+        keep_when: z.ZodOptional<z.ZodObject<{
+            column: z.ZodString;
+        }, z.core.$loose>>;
+    }, z.core.$loose>;
+    status: z.ZodEnum<{
+        ready: "ready";
+        empty: "empty";
+        populating: "populating";
+        failed: "failed";
+        cancelled: "cancelled";
+    }>;
+    target_rows: z.ZodNumber;
+    row_count: z.ZodNumber;
+    run: z.ZodNullable<z.ZodObject<{
+        id: z.ZodUUID;
+        status: z.ZodEnum<{
+            failed: "failed";
+            cancelled: "cancelled";
+            queued: "queued";
+            running: "running";
+            done: "done";
+        }>;
+        started_at: z.ZodString;
+        finished_at: z.ZodOptional<z.ZodString>;
+        error: z.ZodOptional<z.ZodString>;
+    }, z.core.$loose>>;
+    created_at: z.ZodString;
+}, z.core.$loose>;
+export type Dataset = z.infer<typeof datasetSchema>;
+export declare const createRequestSchema: z.ZodObject<{
+    prompt: z.ZodOptional<z.ZodString>;
+    spec: z.ZodOptional<z.ZodObject<{
+        name: z.ZodString;
+        description: z.ZodString;
+        columns: z.ZodArray<z.ZodObject<{
+            name: z.ZodString;
+            label: z.ZodOptional<z.ZodString>;
+            type: z.ZodString;
+            is_primary_key: z.ZodBoolean;
+            retrieval_hint: z.ZodOptional<z.ZodString>;
+        }, z.core.$loose>>;
+        keep_when: z.ZodOptional<z.ZodObject<{
+            column: z.ZodString;
+        }, z.core.$loose>>;
+    }, z.core.$loose>>;
+    target_rows: z.ZodOptional<z.ZodNumber>;
+}, z.core.$strict>;
+export type CreateRequest = z.infer<typeof createRequestSchema>;
+export declare const planRequestSchema: z.ZodObject<{
+    prompt: z.ZodString;
+}, z.core.$strict>;
+export declare const planResponseSchema: z.ZodObject<{
+    spec: z.ZodObject<{
+        name: z.ZodString;
+        description: z.ZodString;
+        columns: z.ZodArray<z.ZodObject<{
+            name: z.ZodString;
+            label: z.ZodOptional<z.ZodString>;
+            type: z.ZodString;
+            is_primary_key: z.ZodBoolean;
+            retrieval_hint: z.ZodOptional<z.ZodString>;
+        }, z.core.$loose>>;
+        keep_when: z.ZodOptional<z.ZodObject<{
+            column: z.ZodString;
+        }, z.core.$loose>>;
+    }, z.core.$loose>;
+}, z.core.$loose>;
+export declare const listQuerySchema: z.ZodObject<{
+    limit: z.ZodOptional<z.ZodNumber>;
+    cursor: z.ZodOptional<z.ZodString>;
+}, z.core.$strict>;
+export type ListQuery = z.infer<typeof listQuerySchema>;
+export declare const listSchema: z.ZodObject<{
+    data: z.ZodArray<z.ZodObject<{
+        id: z.ZodUUID;
+        name: z.ZodString;
+        spec: z.ZodObject<{
+            name: z.ZodString;
+            description: z.ZodString;
+            columns: z.ZodArray<z.ZodObject<{
+                name: z.ZodString;
+                label: z.ZodOptional<z.ZodString>;
+                type: z.ZodString;
+                is_primary_key: z.ZodBoolean;
+                retrieval_hint: z.ZodOptional<z.ZodString>;
+            }, z.core.$loose>>;
+            keep_when: z.ZodOptional<z.ZodObject<{
+                column: z.ZodString;
+            }, z.core.$loose>>;
+        }, z.core.$loose>;
+        status: z.ZodEnum<{
+            ready: "ready";
+            empty: "empty";
+            populating: "populating";
+            failed: "failed";
+            cancelled: "cancelled";
+        }>;
+        target_rows: z.ZodNumber;
+        row_count: z.ZodNumber;
+        run: z.ZodNullable<z.ZodObject<{
+            id: z.ZodUUID;
+            status: z.ZodEnum<{
+                failed: "failed";
+                cancelled: "cancelled";
+                queued: "queued";
+                running: "running";
+                done: "done";
+            }>;
+            started_at: z.ZodString;
+            finished_at: z.ZodOptional<z.ZodString>;
+            error: z.ZodOptional<z.ZodString>;
+        }, z.core.$loose>>;
+        created_at: z.ZodString;
+    }, z.core.$loose>>;
+    pagination: z.ZodObject<{
+        next_cursor: z.ZodNullable<z.ZodString>;
+        has_more: z.ZodBoolean;
+    }, z.core.$loose>;
+}, z.core.$loose>;
+export declare const rowsQuerySchema: z.ZodObject<{
+    limit: z.ZodOptional<z.ZodNumber>;
+    after: z.ZodOptional<z.ZodString>;
+}, z.core.$strict>;
+export type RowsQuery = z.infer<typeof rowsQuerySchema>;
+export declare const rowsSchema: z.ZodObject<{
+    columns: z.ZodArray<z.ZodObject<{
+        name: z.ZodString;
+        label: z.ZodString;
+        type: z.ZodString;
+        is_primary_key: z.ZodBoolean;
+    }, z.core.$loose>>;
+    rows: z.ZodArray<z.ZodObject<{
+        _id: z.ZodString;
+        data: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+    }, z.core.$loose>>;
+    truncated: z.ZodBoolean;
+    next_cursor: z.ZodNullable<z.ZodString>;
+}, z.core.$loose>;
+export type Rows = z.infer<typeof rowsSchema>;
+export declare const populateRequestSchema: z.ZodObject<{
+    reset: z.ZodOptional<z.ZodBoolean>;
+    target_rows: z.ZodOptional<z.ZodNumber>;
+}, z.core.$strict>;
+export type PopulateRequest = z.infer<typeof populateRequestSchema>;
+export declare const cancelSchema: z.ZodObject<{
+    run_id: z.ZodUUID;
+    status: z.ZodString;
+}, z.core.$loose>;
+export declare const deleteSchema: z.ZodObject<{
+    id: z.ZodUUID;
+    deleted: z.ZodLiteral<true>;
+}, z.core.$loose>;
+export declare const frameSchema: z.ZodObject<{
+    event: z.ZodString;
+    data: z.ZodUnknown;
+}, z.core.$strip>;
+export type Frame = z.infer<typeof frameSchema>;
+export declare const statusFrameSchema: z.ZodObject<{
+    status: z.ZodEnum<{
+        failed: "failed";
+        cancelled: "cancelled";
+        queued: "queued";
+        running: "running";
+        done: "done";
+    }>;
+    error: z.ZodOptional<z.ZodString>;
+}, z.core.$loose>;
+export declare const rowFrameSchema: z.ZodObject<{
+    row_id: z.ZodString;
+    data: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+    deleted: z.ZodOptional<z.ZodLiteral<true>>;
+}, z.core.$loose>;
+export declare const logFrameSchema: z.ZodObject<{
+    text: z.ZodString;
+}, z.core.$loose>;
+export declare const errorFrameSchema: z.ZodObject<{
+    message: z.ZodString;
+}, z.core.$loose>;

package/dist/lib/datasets.js

@@ -0,0 +1,119 @@
+import { z } from 'zod';
+// A populate can outlive the SDK's ten-minute timeout; watch reattaches within this.
+export const RUN_TIMEOUT_MS = 1_800_000;
+// Create plans in-request, under the route's 90 s budget.
+export const START_TIMEOUT_MS = 90_000;
+export const READ_TIMEOUT_MS = 15_000;
+export const CANCEL_TIMEOUT_MS = 5_000;
+export const MAX_TARGET_ROWS = 250;
+export const LIST_LIMIT_MAX = 100;
+export const ROWS_LIMIT_MAX = 2000;
+// One stream lasts at most ~10 minutes server-side; a live run is picked up again.
+export const MAX_REATTACHES = 3;
+// Before each reattach: 1 s doubling, capped at 16 s, like the SDK's reconnectDelay.
+export const reattachDelayMs = (attempt) => Math.min(1_000 * 2 ** attempt, 16_000);
+// Resolves after `ms`, or at once when the signal aborts; the next request reports the abort.
+export function pause(ms, signal) {
+    return new Promise((resolve) => {
+        const done = () => {
+            clearTimeout(timer);
+            signal.removeEventListener('abort', done);
+            resolve();
+        };
+        const timer = setTimeout(done, ms);
+        signal.addEventListener('abort', done, { once: true });
+    });
+}
+export const idSchema = z.uuid();
+export const statusSchema = z.enum(['empty', 'populating', 'ready', 'failed', 'cancelled']);
+export const runStatusSchema = z.enum(['queued', 'running', 'done', 'cancelled', 'failed']);
+// Queued and running runs are in progress; the rest have ended.
+export const RUN_IN_PROGRESS = new Set(['queued', 'running']);
+const columnSchema = z.looseObject({
+    name: z.string(),
+    label: z.string().optional(),
+    type: z.string(),
+    is_primary_key: z.boolean(),
+    retrieval_hint: z.string().optional(),
+});
+export const specSchema = z.looseObject({
+    name: z.string().min(1),
+    description: z.string(),
+    columns: z.array(columnSchema).min(1),
+    keep_when: z.looseObject({ column: z.string() }).optional(),
+});
+export const runSchema = z
+    .looseObject({
+    id: idSchema,
+    status: runStatusSchema,
+    started_at: z.string(),
+    finished_at: z.string().optional(),
+    error: z.string().optional(),
+})
+    .nullable();
+export const datasetSchema = z.looseObject({
+    id: idSchema,
+    name: z.string(),
+    spec: specSchema,
+    status: statusSchema,
+    target_rows: z.number().int(),
+    row_count: z.number().int(),
+    run: runSchema,
+    created_at: z.string(),
+});
+const targetRowsSchema = z.number().int().min(1).max(MAX_TARGET_ROWS);
+export const createRequestSchema = z
+    .strictObject({
+    prompt: z.string().trim().min(3).max(2000).optional(),
+    spec: specSchema.optional(),
+    target_rows: targetRowsSchema.optional(),
+})
+    .refine((body) => (body.prompt === undefined) !== (body.spec === undefined), {
+    message: 'Send exactly one of prompt or spec',
+    path: ['prompt'],
+});
+export const planRequestSchema = z.strictObject({ prompt: z.string().trim().min(3).max(2000) });
+export const planResponseSchema = z.looseObject({ spec: specSchema });
+export const listQuerySchema = z.strictObject({
+    limit: z.number().int().min(1).max(LIST_LIMIT_MAX).optional(),
+    cursor: z.string().optional(),
+});
+export const listSchema = z.looseObject({
+    data: z.array(datasetSchema),
+    pagination: z.looseObject({ next_cursor: z.string().nullable(), has_more: z.boolean() }),
+});
+export const rowsQuerySchema = z.strictObject({
+    limit: z.number().int().min(1).max(ROWS_LIMIT_MAX).optional(),
+    after: z.string().max(32).optional(),
+});
+export const rowsSchema = z.looseObject({
+    columns: z.array(z.looseObject({
+        name: z.string(),
+        label: z.string(),
+        type: z.string(),
+        is_primary_key: z.boolean(),
+    })),
+    rows: z.array(z.looseObject({ _id: z.string(), data: z.record(z.string(), z.unknown()) })),
+    truncated: z.boolean(),
+    next_cursor: z.string().nullable(),
+});
+export const populateRequestSchema = z.strictObject({
+    reset: z.boolean().optional(),
+    target_rows: targetRowsSchema.optional(),
+});
+export const cancelSchema = z.looseObject({ run_id: idSchema, status: z.string() });
+export const deleteSchema = z.looseObject({ id: idSchema, deleted: z.literal(true) });
+// The SSE envelope: the event line names the frame, the payload varies by name.
+export const frameSchema = z.object({ event: z.string().min(1), data: z.unknown() });
+export const statusFrameSchema = z.looseObject({
+    status: runStatusSchema,
+    error: z.string().optional(),
+});
+export const rowFrameSchema = z.looseObject({
+    row_id: z.string(),
+    data: z.record(z.string(), z.unknown()),
+    deleted: z.literal(true).optional(),
+});
+export const logFrameSchema = z.looseObject({ text: z.string() });
+// The proxy's own frame: the service went away mid-stream.
+export const errorFrameSchema = z.looseObject({ message: z.string() });

package/dist/program.js

@@ -25,6 +25,7 @@ const GROUPS = {
         (await import('./commands/batch.js')).registerBatch(agentCmd);
     },
     research: async (p) => (await import('./commands/research.js')).registerResearch(p),
+    dataset: async (p) => (await import('./commands/dataset.js')).registerDataset(p),
     search: async (p) => (await import('./commands/search.js')).registerSearch(p),
     fetch: async (p) => (await import('./commands/fetch.js')).registerFetch(p),
     monitor: async (p) => (await import('./commands/monitor.js')).registerMonitor(p),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tiny-fish/cli",
-  "version": "0.51.2-next.417",
+  "version": "0.52.0",
   "description": "TinyFish CLI — run web automations from your terminal",
   "type": "module",
   "bin": {