fauna-api 0.0.0-stage → 1.0.679

package/dist/src/marmot/research.js

@@ -0,0 +1,1138 @@
+import { Fauna } from "../../index.js";
+import { createScraper, getScraper, rebuildScraper, removeScraper, updateScraper } from "./scraper.js";
+export class Schema {
+    static deserialize(data) {
+        switch (data?.type) {
+            case 'json':
+                return new SchemaJSON(deserializeLeaf(data.schema));
+            case 'csv':
+                return new SchemaCSV(Object.fromEntries(Object.entries(data.schema).map(([key, leaf]) => [key, deserializeLeaf(leaf)])), data.minimum, data.maximum, data.prompt ?? '');
+            default:
+                throw new Error(`Invalid schema type: ${data?.type}`);
+        }
+    }
+}
+export class SchemaJSON extends Schema {
+    constructor(schema) {
+        super();
+        Object.defineProperty(this, "type", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: 'json'
+        });
+        Object.defineProperty(this, "schema", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.schema = schema;
+    }
+    serialize() {
+        return { type: this.type, schema: serializeLeaf(this.schema) };
+    }
+}
+export class SchemaCSV extends Schema {
+    constructor(schema, minimum, maximum, prompt = '') {
+        super();
+        Object.defineProperty(this, "type", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: 'csv'
+        });
+        // Columns are keyed by name (mirrors how JSON object scopes are keyed),
+        // so the key becomes the output column header and extraction scope path.
+        Object.defineProperty(this, "schema", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "minimum", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "maximum", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "prompt", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        for (const leaf of Object.values(schema)) {
+            if (!(leaf.value instanceof Fauna.Parrot.Format.Format)) {
+                throw new Error('Nested schema are not supported for CSV');
+            }
+        }
+        if (!Number.isInteger(minimum) || minimum < 0) {
+            throw new Error(`SchemaCSV minimum must be a non-negative integer, got ${minimum}`);
+        }
+        if (!Number.isInteger(maximum) || maximum < 0) {
+            throw new Error(`SchemaCSV maximum must be a non-negative integer, got ${maximum}`);
+        }
+        if (minimum > maximum) {
+            throw new Error(`SchemaCSV minimum (${minimum}) cannot exceed maximum (${maximum})`);
+        }
+        this.schema = schema;
+        this.minimum = minimum;
+        this.maximum = maximum;
+        this.prompt = prompt;
+    }
+    serialize() {
+        return {
+            type: this.type,
+            minimum: this.minimum,
+            maximum: this.maximum,
+            prompt: this.prompt,
+            schema: Object.fromEntries(Object.entries(this.schema).map(([key, leaf]) => [key, serializeLeaf(leaf)]))
+        };
+    }
+}
+function serializeLeaf(leaf) {
+    const { prompt, sources, optional, array, maxIterations, filters } = leaf;
+    const serialized = { prompt, sources, optional, array, maxIterations, filters };
+    if (leaf.value instanceof Fauna.Parrot.Format.Format) {
+        serialized.value = { question: leaf.value.toJSON() };
+    }
+    else if ('value' in leaf.value) {
+        serialized.value = { schema: serializeLeaf(leaf.value) };
+    }
+    else {
+        const children = {};
+        for (const key in leaf.value) {
+            children[key] = serializeLeaf(leaf.value[key]);
+        }
+        serialized.value = { children };
+    }
+    return serialized;
+}
+function deserializeLeaf(data) {
+    const { prompt, sources, optional, array, maxIterations, filters } = data;
+    if (array !== undefined) {
+        if (!Number.isInteger(array.minimum) || array.minimum < 0) {
+            throw new Error(`SchemaLeaf array.minimum must be a non-negative integer, got ${array.minimum}`);
+        }
+        if (!Number.isInteger(array.maximum) || array.maximum < 0) {
+            throw new Error(`SchemaLeaf array.maximum must be a non-negative integer, got ${array.maximum}`);
+        }
+        if (array.minimum > array.maximum) {
+            throw new Error(`SchemaLeaf array.minimum (${array.minimum}) cannot exceed array.maximum (${array.maximum})`);
+        }
+    }
+    return {
+        prompt,
+        sources,
+        optional,
+        array,
+        maxIterations,
+        filters,
+        value: 'question' in data.value
+            ? Fauna.Parrot.Format.Format.fromJSON(data.value.question)
+            : 'children' in data.value
+                ? Object.fromEntries(Object.entries(data.value.children).map(([k, v]) => [k, deserializeLeaf(v)]))
+                : deserializeLeaf(data.value.schema)
+    };
+}
+// ─── Schema Diff (migration) ───────────────────────────────────────────────────
+function formatSignature(value) {
+    if (value instanceof Fauna.Parrot.Format.Format) {
+        const json = value.toJSON();
+        return `format:${json?.type ?? 'unknown'}`;
+    }
+    return 'composite';
+}
+function flattenLeafPaths(leaf, path, out) {
+    if (leaf.array) {
+        // A collection: descend into the inner leaf with an `[]` marker so
+        // every leaf below it is addressed per-element (e.g. "awards[].year").
+        flattenLeafPaths(leaf.value, `${path}[]`, out);
+        return;
+    }
+    const value = leaf.value;
+    if (value instanceof Fauna.Parrot.Format.Format) {
+        out.set(path || 'value', formatSignature(value));
+        return;
+    }
+    if ('value' in value) {
+        // Transparent single-child wrapper (e.g. an array's inner leaf).
+        flattenLeafPaths(value, path, out);
+        return;
+    }
+    for (const key of Object.keys(value)) {
+        const childPath = path ? `${path}.${key}` : key;
+        flattenLeafPaths(value[key], childPath, out);
+    }
+}
+/**
+ * Flatten a schema to a map of `leaf path → type signature`.
+ *   - CSV: each column key is a path, signature is its Format type.
+ *   - JSON: object keys descend with `.`, array leaves append `[]`, Format
+ *     leaves terminate. e.g. "name", "address.city", "awards[].year".
+ */
+export function schemaPaths(schema) {
+    const out = new Map();
+    if (schema instanceof SchemaCSV) {
+        for (const [key, leaf] of Object.entries(schema.schema)) {
+            out.set(key, formatSignature(leaf.value));
+        }
+    }
+    else {
+        flattenLeafPaths(schema.schema, '', out);
+    }
+    return out;
+}
+/**
+ * Diff two schema versions for a dataset migration. A path whose type
+ * signature changed shows up in BOTH lists (drop the old, add the new) — we
+ * deliberately do not attempt rename/retype detection. Renames and tree-shape
+ * changes therefore fall out naturally as drop + add.
+ */
+export function diffSchema(baseline, target) {
+    const a = schemaPaths(baseline);
+    const b = schemaPaths(target);
+    const added = [];
+    const dropped = [];
+    for (const [path, signature] of b) {
+        if (a.get(path) !== signature)
+            added.push(path);
+    }
+    for (const [path, signature] of a) {
+        if (b.get(path) !== signature)
+            dropped.push(path);
+    }
+    return { added, dropped };
+}
+export const FeedRow = {
+    fromJSON(json) {
+        return { seq: Number(json.seq), value: json.value ?? null, sources: json.sources ?? null, created: new Date(json.created) };
+    },
+};
+export const PushResult = {
+    fromJSON(json) {
+        return {
+            added: Number(json.added ?? 0),
+            first: json.first ?? null,
+            last: json.last ?? null,
+            invalid: Array.isArray(json.invalid) ? json.invalid.map((i) => ({ index: Number(i.index), reason: String(i.reason) })) : [],
+        };
+    },
+};
+/** A push nothing in was let through: why, and each row's reason. */
+export class PushRefused extends Error {
+    constructor(status, message, invalid) {
+        super(message);
+        Object.defineProperty(this, "status", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "invalid", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.status = status;
+        this.invalid = invalid;
+    }
+}
+const FEED_PAGE = 500;
+function datasetDataURL(datasetId, options) {
+    const query = new URLSearchParams({ format: options?.format ?? 'json' });
+    if (options?.snapshot !== undefined)
+        query.set('snapshot', options.snapshot);
+    if (options?.sources)
+        query.set('sources', 'true');
+    return `${Fauna.getURL()}/marmot/dataset/${datasetId}/data?${query.toString()}`;
+}
+// ─── Snapshot ────────────────────────────────────────────────────────────────
+export class Snapshot {
+    constructor(authorization, workspaceId, datasetId, id, createdAt, valid = true, group) {
+        Object.defineProperty(this, "id", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "createdAt", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "valid", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "_authorization", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "_workspaceId", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "_datasetId", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "_group", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "_schema", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: null
+        });
+        Object.defineProperty(this, "_data", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: null
+        });
+        this._authorization = authorization;
+        this._workspaceId = workspaceId;
+        this._datasetId = datasetId;
+        this.id = id;
+        this.createdAt = createdAt;
+        this.valid = valid;
+        this._group = group;
+    }
+    async download(assets = false, revalidate = false) {
+        if (!assets && this._data && !revalidate)
+            return this._data;
+        const response = await fetch(`${Fauna.getURL()}/marmot/snapshot/${this._workspaceId}/${this._datasetId}/${this.id}/?assets=${assets}`, {
+            method: "GET",
+            headers: {
+                "Authorization": 'email' in this._authorization ? `Token ${this._authorization.token}` : `ApiKey ${this._authorization.id}`,
+                ...(this._group ? { Group: this._group.id } : {})
+            }
+        });
+        if (!response.ok) {
+            throw new Error(await response.text());
+        }
+        if (assets) {
+            return await response.blob();
+        }
+        const contentType = response.headers.get('content-type') ?? '';
+        const data = contentType.includes('application/json')
+            ? await response.json()
+            : await response.text();
+        this._data = data;
+        return data;
+    }
+    /** This snapshot's rows as CSV text or, by default, a JSON array. */
+    async data(options) {
+        const response = await fetch(datasetDataURL(this._datasetId, { ...options, snapshot: this.id }), {
+            method: "GET",
+            headers: {
+                "Authorization": 'email' in this._authorization ? `Token ${this._authorization.token}` : `ApiKey ${this._authorization.id}`,
+            }
+        });
+        if (!response.ok) {
+            throw new Error(await response.text());
+        }
+        return options?.format === 'csv' ? await response.text() : await response.json();
+    }
+    async downloadSchema(revalidate = false) {
+        if (this._schema && !revalidate)
+            return this._schema;
+        const response = await fetch(`${Fauna.getURL()}/marmot/snapshot/${this._workspaceId}/${this._datasetId}/${this.id}/schema`, {
+            method: "GET",
+            headers: {
+                "Authorization": 'email' in this._authorization ? `Token ${this._authorization.token}` : `ApiKey ${this._authorization.id}`,
+                ...(this._group ? { Group: this._group.id } : {})
+            }
+        });
+        if (!response.ok) {
+            throw new Error(await response.text());
+        }
+        this._schema = Schema.deserialize(await response.json());
+        return this._schema;
+    }
+    async downloadLog() {
+        const response = await fetch(`${Fauna.getURL()}/marmot/snapshot/${this._workspaceId}/${this._datasetId}/${this.id}/log`, {
+            method: "GET",
+            headers: {
+                "Authorization": 'email' in this._authorization ? `Token ${this._authorization.token}` : `ApiKey ${this._authorization.id}`,
+                ...(this._group ? { Group: this._group.id } : {})
+            }
+        });
+        if (!response.ok) {
+            throw new Error(await response.text());
+        }
+        return await response.blob();
+    }
+    async downloadEmbeddings() {
+        const response = await fetch(`${Fauna.getURL()}/marmot/snapshot/${this._workspaceId}/${this._datasetId}/${this.id}/embeddings`, {
+            method: "GET",
+            headers: {
+                "Authorization": 'email' in this._authorization ? `Token ${this._authorization.token}` : `ApiKey ${this._authorization.id}`,
+                ...(this._group ? { Group: this._group.id } : {})
+            }
+        });
+        if (!response.ok) {
+            throw new Error(await response.text());
+        }
+        return await response.blob();
+    }
+    async delete() {
+        const response = await fetch(`${Fauna.getURL()}/marmot/snapshot/${this._workspaceId}/${this._datasetId}/${this.id}`, {
+            method: "DELETE",
+            headers: {
+                "Authorization": 'email' in this._authorization ? `Token ${this._authorization.token}` : `ApiKey ${this._authorization.id}`,
+                ...(this._group ? { Group: this._group.id } : {})
+            }
+        });
+        if (!response.ok) {
+            throw new Error(await response.text());
+        }
+    }
+}
+export class Dataset {
+    constructor(authorization, workspaceId, id, name, schema, termination, contentTypes, proxy, documentModes, snapshots, group) {
+        Object.defineProperty(this, "id", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "workspaceId", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "name", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "schema", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "termination", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "contentTypes", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "proxy", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "documentModes", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "snapshots", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "_authorization", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "_group", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "_ws", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: null
+        });
+        this._authorization = authorization;
+        this.workspaceId = workspaceId;
+        this.id = id;
+        this.name = name;
+        this.schema = schema;
+        this.termination = termination;
+        this.contentTypes = contentTypes;
+        this.proxy = proxy;
+        this.documentModes = documentModes;
+        this.snapshots = snapshots;
+        this._group = group;
+    }
+    async research(callbacks, options) {
+        return this._runSocket({}, callbacks, options);
+    }
+    /** Join a live BrowserRoom opened by a research run. */
+    async joinBrowser(accessCode) {
+        const { BrowserRoom } = await import("./browser.js");
+        return BrowserRoom.join(this._authorization, { accessCode });
+    }
+    /**
+     * Attach to an *already-running* research/migration session for this
+     * dataset and replay its live timeline from the start, then keep streaming.
+     *
+     * Unlike `research`, this NEVER starts a new run: if nothing is in flight
+     * the returned promise rejects, so callers can `try/catch` it on load
+     * (e.g. to passively re-bind a run after a page reload). It otherwise
+     * resolves on the same terminal events as `research` (done / cancelled /
+     * convergence_failed / error).
+     */
+    async reconnect(callbacks, options) {
+        return this._runSocket({}, callbacks, options, true);
+    }
+    _runSocket(extra, callbacks, options, reconnect = false) {
+        return new Promise((resolve, reject) => {
+            const protocol = Fauna.getURL().startsWith('https') ? 'wss' : 'ws';
+            const route = Fauna.getURL().replace(/^https?:\/\//, '');
+            const params = { "Authorization": 'email' in this._authorization ? `Token ${this._authorization.token}` : `ApiKey ${this._authorization.id}`, ...(this._group ? { Group: this._group.id } : {}), Workspace: this.workspaceId, Dataset: this.id, ...extra };
+            if (options?.text)
+                params['Text'] = options.text;
+            if (options?.embedding)
+                params['Embedding'] = options.embedding;
+            // Attach-only: bind to an in-flight run and replay its log, but never
+            // start a new one. The server answers `inactive` if nothing is running.
+            if (reconnect)
+                params['Attach'] = 'true';
+            this._ws = new WebSocket(`${protocol}://${route}/marmot/research?authorization=${encodeURIComponent(JSON.stringify(params))}`);
+            // Guard against settling twice (a terminal event can be followed by
+            // the socket `close`/`error` firing).
+            let settled = false;
+            let attached = false;
+            const finish = () => { if (!settled) {
+                settled = true;
+                resolve();
+            } };
+            const fail = (message) => {
+                if (settled)
+                    return;
+                settled = true;
+                this._ws?.close();
+                this._ws = null;
+                reject(new Error(message));
+            };
+            this._ws.addEventListener('message', (event) => {
+                const packet = JSON.parse(event.data);
+                switch (packet.event) {
+                    case 'connection':
+                        attached = packet.ongoing === true;
+                        callbacks.onConnection?.(packet.ongoing === true);
+                        break;
+                    case 'inactive':
+                        // Attach-only miss: nothing is running for this dataset.
+                        fail('No active research run');
+                        break;
+                    case 'query':
+                        callbacks.onQuery?.(packet.query);
+                        break;
+                    case 'source':
+                        callbacks.onSource?.(packet.url, packet.title, packet.contentType);
+                        break;
+                    case 'answer':
+                        callbacks.onAnswer?.(packet.field, packet.value, packet.converged, packet.answersCount, packet.sources);
+                        break;
+                    case 'nomination_accepted':
+                        callbacks.onNomination?.(packet.scope, packet.row, packet.committedCount, packet.sources);
+                        break;
+                    case 'supervisor_query':
+                        callbacks.onSupervisorQuery?.(packet.question);
+                        break;
+                    case 'supervisor_search':
+                        callbacks.onSupervisorSearch?.(packet.query);
+                        break;
+                    case 'supervisor_answer':
+                        callbacks.onSupervisorAnswer?.(packet.question, packet.answer);
+                        break;
+                    case 'supervisor_refused':
+                        callbacks.onSupervisorRefused?.(packet.question);
+                        break;
+                    case 'browser':
+                        callbacks.onBrowser?.({
+                            url: packet.url,
+                            reference: packet.reference,
+                            code: packet.code,
+                            mode: packet.mode,
+                        });
+                        break;
+                    case 'browser_step':
+                        callbacks.onBrowserStep?.(packet.url, packet.step, packet.action);
+                        break;
+                    case 'compress':
+                        callbacks.onCompress?.();
+                        break;
+                    case 'info':
+                        callbacks.onInfo?.(packet.message);
+                        break;
+                    case 'done': {
+                        const snapshot = new Snapshot(this._authorization, this.workspaceId, this.id, packet.snapshotId, new Date(), packet.valid ?? true, this._group);
+                        this.snapshots.push(snapshot);
+                        callbacks.onDone?.(snapshot);
+                        this._ws?.close();
+                        this._ws = null;
+                        finish();
+                        break;
+                    }
+                    case 'cancelled':
+                        callbacks.onCancelled?.(packet.reason ?? 'user');
+                        this._ws?.close();
+                        this._ws = null;
+                        finish();
+                        break;
+                    case 'convergence_failed':
+                        callbacks.onConvergenceFailed?.(packet.field, packet.reason, packet.candidates ?? []);
+                        this._ws?.close();
+                        this._ws = null;
+                        finish();
+                        break;
+                    case 'error':
+                        callbacks.onError?.(packet.message);
+                        this._ws?.close();
+                        this._ws = null;
+                        finish();
+                        break;
+                }
+            });
+            this._ws.addEventListener('error', () => {
+                this._ws = null;
+                // A connection-level failure before we attached to a run leaves a
+                // reconnect caller with nothing to resume — surface it as a reject.
+                if (reconnect && !attached) {
+                    fail('Failed to connect to research run');
+                    return;
+                }
+                callbacks.onError?.('WebSocket connection failed');
+                finish();
+            });
+            this._ws.addEventListener('close', () => {
+                // Socket closed before we ever attached (reconnect path): treat as
+                // "nothing to resume" so callers can try/catch on load.
+                if (reconnect && !attached) {
+                    fail('No active research run');
+                    return;
+                }
+                finish();
+            });
+        });
+    }
+    async cancel() {
+        // Send the cancel signal but do NOT close the socket. The server
+        // finishes the in-flight operation, breaks out of its loop, and emits
+        // a 'cancelled' event before closing the socket itself. Keeping the
+        // socket open lets the client observe trailing logs and the final
+        // 'cancelled' confirmation.
+        if (this._ws && this._ws.readyState === WebSocket.OPEN) {
+            this._ws.send(JSON.stringify({ event: 'cancel' }));
+        }
+    }
+    async rename(name) {
+        const response = await fetch(`${Fauna.getURL()}/marmot/dataset/update`, {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json",
+                "Authorization": 'email' in this._authorization ? `Token ${this._authorization.token}` : `ApiKey ${this._authorization.id}`,
+                ...(this._group ? { Group: this._group.id } : {})
+            },
+            body: JSON.stringify({ workspaceId: this.workspaceId, id: this.id, name })
+        });
+        if (!response.ok) {
+            throw new Error(await response.text());
+        }
+        this.name = name;
+    }
+    /**
+     * Persist a new schema WITHOUT producing a snapshot. Use for edits that
+     * don't change the set of columns (prompt / sources / optional tweaks);
+     * for column add/drop use `migrate` instead so existing data is reconciled.
+     */
+    async saveSchema(schema) {
+        const response = await fetch(`${Fauna.getURL()}/marmot/dataset/update`, {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json",
+                "Authorization": 'email' in this._authorization ? `Token ${this._authorization.token}` : `ApiKey ${this._authorization.id}`,
+                ...(this._group ? { Group: this._group.id } : {})
+            },
+            body: JSON.stringify({ workspaceId: this.workspaceId, id: this.id, schema: schema.serialize() })
+        });
+        if (!response.ok) {
+            throw new Error(await response.text());
+        }
+        this.schema = schema;
+    }
+    /**
+     * Migrate the dataset to a new schema, producing a fresh snapshot. Runs
+     * over the research WebSocket so the caller sees the same live timeline.
+     *
+     *   - `seedSnapshotId` = a snapshot id: the migration starts from that
+     *     snapshot's rows, drops removed columns, and backfills added columns
+     *     (researching only the new fields per row, reusing each row's stored
+     *     source URLs first).
+     *   - `seedSnapshotId` = null: a from-scratch research run with the new
+     *     schema (no backfill).
+     *
+     * The diff between the seed snapshot's schema and `schema` is recomputed
+     * authoritatively on the server. On completion the dataset's schema is
+     * updated to `schema`.
+     */
+    async migrate(schema, seedSnapshotId, callbacks, options) {
+        await this._runSocket({ Migrate: JSON.stringify({ seedSnapshotId, schema: schema.serialize() }) }, { ...callbacks, onDone: (snapshot) => { this.schema = schema; callbacks.onDone?.(snapshot); } }, options);
+    }
+    async setTermination(termination) {
+        const response = await fetch(`${Fauna.getURL()}/marmot/dataset/update`, {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json",
+                "Authorization": 'email' in this._authorization ? `Token ${this._authorization.token}` : `ApiKey ${this._authorization.id}`,
+                ...(this._group ? { Group: this._group.id } : {})
+            },
+            body: JSON.stringify({ workspaceId: this.workspaceId, id: this.id, termination })
+        });
+        if (!response.ok) {
+            throw new Error(await response.text());
+        }
+        ;
+        this.termination = termination;
+    }
+    async setContentTypes(contentTypes) {
+        const response = await fetch(`${Fauna.getURL()}/marmot/dataset/update`, {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json",
+                "Authorization": 'email' in this._authorization ? `Token ${this._authorization.token}` : `ApiKey ${this._authorization.id}`,
+                ...(this._group ? { Group: this._group.id } : {})
+            },
+            body: JSON.stringify({ workspaceId: this.workspaceId, id: this.id, contentTypes })
+        });
+        if (!response.ok) {
+            throw new Error(await response.text());
+        }
+        this.contentTypes = contentTypes;
+    }
+    async setProxy(proxy) {
+        const response = await fetch(`${Fauna.getURL()}/marmot/dataset/update`, {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json",
+                "Authorization": 'email' in this._authorization ? `Token ${this._authorization.token}` : `ApiKey ${this._authorization.id}`,
+                ...(this._group ? { Group: this._group.id } : {})
+            },
+            body: JSON.stringify({ workspaceId: this.workspaceId, id: this.id, proxy })
+        });
+        if (!response.ok) {
+            throw new Error(await response.text());
+        }
+        this.proxy = proxy;
+    }
+    async setDocumentModes(documentModes) {
+        const response = await fetch(`${Fauna.getURL()}/marmot/dataset/update`, {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json",
+                "Authorization": 'email' in this._authorization ? `Token ${this._authorization.token}` : `ApiKey ${this._authorization.id}`,
+                ...(this._group ? { Group: this._group.id } : {})
+            },
+            body: JSON.stringify({ workspaceId: this.workspaceId, id: this.id, documentModes })
+        });
+        if (!response.ok) {
+            throw new Error(await response.text());
+        }
+        this.documentModes = documentModes;
+    }
+    async remove() {
+        const response = await fetch(`${Fauna.getURL()}/marmot/dataset/remove`, {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json",
+                "Authorization": 'email' in this._authorization ? `Token ${this._authorization.token}` : `ApiKey ${this._authorization.id}`,
+                ...(this._group ? { Group: this._group.id } : {})
+            },
+            body: JSON.stringify({ workspaceId: this.workspaceId, id: this.id })
+        });
+        if (!response.ok) {
+            throw new Error(await response.text());
+        }
+    }
+    /**
+     * The dataset's data as CSV text or, by default, a JSON array: its live
+     * snapshot when new rows are stored into it, else its newest valid snapshot.
+     */
+    async data(options) {
+        const response = await fetch(datasetDataURL(this.id, options), {
+            method: "GET",
+            headers: {
+                "Authorization": 'email' in this._authorization ? `Token ${this._authorization.token}` : `ApiKey ${this._authorization.id}`,
+            }
+        });
+        if (!response.ok) {
+            throw new Error(await response.text());
+        }
+        return options?.format === 'csv' ? await response.text() : await response.json();
+    }
+    /** The `/data` address, for pipelines and curl. Calls to it still need your authorization. */
+    dataURL(options) {
+        return datasetDataURL(this.id, options);
+    }
+    /**
+     * Push rows into the dataset: objects shaped like its rows, or CSV text with a
+     * header row. Rows that don't fit come back in `invalid`; when none fit, it
+     * throws a PushRefused listing them.
+     */
+    async addRows(rows, options) {
+        const query = options?.source ? `?${new URLSearchParams({ source: options.source }).toString()}` : '';
+        const response = await fetch(`${Fauna.getURL()}/marmot/dataset/${this.id}/rows${query}`, {
+            method: "POST",
+            headers: {
+                "Content-Type": typeof rows === 'string' ? "text/csv" : "application/json",
+                "Authorization": 'email' in this._authorization ? `Token ${this._authorization.token}` : `ApiKey ${this._authorization.id}`,
+            },
+            body: typeof rows === 'string' ? rows : JSON.stringify(rows),
+        });
+        if (!response.ok) {
+            const text = await response.text();
+            let body = null;
+            try {
+                body = JSON.parse(text);
+            }
+            catch { /* plain text */ }
+            throw new PushRefused(response.status, body?.message ?? text, PushResult.fromJSON(body ?? {}).invalid);
+        }
+        return PushResult.fromJSON(await response.json());
+    }
+    /** Rows pushed after `since`, oldest first, at most `max`. The next page starts at the last `seq` here. */
+    async updates(query) {
+        const params = new URLSearchParams({ since: String(query.since), max: String(query.max) });
+        const response = await fetch(`${Fauna.getURL()}/marmot/dataset/${this.id}/updates?${params.toString()}`, {
+            method: "GET",
+            headers: {
+                "Authorization": 'email' in this._authorization ? `Token ${this._authorization.token}` : `ApiKey ${this._authorization.id}`,
+            },
+        });
+        if (!response.ok) {
+            throw new Error(await response.text());
+        }
+        return (await response.json()).map(FeedRow.fromJSON);
+    }
+    /**
+     * Every row pushed after `since`, then each new one as it lands. It opens the
+     * live socket first, catches up through `updates`, and skips rows it already
+     * gave, so nothing is missed or repeated. It reconnects by itself until `signal` aborts.
+     */
+    async *live(options) {
+        let since = options?.since ?? 0;
+        const signal = options?.signal;
+        while (!signal?.aborted) {
+            const held = [];
+            let ready = false;
+            let closed = false;
+            let nudge = null;
+            const wake = () => { const n = nudge; nudge = null; n?.(); };
+            const protocol = Fauna.getURL().startsWith('https') ? 'wss' : 'ws';
+            const route = Fauna.getURL().replace(/^https?:\/\//, '');
+            const params = { "Authorization": 'email' in this._authorization ? `Token ${this._authorization.token}` : `ApiKey ${this._authorization.id}`, Dataset: this.id };
+            const ws = new WebSocket(`${protocol}://${route}/marmot/dataset/live?authorization=${encodeURIComponent(JSON.stringify(params))}`);
+            const abort = () => ws.close();
+            signal?.addEventListener('abort', abort, { once: true });
+            ws.addEventListener('message', (event) => {
+                const packet = JSON.parse(event.data);
+                if (packet.event === 'ready')
+                    ready = true;
+                else if (packet.event === 'rows')
+                    held.push(...packet.rows.map(FeedRow.fromJSON));
+                wake();
+            });
+            ws.addEventListener('close', () => { closed = true; wake(); });
+            ws.addEventListener('error', () => { closed = true; wake(); });
+            try {
+                while (!ready && !closed)
+                    await new Promise((resolve) => { nudge = resolve; });
+                if (ready) {
+                    for (;;) {
+                        const page = await this.updates({ since, max: FEED_PAGE });
+                        for (const row of page) {
+                            if (row.seq <= since)
+                                continue;
+                            since = row.seq;
+                            yield row;
+                        }
+                        if (page.length < FEED_PAGE)
+                            break;
+                    }
+                    while (!closed || held.length > 0) {
+                        const row = held.shift();
+                        if (!row) {
+                            await new Promise((resolve) => { nudge = resolve; });
+                            continue;
+                        }
+                        if (row.seq <= since)
+                            continue;
+                        since = row.seq;
+                        yield row;
+                    }
+                }
+            }
+            finally {
+                signal?.removeEventListener('abort', abort);
+                ws.close();
+            }
+            if (!signal?.aborted)
+                await new Promise((r) => setTimeout(r, 2000));
+        }
+    }
+    /** Whether this dataset is kept updated, and how it's going. Null when it isn't. */
+    async scraper() {
+        return getScraper(this._authorization, this.id);
+    }
+    /** Keep this dataset updated by its workspace's scraper agent. You become the person the agent runs as. */
+    async keepUpdated(settings = {}) {
+        return createScraper(this._authorization, this.id, settings);
+    }
+    async updateScraper(settings) {
+        return updateScraper(this._authorization, this.id, settings);
+    }
+    /** Ask the agent to go over this dataset's scraper again, with a note for it. */
+    async rebuildScraper(note) {
+        return rebuildScraper(this._authorization, this.id, note);
+    }
+    /** Stop keeping this dataset updated. */
+    async stopKeepingUpdated() {
+        return removeScraper(this._authorization, this.id);
+    }
+}
+// ─── API Functions ───────────────────────────────────────────────────────────
+export async function listDatasets(authorization, workspaceId, opts) {
+    const response = await fetch(`${Fauna.getURL()}/marmot/dataset/list`, {
+        method: "POST",
+        headers: {
+            "Content-Type": "application/json",
+            "Authorization": 'email' in authorization ? `Token ${authorization.token}` : `ApiKey ${authorization.id}`, ...(opts?.group ? { Group: opts.group.id } : {})
+        },
+        body: JSON.stringify({ workspaceId })
+    });
+    if (!response.ok) {
+        throw new Error(await response.text());
+    }
+    const data = await response.json();
+    return data.map((d) => new Dataset(authorization, workspaceId, d.id, d.name, Schema.deserialize(d.schema), d.termination ?? [], d.contentTypes ?? [], (d.proxy ?? 'datacenter'), (d.documentModes ?? ['extraction']), (d.snapshots ?? []).map((s) => new Snapshot(authorization, workspaceId, d.id, s.id, new Date(s.createdAt), s.valid ?? true, opts?.group)), opts?.group));
+}
+export async function createDataset(authorization, workspaceId, options, opts) {
+    const response = await fetch(`${Fauna.getURL()}/marmot/dataset/create`, {
+        method: "POST",
+        headers: {
+            "Content-Type": "application/json",
+            "Authorization": 'email' in authorization ? `Token ${authorization.token}` : `ApiKey ${authorization.id}`, ...(opts?.group ? { Group: opts.group.id } : {})
+        },
+        body: JSON.stringify({
+            workspaceId,
+            name: options.name,
+            schema: options.schema.serialize(),
+            termination: options.termination ?? [],
+            contentTypes: options.contentTypes ?? [],
+            proxy: options.proxy ?? 'datacenter',
+            documentModes: options.documentModes ?? ['extraction']
+        })
+    });
+    if (!response.ok) {
+        throw new Error(await response.text());
+    }
+    return await response.text();
+}
+function toBase64(data) {
+    let binary = '';
+    const chunk = 0x8000;
+    for (let i = 0; i < data.length; i += chunk) {
+        binary += String.fromCharCode.apply(null, Array.from(data.subarray(i, i + chunk)));
+    }
+    return typeof btoa !== 'undefined' ? btoa(binary) : Buffer.from(data).toString('base64');
+}
+export class Workspace {
+    constructor(authorization, id, name, description, createdAt, datasets = 0, files = 0, group) {
+        Object.defineProperty(this, "id", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "name", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "description", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "createdAt", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "datasets", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "files", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "_authorization", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "_group", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this._authorization = authorization;
+        this.id = id;
+        this.name = name;
+        this.description = description;
+        this.createdAt = createdAt;
+        this.datasets = datasets;
+        this.files = files;
+        this._group = group;
+    }
+    async listDatasets() {
+        return listDatasets(this._authorization, this.id, this._group ? { group: this._group } : undefined);
+    }
+    async createDataset(options) {
+        return createDataset(this._authorization, this.id, options, this._group ? { group: this._group } : undefined);
+    }
+    async rename(name) {
+        await updateWorkspace(this._authorization, this.id, { name }, this._group ? { group: this._group } : undefined);
+        this.name = name;
+    }
+    async setDescription(description) {
+        await updateWorkspace(this._authorization, this.id, { description }, this._group ? { group: this._group } : undefined);
+        this.description = description;
+    }
+    async remove() {
+        const response = await fetch(`${Fauna.getURL()}/marmot/workspace/remove`, {
+            method: "POST",
+            headers: { "Content-Type": "application/json", "Authorization": 'email' in this._authorization ? `Token ${this._authorization.token}` : `ApiKey ${this._authorization.id}`, ...(this._group ? { Group: this._group.id } : {}) },
+            body: JSON.stringify({ id: this.id })
+        });
+        if (!response.ok)
+            throw new Error(await response.text());
+    }
+    async getFiles() {
+        const detail = await getWorkspace(this._authorization, this.id, this._group ? { group: this._group } : undefined);
+        return detail.files;
+    }
+    async uploadFile(name, contentType, data) {
+        const response = await fetch(`${Fauna.getURL()}/marmot/workspace/file/upload`, {
+            method: "POST",
+            headers: { "Content-Type": "application/json", "Authorization": 'email' in this._authorization ? `Token ${this._authorization.token}` : `ApiKey ${this._authorization.id}`, ...(this._group ? { Group: this._group.id } : {}) },
+            body: JSON.stringify({ workspaceId: this.id, name, contentType, data: toBase64(data) })
+        });
+        if (!response.ok)
+            throw new Error(await response.text());
+        return await response.json();
+    }
+    async downloadFile(file) {
+        const response = await fetch(`${Fauna.getURL()}/marmot/workspace/file/get`, {
+            method: "POST",
+            headers: { "Content-Type": "application/json", "Authorization": 'email' in this._authorization ? `Token ${this._authorization.token}` : `ApiKey ${this._authorization.id}`, ...(this._group ? { Group: this._group.id } : {}) },
+            body: JSON.stringify({ key: file.key, contentType: file.contentType, workspaceId: this.id })
+        });
+        if (!response.ok)
+            throw new Error(await response.text());
+        return await response.blob();
+    }
+    async removeFile(key) {
+        const response = await fetch(`${Fauna.getURL()}/marmot/workspace/file/remove`, {
+            method: "POST",
+            headers: { "Content-Type": "application/json", "Authorization": 'email' in this._authorization ? `Token ${this._authorization.token}` : `ApiKey ${this._authorization.id}`, ...(this._group ? { Group: this._group.id } : {}) },
+            body: JSON.stringify({ workspaceId: this.id, key })
+        });
+        if (!response.ok)
+            throw new Error(await response.text());
+    }
+}
+export async function listWorkspaces(authorization, opts) {
+    const response = await fetch(`${Fauna.getURL()}/marmot/workspace/list`, {
+        method: "POST",
+        headers: { "Content-Type": "application/json", "Authorization": 'email' in authorization ? `Token ${authorization.token}` : `ApiKey ${authorization.id}`, ...(opts?.group ? { Group: opts.group.id } : {}) }
+    });
+    if (!response.ok)
+        throw new Error(await response.text());
+    const data = await response.json();
+    return data.map((w) => new Workspace(authorization, w.id, w.name, w.description ?? '', w.createdAt ?? 0, w.datasets ?? 0, w.files ?? 0, opts?.group));
+}
+export async function getWorkspace(authorization, id, opts) {
+    const response = await fetch(`${Fauna.getURL()}/marmot/workspace/get`, {
+        method: "POST",
+        headers: { "Content-Type": "application/json", "Authorization": 'email' in authorization ? `Token ${authorization.token}` : `ApiKey ${authorization.id}`, ...(opts?.group ? { Group: opts.group.id } : {}) },
+        body: JSON.stringify({ id })
+    });
+    if (!response.ok)
+        throw new Error(await response.text());
+    return await response.json();
+}
+export async function createWorkspace(authorization, name, description = '', opts) {
+    const response = await fetch(`${Fauna.getURL()}/marmot/workspace/create`, {
+        method: "POST",
+        headers: { "Content-Type": "application/json", "Authorization": 'email' in authorization ? `Token ${authorization.token}` : `ApiKey ${authorization.id}`, ...(opts?.group ? { Group: opts.group.id } : {}) },
+        body: JSON.stringify({ name, description })
+    });
+    if (!response.ok)
+        throw new Error(await response.text());
+    return await response.text();
+}
+export async function updateWorkspace(authorization, id, update, opts) {
+    const response = await fetch(`${Fauna.getURL()}/marmot/workspace/update`, {
+        method: "POST",
+        headers: { "Content-Type": "application/json", "Authorization": 'email' in authorization ? `Token ${authorization.token}` : `ApiKey ${authorization.id}`, ...(opts?.group ? { Group: opts.group.id } : {}) },
+        body: JSON.stringify({ id, ...update })
+    });
+    if (!response.ok)
+        throw new Error(await response.text());
+}