fauna-api 0.0.0-stage → 1.0.676

package/dist/src/marmot/media.js

@@ -0,0 +1,265 @@
+import { fromBytes } from "../drive.js";
+import { Marmot } from "./index.js";
+export const ATTACHMENT_TYPES = {
+    'image/jpeg': 'image',
+    'image/png': 'image',
+    'image/webp': 'image',
+    'image/heic': 'image',
+    'image/heif': 'image',
+    'audio/mpeg': 'audio_file',
+    'audio/wav': 'audio_file',
+    'video/mp4': 'movie',
+    'video/webm': 'movie',
+    'video/quicktime': 'movie',
+    'application/pdf': 'picture_as_pdf',
+    'text/plain': 'description',
+    'text/markdown': 'description',
+    'text/csv': 'table_chart',
+    'text/html': 'language',
+    'application/json': 'data_object',
+    'application/xml': 'code',
+};
+/** Resolve a usable MIME when the browser leaves `File.type` empty (common for HEIC). */
+export function mimeFromFileName(name) {
+    const ext = name.includes('.') ? name.split('.').pop().toLowerCase() : '';
+    switch (ext) {
+        case 'jpg':
+        case 'jpeg': return 'image/jpeg';
+        case 'png': return 'image/png';
+        case 'webp': return 'image/webp';
+        case 'heic': return 'image/heic';
+        case 'heif': return 'image/heif';
+        case 'mp3': return 'audio/mpeg';
+        case 'wav': return 'audio/wav';
+        case 'mp4': return 'video/mp4';
+        case 'webm': return 'video/webm';
+        case 'mov':
+        case 'qt': return 'video/quicktime';
+        case 'pdf': return 'application/pdf';
+        case 'txt': return 'text/plain';
+        case 'md':
+        case 'markdown': return 'text/markdown';
+        case 'csv': return 'text/csv';
+        case 'html':
+        case 'htm': return 'text/html';
+        case 'json': return 'application/json';
+        case 'xml': return 'application/xml';
+        default: return undefined;
+    }
+}
+let TYPES_CACHE = null;
+export async function getTypes() {
+    if (TYPES_CACHE)
+        return TYPES_CACHE;
+    const response = await fetch(`${Marmot.getURL()}/media/types`, { method: "GET" });
+    if (!response.ok)
+        throw new Error(await response.text());
+    TYPES_CACHE = await response.json();
+    return TYPES_CACHE;
+}
+export async function query(authorization, query, options) {
+    const response = await fetch(`${Marmot.getURL()}/media/query`, {
+        method: "POST",
+        headers: {
+            "Content-Type": "application/json",
+            "Authorization": 'email' in authorization ? `Token ${authorization.token}` : `ApiKey ${authorization.id}`
+        },
+        body: JSON.stringify({
+            query: query,
+            cursor: options.cursor,
+            max: options.max,
+            ...(options.start ? { start: options.start.toISOString() } : {}),
+            ...(options.end ? { end: options.end.toISOString() } : {}),
+        })
+    });
+    return await response.json();
+}
+export async function pull(authorization, url, request, options = {}) {
+    const effectiveOptions = { ...options, proxy: options.proxy ?? 'datacenter' };
+    const response = await fetch(`${Marmot.getURL()}/media/pull`, {
+        method: "POST",
+        headers: {
+            "Content-Type": "application/json",
+            "Authorization": 'email' in authorization ? `Token ${authorization.token}` : `ApiKey ${authorization.id}`,
+            "Url": url,
+            ...(request ? { "Request": JSON.stringify(request) } : {}),
+            "Options": JSON.stringify(effectiveOptions)
+        }
+    });
+    if (!response.ok)
+        throw new Error(`Failed to fetch website at ${url}.`);
+    const contentType = response.headers.get('Content-Type') || 'application/octet-stream';
+    const buffer = await parseStream(response);
+    return { contentType, data: Array.isArray(buffer) ? buffer[0] : buffer };
+}
+function chatParts(authorization, raw) {
+    return raw.map((part) => part?.type === 'FILE'
+        ? { type: 'FILE', file: fromBytes(authorization, String(part.name ?? 'file'), String(part.contentType ?? 'application/octet-stream'), Uint8Array.from(atob(String(part.data ?? '')), (c) => c.charCodeAt(0))) }
+        : { type: 'TEXT', text: String(part?.text ?? '') });
+}
+/**
+ * A media file as message parts for a model that takes `takes` natively (images only
+ * by default). What it takes passes through; the rest becomes what it can take: a
+ * video becomes a line, its frames and its sound (audio, or a description of it),
+ * audio becomes a description, and documents become their text. With `split`, one
+ * list of parts per segment of that many milliseconds. Undefined when it can't be read.
+ */
+export async function chat(authorization, contentType, data, options = {}) {
+    const response = await fetch(`${Marmot.getURL()}/media/chat`, {
+        method: "POST",
+        headers: {
+            "Content-Type": contentType,
+            "Authorization": 'email' in authorization ? `Token ${authorization.token}` : `ApiKey ${authorization.id}`,
+            "Takes": (options.takes ?? ['image']).join(','),
+            ...(options.name ? { "Name": encodeURIComponent(options.name) } : {}),
+            ...(options.segment ? { "Segment": `${options.segment[0]},${options.segment[1]}` } : {}),
+            ...(options.split ? { "Split": options.split.toString() } : {})
+        },
+        body: data.buffer
+    });
+    if (!response.ok) {
+        return undefined;
+    }
+    const json = await response.json();
+    if (!Array.isArray(json))
+        return undefined;
+    if (options.split)
+        return json.map((segment) => chatParts(authorization, Array.isArray(segment) ? segment : []));
+    return chatParts(authorization, json);
+}
+export async function parse(authorization, contentType, data, onProgress) {
+    return new Promise((resolve, reject) => {
+        const xhr = new XMLHttpRequest();
+        xhr.open("POST", `${Marmot.getURL()}/media/parse`);
+        xhr.setRequestHeader("Authorization", 'email' in authorization ? `Token ${authorization.token}` : `ApiKey ${authorization.id}`);
+        xhr.setRequestHeader("Content-Type", contentType || "application/octet-stream");
+        xhr.upload.onprogress = (event) => {
+            if (event.lengthComputable && onProgress) {
+                onProgress(event.loaded / event.total);
+            }
+        };
+        xhr.onload = () => {
+            if (xhr.status >= 200 && xhr.status < 300) {
+                resolve(xhr.responseText);
+            }
+            else {
+                reject(new Error(`Failed to parse media: ${xhr.status}`));
+            }
+        };
+        xhr.onerror = () => reject(new Error(`Failed to parse media.`));
+        xhr.send(data.buffer);
+    });
+}
+export async function slice(authorization, contentType, data, options) {
+    const response = await fetch(`${Marmot.getURL()}/media/slice`, {
+        method: "POST",
+        headers: {
+            "Content-Type": contentType || "application/octet-stream",
+            "Authorization": 'email' in authorization ? `Token ${authorization.token}` : `ApiKey ${authorization.id}`,
+            ...(options.segment ? { "Segment": `${options.segment[0]},${options.segment[1]}` } : {}),
+            ...(options.split ? { "Split": options.split.toString() } : {})
+        },
+        body: data.buffer
+    });
+    if (!response.ok) {
+        return undefined;
+    }
+    return await parseStream(response);
+}
+export async function duration(authorization, contentType, data, onProgress) {
+    return new Promise((resolve, reject) => {
+        const xhr = new XMLHttpRequest();
+        xhr.open("POST", `${Marmot.getURL()}/media/duration`);
+        xhr.setRequestHeader("Authorization", 'email' in authorization ? `Token ${authorization.token}` : `ApiKey ${authorization.id}`);
+        xhr.setRequestHeader("Content-Type", contentType || "application/octet-stream");
+        xhr.upload.onprogress = (event) => {
+            if (event.lengthComputable && onProgress) {
+                onProgress(event.loaded / event.total);
+            }
+        };
+        xhr.onload = () => {
+            if (xhr.status >= 200 && xhr.status < 300) {
+                resolve(Number.parseFloat(xhr.responseText));
+            }
+            else {
+                reject(new Error(`Failed to duration media: ${xhr.status}`));
+            }
+        };
+        xhr.onerror = () => reject(new Error(`Failed to duration media.`));
+        xhr.send(data.buffer);
+    });
+}
+async function parseStream(response) {
+    const encodingSize = response.headers.get('Encoding-Size');
+    if (encodingSize && parseInt(encodingSize) > 1) {
+        const reader = response.body.getReader();
+        let buffer = new Uint8Array(0);
+        const boundaryBytes = new TextEncoder().encode('--BOUNDARY');
+        const parts = [];
+        while (true) {
+            const { done, value } = await reader.read();
+            if (done)
+                break;
+            if (value) {
+                const tmp = new Uint8Array(buffer.length + value.length);
+                tmp.set(buffer, 0);
+                tmp.set(value, buffer.length);
+                buffer = tmp;
+            }
+        }
+        function indexOfBoundary(buf, startPos = 0) {
+            for (let i = startPos; i <= buf.length - boundaryBytes.length; i++) {
+                let match = true;
+                for (let j = 0; j < boundaryBytes.length; j++) {
+                    if (buf[i + j] !== boundaryBytes[j]) {
+                        match = false;
+                        break;
+                    }
+                }
+                if (match) {
+                    return i;
+                }
+            }
+            return -1;
+        }
+        let currentPos = 0;
+        let boundaryCount = 0;
+        while (true) {
+            const idx = indexOfBoundary(buffer, currentPos);
+            if (idx === -1) {
+                if (currentPos < buffer.length) {
+                    const part = buffer.slice(currentPos);
+                    if (part.length > 0) {
+                        parts.push(part);
+                    }
+                }
+                break;
+            }
+            else {
+                boundaryCount++;
+                const part = buffer.slice(currentPos, idx);
+                if (part.length > 0) {
+                    parts.push(part);
+                }
+                currentPos = idx + boundaryBytes.length;
+            }
+        }
+        return parts;
+    }
+    else {
+        const reader = response.body.getReader();
+        let buffer = new Uint8Array(0);
+        while (true) {
+            const { done, value } = await reader.read();
+            if (done)
+                break;
+            if (value) {
+                const tmp = new Uint8Array(buffer.length + value.length);
+                tmp.set(buffer, 0);
+                tmp.set(value, buffer.length);
+                buffer = tmp;
+            }
+        }
+        return buffer;
+    }
+}

package/dist/src/marmot/research.d.ts

@@ -0,0 +1,337 @@
+import { Fauna } from "../../index.js";
+import { ApiKey, Credentials } from "../authentification.js";
+import { Group } from "../groups.js";
+import type { ProxyMode, DocumentMode } from "./media.js";
+import type { BrowserTool } from "./browser.js";
+import { Scraper, ScraperSettings } from "./scraper.js";
+export type SchemaLeaf = {
+    value: Fauna.Parrot.Format.Format<any> | SchemaLeaf | Record<string, SchemaLeaf>;
+    prompt?: string;
+    sources?: number;
+    optional?: boolean;
+    array?: {
+        minimum: number;
+        maximum: number;
+    };
+    maxIterations?: number;
+    filters?: string | string[];
+};
+export declare abstract class Schema {
+    abstract readonly type: string;
+    abstract readonly schema: SchemaLeaf | Record<string, SchemaLeaf>;
+    abstract serialize(): any;
+    static deserialize(data: any): Schema;
+}
+export declare class SchemaJSON extends Schema {
+    readonly type = "json";
+    readonly schema: SchemaLeaf;
+    constructor(schema: SchemaLeaf);
+    serialize(): any;
+}
+export declare class SchemaCSV extends Schema {
+    readonly type = "csv";
+    readonly schema: Record<string, SchemaLeaf>;
+    readonly minimum: number;
+    readonly maximum: number;
+    readonly prompt: string;
+    constructor(schema: Record<string, SchemaLeaf>, minimum: number, maximum: number, prompt?: string);
+    serialize(): any;
+}
+/**
+ * Flatten a schema to a map of `leaf path → type signature`.
+ *   - CSV: each column key is a path, signature is its Format type.
+ *   - JSON: object keys descend with `.`, array leaves append `[]`, Format
+ *     leaves terminate. e.g. "name", "address.city", "awards[].year".
+ */
+export declare function schemaPaths(schema: Schema): Map<string, string>;
+/**
+ * Diff two schema versions for a dataset migration. A path whose type
+ * signature changed shows up in BOTH lists (drop the old, add the new) — we
+ * deliberately do not attempt rename/retype detection. Renames and tree-shape
+ * changes therefore fall out naturally as drop + add.
+ */
+export declare function diffSchema(baseline: Schema, target: Schema): {
+    added: string[];
+    dropped: string[];
+};
+export interface Termination {
+    type: string;
+}
+export interface TokenTermination extends Termination {
+    type: 'tokens';
+    max: number;
+}
+export interface TimeTermination extends Termination {
+    type: 'time';
+    max: number;
+}
+export interface CostTermination extends Termination {
+    type: 'cost';
+    max: number;
+}
+/** How `/data` writes rows: CSV with a header row, or a JSON array. */
+export type DataFormat = 'csv' | 'json';
+export type DataOptions = {
+    format?: DataFormat;
+    /** A snapshot id. The dataset's current data when absent. */
+    snapshot?: string;
+    /** Add each row's sources. */
+    sources?: boolean;
+};
+/** A row pushed into a dataset, numbered by `seq`, which rises within the dataset. */
+export type FeedRow = {
+    seq: number;
+    value: any;
+    sources: any;
+    created: Date;
+};
+export declare const FeedRow: {
+    fromJSON(json: any): FeedRow;
+};
+/** What a push stored, and the rows it left out with why. */
+export type PushResult = {
+    added: number;
+    first: number | null;
+    last: number | null;
+    invalid: {
+        index: number;
+        reason: string;
+    }[];
+};
+export declare const PushResult: {
+    fromJSON(json: any): PushResult;
+};
+/** A push nothing in was let through: why, and each row's reason. */
+export declare class PushRefused extends Error {
+    readonly status: number;
+    readonly invalid: {
+        index: number;
+        reason: string;
+    }[];
+    constructor(status: number, message: string, invalid: {
+        index: number;
+        reason: string;
+    }[]);
+}
+export declare class Snapshot {
+    readonly id: string;
+    readonly createdAt: Date;
+    readonly valid: boolean;
+    protected _authorization: Credentials | ApiKey;
+    protected _workspaceId: string;
+    protected _datasetId: string;
+    protected _group?: Group;
+    protected _schema: Schema | null;
+    protected _data: any | null;
+    constructor(authorization: Credentials | ApiKey, workspaceId: string, datasetId: string, id: string, createdAt: Date, valid?: boolean, group?: Group);
+    download(assets?: boolean, revalidate?: boolean): Promise<any>;
+    /** This snapshot's rows as CSV text or, by default, a JSON array. */
+    data(options?: {
+        format?: DataFormat;
+        sources?: boolean;
+    }): Promise<any>;
+    downloadSchema(revalidate?: boolean): Promise<Schema>;
+    downloadLog(): Promise<Blob>;
+    downloadEmbeddings(): Promise<Blob>;
+    delete(): Promise<void>;
+}
+export type ResearchCallbacks = {
+    onConnection?: (ongoing: boolean) => void;
+    onQuery?: (query: string) => void;
+    onSource?: (url: string, title: string, contentType: string) => void;
+    onAnswer?: (field: string, value: any, converged: boolean, answersCount: number, sources?: string[]) => void;
+    onNomination?: (scope: string, row: any, committedCount: number, sources?: string[]) => void;
+    onSupervisorQuery?: (question: string) => void;
+    onSupervisorSearch?: (query: string) => void;
+    onSupervisorAnswer?: (question: string, answer: string) => void;
+    onSupervisorRefused?: (question: string) => void;
+    onBrowser?: (frame: {
+        url: string;
+        reference: string;
+        code: string;
+        mode: 'render' | 'virtualized';
+    }) => void;
+    onBrowserStep?: (url: string, step: number, action: BrowserTool) => void;
+    onCompress?: () => void;
+    onInfo?: (message: string) => void;
+    onDone?: (snapshot: Snapshot) => void;
+    onCancelled?: (reason: 'user' | 'tokens' | 'time' | 'cost') => void;
+    onConvergenceFailed?: (field: string, reason: string, candidates: {
+        value: any;
+        source: string;
+    }[]) => void;
+    onError?: (message: string) => void;
+};
+export type ResearchOptions = {
+    text?: Fauna.Parrot.Generative.Text;
+    embedding?: Fauna.Parrot.Generative.Embedding;
+};
+export type { ProxyMode, DocumentMode };
+export type { BrowserTool };
+export declare class Dataset {
+    readonly id: string;
+    readonly workspaceId: string;
+    name: string;
+    schema: Schema;
+    readonly termination: Termination[];
+    contentTypes: string[];
+    proxy: ProxyMode;
+    documentModes: DocumentMode[];
+    snapshots: Snapshot[];
+    private _authorization;
+    private _group?;
+    private _ws;
+    constructor(authorization: Credentials | ApiKey, workspaceId: string, id: string, name: string, schema: Schema, termination: Termination[], contentTypes: string[], proxy: ProxyMode, documentModes: DocumentMode[], snapshots: Snapshot[], group?: Group);
+    research(callbacks: ResearchCallbacks, options?: ResearchOptions): Promise<void>;
+    /** Join a live BrowserRoom opened by a research run. */
+    joinBrowser(accessCode: string): Promise<{
+        room: Fauna.Marmot.Browser.BrowserRoom;
+        self: Fauna.Live.Presence;
+    }>;
+    /**
+     * Attach to an *already-running* research/migration session for this
+     * dataset and replay its live timeline from the start, then keep streaming.
+     *
+     * Unlike `research`, this NEVER starts a new run: if nothing is in flight
+     * the returned promise rejects, so callers can `try/catch` it on load
+     * (e.g. to passively re-bind a run after a page reload). It otherwise
+     * resolves on the same terminal events as `research` (done / cancelled /
+     * convergence_failed / error).
+     */
+    reconnect(callbacks: ResearchCallbacks, options?: ResearchOptions): Promise<void>;
+    private _runSocket;
+    cancel(): Promise<void>;
+    rename(name: string): Promise<void>;
+    /**
+     * Persist a new schema WITHOUT producing a snapshot. Use for edits that
+     * don't change the set of columns (prompt / sources / optional tweaks);
+     * for column add/drop use `migrate` instead so existing data is reconciled.
+     */
+    saveSchema(schema: Schema): Promise<void>;
+    /**
+     * Migrate the dataset to a new schema, producing a fresh snapshot. Runs
+     * over the research WebSocket so the caller sees the same live timeline.
+     *
+     *   - `seedSnapshotId` = a snapshot id: the migration starts from that
+     *     snapshot's rows, drops removed columns, and backfills added columns
+     *     (researching only the new fields per row, reusing each row's stored
+     *     source URLs first).
+     *   - `seedSnapshotId` = null: a from-scratch research run with the new
+     *     schema (no backfill).
+     *
+     * The diff between the seed snapshot's schema and `schema` is recomputed
+     * authoritatively on the server. On completion the dataset's schema is
+     * updated to `schema`.
+     */
+    migrate(schema: Schema, seedSnapshotId: string | null, callbacks: ResearchCallbacks, options?: ResearchOptions): Promise<void>;
+    setTermination(termination: Termination[]): Promise<void>;
+    setContentTypes(contentTypes: string[]): Promise<void>;
+    setProxy(proxy: ProxyMode): Promise<void>;
+    setDocumentModes(documentModes: DocumentMode[]): Promise<void>;
+    remove(): Promise<void>;
+    /**
+     * The dataset's data as CSV text or, by default, a JSON array: its live
+     * snapshot when new rows are stored into it, else its newest valid snapshot.
+     */
+    data(options?: DataOptions): Promise<any>;
+    /** The `/data` address, for pipelines and curl. Calls to it still need your authorization. */
+    dataURL(options?: DataOptions): string;
+    /**
+     * Push rows into the dataset: objects shaped like its rows, or CSV text with a
+     * header row. Rows that don't fit come back in `invalid`; when none fit, it
+     * throws a PushRefused listing them.
+     */
+    addRows(rows: any[] | string, options?: {
+        source?: string;
+    }): Promise<PushResult>;
+    /** Rows pushed after `since`, oldest first, at most `max`. The next page starts at the last `seq` here. */
+    updates(query: {
+        since: number;
+        max: number;
+    }): Promise<FeedRow[]>;
+    /**
+     * Every row pushed after `since`, then each new one as it lands. It opens the
+     * live socket first, catches up through `updates`, and skips rows it already
+     * gave, so nothing is missed or repeated. It reconnects by itself until `signal` aborts.
+     */
+    live(options?: {
+        since?: number;
+        signal?: AbortSignal;
+    }): AsyncGenerator<FeedRow>;
+    /** Whether this dataset is kept updated, and how it's going. Null when it isn't. */
+    scraper(): Promise<Scraper | null>;
+    /** Keep this dataset updated by its workspace's scraper agent. You become the person the agent runs as. */
+    keepUpdated(settings?: ScraperSettings): Promise<Scraper>;
+    updateScraper(settings: ScraperSettings): Promise<Scraper>;
+    /** Ask the agent to go over this dataset's scraper again, with a note for it. */
+    rebuildScraper(note?: string): Promise<void>;
+    /** Stop keeping this dataset updated. */
+    stopKeepingUpdated(): Promise<void>;
+}
+export declare function listDatasets(authorization: Credentials | ApiKey, workspaceId: string, opts?: {
+    group?: Group;
+}): Promise<Dataset[]>;
+export declare function createDataset(authorization: Credentials | ApiKey, workspaceId: string, options: {
+    name: string;
+    schema: Schema;
+    termination?: Termination[];
+    contentTypes?: string[];
+    proxy?: ProxyMode;
+    documentModes?: DocumentMode[];
+}, opts?: {
+    group?: Group;
+}): Promise<string>;
+export type WorkspaceFile = {
+    name: string;
+    contentType: string;
+    size: number;
+    key: string;
+};
+export declare class Workspace {
+    readonly id: string;
+    name: string;
+    description: string;
+    readonly createdAt: number;
+    datasets: number;
+    files: number;
+    private _authorization;
+    private _group?;
+    constructor(authorization: Credentials | ApiKey, id: string, name: string, description: string, createdAt: number, datasets?: number, files?: number, group?: Group);
+    listDatasets(): Promise<Dataset[]>;
+    createDataset(options: {
+        name: string;
+        schema: Schema;
+        termination?: Termination[];
+        contentTypes?: string[];
+        proxy?: ProxyMode;
+        documentModes?: DocumentMode[];
+    }): Promise<string>;
+    rename(name: string): Promise<void>;
+    setDescription(description: string): Promise<void>;
+    remove(): Promise<void>;
+    getFiles(): Promise<WorkspaceFile[]>;
+    uploadFile(name: string, contentType: string, data: Uint8Array): Promise<WorkspaceFile>;
+    downloadFile(file: WorkspaceFile): Promise<Blob>;
+    removeFile(key: string): Promise<void>;
+}
+export declare function listWorkspaces(authorization: Credentials | ApiKey, opts?: {
+    group?: Group;
+}): Promise<Workspace[]>;
+export declare function getWorkspace(authorization: Credentials | ApiKey, id: string, opts?: {
+    group?: Group;
+}): Promise<{
+    id: string;
+    name: string;
+    description: string;
+    createdAt: number;
+    files: WorkspaceFile[];
+}>;
+export declare function createWorkspace(authorization: Credentials | ApiKey, name: string, description?: string, opts?: {
+    group?: Group;
+}): Promise<string>;
+export declare function updateWorkspace(authorization: Credentials | ApiKey, id: string, update: Partial<{
+    name: string;
+    description: string;
+}>, opts?: {
+    group?: Group;
+}): Promise<void>;