fauna-api 0.0.0-stage → 1.0.676

package/dist/src/marmot/browser.d.ts

@@ -0,0 +1,139 @@
+/**
+ * Server-hosted live browser room. Contains a WebRoom for HTML replay on the
+ * same socket. Tools run on the Marmot host (Puppeteer), not a client tab.
+ */
+import { ApiKey, Credentials } from "../authentification.js";
+import { Group } from "../groups.js";
+import { Authorization, Presence, Room } from "../shared/server.js";
+import type { Tool } from "../parrot/generative.js";
+import { Marmot } from "./index.js";
+import { WebRoom, type WebRoomHooks, type HtmlPointer, type BootstrapMeta } from "./web.js";
+export type MouseButton = "left" | "right" | "middle";
+export type BrowserTool = {
+    kind: "move";
+    x: number;
+    y: number;
+}
+/** `count` clicks in a row (2 is a double click). A click on the same spot right after
+ *  another continues its run, the way a person's quick second click does. */
+ | {
+    kind: "click";
+    x?: number;
+    y?: number;
+    downMs?: number;
+    button?: MouseButton;
+    count?: number;
+} | {
+    kind: "drag";
+    to: {
+        x: number;
+        y: number;
+    };
+    from?: {
+        x: number;
+        y: number;
+    };
+    downMs?: number;
+} | {
+    kind: "scroll";
+    deltaX?: number;
+    deltaY?: number;
+} | {
+    kind: "type";
+    text: string;
+} | {
+    kind: "executeJavascript";
+    code: string;
+} | {
+    kind: "search";
+    query: string;
+} | {
+    kind: "exit";
+    answer?: string;
+    reason?: string;
+};
+export type BrowserRoomHooks<T extends BrowserTool = BrowserTool> = Partial<{
+    onCursor(room: PageRoom<T>, cursor: HtmlPointer): void;
+    onTool(room: PageRoom<T>, tool: T): void;
+    onClosed(room: PageRoom<T>): void;
+    onPresence(room: PageRoom<T>): void;
+    /** Control changed hands, or an agent took or released the room. */
+    onControl(room: PageRoom<T>): void;
+    web?: WebRoomHooks;
+}>;
+/** A server agent driving the room. While one does, nobody else can act. */
+export type RoomAgent = {
+    id: string;
+    name: string;
+    icon?: string;
+};
+export declare const RoomAgent: {
+    fromJSON(json: any): RoomAgent;
+};
+/** A model's tools for the `enabled` kinds. A room that takes only some kinds (a machine's
+ *  desktop) gets calls of only those kinds. */
+export declare function parrotTools<T extends BrowserTool = BrowserTool>(enabled: ReadonlySet<T["kind"]>, invoke: (tool: T) => Promise<string>): Tool[];
+/**
+ * A room around one live page: its HTML stream, cursor, control and tools. `T` is the
+ * tools it takes: every browser tool for a web page (BrowserRoom), fewer for a machine's
+ * desktop (MachineRoom). Each subclass brings its own ways to open one.
+ */
+export declare class PageRoom<T extends BrowserTool = BrowserTool> extends Room {
+    readonly web: WebRoom;
+    cursor: HtmlPointer;
+    readonly enabledTools: ReadonlySet<T["kind"]>;
+    /** The seat (presence id) whose actions run. */
+    controller: string | null;
+    /** Set while a server agent holds the room. */
+    agent: RoomAgent | null;
+    private hooks;
+    private pending;
+    constructor(ws: WebSocket, authorization: Authorization, id: string, hooks?: BrowserRoomHooks<T>, state?: {
+        bootstrap?: BootstrapMeta;
+        cursor?: HtmlPointer;
+        enabledTools?: T["kind"][];
+        control?: {
+            controller?: string | null;
+            agent?: any;
+        };
+    }, endpoint?: string);
+    watch(hooks: BrowserRoomHooks<T>): void;
+    /** Whether your actions run: you control it and no agent holds it. */
+    controlling(self: Presence): boolean;
+    /** Hand control to a seat, or take it yourself without one. While an agent holds the room this stops the agent. */
+    setController(seat?: string): Promise<void>;
+    /** Presence changed: tell whoever is watching now. */
+    emitPresence(): void;
+    emitClosed(): void;
+    protected applyControl(control: {
+        controller?: string | null;
+        agent?: any;
+    } | undefined): void;
+    invokeTool(tool: T): Promise<string>;
+    parrotTools(): Tool[];
+    handleEvent(packet: any): boolean;
+}
+/** A live web page in a Marmot (or Ant) browser: every browser tool. */
+export declare class BrowserRoom extends PageRoom<BrowserTool> {
+    static connect(authorization: Credentials | ApiKey, target: {
+        reference: string;
+    }, opts?: {
+        tools?: BrowserTool["kind"][];
+        hooks?: BrowserRoomHooks;
+        group?: Group;
+    }): Promise<{
+        room: BrowserRoom;
+        self: Presence;
+    }>;
+    /** `endpoint` is where the page's room lives: Marmot's by default, or e.g. Ant's /timeline/browser/live. */
+    static join(authorization: Credentials | ApiKey, target: {
+        accessCode: string;
+    }, opts?: {
+        tools?: BrowserTool["kind"][];
+        hooks?: BrowserRoomHooks;
+        endpoint?: string;
+    }): Promise<{
+        room: BrowserRoom;
+        self: Presence;
+    }>;
+}

package/dist/src/marmot/browser.js

@@ -0,0 +1,389 @@
+/**
+ * Server-hosted live browser room. Contains a WebRoom for HTML replay on the
+ * same socket. Tools run on the Marmot host (Puppeteer), not a client tab.
+ */
+import { Presence, Room, connect as connectServer } from "../shared/server.js";
+import { Marmot } from "./index.js";
+import { WebRoom } from "./web.js";
+const ALL_TOOLS = [
+    "move", "click", "drag", "scroll", "type",
+    "executeJavascript", "search", "exit",
+];
+export const RoomAgent = {
+    fromJSON(json) {
+        return {
+            id: String(json.id),
+            name: String(json.name ?? 'Agent'),
+            ...(json.icon ? { icon: String(json.icon) } : {}),
+        };
+    },
+};
+/** A model's tools for the `enabled` kinds. A room that takes only some kinds (a machine's
+ *  desktop) gets calls of only those kinds. */
+export function parrotTools(enabled, invoke) {
+    // Only enabled kinds become tools, so every call below is a T.
+    const run = invoke;
+    const tools = [];
+    const add = (tool) => {
+        if (enabled.has(tool.function.name))
+            tools.push(tool);
+    };
+    add({
+        function: {
+            name: "move",
+            description: "Move the pointer to viewport coordinates. Does not click.",
+            parameters: {
+                type: "OBJECT",
+                properties: {
+                    x: { type: "NUMBER", description: "X pixel" },
+                    y: { type: "NUMBER", description: "Y pixel" },
+                },
+                required: ["x", "y"],
+            },
+        },
+        evaluate: async (params) => run({ kind: "move", x: Number(params?.x), y: Number(params?.y) }),
+    });
+    add({
+        function: {
+            name: "click",
+            description: "Click. Optional coordinates move there first. downMs is how long the button stays down. button right opens context menus; count 2 double clicks.",
+            parameters: {
+                type: "OBJECT",
+                properties: {
+                    x: { type: "NUMBER", description: "X pixel" },
+                    y: { type: "NUMBER", description: "Y pixel" },
+                    downMs: { type: "NUMBER", description: "Hold duration in ms" },
+                    button: { type: "STRING", enum: ["left", "right", "middle"], description: "Mouse button, default left" },
+                    count: { type: "NUMBER", description: "Clicks in a row: 2 double clicks, 3 triple clicks. Default 1" },
+                },
+            },
+        },
+        evaluate: async (params) => run({
+            kind: "click",
+            ...(params?.x != null ? { x: Number(params.x) } : {}),
+            ...(params?.y != null ? { y: Number(params.y) } : {}),
+            ...(params?.downMs != null ? { downMs: Number(params.downMs) } : {}),
+            ...(params?.button === "right" || params?.button === "middle" ? { button: params.button } : {}),
+            ...(params?.count != null ? { count: Number(params.count) } : {}),
+        }),
+    });
+    add({
+        function: {
+            name: "drag",
+            description: "Drag from an optional start point to a destination.",
+            parameters: {
+                type: "OBJECT",
+                properties: {
+                    to: { type: "OBJECT", description: "Destination {x,y}" },
+                    from: { type: "OBJECT", description: "Start {x,y}" },
+                    downMs: { type: "NUMBER", description: "Hold at the end in ms" },
+                },
+                required: ["to"],
+            },
+        },
+        evaluate: async (params) => run({
+            kind: "drag",
+            to: { x: Number(params?.to?.x), y: Number(params?.to?.y) },
+            ...(params?.from ? { from: { x: Number(params.from.x), y: Number(params.from.y) } } : {}),
+            ...(params?.downMs != null ? { downMs: Number(params.downMs) } : {}),
+        }),
+    });
+    add({
+        function: {
+            name: "scroll",
+            description: "Scroll the page with the wheel at the current pointer.",
+            parameters: {
+                type: "OBJECT",
+                properties: {
+                    deltaX: { type: "NUMBER", description: "Horizontal wheel delta" },
+                    deltaY: { type: "NUMBER", description: "Vertical wheel delta" },
+                },
+            },
+        },
+        evaluate: async (params) => run({
+            kind: "scroll",
+            ...(params?.deltaX != null ? { deltaX: Number(params.deltaX) } : {}),
+            ...(params?.deltaY != null ? { deltaY: Number(params.deltaY) } : {}),
+        }),
+    });
+    add({
+        function: {
+            name: "type",
+            description: "Type text at the current focus, one key at a time.",
+            parameters: {
+                type: "OBJECT",
+                properties: { text: { type: "STRING", description: "Text to type" } },
+                required: ["text"],
+            },
+        },
+        evaluate: async (params) => run({ kind: "type", text: String(params?.text ?? "") }),
+    });
+    add({
+        function: {
+            name: "executeJavascript",
+            description: "Evaluate JavaScript in the page and return the stringified result.",
+            parameters: {
+                type: "OBJECT",
+                properties: { code: { type: "STRING", description: "JavaScript to evaluate" } },
+                required: ["code"],
+            },
+        },
+        evaluate: async (params) => run({ kind: "executeJavascript", code: String(params?.code ?? "") }),
+    });
+    add({
+        function: {
+            name: "search",
+            description: "Search the web. Returns matching websites.",
+            parameters: {
+                type: "OBJECT",
+                properties: { query: { type: "STRING", description: "Search query" } },
+                required: ["query"],
+            },
+        },
+        evaluate: async (params) => run({ kind: "search", query: String(params?.query ?? "") }),
+    });
+    add({
+        function: {
+            name: "exit",
+            description: "Leave the page. Pass answer when the goal is done, or reason when giving up.",
+            parameters: {
+                type: "OBJECT",
+                properties: {
+                    answer: { type: "STRING", description: "What was found" },
+                    reason: { type: "STRING", description: "Why the goal cannot be met" },
+                },
+            },
+        },
+        evaluate: async (params) => run({
+            kind: "exit",
+            ...(params?.answer != null ? { answer: String(params.answer) } : {}),
+            ...(params?.reason != null ? { reason: String(params.reason) } : {}),
+        }),
+    });
+    return tools;
+}
+/**
+ * A room around one live page: its HTML stream, cursor, control and tools. `T` is the
+ * tools it takes: every browser tool for a web page (BrowserRoom), fewer for a machine's
+ * desktop (MachineRoom). Each subclass brings its own ways to open one.
+ */
+export class PageRoom extends Room {
+    constructor(ws, authorization, id, hooks = {}, state, endpoint = `${Marmot.getURL()}/browser/live`) {
+        super(endpoint, ws, authorization, id);
+        Object.defineProperty(this, "web", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "cursor", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: { x: 0, y: 0, scrollX: 0, scrollY: 0 }
+        });
+        Object.defineProperty(this, "enabledTools", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        /** The seat (presence id) whose actions run. */
+        Object.defineProperty(this, "controller", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: null
+        });
+        /** Set while a server agent holds the room. */
+        Object.defineProperty(this, "agent", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: null
+        });
+        Object.defineProperty(this, "hooks", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "pending", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: new Map()
+        });
+        this.hooks = hooks;
+        this.enabledTools = new Set(state?.enabledTools?.length ? state.enabledTools : ALL_TOOLS);
+        this.web = WebRoom.fromParent(this, hooks.web ?? {}, state);
+        if (state?.bootstrap)
+            this.web.ingestBootstrap(state.bootstrap);
+        if (state?.cursor)
+            this.cursor = state.cursor;
+        this.applyControl(state?.control);
+    }
+    watch(hooks) {
+        this.hooks = { ...this.hooks, ...hooks };
+        if (hooks.web)
+            this.web.setHooks(hooks.web);
+    }
+    /** Whether your actions run: you control it and no agent holds it. */
+    controlling(self) {
+        return !this.agent && this.controller === self.id;
+    }
+    /** Hand control to a seat, or take it yourself without one. While an agent holds the room this stops the agent. */
+    setController(seat) {
+        const id = crypto.randomUUID();
+        return new Promise((resolve, reject) => {
+            const timer = setTimeout(() => {
+                this.pending.delete(id);
+                reject(new Error("Control timed out."));
+            }, 30000);
+            this.pending.set(id, {
+                resolve: (v) => { clearTimeout(timer); resolve(v); },
+                reject: (e) => { clearTimeout(timer); reject(e); },
+            });
+            this.post({ event: "control-call", data: { id, ...(seat ? { seat } : {}) } });
+        }).then(() => undefined);
+    }
+    /** Presence changed: tell whoever is watching now. */
+    emitPresence() {
+        this.hooks.onPresence?.(this);
+    }
+    emitClosed() {
+        this.hooks.onClosed?.(this);
+    }
+    applyControl(control) {
+        if (!control)
+            return;
+        this.controller = control.controller ?? null;
+        this.agent = control.agent ? RoomAgent.fromJSON(control.agent) : null;
+    }
+    invokeTool(tool) {
+        if (!this.enabledTools.has(tool.kind)) {
+            return Promise.reject(new Error(`Tool is not enabled: ${tool.kind}`));
+        }
+        const id = crypto.randomUUID();
+        return new Promise((resolve, reject) => {
+            const timer = setTimeout(() => {
+                this.pending.delete(id);
+                reject(new Error(`Tool timed out: ${tool.kind}`));
+            }, 30000);
+            this.pending.set(id, {
+                resolve: (v) => { clearTimeout(timer); resolve(v); },
+                reject: (e) => { clearTimeout(timer); reject(e); },
+            });
+            this.post({ event: "browser-tool-call", data: { id, tool } });
+        });
+    }
+    parrotTools() {
+        return parrotTools(this.enabledTools, (tool) => this.invokeTool(tool));
+    }
+    handleEvent(packet) {
+        if (packet.event === "browser-closed" || packet.event === "server-closed") {
+            this.web.handleEvent(packet);
+            this.hooks.onClosed?.(this);
+            return true;
+        }
+        if (this.web.handleEvent(packet))
+            return true;
+        switch (packet.event) {
+            case "room-control": {
+                this.applyControl(packet.data ?? {});
+                this.hooks.onControl?.(this);
+                return true;
+            }
+            case "control-result": {
+                const id = packet.data?.id;
+                const pending = id ? this.pending.get(id) : undefined;
+                if (!pending)
+                    return true;
+                this.pending.delete(id);
+                if (packet.data?.error)
+                    pending.reject(new Error(String(packet.data.error)));
+                else
+                    pending.resolve("ok");
+                return true;
+            }
+            case "browser-cursor": {
+                if (packet.data?.cursor) {
+                    this.cursor = packet.data.cursor;
+                    this.hooks.onCursor?.(this, this.cursor);
+                }
+                return true;
+            }
+            case "browser-tool-call": {
+                if (packet.data?.tool)
+                    this.hooks.onTool?.(this, packet.data.tool);
+                return true;
+            }
+            case "browser-tool-result": {
+                const id = packet.data?.id;
+                if (!id)
+                    return true;
+                const pending = this.pending.get(id);
+                if (!pending)
+                    return true;
+                this.pending.delete(id);
+                if (packet.data?.error)
+                    pending.reject(new Error(String(packet.data.error)));
+                else
+                    pending.resolve(String(packet.data?.result ?? ""));
+                return true;
+            }
+            default:
+                return false;
+        }
+    }
+}
+/** A live web page in a Marmot (or Ant) browser: every browser tool. */
+export class BrowserRoom extends PageRoom {
+    static async connect(authorization, target, opts) {
+        return openBrowser(authorization, { server: target.reference, group: opts?.group?.id }, opts);
+    }
+    /** `endpoint` is where the page's room lives: Marmot's by default, or e.g. Ant's /timeline/browser/live. */
+    static async join(authorization, target, opts) {
+        return openBrowser(authorization, { accessCode: target.accessCode }, opts);
+    }
+}
+async function openBrowser(authorization, target, opts) {
+    const hooks = opts?.hooks ?? {};
+    const auth = "accessCode" in target
+        ? { authorization, accessCode: target.accessCode }
+        : { authorization, server: target.server, ...(target.group ? { group: target.group } : {}) };
+    const endpoint = opts?.endpoint ?? `${Marmot.getURL()}/browser/live`;
+    let room;
+    const notify = () => { if (room)
+        room.emitPresence(); };
+    const self = await connectServer(auth, endpoint, {
+        createRoom: (_endpoint, ws, a, id, state) => {
+            room = new BrowserRoom(ws, a, id, hooks, {
+                bootstrap: state?.bootstrap,
+                cursor: state?.cursor,
+                enabledTools: opts?.tools ?? state?.enabledTools,
+                control: state?.control,
+            }, endpoint);
+            return room;
+        },
+        createPresence: (r, data) => {
+            const presence = new Presence(r, data.id, data.profile);
+            presence.update(data);
+            return presence;
+        },
+        onPresence: notify,
+        onConnection: (r, state) => {
+            const next = r;
+            if (state?.bootstrap)
+                next.web.ingestBootstrap(state.bootstrap);
+            if (state?.cursor) {
+                next.cursor = state.cursor;
+                hooks.onCursor?.(next, next.cursor);
+            }
+        },
+        onEvent: (r, packet) => r.handleEvent(packet),
+        onClosed: (r) => r.emitClosed(),
+    });
+    notify();
+    return { room, self };
+}

package/dist/src/marmot/index.d.ts

@@ -0,0 +1,20 @@
+import * as MediaAPI from "./media.js";
+import * as ResearchAPI from './research.js';
+import * as WorkspaceAPI from './workspace.js';
+import * as WebAPI from "./web.js";
+import * as BrowserAPI from "./browser.js";
+import * as ScraperAPI from "./scraper.js";
+export declare namespace Marmot {
+    function setRoute(route: string): void;
+    function getIP(): string;
+    function getPort(): number;
+    function getRoute(): string;
+    function getURL(): string;
+    function ping(): Promise<number>;
+    export import Media = MediaAPI;
+    export import Research = ResearchAPI;
+    export import Workspace = WorkspaceAPI;
+    export import Web = WebAPI;
+    export import Browser = BrowserAPI;
+    export import Scraper = ScraperAPI;
+}

package/dist/src/marmot/index.js

@@ -0,0 +1,65 @@
+import * as MediaAPI from "./media.js";
+import * as ResearchAPI from './research.js';
+import * as WorkspaceAPI from './workspace.js';
+import * as WebAPI from "./web.js";
+import * as BrowserAPI from "./browser.js";
+import * as ScraperAPI from "./scraper.js";
+export var Marmot;
+(function (Marmot) {
+    var PREFIX = 'https://';
+    var IP = 'api.faunakit.com';
+    var PORT = 443;
+    function setRoute(route) {
+        const parts_0 = route.split('//');
+        if (parts_0.length > 1) {
+            PREFIX = parts_0[0] + '//';
+            const parts_1 = parts_0[1].split(':');
+            if (parts_1.length > 1) {
+                IP = parts_1[0];
+                PORT = Number(parts_1[1]);
+            }
+            else {
+                IP = parts_1[0];
+            }
+        }
+        else {
+            const parts_1 = parts_0[0].split(':');
+            if (parts_1.length > 1) {
+                IP = parts_1[0];
+                PORT = Number(parts_1[1]);
+            }
+            else {
+                IP = parts_1[0];
+            }
+        }
+    }
+    Marmot.setRoute = setRoute;
+    function getIP() {
+        return IP;
+    }
+    Marmot.getIP = getIP;
+    function getPort() {
+        return PORT;
+    }
+    Marmot.getPort = getPort;
+    function getRoute() {
+        return IP + ':' + PORT;
+    }
+    Marmot.getRoute = getRoute;
+    function getURL() {
+        return PREFIX + IP + ':' + PORT;
+    }
+    Marmot.getURL = getURL;
+    async function ping() {
+        const start = Date.now();
+        await fetch(`${getURL()}/`);
+        return Date.now() - start;
+    }
+    Marmot.ping = ping;
+    Marmot.Media = MediaAPI;
+    Marmot.Research = ResearchAPI;
+    Marmot.Workspace = WorkspaceAPI;
+    Marmot.Web = WebAPI;
+    Marmot.Browser = BrowserAPI;
+    Marmot.Scraper = ScraperAPI;
+})(Marmot || (Marmot = {}));

package/dist/src/marmot/media.d.ts

@@ -0,0 +1,56 @@
+import { ApiKey, Credentials } from "../authentification.js";
+import { Part } from "../parrot/generative.js";
+export declare const ATTACHMENT_TYPES: Record<string, string>;
+/** Resolve a usable MIME when the browser leaves `File.type` empty (common for HEIC). */
+export declare function mimeFromFileName(name: string): string | undefined;
+export type Website = {
+    url: string;
+    contentType: string;
+    title: string;
+    description: string;
+};
+export type ProxyMode = 'residential' | 'datacenter';
+export type DocumentMode = 'extraction' | 'render' | 'virtualized';
+export type ContentTypeInfo = {
+    contentType: string;
+    name: string;
+    extensions: string[];
+};
+export declare function getTypes(): Promise<Record<string, ContentTypeInfo[]>>;
+export type PullResponse = {
+    data: Uint8Array | Uint8Array[];
+    contentType: string;
+};
+export declare function query(authorization: Credentials | ApiKey, query: string, options: {
+    cursor: number;
+    max: number;
+    start?: Date;
+    end?: Date;
+}): Promise<Website[]>;
+export declare function pull(authorization: Credentials | ApiKey, url: string, request?: RequestInit, options?: Partial<{
+    modals: string[];
+    segment: [number, number];
+    split: number;
+    proxy: ProxyMode;
+}>): Promise<PullResponse | undefined>;
+/** A kind of file a text model reads itself. */
+export type Input = 'image' | 'video' | 'audio';
+/**
+ * A media file as message parts for a model that takes `takes` natively (images only
+ * by default). What it takes passes through; the rest becomes what it can take: a
+ * video becomes a line, its frames and its sound (audio, or a description of it),
+ * audio becomes a description, and documents become their text. With `split`, one
+ * list of parts per segment of that many milliseconds. Undefined when it can't be read.
+ */
+export declare function chat(authorization: Credentials | ApiKey, contentType: string, data: Uint8Array, options?: Partial<{
+    name: string;
+    takes: Input[];
+    segment: [number, number];
+    split: number;
+}>): Promise<Part[] | Part[][] | undefined>;
+export declare function parse(authorization: Credentials | ApiKey, contentType: string, data: Uint8Array, onProgress?: (percent: number) => void): Promise<string>;
+export declare function slice(authorization: Credentials | ApiKey, contentType: string, data: Uint8Array, options: Partial<{
+    segment: [number, number];
+    split: number;
+}>): Promise<undefined | Uint8Array | Uint8Array[]>;
+export declare function duration(authorization: Credentials | ApiKey, contentType: string, data: Uint8Array, onProgress?: (percent: number) => void): Promise<number>;