fauna-api 0.0.0-stage → 1.0.679

package/dist/src/marmot/scraper.d.ts

@@ -0,0 +1,59 @@
+/**
+ * Keep updated: a workspace's one Termite repository, where its machine agent
+ * keeps a scraper per kept dataset running nonstop. Scrapers push rows into
+ * their dataset as they scrape; with `store` on, the rows merge into it.
+ */
+import { Fauna } from "../../index.js";
+import { ApiKey, Credentials } from "../authentification.js";
+import { Wake } from "../termite/os.js";
+export type ScraperStatus = 'building' | 'active' | 'off';
+/** A dataset kept updated by its workspace's scraper agent. */
+export type Scraper = {
+    datasetId: string;
+    workspaceId: string;
+    /** `building` until its first rows arrive, then `active`. `off` when it stopped and needs turning on again. */
+    status: ScraperStatus;
+    /** Pushed rows merge into the dataset's live snapshot, so its data includes them. */
+    store: boolean;
+    /** Fields a row is known by: a pushed row with the same key replaces the old one. None means rows append. */
+    key: string[];
+    instructions: string;
+    /** The snapshot pushed rows merge into, once there is one. */
+    liveSnapshotId: string | null;
+    lastRowsAt: Date | null;
+    rowsTotal: number;
+    /** The last thing that went wrong, or ''. */
+    lastError: string;
+    createdAt: Date | null;
+    updatedAt: Date | null;
+};
+export declare const Scraper: {
+    fromJSON(json: any): Scraper;
+};
+export type ScraperSettings = {
+    store?: boolean;
+    key?: string[];
+    /** What the agent should know about this dataset. */
+    instructions?: string;
+    /** The model the workspace's agent thinks with. */
+    text?: Fauna.Parrot.Generative.Text;
+};
+/** Whether a dataset is kept updated, and how it's going. Null when it isn't. */
+export declare function getScraper(authorization: Credentials | ApiKey, datasetId: string): Promise<Scraper | null>;
+/**
+ * Keep a dataset updated. The workspace's machine agent writes its scraper, and
+ * the workspace's machine runs it nonstop; the workspace's owner pays for both.
+ * You become the person the agent runs as.
+ */
+export declare function createScraper(authorization: Credentials | ApiKey, datasetId: string, settings?: ScraperSettings): Promise<Scraper>;
+/** Change a kept dataset's settings. Key and instruction changes go to the agent. */
+export declare function updateScraper(authorization: Credentials | ApiKey, datasetId: string, settings: ScraperSettings): Promise<Scraper>;
+/** Ask the agent to go over a kept dataset's scraper again, with a note for it. */
+export declare function rebuildScraper(authorization: Credentials | ApiKey, datasetId: string, note?: string): Promise<void>;
+/** Stop keeping a dataset updated. Its scraper goes; the machine stops with the workspace's last one. */
+export declare function removeScraper(authorization: Credentials | ApiKey, datasetId: string): Promise<void>;
+/** The times a workspace's scraper agent woke, newest first. */
+export declare function listWakes(authorization: Credentials | ApiKey, workspaceId: string, query: {
+    cursor: number;
+    max: number;
+}): Promise<Wake[]>;

package/dist/src/marmot/scraper.js

@@ -0,0 +1,94 @@
+/**
+ * Keep updated: a workspace's one Termite repository, where its machine agent
+ * keeps a scraper per kept dataset running nonstop. Scrapers push rows into
+ * their dataset as they scrape; with `store` on, the rows merge into it.
+ */
+import { Fauna } from "../../index.js";
+import { Wake } from "../termite/os.js";
+export const Scraper = {
+    fromJSON(json) {
+        return {
+            datasetId: String(json.datasetId),
+            workspaceId: String(json.workspaceId),
+            status: json.status === 'active' || json.status === 'off' ? json.status : 'building',
+            store: Boolean(json.store),
+            key: Array.isArray(json.key) ? json.key.map(String) : [],
+            instructions: String(json.instructions ?? ''),
+            liveSnapshotId: json.liveSnapshotId ?? null,
+            lastRowsAt: json.lastRowsAt ? new Date(json.lastRowsAt) : null,
+            rowsTotal: Number(json.rowsTotal ?? 0),
+            lastError: String(json.lastError ?? ''),
+            createdAt: json.createdAt ? new Date(json.createdAt) : null,
+            updatedAt: json.updatedAt ? new Date(json.updatedAt) : null,
+        };
+    },
+};
+/** Whether a dataset is kept updated, and how it's going. Null when it isn't. */
+export async function getScraper(authorization, datasetId) {
+    const response = await fetch(`${Fauna.getURL()}/marmot/dataset/scraper/get`, {
+        method: "POST",
+        headers: { "Content-Type": "application/json", "Authorization": 'email' in authorization ? `Token ${authorization.token}` : `ApiKey ${authorization.id}` },
+        body: JSON.stringify({ datasetId }),
+    });
+    if (!response.ok)
+        throw new Error(await response.text());
+    const json = await response.json();
+    return json ? Scraper.fromJSON(json) : null;
+}
+/**
+ * Keep a dataset updated. The workspace's machine agent writes its scraper, and
+ * the workspace's machine runs it nonstop; the workspace's owner pays for both.
+ * You become the person the agent runs as.
+ */
+export async function createScraper(authorization, datasetId, settings = {}) {
+    const response = await fetch(`${Fauna.getURL()}/marmot/dataset/scraper/create`, {
+        method: "POST",
+        headers: { "Content-Type": "application/json", "Authorization": 'email' in authorization ? `Token ${authorization.token}` : `ApiKey ${authorization.id}` },
+        body: JSON.stringify({ datasetId, ...settings }),
+    });
+    if (!response.ok)
+        throw new Error(await response.text());
+    return Scraper.fromJSON(await response.json());
+}
+/** Change a kept dataset's settings. Key and instruction changes go to the agent. */
+export async function updateScraper(authorization, datasetId, settings) {
+    const response = await fetch(`${Fauna.getURL()}/marmot/dataset/scraper/update`, {
+        method: "POST",
+        headers: { "Content-Type": "application/json", "Authorization": 'email' in authorization ? `Token ${authorization.token}` : `ApiKey ${authorization.id}` },
+        body: JSON.stringify({ datasetId, ...settings }),
+    });
+    if (!response.ok)
+        throw new Error(await response.text());
+    return Scraper.fromJSON(await response.json());
+}
+/** Ask the agent to go over a kept dataset's scraper again, with a note for it. */
+export async function rebuildScraper(authorization, datasetId, note) {
+    const response = await fetch(`${Fauna.getURL()}/marmot/dataset/scraper/rebuild`, {
+        method: "POST",
+        headers: { "Content-Type": "application/json", "Authorization": 'email' in authorization ? `Token ${authorization.token}` : `ApiKey ${authorization.id}` },
+        body: JSON.stringify({ datasetId, note }),
+    });
+    if (!response.ok)
+        throw new Error(await response.text());
+}
+/** Stop keeping a dataset updated. Its scraper goes; the machine stops with the workspace's last one. */
+export async function removeScraper(authorization, datasetId) {
+    const response = await fetch(`${Fauna.getURL()}/marmot/dataset/scraper/remove`, {
+        method: "POST",
+        headers: { "Content-Type": "application/json", "Authorization": 'email' in authorization ? `Token ${authorization.token}` : `ApiKey ${authorization.id}` },
+        body: JSON.stringify({ datasetId }),
+    });
+    if (!response.ok)
+        throw new Error(await response.text());
+}
+/** The times a workspace's scraper agent woke, newest first. */
+export async function listWakes(authorization, workspaceId, query) {
+    const response = await fetch(`${Fauna.getURL()}/marmot/workspace/wakes`, {
+        method: "POST",
+        headers: { "Content-Type": "application/json", "Authorization": 'email' in authorization ? `Token ${authorization.token}` : `ApiKey ${authorization.id}` },
+        body: JSON.stringify({ workspaceId, cursor: query.cursor, max: query.max }),
+    });
+    if (!response.ok)
+        throw new Error(await response.text());
+    return (await response.json()).map(Wake.fromJSON);
+}

package/dist/src/marmot/web.d.ts

@@ -0,0 +1,402 @@
+/**
+ * Marmot-owned live web-room client: DOM capture/replay, HTTP bootstrap/assets,
+ * and the WebRoom socket. Transport-agnostic capture plus the fauna share
+ * routes (`/extensions/web/live`, `/bootstrap`, `/asset`). Session orchestration
+ * (startWebShare, invite, chat pairing) stays in extensions/web.
+ */
+import { Credentials, ApiKey } from "../authentification.js";
+import { Room, Presence, Authorization } from "../shared/server.js";
+/** A single serialized DOM node. `id` is a synthetic stable key assigned by the
+ *  recorder; it is NOT the element's DOM id attribute. */
+export type WebNode = {
+    id: number;
+    kind: "element";
+    tag: string;
+    attributes: Record<string, string>;
+    children: number[];
+} | {
+    id: number;
+    kind: "text";
+    text: string;
+};
+/** A full document snapshot: the root node id plus every node reachable from
+ *  it, flattened into an id->node map for O(1) patching on the viewer. */
+export type WebSnapshot = {
+    root: number;
+    nodes: Record<number, WebNode>;
+    /** Viewport the host captured at (viewers use it only for reference — they
+     *  render at their own size). */
+    viewport: {
+        width: number;
+        height: number;
+    };
+};
+/** An incremental change to a previously-snapshotted tree. */
+export type WebMutation = {
+    kind: "insert";
+    parent: number;
+    node: WebNode;
+    subtree: WebNode[];
+    before: number | null;
+} | {
+    kind: "remove";
+    id: number;
+} | {
+    kind: "text";
+    id: number;
+    text: string;
+} | {
+    kind: "attribute";
+    id: number;
+    name: string;
+    value: string | null;
+} | {
+    kind: "viewport";
+    width: number;
+    height: number;
+};
+/** Agent or host pointer. Sibling of WebMutation — not applied by DomReplayer. */
+export type HtmlPointer = {
+    x: number;
+    y: number;
+    scrollX: number;
+    scrollY: number;
+    visible?: boolean;
+    /** The CSS cursor under the pointer (`pointer`, `text`, `nwse-resize`...), `default` for `auto`. */
+    shape?: string;
+};
+/** Placeholder scheme rewritten into guest blob: URLs after asset download. */
+export declare const WEB_ASSET_SCHEME = "fauna-web-asset:";
+/** Optional privileged fetcher (extension background). */
+export type MediaFetcher = (url: string) => Promise<{
+    data: Uint8Array;
+    contentType: string;
+} | null>;
+/** Install a privileged media fetcher (e.g. chrome.runtime fetch-bytes). */
+export declare function setMediaFetcher(fetcher: MediaFetcher | null): void;
+/** Pull url(...) and @import targets out of CSS text (preserves @media as-is). */
+export declare function extractCssUrls(css: string): string[];
+/** Rewrite url()/ @import targets in CSS using fauna-web-asset codes. */
+export declare function rewriteCssUrls(css: string, map: Map<string, string>): string;
+/**
+ * Serialize readable document stylesheets as rule text (keeps @media / @supports).
+ * Opaque cross-origin sheets are skipped unless their <style> text is available.
+ */
+export declare function serializeDocumentStyles(doc?: Document): {
+    css: string;
+    urls: string[];
+};
+/**
+ * Deep-serialize an element tree: strip scripts/extension UI, flatten open
+ * shadow roots into light DOM, replace opaque iframes with placeholders.
+ */
+export declare function deepFreezeClone(source: Element): Element;
+/** Collect absolute/resolvable media URLs from a cleaned element tree. */
+export declare function collectMediaUrls(root: Element): string[];
+/** Rewrite media URLs in-place using `map` (original URL → fauna-web-asset:code). */
+export declare function rewriteMediaUrls(root: Element, map: Map<string, string>): void;
+export declare function extractAssetCodes(html: string): string[];
+/** Replace `fauna-web-asset:CODE` markers with blob/object URLs from `blobs`. */
+export declare function materializeAssetUrls(html: string, blobs: Map<string, string>): string;
+/**
+ * Deep-freeze `root` + inject serialized stylesheets. Does not rewrite assets —
+ * call `collectMediaUrls` / `rewriteMediaUrls` (and CSS urls) afterward.
+ */
+export declare function toHtml(root?: Node): {
+    html: string;
+    viewport: {
+        width: number;
+        height: number;
+    };
+    clone: Element;
+    cssUrls: string[];
+};
+/** Best-effort fetch of a media URL — prefers privileged `setMediaFetcher`. */
+export declare function fetchMediaBytes(url: string): Promise<{
+    data: Uint8Array;
+    contentType: string;
+} | null>;
+/** Absolute-resolve a URL against the current page (or a provided base). */
+export declare function absoluteUrl(url: string, base?: string): string;
+/** Collect media + CSS urls from a frozen clone / stylesheet list. */
+export declare function collectAllAssetUrls(root: Element, cssUrls?: string[]): string[];
+/**
+ * Serialize a live DOM subtree into a flat `WebSnapshot`. The recorder tags each
+ * captured node with a synthetic id via a WeakMap so later mutations can
+ * reference nodes by that id without relying on DOM identity crossing the wire.
+ */
+export declare class DomRecorder {
+    private readonly root;
+    private nextId;
+    private ids;
+    private observer;
+    private frames;
+    private onMutation;
+    constructor(root?: Node);
+    /**
+     * Reset and DFS-assign synthetic ids on the live tree (same order as
+     * `serializeNode`, descending into framed documents). Call after publishing
+     * HTML bootstrap so mutations align with the guest's `applyHtml` bind.
+     */
+    bindIds(): void;
+    private idFor;
+    private serializeNode;
+    /** The one child a frame's wrapper holds: its document root, or a placeholder text. */
+    private serializeFrameContent;
+    /** Track a frame: observe its document and re-inline it when it navigates. */
+    private watchFrame;
+    private reinline;
+    /** Serialize a single node and its whole subtree into a flat node list (used
+     *  to describe inserted subtrees in mutations). */
+    private serializeSubtree;
+    /** Capture the current document as a snapshot. */
+    snapshot(): WebSnapshot;
+    private handle;
+    /**
+     * Start observing the document, and every framed document it inlines.
+     * `onMutation` fires with a batched-flattened `WebMutation` for every DOM
+     * change. Also emits `viewport` mutations on resize. Call `stop()` to detach.
+     */
+    start(onMutation: (mutation: WebMutation) => void): void;
+    private detachResize;
+    stop(): void;
+}
+/**
+ * Re-render a `WebSnapshot` into a target document (typically an isolated iframe
+ * or shadow root on the viewer) and apply incoming `WebMutation`s. The replayer
+ * keeps its own id->Node map mirroring the recorder's synthetic ids.
+ */
+export declare class DomReplayer {
+    private readonly mount;
+    private nodes;
+    private _viewport;
+    private iframe;
+    private queued;
+    private loads;
+    private frameScopes;
+    private scopedSheets;
+    private boxSheet;
+    private boxes;
+    constructor(mount: Element);
+    /** Host viewport from the last snapshot / viewport mutation (advisory). */
+    getViewport(): {
+        width: number;
+        height: number;
+    } | null;
+    private docRoot;
+    /**
+     * Apply a full cleaned HTML document (or fragment). Uses an iframe when the
+     * payload looks like a full document so `<html>/<head>/<body>` structure and
+     * DFS ids match the host.
+     */
+    applyHtml(html: string, viewport?: {
+        width: number;
+        height: number;
+    }): Promise<void>;
+    /** Bind synthetic ids: from the ids a recorded snapshot carries, else in DFS order to
+     *  match a recorder that numbers in that order. */
+    bindIds(): void;
+    /** Elements carry `data-fauna-node`, text-only elements name their text with
+     *  `data-fauna-text`, and every other text node follows a `fauna:N` comment. A marker
+     *  with no text after it was an empty text node on the host, so one is made here. */
+    private bindMarked;
+    /** The document the replay renders into: its iframe's for full documents, else the mount's. */
+    getDocument(): Document;
+    /** Scope inlined frames' styles, mirror hover and focus, and size frame boxes, for a full document's frame. */
+    private restyle;
+    /**
+     * Inlined frames are divs, so they get an iframe's box: inline block, 300 by 150 and
+     * clipped unless the page says otherwise, their `width`, `height` and `frameborder`
+     * attributes as sizes and border, and the framed document filling them. All at zero
+     * specificity in a sheet of its own, so page CSS wins, like a real iframe's
+     * presentational hints. No nodes join the replay, so ids stay aligned.
+     */
+    private frameBoxes;
+    /** Scroll a replayed element, or a framed document's html, to the host's `data-fauna-scroll`. */
+    private applyScroll;
+    /**
+     * A framed document is inlined into the replay, so its stylesheets would style the
+     * whole copy: a site's `body { background }` repaints the page around its frame. Each
+     * of its sheets goes inside `@scope` at the frame's html div instead. Sheets from other
+     * origins can't be read, so they stay as they are.
+     */
+    private scopeFrames;
+    /**
+     * `:hover` and `:focus` never apply in a replay. The host marks what they would match
+     * with `data-fauna-hover` and the focus attributes, so point those selectors at the
+     * attributes instead; both weigh the same in the cascade. Only a full document's own
+     * frame is rewritten.
+     */
+    private mirrorStates;
+    private ownerDoc;
+    private build;
+    private buildFromList;
+    /** Render a fresh snapshot, replacing whatever the mount currently holds. */
+    apply(snapshot: WebSnapshot): void;
+    /** Apply one incremental mutation to the rendered tree. */
+    mutate(mutation: WebMutation): void;
+}
+/**
+ * Self-contained freeze + style serialize for `page.evaluate`. Closures over
+ * module helpers break Puppeteer serialization, so helpers live inside.
+ */
+export declare function pageToHtml(root?: Node): {
+    html: string;
+    viewport: {
+        width: number;
+        height: number;
+    };
+    clone: Element;
+    cssUrls: string[];
+};
+/** Install a MutationObserver that posts WebMutations to `window[pushName]`.
+ *  Framed documents are walked, observed, and re-inlined on navigation, with
+ *  their html, head, and body addressed as the divs `pageToHtml` emits. */
+export declare function pageInstallRecorder(pushName: string): void;
+export declare function pageStopRecorder(): void;
+/** Runs in the page: the pointer at (x, y) with the CSS cursor of whatever is under it,
+ *  looking inside frames the recorder inlines. */
+export declare function pageReadPointer(x?: number, y?: number, visible?: boolean): HtmlPointer;
+/** In-page media fetch for Puppeteer evaluate (cookies, no chrome.runtime). */
+export declare function pageFetchMedia(url: string): Promise<{
+    dataBase64: string;
+    contentType: string;
+} | null>;
+export type WebTarget = {
+    code: string;
+} | {
+    reference: string;
+};
+export type BootstrapMeta = {
+    version: number;
+    viewport?: {
+        width: number;
+        height: number;
+    };
+    contentLength?: number;
+    contentType?: string;
+    /** In-band HTML for server-hosted rooms (BrowserRoom). Extension share uses HTTP. */
+    html?: string;
+};
+export type AssetUploadResult = {
+    assetId: string;
+    accessCode: string;
+    contentType: string;
+};
+/**
+ * Upload cleaned HTML as a raw stream (XHR for upload progress, Drive-style).
+ */
+export declare function uploadBootstrap(authorization: Credentials | ApiKey, target: WebTarget, html: string | Uint8Array, options?: {
+    version?: number;
+    viewport?: {
+        width: number;
+        height: number;
+    };
+    contentType?: string;
+    onProgress?: (percent: number) => void;
+}): Promise<{
+    ok: boolean;
+    version: number;
+}>;
+/**
+ * Download the room HTML bootstrap with download progress when Content-Length is known.
+ */
+export declare function downloadBootstrap(authorization: Credentials | ApiKey, target: WebTarget, onProgress?: (percent: number) => void): Promise<{
+    html: string;
+    contentLength: number;
+    contentType: string;
+}>;
+/**
+ * Upload one media asset; returns a per-asset access code for guest download.
+ */
+export declare function uploadAsset(authorization: Credentials | ApiKey, target: WebTarget, data: Uint8Array, contentType: string, options?: {
+    assetId?: string;
+    onProgress?: (percent: number) => void;
+}): Promise<AssetUploadResult>;
+/** Download one asset by its file access code. */
+export declare function downloadAsset(authorization: Credentials | ApiKey, accessCode: string, onProgress?: (percent: number) => void): Promise<{
+    data: Uint8Array;
+    contentType: string;
+}>;
+/** Aggregate sequential upload/download stages into a single 0..1 progress. */
+export declare function weightedProgress(stages: {
+    weight: number;
+    run: (onProgress: (p: number) => void) => Promise<void>;
+}[], onProgress?: (percent: number) => void): Promise<void>;
+export type WebRoomHooks = Partial<{
+    /** Host published bootstrap meta — guests should HTTP GET the HTML. */
+    onBootstrap(room: WebRoom, bootstrap: BootstrapMeta): void;
+    /** The host published a fresh full-page snapshot (legacy / fallback). */
+    onSnapshot(room: WebRoom, snapshot: WebSnapshot): void;
+    /** The host emitted one incremental DOM mutation. */
+    onMutation(room: WebRoom, mutation: WebMutation): void;
+    /** The host disconnected / stopped sharing — the room is closing. */
+    onClosed(room: WebRoom): void;
+    /** Presence roster changed (join/leave/update). */
+    onPresence(room: WebRoom): void;
+}>;
+/**
+ * A live web room for page sharing. `WebRoom extends Room` and carries presence
+ * + small DOM mutations. Initial HTML and media bytes travel over HTTP
+ * bootstrap/asset routes, not this socket.
+ */
+export declare class WebRoom extends Room {
+    private hooks;
+    private latest;
+    private bootstrap;
+    private toolHandler;
+    private pendingTools;
+    /** Tagged host principal from the live connection (`credentials:` / `group:` / …). */
+    private hostPrincipal;
+    /** Server-computed: whether this caller may invite others to the share. */
+    private _canInvite;
+    constructor(ws: WebSocket, authorization: Authorization, id: string, hooks: WebRoomHooks, connectionState?: {
+        room?: {
+            hostPrincipal?: string;
+            canInvite?: boolean;
+        };
+    }, opts?: {
+        owned?: boolean;
+        endpoint?: string;
+    });
+    private readonly owned;
+    setHooks(hooks: WebRoomHooks): void;
+    /** Ingest-only child of a BrowserRoom (same socket). */
+    static fromParent(parent: Room, hooks?: WebRoomHooks, connectionState?: {
+        room?: {
+            hostPrincipal?: string;
+            canInvite?: boolean;
+        };
+    }): WebRoom;
+    /** Host principal tagged string, if the connection provided it. */
+    getHostPrincipal(): string | null;
+    /** Whether this signed-in caller may invite others (server-gated). */
+    canInvite(): boolean;
+    getSnapshot(): WebSnapshot | null;
+    getBootstrap(): BootstrapMeta | null;
+    disconnect(): void;
+    ingestSnapshot(snapshot: WebSnapshot): void;
+    ingestBootstrap(meta: BootstrapMeta): void;
+    publishBootstrap(meta: BootstrapMeta): void;
+    publishSnapshot(snapshot: WebSnapshot): void;
+    publishMutation(mutation: WebMutation): void;
+    publishClosed(): void;
+    /** Host: fulfill inbound `web-tool-call` frames. */
+    setToolHandler(handler: (name: string, params: Record<string, unknown>) => Promise<string>): () => void;
+    /** Any member: ask the host to run a page tool. */
+    invokeTool(name: string, params?: Record<string, unknown>, timeoutMs?: number): Promise<string>;
+    handleEvent(packet: any): boolean;
+    private handleToolCall;
+}
+/**
+ * Connect to a web room. `target` is an access code (viewer joining a shared
+ * page) or a room reference (host opening its own room).
+ */
+export declare function connectWebRoom(authorization: Credentials | ApiKey, target: {
+    code: string;
+} | {
+    reference: string;
+}, hooks?: WebRoomHooks): Promise<{
+    room: WebRoom;
+    self: Presence;
+}>;