@wirebench/engine 2.2.1 → 3.1.0-rc.1

package/dist/rest/send.js

@@ -10,10 +10,14 @@
  */
 import { WirebenchError } from '../errors.js';
 import { sendWithAuth } from '../http/auth/apply.js';
-import { applyAuth } from './auth.js';
+import { signWebhook } from '../http/webhook-signature.js';
+import { applyAuth } from '../http/auth/apply-auth.js';
 import { encodeRestBody } from './body.js';
+import { mergeCookieHeader } from './cookie-jar.js';
 import { cookieHeader } from './cookies.js';
 import { decodeResponseText, detectLanguage, parseSetCookie } from './response.js';
+import { createSseParser, isEventStream } from './sse.js';
+import { createSseRowStore } from './sse-transcript.js';
 import { composeUrl } from './url.js';
 /**
  * Merges the header sources in precedence order, lowest first: host defaults, then what the body
@@ -47,6 +51,31 @@ function mergeRequestHeaders(input) {
     }
     return Object.fromEntries([...merged.values()]);
 }
+/** `headers` with the signing headers set over `body`, replacing any header of the same name. */
+function withSignature(headers, sign, body) {
+    const out = { ...headers };
+    for (const [name, value] of signWebhook(sign.scheme, sign.secret, body)) {
+        for (const key of Object.keys(out)) {
+            if (key.toLowerCase() === name.toLowerCase())
+                delete out[key];
+        }
+        out[name] = value;
+    }
+    return out;
+}
+/**
+ * The credentials `applyAuth` put on the request, for `sendHttp` to keep to the request's origin
+ * across redirects. `Authorization` is dropped on a cross-origin hop anyway; an API key's custom
+ * header or query parameter is only known here.
+ */
+function originCredentialsFor(applied) {
+    const headers = Object.keys(applied.headers);
+    const query = applied.query.map(({ name, value }) => ({ name, value }));
+    if (headers.length === 0 && query.length === 0) {
+        return {};
+    }
+    return { originCredentials: { headers, query } };
+}
 /** The methods this transport will send. A custom method is passed through as given. */
 function methodFor(method) {
     return method.toUpperCase();
@@ -64,6 +93,7 @@ function methodFor(method) {
  * status is not a failure: it is a response, and the caller inspects it.
  *
  * @throws WirebenchError `rest-url-incomplete` when the URL could not be completed
+ * @throws WirebenchError `webhook-signing-secret` when a `standard` signing secret cannot be decoded
  */
 export async function sendRest(input) {
     const { request, settings } = input;
@@ -80,8 +110,8 @@ export async function sendRest(input) {
         ...(input.boundary !== undefined ? { boundary: input.boundary } : {}),
         ...(input.signal !== undefined ? { signal: input.signal } : {}),
     });
-    const cookies = cookieHeader(input.cookies ?? []);
-    const headers = mergeRequestHeaders({
+    const cookies = input.jar === undefined ? cookieHeader(input.cookies ?? []) : undefined;
+    const merged = mergeRequestHeaders({
         defaults: input.defaultHeaders ?? {},
         computed: {
             ...applied.headers,
@@ -90,6 +120,21 @@ export async function sendRest(input) {
         },
         typed: request.headers,
     });
+    // Signing runs last (§5.2): after auth, encoding and merging, so it signs the bytes that go out.
+    const headers = input.sign === undefined ? merged : withSignature(merged, input.sign, encoded.bytes ?? new Uint8Array());
+    const onStream = input.onStream;
+    const sseState = onStream !== undefined ? createSseState() : undefined;
+    // The jar stores every hop's cookies whatever the setting; it is read only when `send` is on.
+    const jar = input.jar;
+    const received = {};
+    const cookieHook = jar === undefined
+        ? undefined
+        : {
+            header: (url, handSet) => (jar.send ? mergeCookieHeader(jar.host.cookiesFor(url), handSet) : handSet),
+            received: (url, rawHeaders) => {
+                received.verdicts = jar.host.remember(url, parseSetCookie(rawHeaders));
+            },
+        };
     const httpRequest = {
         url: composed.url,
         method: methodFor(request.method),
@@ -100,21 +145,68 @@ export async function sendRest(input) {
         ...(settings.maxRedirects !== undefined ? { maxRedirects: settings.maxRedirects } : {}),
         ...(settings.keepBodyOnRedirect === true ? { preserveMethodOnRedirect: true } : {}),
         ...(settings.maxSizeBytes !== undefined ? { maxSizeBytes: settings.maxSizeBytes } : {}),
+        ...originCredentialsFor(applied),
         ...(settings.localAddress !== undefined ? { localAddress: settings.localAddress } : {}),
         ...(settings.allowH2 === true ? { allowH2: true } : {}),
         ...(settings.decompress !== undefined ? { decompress: settings.decompress } : {}),
         ...(input.tls !== undefined ? { tls: input.tls } : {}),
         ...(input.proxy !== undefined ? { proxy: input.proxy } : {}),
         ...(input.signal !== undefined ? { signal: input.signal } : {}),
+        ...(sseState !== undefined && onStream !== undefined ? { stream: sseState.hook(onStream) } : {}),
+        ...(cookieHook !== undefined ? { cookies: cookieHook } : {}),
     };
     // Basic and NTLM may need a round trip of their own; every other scheme is already in `headers`.
     const authenticated = await sendWithAuth(httpRequest, applied.transportAuth);
     return {
-        ...decodeRestResponse(authenticated.http, methodFor(request.method)),
+        ...decodeRestResponse(authenticated.http, methodFor(request.method), sseState),
         ...(authenticated.auth !== undefined ? { auth: authenticated.auth } : {}),
+        // The last hop reported is the final response, whose cookies `cookies` lists.
+        ...(received.verdicts !== undefined ? { cookieVerdicts: received.verdicts } : {}),
         durationMs: authenticated.durationMs,
     };
 }
+function createSseState() {
+    const store = createSseRowStore();
+    const counts = { events: 0, comments: 0, retries: 0, bytes: 0 };
+    const state = {
+        store,
+        counts,
+        lastEventId: '',
+        hook(onStream) {
+            return {
+                accept(status, headers) {
+                    if (!isEventStream(headers['content-type']))
+                        return undefined;
+                    onStream.onOpen?.(status, headers);
+                    const headersAt = performance.now();
+                    const parser = createSseParser((row) => {
+                        store.add(row);
+                        counts.bytes += row.size;
+                        if (row.kind === 'event') {
+                            counts.events++;
+                            state.lastEventId = row.lastEventId;
+                        }
+                        else if (row.kind === 'comment') {
+                            counts.comments++;
+                        }
+                        else {
+                            counts.retries++;
+                            state.retryMs = row.ms;
+                        }
+                        onStream.onRow?.(row);
+                    });
+                    state.parser = parser;
+                    return {
+                        onChunk(bytes) {
+                            parser.push(bytes, performance.now() - headersAt);
+                        },
+                    };
+                },
+            };
+        },
+    };
+    return state;
+}
 /** The error a URL that cannot be completed raises, naming every problem at once. */
 function urlIncomplete(problems, url) {
     const missing = problems.filter((problem) => problem.code === 'missing-path-param').map((problem) => problem.name);
@@ -130,12 +222,26 @@ function urlIncomplete(problems, url) {
  * Split out so a response replayed from history or from the exchange cache is presented exactly as
  * a live one.
  */
-export function decodeRestResponse(exchange, sentMethod) {
+export function decodeRestResponse(exchange, sentMethod, sse) {
+    const stream = buildEventStream(exchange, sse);
+    // Worked out once, for a streamed exchange and a buffered one alike: the redirect chain is the same.
+    const methodChanged = sentMethod !== 'GET' &&
+        exchange.redirects.some((hop) => hop.status === 303 || ((hop.status === 301 || hop.status === 302) && sentMethod === 'POST'));
+    if (stream !== undefined) {
+        // A streamed response has an empty body and no text form of its own; the rows are the content.
+        return {
+            ...exchange,
+            durationMs: exchange.timings.totalMs,
+            text: '',
+            language: 'text',
+            cookies: parseSetCookie(exchange.rawHeaders),
+            methodChanged,
+            stream,
+        };
+    }
     const contentType = exchange.headers['content-type'];
     const language = detectLanguage(contentType, exchange.body);
     const decoded = language === 'image' || language === 'binary' ? { text: '' } : decodeResponseText(exchange.body, contentType);
-    const methodChanged = sentMethod !== 'GET' &&
-        exchange.redirects.some((hop) => hop.status === 303 || ((hop.status === 301 || hop.status === 302) && sentMethod === 'POST'));
     return {
         ...exchange,
         durationMs: exchange.timings.totalMs,
@@ -146,3 +252,18 @@ export function decodeRestResponse(exchange, sentMethod) {
         methodChanged,
     };
 }
+/** Finalizes the parser and builds the `RestEventStream`, when the response was accepted as one. */
+function buildEventStream(exchange, sse) {
+    if (sse?.parser === undefined || exchange.streamEnd === undefined)
+        return undefined;
+    sse.parser.end();
+    return {
+        rows: sse.store.rows,
+        counts: { ...sse.counts },
+        lastEventId: sse.lastEventId,
+        ...(sse.retryMs !== undefined ? { retryMs: sse.retryMs } : {}),
+        endedBy: exchange.streamEnd.by,
+        ...(exchange.streamEnd.error !== undefined ? { error: exchange.streamEnd.error } : {}),
+        droppedRows: sse.store.droppedRows,
+    };
+}

package/dist/rest/sse-transcript.d.ts

@@ -0,0 +1,25 @@
+/**
+ * Caps on an SSE row list: the same head/tail/byte-budget shape as the WebSocket transcript's
+ * `capFrames`, shared via `capByEnds`, plus an in-memory store for a live stream that keeps growing.
+ */
+import type { CapLimits } from '../http/transcript-cap.js';
+import type { SseRow } from './sse.js';
+export declare const SSE_MEMORY_ROWS = 10000;
+export declare const SSE_MEMORY_BYTES = 33554432;
+export declare const SSE_SUMMARY_LIMITS: CapLimits;
+export declare const SSE_HISTORY_LIMITS: CapLimits;
+export interface SseTranscript {
+    readonly rows: readonly SseRow[];
+    readonly truncated: boolean;
+    readonly omittedRows: number;
+}
+export declare function capSseRows(rows: readonly SseRow[], limits: CapLimits): SseTranscript;
+export interface SseRowStore {
+    add(row: SseRow): void;
+    readonly rows: readonly SseRow[];
+    readonly droppedRows: number;
+}
+export declare function createSseRowStore(limits?: {
+    rows: number;
+    bytes: number;
+}): SseRowStore;

package/dist/rest/sse-transcript.js

@@ -0,0 +1,95 @@
+import { capByEnds, WS_HISTORY_HEAD, WS_HISTORY_MAX_BYTES, WS_HISTORY_TAIL } from '../http/transcript-cap.js';
+export const SSE_MEMORY_ROWS = 10_000;
+export const SSE_MEMORY_BYTES = 33_554_432;
+export const SSE_SUMMARY_LIMITS = { head: 400, tail: 4_600, maxBytes: SSE_MEMORY_BYTES };
+export const SSE_HISTORY_LIMITS = {
+    head: WS_HISTORY_HEAD,
+    tail: WS_HISTORY_TAIL,
+    maxBytes: WS_HISTORY_MAX_BYTES,
+};
+function stripRow(row) {
+    if (row.kind !== 'event')
+        return undefined; // comment/retry rows keep their (small) text
+    return { ...row, data: '', payloadTruncated: true };
+}
+export function capSseRows(rows, limits) {
+    const r = capByEnds(rows, limits, (row) => row.size, stripRow);
+    return { rows: r.items, truncated: r.truncated, omittedRows: r.omitted };
+}
+const HEAD_CAP = 400;
+/**
+ * A FIFO queue backed by a circular array, so pushing and evicting the oldest element are both O(1)
+ * (amortized for push, which occasionally doubles the backing array — never an O(n) shift). Used for
+ * the store's tail, which can hold thousands of rows on a long-running stream.
+ */
+class RingBuffer {
+    buf = new Array(8).fill(undefined);
+    start = 0;
+    len = 0;
+    get length() {
+        return this.len;
+    }
+    push(item) {
+        if (this.len === this.buf.length)
+            this.grow();
+        this.buf[(this.start + this.len) % this.buf.length] = item;
+        this.len++;
+    }
+    evictFront() {
+        if (this.len === 0)
+            return undefined;
+        const item = this.buf[this.start];
+        this.buf[this.start] = undefined;
+        this.start = (this.start + 1) % this.buf.length;
+        this.len--;
+        return item;
+    }
+    toArray() {
+        const out = [];
+        for (let i = 0; i < this.len; i++)
+            out.push(this.buf[(this.start + i) % this.buf.length]);
+        return out;
+    }
+    grow() {
+        const next = new Array(this.buf.length * 2).fill(undefined);
+        for (let i = 0; i < this.len; i++)
+            next[i] = this.buf[(this.start + i) % this.buf.length];
+        this.buf = next;
+        this.start = 0;
+    }
+}
+export function createSseRowStore(limits = { rows: SSE_MEMORY_ROWS, bytes: SSE_MEMORY_BYTES }) {
+    const head = [];
+    const tail = new RingBuffer();
+    let headBytes = 0;
+    let tailBytes = 0;
+    let droppedRows = 0;
+    const headCap = Math.min(HEAD_CAP, limits.rows);
+    return {
+        add(row) {
+            if (head.length < headCap) {
+                // The head is never evicted: a byte budget smaller than the head's own bytes leaves the
+                // store permanently over budget rather than losing head rows (see the test that pins this).
+                head.push(row);
+                headBytes += row.size;
+            }
+            else {
+                tail.push(row);
+                tailBytes += row.size;
+            }
+            while (tail.length > 0 && (head.length + tail.length > limits.rows || headBytes + tailBytes > limits.bytes)) {
+                const dropped = tail.evictFront();
+                if (dropped === undefined)
+                    break;
+                tailBytes -= dropped.size;
+                droppedRows++;
+            }
+        },
+        get rows() {
+            return [...head, ...tail.toArray()];
+        },
+        get droppedRows() {
+            return droppedRows;
+        },
+    };
+}

package/dist/rest/sse.d.ts

@@ -0,0 +1,43 @@
+/**
+ * An in-house parser for the `text/event-stream` format (the WHATWG event-stream rules), browser-safe
+ * and dependency-free so it can be shared between the desktop send path and the renderer's own
+ * decoding needs. Re-exported from `rest/browser.ts`, so this module must not import from `node:*`.
+ *
+ * A stream arrives as arbitrary byte chunks — a line, a field, even a single UTF-8 code point can be
+ * split across chunk boundaries — and the parser has to produce the same rows regardless of how the
+ * bytes were sliced. `createSseParser` does the incremental decode; `isEventStream` decides whether a
+ * response should go through it at all; `eventStreamDocument` and `serializeEventStream` turn a row
+ * list back into, respectively, the JSON document the Query view runs over and the event-stream text
+ * used for HAR export.
+ */
+export type SseRow = {
+    readonly kind: 'event';
+    readonly index: number;
+    readonly at: number;
+    readonly size: number;
+    readonly event: string;
+    readonly data: string;
+    readonly id?: string;
+    readonly lastEventId: string;
+    readonly payloadTruncated?: true;
+} | {
+    readonly kind: 'comment';
+    readonly index: number;
+    readonly at: number;
+    readonly size: number;
+    readonly text: string;
+} | {
+    readonly kind: 'retry';
+    readonly index: number;
+    readonly at: number;
+    readonly size: number;
+    readonly ms: number;
+};
+export interface SseParser {
+    push(chunk: Uint8Array, at: number): void;
+    end(): void;
+}
+export declare function createSseParser(onRow: (row: SseRow) => void): SseParser;
+export declare function isEventStream(contentType: string | undefined): boolean;
+export declare function eventStreamDocument(rows: readonly SseRow[]): string;
+export declare function serializeEventStream(rows: readonly SseRow[]): string;

package/dist/rest/sse.js

@@ -0,0 +1,195 @@
+/**
+ * An in-house parser for the `text/event-stream` format (the WHATWG event-stream rules), browser-safe
+ * and dependency-free so it can be shared between the desktop send path and the renderer's own
+ * decoding needs. Re-exported from `rest/browser.ts`, so this module must not import from `node:*`.
+ *
+ * A stream arrives as arbitrary byte chunks — a line, a field, even a single UTF-8 code point can be
+ * split across chunk boundaries — and the parser has to produce the same rows regardless of how the
+ * bytes were sliced. `createSseParser` does the incremental decode; `isEventStream` decides whether a
+ * response should go through it at all; `eventStreamDocument` and `serializeEventStream` turn a row
+ * list back into, respectively, the JSON document the Query view runs over and the event-stream text
+ * used for HAR export.
+ */
+const enc = new TextEncoder();
+/** UTF-8 byte length of a row's lines, each counted with its trailing LF (spec decision 4). */
+function lineSize(...lines) {
+    return lines.reduce((sum, line) => sum + enc.encode(line).length + 1, 0);
+}
+export function createSseParser(onRow) {
+    const decoder = new TextDecoder('utf-8', { fatal: false });
+    let carry = '';
+    let sawText = false;
+    let pendingCr = false;
+    let index = 0;
+    /** The arrival time of the last chunk pushed, which is when anything `end()` flushes arrived. */
+    let lastAt = 0;
+    let eventType = '';
+    let dataLines = [];
+    let eventId;
+    let eventLines = [];
+    let lastEventId = '';
+    let hasData = false;
+    function resetEvent() {
+        eventType = '';
+        dataLines = [];
+        eventLines = [];
+        eventId = undefined;
+        hasData = false;
+    }
+    resetEvent();
+    function handleField(line, at) {
+        if (line === '') {
+            if (hasData) {
+                const data = dataLines.join('\n');
+                onRow({
+                    kind: 'event',
+                    index: index++,
+                    at,
+                    size: lineSize(...eventLines),
+                    event: eventType === '' ? 'message' : eventType,
+                    data,
+                    ...(eventId !== undefined ? { id: eventId } : {}),
+                    lastEventId,
+                });
+            }
+            resetEvent();
+            return;
+        }
+        if (line.startsWith(':')) {
+            const text = line.slice(1);
+            onRow({ kind: 'comment', index: index++, at, size: lineSize(line), text });
+            return;
+        }
+        const colon = line.indexOf(':');
+        let field;
+        let value;
+        if (colon === -1) {
+            field = line;
+            value = '';
+        }
+        else {
+            field = line.slice(0, colon);
+            value = line.slice(colon + 1);
+            if (value.startsWith(' '))
+                value = value.slice(1);
+        }
+        switch (field) {
+            case 'event':
+                eventType = value;
+                eventLines.push(line);
+                break;
+            case 'data':
+                dataLines.push(value);
+                eventLines.push(line);
+                hasData = true;
+                break;
+            case 'id':
+                if (!value.includes('\u0000')) {
+                    eventId = value;
+                    lastEventId = value;
+                }
+                eventLines.push(line);
+                break;
+            case 'retry':
+                if (/^[0-9]+$/.test(value)) {
+                    onRow({ kind: 'retry', index: index++, at, size: lineSize(line), ms: Number(value) });
+                }
+                break;
+            default:
+                // unknown field: not dispatched, but its bytes still belong to the event they arrived in
+                eventLines.push(line);
+                break;
+        }
+    }
+    function consumeText(text, at) {
+        if (!sawText) {
+            sawText = true;
+            if (text.startsWith('\uFEFF'))
+                text = text.slice(1);
+        }
+        let buf = carry + text;
+        carry = '';
+        if (pendingCr) {
+            pendingCr = false;
+            if (buf.startsWith('\n'))
+                buf = buf.slice(1);
+        }
+        // Normalize CR/CRLF to LF, but hold back a trailing CR in case it's split from its LF: it already
+        // ends the line it closes, so a synthetic LF stands in for it once split.
+        let endsWithCr = false;
+        if (buf.endsWith('\r')) {
+            endsWithCr = true;
+            buf = buf.slice(0, -1);
+        }
+        let normalized = buf.replace(/\r\n|\r/g, '\n');
+        if (endsWithCr) {
+            pendingCr = true;
+            normalized += '\n';
+        }
+        const lines = normalized.split('\n');
+        carry = lines.pop() ?? '';
+        for (const line of lines)
+            handleField(line, at);
+    }
+    return {
+        push(chunk, at) {
+            lastAt = at;
+            const text = decoder.decode(chunk, { stream: true });
+            consumeText(text, at);
+        },
+        end() {
+            const text = decoder.decode();
+            if (text !== '')
+                consumeText(text, lastAt);
+            // An unterminated event at end of stream is discarded per spec.
+        },
+    };
+}
+export function isEventStream(contentType) {
+    if (contentType === undefined)
+        return false;
+    const mediaType = contentType.split(';', 1)[0]?.trim().toLowerCase();
+    return mediaType === 'text/event-stream';
+}
+export function eventStreamDocument(rows) {
+    const entries = rows
+        .filter((row) => row.kind === 'event')
+        .map((row) => {
+        let data = row.data;
+        try {
+            data = JSON.parse(row.data);
+        }
+        catch {
+            data = row.data;
+        }
+        return {
+            event: row.event,
+            ...(row.id !== undefined ? { id: row.id } : {}),
+            data,
+            at: row.at,
+        };
+    });
+    return JSON.stringify(entries);
+}
+export function serializeEventStream(rows) {
+    let out = '';
+    for (const row of rows) {
+        if (row.kind === 'comment') {
+            out += `:${row.text}\n\n`;
+        }
+        else if (row.kind === 'retry') {
+            out += `retry: ${row.ms}\n\n`;
+        }
+        else {
+            if (row.event !== 'message')
+                out += `event: ${row.event}\n`;
+            if (row.id !== undefined)
+                out += `id: ${row.id}\n`;
+            const dataLines = row.data.split('\n');
+            for (const line of dataLines)
+                out += `data: ${line}\n`;
+            out += '\n';
+        }
+    }
+    return out;
+}

package/dist/rest/storage.d.ts

@@ -0,0 +1,26 @@
+/**
+ * How REST APIs are stored in a project folder (spec §3.2): `apis/<slug>/api.yaml` with
+ * `kind: rest`, and a tree of folders and `*.request.yaml` files under `requests/`, a raw body in
+ * the sibling `<slug>.body.<ext>`.
+ *
+ * `restRequestReader` and `signingOf` are exported for core's loader, which reads the project's
+ * webhook collection (REST requests in a tree of their own) until phase 3 of #184 moves it here.
+ */
+import type { FsLike } from '../project/fs.js';
+import type { RequestReader } from '../project/load-helpers.js';
+import type { ProjectProblem } from '../project/load.js';
+import type { WebhookSigningFile } from '../project/schema-parts.js';
+import type { RequestWriter } from '../project/serialize-helpers.js';
+import type { ProtocolStorage } from '../protocol/module.js';
+import type { WebhookSigning } from '../webhooks/model.js';
+import type { RestApi, RestRequestDef } from './model.js';
+/** A parsed `signing` key as the model holds it. */
+export declare function signingOf(parsed: WebhookSigningFile): WebhookSigning;
+/** Reads a REST request and its raw body file. */
+export declare function restRequestReader(fs: FsLike, root: string, problems: ProjectProblem[]): RequestReader<RestRequestDef>;
+/** A `signing` key as written: the scheme's own fields, and references only. */
+export declare function signingDocument(signing: WebhookSigning | undefined): Record<string, unknown> | undefined;
+/** A REST request as written: its document, its raw body file and its script files. Core writes the webhook collection's items with it. */
+export declare const writeRestRequest: RequestWriter<RestRequestDef>;
+/** REST's storage facet. */
+export declare const restStorage: ProtocolStorage<RestApi>;