@oxygen-agent/cli 1.226.15 → 1.242.6

package/dist/local-custom-http-column.d.ts

@@ -1,8 +1,10 @@
+import { type CustomHttpResolveHostname } from "@oxygen/shared/custom-http-safety";
 type LocalColumnRunOptions = {
     rowId: string | null;
     limit?: number;
     concurrency?: number;
     force: boolean;
+    resolveHostname?: CustomHttpResolveHostname;
 };
 export declare function runLocalCustomHttpColumn(table: string, columnKeyOrId: string, options: LocalColumnRunOptions): Promise<Record<string, unknown>>;
 export {};

package/dist/local-custom-http-column.js

@@ -1,5 +1,9 @@
+import { Buffer } from "node:buffer";
 import { OxygenError } from "@oxygen/shared";
+import { CustomHttpUrlSafetyError, assertCustomHttpPublicUrlSyntax, assertCustomHttpResolvedHostAllowed, } from "@oxygen/shared/custom-http-safety";
 import { requestOxygen } from "./http-client.js";
+const LOCAL_CUSTOM_HTTP_TIMEOUT_MS = 10_000;
+const LOCAL_CUSTOM_HTTP_MAX_RESPONSE_BYTES = 1_000_000;
 export async function runLocalCustomHttpColumn(table, columnKeyOrId, options) {
     const described = await requestOxygen("/api/cli/tables/describe", {
         method: "POST",
@@ -17,83 +21,20 @@ export async function runLocalCustomHttpColumn(table, columnKeyOrId, options) {
     });
     const rows = Array.isArray(queried.rows) ? queried.rows.filter(isRecord) : [];
     const concurrency = normalizeLocalConcurrency(options.concurrency);
-    const results = await mapLocalConcurrent(rows, concurrency, async (row) => {
-        const rowId = readLocalRowId(row);
-        if (!options.force && isNonEmptyLocalValue(row[column.key])) {
-            return {
-                status: "skipped",
-                rowId,
-                reason: "existing_value",
-                toolId: "custom_http",
-            };
-        }
-        try {
-            const execution = await executeLocalCustomHttpColumn(definition, row, secrets);
-            const update = await requestOxygen("/api/cli/tables/rows/update", {
-                method: "POST",
-                body: {
-                    table,
-                    row_id: rowId,
-                    values: { [column.key]: execution.output },
-                    metadata: {
-                        command: "columns run --local",
-                        column_key: column.key,
-                        column_kind: "tool",
-                        tool_mode: "custom_http",
-                        request: execution.request,
-                        response: execution.response,
-                        http_status: execution.response.status,
-                    },
-                },
-            });
-            return {
-                status: "completed",
-                rowId,
-                toolId: "custom_http",
-                request: execution.request,
-                response: execution.response,
-                output: execution.output,
-                run: isRecord(update.run) ? update.run : null,
-            };
-        }
-        catch (error) {
-            const serialized = serializeLocalColumnError(error, "custom_http_run_failed");
-            const update = await requestOxygen("/api/cli/tables/rows/update", {
-                method: "POST",
-                body: {
-                    table,
-                    row_id: rowId,
-                    values: { [column.key]: serialized },
-                    metadata: {
-                        command: "columns run --local",
-                        column_key: column.key,
-                        column_kind: "tool",
-                        tool_mode: "custom_http",
-                        error: serialized.error,
-                    },
-                },
-            });
-            return {
-                status: "failed",
-                rowId,
-                toolId: "custom_http",
-                output: serialized,
-                error: serialized.error,
-                run: isRecord(update.run) ? update.run : null,
-            };
-        }
-    });
-    const completedCount = results.filter((result) => result.status === "completed").length;
-    const failedCount = results.filter((result) => result.status === "failed").length;
-    const skippedCount = results.filter((result) => result.status === "skipped").length;
-    const first = results[0];
+    const results = await mapLocalConcurrent(rows, concurrency, (row) => runLocalCustomHttpRow({
+        table,
+        columnKey: column.key,
+        definition,
+        row,
+        secrets,
+        force: options.force,
+        ...(options.resolveHostname ? { resolveHostname: options.resolveHostname } : {}),
+    }));
+    const { completedCount, failedCount, skippedCount } = countLocalCustomHttpResults(results);
     return {
-        status: failedCount > 0
-            ? "completed_with_errors"
-            : completedCount > 0
-                ? "completed"
-                : "skipped",
+        status: localCustomHttpRunStatus({ completedCount, failedCount, skippedCount }),
         table: described.table ?? queried.table ?? null,
+        web_url: localCustomHttpTableWebUrl(table, described.table, queried.table),
         column,
         toolId: "custom_http",
         requestedRowCount: rows.length,
@@ -103,13 +44,123 @@ export async function runLocalCustomHttpColumn(table, columnKeyOrId, options) {
         failedCount,
         skippedCount,
         results,
-        ...(first ? { rowId: first.rowId } : {}),
-        ...(first && "request" in first ? { request: first.request } : {}),
-        ...(first && "output" in first ? { output: first.output } : {}),
-        ...(first && "error" in first ? { error: first.error } : {}),
-        ...(first && "run" in first ? { run: first.run } : {}),
+        ...localFirstRowFields(results[0]),
+    };
+}
+async function runLocalCustomHttpRow(input) {
+    const rowId = readLocalRowId(input.row);
+    if (!input.force && isNonEmptyLocalValue(input.row[input.columnKey])) {
+        return {
+            status: "skipped",
+            rowId,
+            reason: "existing_value",
+            toolId: "custom_http",
+        };
+    }
+    try {
+        const execution = await executeLocalCustomHttpColumn(input.definition, input.row, input.secrets, input.resolveHostname);
+        return await persistLocalCustomHttpSuccess({ ...input, rowId, execution });
+    }
+    catch (error) {
+        return await persistLocalCustomHttpFailure({ ...input, rowId, error });
+    }
+}
+async function persistLocalCustomHttpSuccess(input) {
+    const update = await requestOxygen("/api/cli/tables/rows/update", {
+        method: "POST",
+        body: {
+            table: input.table,
+            row_id: input.rowId,
+            values: { [input.columnKey]: input.execution.output },
+            metadata: {
+                command: "columns run --local",
+                column_key: input.columnKey,
+                column_kind: "tool",
+                tool_mode: "custom_http",
+                request: input.execution.request,
+                response: input.execution.response,
+                http_status: input.execution.response.status,
+            },
+        },
+    });
+    return {
+        status: "completed",
+        rowId: input.rowId,
+        toolId: "custom_http",
+        request: input.execution.request,
+        response: input.execution.response,
+        output: input.execution.output,
+        run: isRecord(update.run) ? update.run : null,
+    };
+}
+async function persistLocalCustomHttpFailure(input) {
+    const serialized = serializeLocalColumnError(input.error, "custom_http_run_failed");
+    const update = await requestOxygen("/api/cli/tables/rows/update", {
+        method: "POST",
+        body: {
+            table: input.table,
+            row_id: input.rowId,
+            values: { [input.columnKey]: serialized },
+            metadata: {
+                command: "columns run --local",
+                column_key: input.columnKey,
+                column_kind: "tool",
+                tool_mode: "custom_http",
+                error: serialized.error,
+            },
+        },
+    });
+    return {
+        status: "failed",
+        rowId: input.rowId,
+        toolId: "custom_http",
+        output: serialized,
+        error: serialized.error,
+        run: isRecord(update.run) ? update.run : null,
     };
 }
+function countLocalCustomHttpResults(results) {
+    return {
+        completedCount: results.filter((result) => result.status === "completed").length,
+        failedCount: results.filter((result) => result.status === "failed").length,
+        skippedCount: results.filter((result) => result.status === "skipped").length,
+    };
+}
+function localCustomHttpRunStatus(counts) {
+    if (counts.failedCount > 0)
+        return "completed_with_errors";
+    if (counts.completedCount > 0)
+        return "completed";
+    return "skipped";
+}
+function localFirstRowFields(first) {
+    if (!first)
+        return {};
+    const fields = { rowId: first.rowId };
+    if ("request" in first)
+        fields.request = first.request;
+    if ("output" in first)
+        fields.output = first.output;
+    if ("error" in first)
+        fields.error = first.error;
+    if ("run" in first)
+        fields.run = first.run;
+    return fields;
+}
+function localCustomHttpTableWebUrl(table, described, queried) {
+    const identifier = readLocalTableIdentifier(described) ?? readLocalTableIdentifier(queried) ?? table;
+    return `https://oxygen-agent.com/tables/${encodeURIComponent(identifier)}`;
+}
+function readLocalTableIdentifier(table) {
+    if (!isRecord(table))
+        return null;
+    for (const key of ["id", "slug", "key"]) {
+        const value = table[key];
+        if (typeof value === "string" && value.trim())
+            return value.trim();
+    }
+    return null;
+}
 function findLocalColumn(columns, columnKeyOrId) {
     const column = columns.find((entry) => entry.id === columnKeyOrId || entry.key === columnKeyOrId);
     if (!column) {
@@ -283,47 +334,143 @@ async function mapLocalConcurrent(values, concurrency, action) {
     }));
     return results;
 }
-async function executeLocalCustomHttpColumn(definition, row, secrets) {
-    const secretValues = Object.values(secrets);
+async function executeLocalCustomHttpColumn(definition, row, secrets, resolveHostname) {
+    const prepared = prepareLocalCustomHttpRequest(definition, row, secrets);
+    const response = await fetchLocalCustomHttpResponse(prepared, resolveHostname);
+    return await readLocalCustomHttpExecution(response, prepared, definition.outputPath ?? null);
+}
+function prepareLocalCustomHttpRequest(definition, row, secrets) {
+    const secretValues = expandLocalSecretValues(secrets);
     const method = definition.request.method;
-    const resolvedUrl = resolveLocalTemplateString(definition.request.url, row, secrets, "request.url");
-    if (typeof resolvedUrl !== "string" || !resolvedUrl.trim()) {
-        throw new OxygenError("invalid_column_run", "Custom HTTP request url must resolve to a string.", {
-            details: { field: "request.url" },
-            exitCode: 1,
-        });
-    }
-    const url = validateLocalHttpUrl(resolvedUrl, secretValues);
+    const url = resolveLocalHttpUrl(definition.request.url, row, secrets, secretValues);
     const headers = resolveLocalHeaders(definition.request.headers ?? {}, row, secrets);
-    const bodyValue = Object.hasOwn(definition.request, "body")
+    const bodyValue = resolveLocalHttpBodyValue(definition, row, secrets);
+    assertLocalHttpBodyAllowed(method, bodyValue);
+    const body = bodyValue === undefined ? undefined : serializeLocalHttpBody(bodyValue, headers);
+    return { method, url, headers, bodyValue, body, secretValues };
+}
+function expandLocalSecretValues(secrets) {
+    const values = new Set();
+    for (const secret of Object.values(secrets)) {
+        if (!secret)
+            continue;
+        values.add(secret);
+        try {
+            values.add(encodeURIComponent(secret));
+        }
+        catch {
+            // Ignore values that cannot be URL-encoded.
+        }
+        try {
+            values.add(encodeURI(secret));
+        }
+        catch {
+            // Ignore values that cannot be URL-encoded.
+        }
+    }
+    return [...values];
+}
+function resolveLocalHttpUrl(template, row, secrets, secretValues) {
+    const resolvedUrl = resolveLocalTemplateString(template, row, secrets, "request.url");
+    if (typeof resolvedUrl === "string" && resolvedUrl.trim())
+        return validateLocalHttpUrl(resolvedUrl, secretValues);
+    throw new OxygenError("invalid_column_run", "Custom HTTP request url must resolve to a string.", {
+        details: { field: "request.url" },
+        exitCode: 1,
+    });
+}
+function resolveLocalHttpBodyValue(definition, row, secrets) {
+    return Object.hasOwn(definition.request, "body")
         ? resolveLocalTemplateValue(definition.request.body, row, secrets, "request.body")
         : undefined;
+}
+function assertLocalHttpBodyAllowed(method, bodyValue) {
     if (method === "GET" && bodyValue !== undefined) {
         throw new OxygenError("invalid_column_run", "GET custom HTTP requests cannot include a body.", {
             details: { method },
             exitCode: 1,
         });
     }
-    const body = bodyValue === undefined ? undefined : serializeLocalHttpBody(bodyValue, headers);
-    const response = await fetch(url, {
-        method,
-        headers,
-        ...(body !== undefined ? { body } : {}),
+}
+async function fetchLocalCustomHttpResponse(prepared, resolveHostname) {
+    await assertLocalResolvedHttpUrlAllowed(prepared, resolveHostname);
+    return await fetch(prepared.url, {
+        method: prepared.method,
+        headers: prepared.headers,
+        ...(prepared.body !== undefined ? { body: prepared.body } : {}),
+        redirect: "manual",
+        signal: AbortSignal.timeout(LOCAL_CUSTOM_HTTP_TIMEOUT_MS),
     }).catch((error) => {
         throw new OxygenError("custom_http_network_error", "Custom HTTP request failed before receiving a response.", {
             details: {
-                reason: redactLocalSecrets(error instanceof Error ? error.message : String(error), secretValues),
+                reason: redactLocalSecrets(error instanceof Error ? error.message : String(error), prepared.secretValues),
             },
             exitCode: 1,
         });
     });
-    const responseBody = await readLocalHttpResponseBody(response);
-    const redactedResponseBody = redactLocalSecrets(responseBody, secretValues);
-    const responseSummary = {
+}
+async function assertLocalResolvedHttpUrlAllowed(prepared, resolveHostname) {
+    try {
+        await assertCustomHttpResolvedHostAllowed({
+            url: prepared.url,
+            ...(resolveHostname ? { resolveHostname } : {}),
+        });
+    }
+    catch (error) {
+        if (!(error instanceof CustomHttpUrlSafetyError))
+            throw error;
+        if (error.reason === "dns_lookup_failed") {
+            throw new OxygenError("custom_http_network_error", "Custom HTTP request failed before receiving a response.", {
+                details: {
+                    reason: redactLocalSecrets(error.message, prepared.secretValues),
+                },
+                exitCode: 1,
+            });
+        }
+        throw new OxygenError("invalid_column_run", "Custom HTTP request url is invalid.", {
+            details: {
+                reason: redactLocalSecrets(error.message, prepared.secretValues),
+            },
+            exitCode: 1,
+        });
+    }
+}
+async function readLocalCustomHttpExecution(response, prepared, outputPath) {
+    assertLocalNoRedirect(response, prepared.secretValues);
+    const responseBody = await readLocalHttpResponseBody(response, LOCAL_CUSTOM_HTTP_MAX_RESPONSE_BYTES);
+    const redactedResponseBody = redactLocalSecrets(responseBody, prepared.secretValues);
+    const responseSummary = summarizeLocalHttpResponse(response, prepared.secretValues);
+    assertLocalOkResponse(response, redactedResponseBody);
+    return {
+        output: applyLocalOutputPath(redactedResponseBody, outputPath),
+        request: redactLocalSecrets({
+            method: prepared.method,
+            url: prepared.url,
+            ...(Object.keys(prepared.headers).length > 0 ? { headers: prepared.headers } : {}),
+            ...(prepared.bodyValue !== undefined ? { body: prepared.bodyValue } : {}),
+        }, prepared.secretValues),
+        response: responseSummary,
+    };
+}
+function assertLocalNoRedirect(response, secretValues) {
+    if (response.status >= 300 && response.status < 400) {
+        throw new OxygenError("custom_http_redirect_blocked", "Custom HTTP redirects are blocked for safety.", {
+            details: {
+                status: response.status,
+                location: redactLocalSecrets(response.headers.get("location") ?? null, secretValues),
+            },
+            exitCode: 1,
+        });
+    }
+}
+function summarizeLocalHttpResponse(response, secretValues) {
+    return {
         status: response.status,
         statusText: response.statusText,
         headers: redactLocalSecrets(localHeadersToObject(response.headers), secretValues),
     };
+}
+function assertLocalOkResponse(response, redactedResponseBody) {
     if (!response.ok) {
         throw new OxygenError("custom_http_request_failed", "Custom HTTP request returned a non-2xx response.", {
             details: {
@@ -334,22 +481,11 @@ async function executeLocalCustomHttpColumn(definition, row, secrets) {
             exitCode: 1,
         });
     }
-    return {
-        output: applyLocalOutputPath(redactedResponseBody, definition.outputPath ?? null),
-        request: redactLocalSecrets({
-            method,
-            url,
-            ...(Object.keys(headers).length > 0 ? { headers } : {}),
-            ...(bodyValue !== undefined ? { body: bodyValue } : {}),
-        }, secretValues),
-        response: responseSummary,
-    };
 }
 function validateLocalHttpUrl(rawUrl, secretValues) {
     try {
         const parsed = new URL(rawUrl);
-        if (parsed.protocol !== "http:" && parsed.protocol !== "https:")
-            throw new Error("Unsupported protocol.");
+        validateLocalPublicHttpsUrl(parsed);
         return parsed.toString();
     }
     catch (error) {
@@ -362,6 +498,16 @@ function validateLocalHttpUrl(rawUrl, secretValues) {
         });
     }
 }
+function validateLocalPublicHttpsUrl(parsed) {
+    try {
+        assertCustomHttpPublicUrlSyntax(parsed);
+    }
+    catch (error) {
+        if (error instanceof CustomHttpUrlSafetyError)
+            throw new Error(error.message);
+        throw error;
+    }
+}
 function resolveLocalHeaders(headers, row, secrets) {
     const resolved = {};
     for (const [key, value] of Object.entries(headers)) {
@@ -382,8 +528,17 @@ function hasLocalHeader(headers, name) {
     const normalized = name.toLowerCase();
     return Object.keys(headers).some((key) => key.toLowerCase() === normalized);
 }
-async function readLocalHttpResponseBody(response) {
-    const text = await response.text();
+async function readLocalHttpResponseBody(response, maxBytes) {
+    const contentLength = response.headers.get("content-length");
+    const declaredLength = contentLength === null ? null : Number(contentLength);
+    if (declaredLength !== null && Number.isFinite(declaredLength) && declaredLength > maxBytes) {
+        throw new OxygenError("custom_http_response_too_large", "Custom HTTP response exceeds the configured size cap.", {
+            details: { max_response_bytes: maxBytes, content_length: declaredLength },
+            exitCode: 1,
+        });
+    }
+    const buffer = await readLocalHttpResponseBuffer(response, maxBytes);
+    const text = buffer.toString("utf8");
     if (!text)
         return null;
     const contentType = response.headers.get("content-type") ?? "";
@@ -403,6 +558,41 @@ async function readLocalHttpResponseBody(response) {
         return text;
     }
 }
+async function readLocalHttpResponseBuffer(response, maxBytes) {
+    if (!response.body) {
+        const buffer = await response.arrayBuffer();
+        if (buffer.byteLength > maxBytes) {
+            throw new OxygenError("custom_http_response_too_large", "Custom HTTP response exceeds the configured size cap.", {
+                details: { max_response_bytes: maxBytes, actual_bytes: buffer.byteLength },
+                exitCode: 1,
+            });
+        }
+        return Buffer.from(buffer);
+    }
+    const reader = response.body.getReader();
+    const chunks = [];
+    let actualBytes = 0;
+    try {
+        while (true) {
+            const { done, value } = await reader.read();
+            if (done)
+                break;
+            actualBytes += value.byteLength;
+            if (actualBytes > maxBytes) {
+                await reader.cancel().catch(() => undefined);
+                throw new OxygenError("custom_http_response_too_large", "Custom HTTP response exceeds the configured size cap.", {
+                    details: { max_response_bytes: maxBytes, actual_bytes: actualBytes },
+                    exitCode: 1,
+                });
+            }
+            chunks.push(Buffer.from(value));
+        }
+    }
+    finally {
+        reader.releaseLock();
+    }
+    return Buffer.concat(chunks, actualBytes);
+}
 function localHeadersToObject(headers) {
     const result = {};
     headers.forEach((value, key) => {
@@ -434,10 +624,17 @@ function resolveLocalTemplateString(value, row, secrets, inputName) {
     });
 }
 function resolveLocalTemplatePath(rawPath, row, secrets, inputName) {
-    const segments = rawPath
+    const segments = splitLocalTemplatePath(rawPath);
+    const root = resolveLocalTemplateRoot(segments, row, secrets, inputName, rawPath);
+    return resolveLocalTemplateSegments(root.current, root.segments);
+}
+function splitLocalTemplatePath(rawPath) {
+    return rawPath
         .split(".")
         .map((segment) => segment.trim())
         .filter(Boolean);
+}
+function resolveLocalTemplateRoot(segments, row, secrets, inputName, rawPath) {
     const root = segments.shift();
     if (!root) {
         throw new OxygenError("invalid_column_run", "Template reference is empty.", {
@@ -445,40 +642,47 @@ function resolveLocalTemplatePath(rawPath, row, secrets, inputName) {
             exitCode: 1,
         });
     }
-    let current;
     if (root === "secrets") {
-        const secretName = segments.shift();
-        if (!secretName || !Object.hasOwn(secrets, secretName)) {
-            throw new OxygenError("invalid_column_run", "Template references an unknown custom HTTP secret.", {
-                details: { input: inputName, secret: secretName ?? null, template: rawPath },
-                exitCode: 1,
-            });
-        }
-        current = secrets[secretName];
+        return resolveLocalTemplateSecretRoot(segments, secrets, inputName, rawPath);
     }
-    else {
-        if (!Object.hasOwn(row, root)) {
-            throw new OxygenError("invalid_column_run", "Template references an unknown row column.", {
-                details: { input: inputName, column_key: root, template: rawPath },
-                exitCode: 1,
-            });
-        }
-        current = row[root];
+    return resolveLocalTemplateRowRoot(root, segments, row, inputName, rawPath);
+}
+function resolveLocalTemplateSecretRoot(segments, secrets, inputName, rawPath) {
+    const secretName = segments.shift();
+    if (!secretName || !Object.hasOwn(secrets, secretName)) {
+        throw new OxygenError("invalid_column_run", "Template references an unknown custom HTTP secret.", {
+            details: { input: inputName, secret: secretName ?? null, template: rawPath },
+            exitCode: 1,
+        });
     }
+    return { current: secrets[secretName], segments };
+}
+function resolveLocalTemplateRowRoot(root, segments, row, inputName, rawPath) {
+    if (!Object.hasOwn(row, root)) {
+        throw new OxygenError("invalid_column_run", "Template references an unknown row column.", {
+            details: { input: inputName, column_key: root, template: rawPath },
+            exitCode: 1,
+        });
+    }
+    return { current: row[root], segments };
+}
+function resolveLocalTemplateSegments(current, segments) {
+    let value = current;
     for (const segment of segments) {
-        if (current === null || current === undefined)
+        value = resolveLocalTemplateSegment(value, segment);
+        if (value === null)
             return null;
-        if (Array.isArray(current) && /^\d+$/.test(segment)) {
-            current = current[Number(segment)] ?? null;
-            continue;
-        }
-        if (isRecord(current) && Object.hasOwn(current, segment)) {
-            current = current[segment];
-            continue;
-        }
-        return null;
     }
-    return current ?? null;
+    return value ?? null;
+}
+function resolveLocalTemplateSegment(current, segment) {
+    if (current === null || current === undefined)
+        return null;
+    if (Array.isArray(current) && /^\d+$/.test(segment))
+        return current[Number(segment)] ?? null;
+    if (isRecord(current) && Object.hasOwn(current, segment))
+        return current[segment];
+    return null;
 }
 function applyLocalOutputPath(value, outputPath) {
     const path = typeof outputPath === "string" ? outputPath.trim() : "";

package/node_modules/@oxygen/shared/dist/custom-http-safety.d.ts

@@ -0,0 +1,18 @@
+export type CustomHttpResolvedAddress = {
+    address: string;
+    family?: number;
+};
+export type CustomHttpResolveHostname = (hostname: string) => Promise<CustomHttpResolvedAddress[]>;
+export type CustomHttpUrlSafetyErrorReason = "blocked_host" | "blocked_resolved_address" | "credentials" | "dns_lookup_failed" | "protocol";
+export declare class CustomHttpUrlSafetyError extends Error {
+    readonly reason: CustomHttpUrlSafetyErrorReason;
+    readonly details: Record<string, unknown>;
+    constructor(reason: CustomHttpUrlSafetyErrorReason, message: string, details?: Record<string, unknown>);
+}
+export declare function assertCustomHttpPublicUrlSyntax(url: string | URL): URL;
+export declare function assertCustomHttpResolvedHostAllowed(input: {
+    url: string | URL;
+    resolveHostname?: CustomHttpResolveHostname;
+}): Promise<void>;
+export declare function normalizeCustomHttpUrlHost(hostname: string): string;
+export declare function isBlockedCustomHttpHost(host: string): boolean;