lakebed 0.0.8 → 0.0.10

package/src/source-runtime.js

@@ -0,0 +1,668 @@
+import { fork } from "node:child_process";
+import { lookup } from "node:dns/promises";
+import { request as httpRequest } from "node:http";
+import { request as httpsRequest } from "node:https";
+import { isIP } from "node:net";
+import { dirname, resolve } from "node:path";
+import { fileURLToPath, pathToFileURL } from "node:url";
+import {
+  DEFAULT_ANONYMOUS_LIMITS,
+  prepareAnonymousPatch,
+  stableStringify
+} from "./anonymous.js";
+
+const moduleDir = dirname(fileURLToPath(import.meta.url));
+const defaultWorkerPath = resolve(moduleDir, "source-runtime-worker.js");
+const defaultLoaderPath = resolve(moduleDir, "source-runtime-loader.mjs");
+const defaultTimeoutMs = 5000;
+const defaultFetchTimeoutMs = 5000;
+const defaultMaxOldSpaceSizeMb = 64;
+const defaultMaxFetchResponseBytes = 1024 * 1024;
+const defaultMaxRequestBytes = 128 * 1024;
+const maxSourceOperations = 10000;
+
+function isPlainObject(value) {
+  return Boolean(value) && typeof value === "object" && Object.getPrototypeOf(value) === Object.prototype;
+}
+
+function byteLength(value) {
+  return Buffer.byteLength(typeof value === "string" ? value : stableStringify(value), "utf8");
+}
+
+function positiveNumber(value, fallback) {
+  const number = Number(value);
+  return Number.isFinite(number) && number > 0 ? number : fallback;
+}
+
+function assertFieldValue(tableName, fieldName, field, value, limits = DEFAULT_ANONYMOUS_LIMITS) {
+  if (value === undefined) {
+    throw new Error(`Missing value for ${tableName}.${fieldName}`);
+  }
+
+  if (field.kind === "string" && typeof value !== "string") {
+    throw new Error(`Expected ${tableName}.${fieldName} to be a string.`);
+  }
+
+  if (field.kind === "boolean" && typeof value !== "boolean") {
+    throw new Error(`Expected ${tableName}.${fieldName} to be a boolean.`);
+  }
+
+  const maxValueBytes = limits.maxValueBytes ?? DEFAULT_ANONYMOUS_LIMITS.maxValueBytes;
+  if (byteLength(value) > maxValueBytes) {
+    throw new Error(`Value for ${tableName}.${fieldName} exceeds ${maxValueBytes} bytes.`);
+  }
+}
+
+function validateSourceInsertRow(schema, tableName, row, limits = DEFAULT_ANONYMOUS_LIMITS) {
+  const table = schema[tableName];
+  if (!table) {
+    throw new Error(`Unknown table: ${tableName}`);
+  }
+  if (!isPlainObject(row)) {
+    throw new Error(`Invalid insert row for ${tableName}.`);
+  }
+
+  const fields = table.fields ?? {};
+  const metadata = new Set(["id", "createdAt", "updatedAt"]);
+  const cleanRow = {};
+
+  for (const key of Object.keys(row)) {
+    if (!fields[key] && !metadata.has(key)) {
+      throw new Error(`Unknown field for ${tableName}: ${key}`);
+    }
+  }
+
+  for (const key of metadata) {
+    if (typeof row[key] !== "string" || row[key].length === 0) {
+      throw new Error(`Invalid insert row metadata for ${tableName}.${key}.`);
+    }
+    cleanRow[key] = row[key];
+  }
+
+  for (const [fieldName, field] of Object.entries(fields)) {
+    const value = row[fieldName] ?? field.defaultValue;
+    assertFieldValue(tableName, fieldName, field, value, limits);
+    cleanRow[fieldName] = value;
+  }
+
+  return cleanRow;
+}
+
+function validateSourcePatch(schema, tableName, patch, limits = DEFAULT_ANONYMOUS_LIMITS) {
+  if (!isPlainObject(patch)) {
+    throw new Error(`Invalid update patch for ${tableName}.`);
+  }
+
+  const appPatch = {};
+  for (const [key, value] of Object.entries(patch)) {
+    if (key === "updatedAt") {
+      continue;
+    }
+    appPatch[key] = value;
+  }
+
+  return prepareAnonymousPatch(schema, tableName, appPatch, limits);
+}
+
+async function snapshotSourceState({ artifact, deployId, state }) {
+  const rows = {};
+  for (const tableName of Object.keys(artifact.server.schema ?? {})) {
+    rows[tableName] = await state.listRows(deployId, tableName);
+  }
+
+  return {
+    env: typeof state.getServerEnv === "function" ? await state.getServerEnv(deployId) : {},
+    rows
+  };
+}
+
+function validateSourceOperations({ artifact, operations }) {
+  if (!Array.isArray(operations)) {
+    throw new Error("Source runtime returned invalid mutation operations.");
+  }
+  if (operations.length > maxSourceOperations) {
+    throw new Error(`Source runtime returned more than ${maxSourceOperations} mutation operations.`);
+  }
+
+  const schema = artifact.server.schema ?? {};
+  const limits = artifact.limits ?? DEFAULT_ANONYMOUS_LIMITS;
+  const cleanOperations = [];
+  for (const operation of operations) {
+    if (!isPlainObject(operation) || typeof operation.table !== "string" || !schema[operation.table]) {
+      throw new Error("Source runtime returned an invalid mutation operation.");
+    }
+
+    if (operation.op === "insert") {
+      cleanOperations.push({
+        op: "insert",
+        row: validateSourceInsertRow(schema, operation.table, operation.row, limits),
+        table: operation.table
+      });
+      continue;
+    }
+
+    if (operation.op === "update") {
+      if (typeof operation.id !== "string") {
+        throw new Error(`Source runtime returned an invalid update id for ${operation.table}.`);
+      }
+      cleanOperations.push({
+        id: operation.id,
+        op: "update",
+        patch: validateSourcePatch(schema, operation.table, operation.patch, limits),
+        table: operation.table
+      });
+      continue;
+    }
+
+    if (operation.op === "delete") {
+      if (typeof operation.id !== "string") {
+        throw new Error(`Source runtime returned an invalid delete id for ${operation.table}.`);
+      }
+      cleanOperations.push({ id: operation.id, op: "delete", table: operation.table });
+      continue;
+    }
+
+    throw new Error(`Source runtime returned unsupported mutation operation: ${operation.op}`);
+  }
+  return cleanOperations;
+}
+
+async function applySourceOperations({ artifact, deployId, operations, tx }) {
+  const cleanOperations = validateSourceOperations({ artifact, operations });
+  for (const operation of cleanOperations) {
+    if (operation.op === "insert") {
+      await tx.insertRow(deployId, operation.table, operation.row);
+    } else if (operation.op === "update") {
+      await tx.updateRow(deployId, operation.table, operation.id, operation.patch);
+    } else if (operation.op === "delete") {
+      await tx.deleteRow(deployId, operation.table, operation.id);
+    }
+  }
+}
+
+function permissionFlag() {
+  if (process.allowedNodeEnvironmentFlags?.has("--permission")) {
+    return "--permission";
+  }
+  if (process.allowedNodeEnvironmentFlags?.has("--experimental-permission")) {
+    return "--experimental-permission";
+  }
+  return null;
+}
+
+function sourceRuntimeExecArgv({ loaderPath, maxOldSpaceSizeMb, workerPath }) {
+  const execArgv = [
+    `--max-old-space-size=${maxOldSpaceSizeMb}`,
+    "--disallow-code-generation-from-strings",
+    "--loader",
+    pathToFileURL(loaderPath).href
+  ];
+  const flag = permissionFlag();
+  if (flag) {
+    execArgv.unshift(flag, "--allow-worker", `--allow-fs-read=${workerPath}`, `--allow-fs-read=${loaderPath}`);
+  }
+  return execArgv;
+}
+
+function normalizeHeaderObject(headers = {}) {
+  if (!headers) {
+    return {};
+  }
+
+  if (Array.isArray(headers)) {
+    return Object.fromEntries(headers.map(([key, value]) => [String(key), String(value)]));
+  }
+
+  if (typeof headers.entries === "function") {
+    return Object.fromEntries(Array.from(headers.entries()).map(([key, value]) => [String(key), String(value)]));
+  }
+
+  if (isPlainObject(headers)) {
+    return Object.fromEntries(Object.entries(headers).map(([key, value]) => [key, String(value)]));
+  }
+
+  return {};
+}
+
+function sanitizeRequestHeaders(headers) {
+  const blocked = new Set([
+    "connection",
+    "content-length",
+    "host",
+    "proxy-authenticate",
+    "proxy-authorization",
+    "te",
+    "trailer",
+    "transfer-encoding",
+    "upgrade"
+  ]);
+  const clean = {};
+  for (const [rawName, rawValue] of Object.entries(normalizeHeaderObject(headers))) {
+    const name = rawName.toLowerCase();
+    if (!/^[a-z0-9!#$%&'*+.^_`|~-]+$/.test(name) || blocked.has(name) || name.startsWith("sec-")) {
+      continue;
+    }
+    clean[name] = rawValue;
+  }
+  return clean;
+}
+
+function stripIpv6Brackets(hostname) {
+  return hostname.startsWith("[") && hostname.endsWith("]") ? hostname.slice(1, -1) : hostname;
+}
+
+function isPrivateIpv4(address) {
+  const parts = address.split(".").map((part) => Number(part));
+  if (parts.length !== 4 || parts.some((part) => !Number.isInteger(part) || part < 0 || part > 255)) {
+    return true;
+  }
+  const [a, b] = parts;
+  return (
+    a === 0 ||
+    a === 10 ||
+    a === 127 ||
+    (a === 100 && b >= 64 && b <= 127) ||
+    (a === 169 && b === 254) ||
+    (a === 172 && b >= 16 && b <= 31) ||
+    (a === 192 && b === 168) ||
+    (a === 198 && (b === 18 || b === 19)) ||
+    a >= 224
+  );
+}
+
+function isPrivateIpv6(address) {
+  const normalized = address.toLowerCase();
+  if (
+    normalized === "::1" ||
+    normalized === "::" ||
+    normalized.startsWith("fc") ||
+    normalized.startsWith("fd") ||
+    normalized.startsWith("fe8") ||
+    normalized.startsWith("fe9") ||
+    normalized.startsWith("fea") ||
+    normalized.startsWith("feb")
+  ) {
+    return true;
+  }
+
+  const mapped = normalized.match(/^::ffff:(\d+\.\d+\.\d+\.\d+)$/);
+  return mapped ? isPrivateIpv4(mapped[1]) : false;
+}
+
+function isPrivateAddress(address) {
+  const normalized = stripIpv6Brackets(address);
+  const family = isIP(normalized);
+  if (family === 4) {
+    return isPrivateIpv4(normalized);
+  }
+  if (family === 6) {
+    return isPrivateIpv6(normalized);
+  }
+  return false;
+}
+
+async function resolveFetchTarget(url, { allowPrivateNetwork }) {
+  if (url.protocol !== "https:" && url.protocol !== "http:") {
+    throw new Error("Source fetch only supports http and https URLs.");
+  }
+  if (url.username || url.password) {
+    throw new Error("Source fetch URLs cannot include credentials.");
+  }
+
+  const hostname = stripIpv6Brackets(url.hostname).toLowerCase();
+  if (!hostname) {
+    throw new Error("Source fetch requires a hostname.");
+  }
+
+  if (isIP(hostname)) {
+    if (!allowPrivateNetwork && isPrivateAddress(hostname)) {
+      throw new Error("Source fetch cannot access private network addresses.");
+    }
+    return {
+      address: hostname,
+      family: isIP(hostname),
+      hostHeader: url.host,
+      originalHostname: hostname
+    };
+  }
+
+  if (!allowPrivateNetwork && (hostname === "localhost" || hostname.endsWith(".localhost"))) {
+    throw new Error("Source fetch cannot access localhost.");
+  }
+
+  const addresses = await lookup(hostname, { all: true });
+  if (addresses.length === 0) {
+    throw new Error("Source fetch hostname did not resolve.");
+  }
+  if (!allowPrivateNetwork && addresses.some((entry) => isPrivateAddress(entry.address))) {
+    throw new Error("Source fetch cannot access private network addresses.");
+  }
+
+  return {
+    address: addresses[0].address,
+    family: addresses[0].family,
+    hostHeader: url.host,
+    originalHostname: hostname
+  };
+}
+
+function normalizeFetchRequest(request, { maxRequestBytes }) {
+  const url = new URL(String(request.url));
+  const method = String(request.method ?? "GET").toUpperCase();
+  if (!/^[A-Z]+$/.test(method)) {
+    throw new Error("Invalid source fetch method.");
+  }
+
+  const body = request.bodyBase64 ? Buffer.from(String(request.bodyBase64), "base64") : undefined;
+  if ((method === "GET" || method === "HEAD") && body?.byteLength) {
+    throw new Error("Source fetch GET and HEAD requests cannot include a body.");
+  }
+  if ((body?.byteLength ?? 0) > maxRequestBytes) {
+    throw new Error(`Source fetch request body exceeds ${maxRequestBytes} bytes.`);
+  }
+
+  return {
+    body,
+    headers: sanitizeRequestHeaders(request.headers),
+    method,
+    url
+  };
+}
+
+function headersForBroker(headers) {
+  const clean = {};
+  for (const [key, value] of Object.entries(headers)) {
+    clean[key] = Array.isArray(value) ? value.join(", ") : String(value ?? "");
+  }
+  return clean;
+}
+
+function requestPathForUrl(url) {
+  return `${url.pathname || "/"}${url.search}`;
+}
+
+function requestWithPinnedAddress(current, options, target) {
+  return new Promise((resolveRequest, rejectRequest) => {
+    let settled = false;
+    const requestFn = current.url.protocol === "https:" ? httpsRequest : httpRequest;
+    const request = requestFn({
+      family: target.family,
+      headers: {
+        ...current.headers,
+        host: target.hostHeader
+      },
+      hostname: target.address,
+      method: current.method,
+      path: requestPathForUrl(current.url),
+      port: current.url.port || (current.url.protocol === "https:" ? 443 : 80),
+      ...(current.url.protocol === "https:" && !isIP(target.originalHostname) ? { servername: target.originalHostname } : {})
+    });
+
+    const finish = (callback, value) => {
+      if (settled) {
+        return;
+      }
+      settled = true;
+      clearTimeout(timeout);
+      callback(value);
+    };
+
+    const timeout = setTimeout(() => {
+      request.destroy(new Error(`Source fetch exceeded ${options.fetchTimeoutMs}ms.`));
+    }, options.fetchTimeoutMs);
+
+    request.on("response", (response) => {
+      const location = response.headers.location;
+      if ([301, 302, 303, 307, 308].includes(response.statusCode) && location) {
+        response.resume();
+        finish(resolveRequest, {
+          location,
+          redirect: true,
+          status: response.statusCode
+        });
+        return;
+      }
+
+      const chunks = [];
+      let total = 0;
+      response.on("data", (chunk) => {
+        total += chunk.byteLength;
+        if (total > options.maxResponseBytes) {
+          const error = new Error(`Source fetch response exceeds ${options.maxResponseBytes} bytes.`);
+          response.destroy();
+          request.destroy();
+          finish(rejectRequest, error);
+          return;
+        }
+        chunks.push(Buffer.from(chunk));
+      });
+      response.on("end", () => {
+        const buffer = Buffer.concat(chunks, total);
+        finish(resolveRequest, {
+          bodyBase64: buffer.toString("base64"),
+          headers: headersForBroker(response.headers),
+          ok: response.statusCode >= 200 && response.statusCode < 300,
+          status: response.statusCode,
+          statusText: response.statusMessage,
+          url: current.url.toString()
+        });
+      });
+      response.on("error", (error) => finish(rejectRequest, error));
+    });
+    request.on("error", (error) => finish(rejectRequest, error));
+    if (current.body) {
+      request.write(current.body);
+    }
+    request.end();
+  });
+}
+
+function shouldRewriteRedirectToGet(status, method) {
+  if (status === 303) {
+    return method !== "GET" && method !== "HEAD";
+  }
+  return (status === 301 || status === 302) && method === "POST";
+}
+
+async function brokeredFetch(request, options) {
+  let current = normalizeFetchRequest(request, options);
+  for (let redirectCount = 0; redirectCount <= options.maxRedirects; redirectCount += 1) {
+    const target = await resolveFetchTarget(current.url, options);
+    const response = await requestWithPinnedAddress(current, options, target);
+    if (response.redirect) {
+      if (redirectCount === options.maxRedirects) {
+        throw new Error("Source fetch exceeded redirect limit.");
+      }
+      const rewriteToGet = shouldRewriteRedirectToGet(response.status, current.method);
+      current = {
+        body: rewriteToGet ? undefined : current.body,
+        headers: current.headers,
+        method: rewriteToGet ? "GET" : current.method,
+        url: new URL(response.location, current.url)
+      };
+      continue;
+    }
+
+    return response;
+  }
+
+  throw new Error("Source fetch exceeded redirect limit.");
+}
+
+function workerError(payload) {
+  const error = new Error(payload?.message ?? "Source runtime failed.");
+  error.name = payload?.name ?? "SourceRuntimeError";
+  return error;
+}
+
+export class ChildProcessSourceRuntime {
+  constructor({
+    allowPrivateNetwork = false,
+    fetchTimeoutMs = defaultFetchTimeoutMs,
+    loaderPath = defaultLoaderPath,
+    maxFetchResponseBytes = defaultMaxFetchResponseBytes,
+    maxOldSpaceSizeMb = defaultMaxOldSpaceSizeMb,
+    maxRedirects = 5,
+    maxRequestBytes = defaultMaxRequestBytes,
+    timeoutMs = defaultTimeoutMs,
+    workerPath = defaultWorkerPath
+  } = {}) {
+    this.allowPrivateNetwork = allowPrivateNetwork;
+    this.fetchTimeoutMs = positiveNumber(fetchTimeoutMs, defaultFetchTimeoutMs);
+    this.loaderPath = loaderPath;
+    this.maxFetchResponseBytes = positiveNumber(maxFetchResponseBytes, defaultMaxFetchResponseBytes);
+    this.maxOldSpaceSizeMb = positiveNumber(maxOldSpaceSizeMb, defaultMaxOldSpaceSizeMb);
+    this.maxRedirects = Math.max(0, Math.floor(positiveNumber(maxRedirects, 5)));
+    this.maxRequestBytes = positiveNumber(maxRequestBytes, defaultMaxRequestBytes);
+    this.timeoutMs = positiveNumber(timeoutMs, defaultTimeoutMs);
+    this.workerPath = workerPath;
+  }
+
+  async executeQuery({ args = [], artifact, auth, deployId, name, state }) {
+    const snapshot = await snapshotSourceState({ artifact, deployId, state });
+    const response = await this.runWorker({
+      args,
+      artifact,
+      auth,
+      env: snapshot.env,
+      name,
+      op: "query",
+      rows: snapshot.rows
+    });
+    return response.result ?? null;
+  }
+
+  async executeMutation({ args = [], artifact, auth, deployId, limits = DEFAULT_ANONYMOUS_LIMITS, name, state }) {
+    return state.transaction(deployId, async (tx) => {
+      const snapshot = await snapshotSourceState({ artifact, deployId, state: tx });
+      const response = await this.runWorker({
+        args,
+        artifact,
+        auth,
+        env: snapshot.env,
+        name,
+        op: "mutation",
+        rows: snapshot.rows
+      });
+      await applySourceOperations({
+        artifact,
+        deployId,
+        operations: response.operations ?? [],
+        tx
+      });
+      return response.result ?? null;
+    }, { stateBytesLimit: limits.stateBytes ?? DEFAULT_ANONYMOUS_LIMITS.stateBytes });
+  }
+
+  runWorker(request) {
+    return new Promise((resolveRun, rejectRun) => {
+      const child = fork(this.workerPath, [], {
+        env: {},
+        execArgv: sourceRuntimeExecArgv({
+          loaderPath: this.loaderPath,
+          maxOldSpaceSizeMb: this.maxOldSpaceSizeMb,
+          workerPath: this.workerPath
+        }),
+        serialization: "json",
+        stdio: ["ignore", "ignore", "pipe", "ipc"]
+      });
+      let stderr = "";
+      let settled = false;
+      let timeout;
+
+      const finish = (callback, value) => {
+        if (settled) {
+          return;
+        }
+        settled = true;
+        clearTimeout(timeout);
+        child.off("error", onError);
+        child.off("exit", onExit);
+        child.off("message", onMessage);
+        if (!child.killed) {
+          child.kill();
+        }
+        callback(value);
+      };
+
+      const onError = (error) => finish(rejectRun, error);
+      const onExit = (code, signal) => {
+        if (!settled) {
+          finish(rejectRun, new Error(`Source runtime exited before returning a result. code=${code ?? "null"} signal=${signal ?? "null"}${stderr ? ` stderr=${stderr}` : ""}`));
+        }
+      };
+      const onMessage = (message) => {
+        if (!isPlainObject(message)) {
+          return;
+        }
+
+        if (message.type === "source-runtime.result") {
+          if (message.ok) {
+            finish(resolveRun, {
+              operations: message.operations,
+              result: message.result
+            });
+          } else {
+            finish(rejectRun, workerError(message.error));
+          }
+          return;
+        }
+
+        if (message.type === "source-runtime.fetch") {
+          brokeredFetch(message.request, {
+            allowPrivateNetwork: this.allowPrivateNetwork,
+            fetchTimeoutMs: this.fetchTimeoutMs,
+            maxRedirects: this.maxRedirects,
+            maxRequestBytes: this.maxRequestBytes,
+            maxResponseBytes: this.maxFetchResponseBytes
+          })
+            .then((response) => {
+              if (child.connected) {
+                child.send({ id: message.id, ok: true, response, type: "source-runtime.fetch.result" });
+              }
+            })
+            .catch((error) => {
+              if (child.connected) {
+                child.send({
+                  error: { message: error instanceof Error ? error.message : String(error), name: error instanceof Error ? error.name : "Error" },
+                  id: message.id,
+                  ok: false,
+                  type: "source-runtime.fetch.result"
+                });
+              }
+            });
+        }
+      };
+
+      timeout = setTimeout(() => {
+        finish(rejectRun, new Error(`Source runtime exceeded ${this.timeoutMs}ms.`));
+      }, this.timeoutMs);
+
+      child.stderr?.on("data", (chunk) => {
+        stderr = `${stderr}${String(chunk)}`.slice(-4096);
+      });
+      child.on("error", onError);
+      child.on("exit", onExit);
+      child.on("message", onMessage);
+      child.send({ request, type: "source-runtime.run" });
+    });
+  }
+}
+
+export function createSourceRuntimeFromEnv(env = process.env) {
+  const mode = String(env.LAKEBED_SOURCE_RUNTIME ?? "child-process");
+  if (env.LAKEBED_UNSAFE_IN_PROCESS_SOURCE === "1" || mode === "unsafe-in-process") {
+    return null;
+  }
+  if (mode !== "child-process") {
+    throw new Error(`Unsupported LAKEBED_SOURCE_RUNTIME: ${mode}`);
+  }
+
+  return new ChildProcessSourceRuntime({
+    allowPrivateNetwork: env.LAKEBED_SOURCE_RUNTIME_ALLOW_PRIVATE_NETWORK === "1",
+    fetchTimeoutMs: Number(env.LAKEBED_SOURCE_FETCH_TIMEOUT_MS ?? defaultFetchTimeoutMs),
+    maxFetchResponseBytes: Number(env.LAKEBED_SOURCE_FETCH_RESPONSE_BYTES ?? defaultMaxFetchResponseBytes),
+    maxOldSpaceSizeMb: Number(env.LAKEBED_SOURCE_RUNTIME_MAX_OLD_SPACE_MB ?? defaultMaxOldSpaceSizeMb),
+    timeoutMs: Number(env.LAKEBED_SOURCE_RUNTIME_TIMEOUT_MS ?? defaultTimeoutMs)
+  });
+}

package/src/version.js

@@ -1 +1 @@
-export const LAKEBED_VERSION = "0.0.8";
+export const LAKEBED_VERSION = "0.0.10";