@firedrill-tools/unstructured 0.1.4

package/firedrill/tools/unstructured/behavior.mjs

@@ -0,0 +1,69 @@
+// Synthetic Unstructured account for Firedrill: the legacy Partition Endpoint (`POST /general/v0/general`) over a
+// deterministic text partitioner, and the Workflow Endpoint subset (sources, destinations, workflows, jobs).
+// Every operation computes from `context.state`; ids come from the seeded random source and `meta/counters`,
+// timestamps from virtual time. No Unstructured service is contacted.
+import { DESTINATION_TYPES, SOURCE_TYPES } from "./lib/connectors.mjs";
+import { Q, decoder, encodeError, encoder } from "./lib/wire.mjs";
+import { decodePartition, decodeRun } from "./lib/wire-multipart.mjs";
+import { connectorOps } from "./ops/connectors.mjs";
+import * as jobs from "./ops/jobs.mjs";
+import { partition } from "./ops/partition.mjs";
+import * as workflows from "./ops/workflows.mjs";
+
+const sources = connectorOps({ namespace: "sources", idField: "source_id", types: SOURCE_TYPES, typeFilter: "source_type", label: "Source connector" });
+const destinations = connectorOps({ namespace: "destinations", idField: "destination_id", types: DESTINATION_TYPES, typeFilter: "destination_type", label: "Destination connector" });
+
+const CONNECTOR_FIELDS = ["name", "type", "config", "key"];
+const WORKFLOW_FIELDS = ["name", "workflow_type", "source_id", "destination_id", "workflow_nodes", "schedule", "reprocess_all", "key", "template_id", "source_ids", "destination_ids", "skip_preflight"];
+const WORKFLOW_UPDATE_FIELDS = [...WORKFLOW_FIELDS.filter((f) => f !== "key"), "status"];
+const WORKFLOW_QUERY = { source_id: Q.str, destination_id: Q.str, status: Q.str, name: Q.str, page: Q.int, page_size: Q.int, sort_by: Q.str, sort_direction: Q.str, show_only_soft_deleted: Q.str, show_recommender_workflows: Q.str, dag_node_configuration_id: Q.str, created_since: Q.str, created_before: Q.str };
+
+const API_VERSION = { "unstructured-api-version": "0.1.0-firedrill" };
+const encodePartition = (result) => {
+  if (result.outcome.status !== "ok") return { headers: API_VERSION, ...encodeError(result) };
+  const value = result.outcome.value;
+  if (value !== null && typeof value === "object" && !Array.isArray(value) && typeof value.csv === "string") return { headers: API_VERSION, body: { kind: "text", value: value.csv, contentType: "text/csv; charset=utf-8" } };
+  return { headers: API_VERSION, body: { kind: "json", value } };
+};
+
+/** [operation id, handler, [route id, decode, encode?]...] */
+const TABLE = [
+  ["general.partition", partition, ["partition", decodePartition, encodePartition]],
+
+  ["sources.list", sources.list, ["list-sources", decoder({ query: { source_type: Q.str }, idem: false })]],
+  ["sources.create", sources.create, ["create-source", decoder({ body: "json", fields: CONNECTOR_FIELDS })]],
+  ["sources.get", sources.get, ["get-source", decoder({ path: ["source_id"], idem: false })]],
+  ["sources.update", sources.update, ["update-source", decoder({ path: ["source_id"], body: "json", fields: CONNECTOR_FIELDS })]],
+  ["sources.delete", sources.remove, ["delete-source", decoder({ path: ["source_id"] })]],
+  ["sources.check_connection", sources.checkConnection, ["create-source-connection-check", decoder({ path: ["source_id"] })]],
+  ["sources.get_connection_check", sources.getConnectionCheck, ["get-source-connection-check", decoder({ path: ["source_id"], idem: false })]],
+
+  ["destinations.list", destinations.list, ["list-destinations", decoder({ query: { destination_type: Q.str }, idem: false })]],
+  ["destinations.create", destinations.create, ["create-destination", decoder({ body: "json", fields: CONNECTOR_FIELDS })]],
+  ["destinations.get", destinations.get, ["get-destination", decoder({ path: ["destination_id"], idem: false })]],
+  ["destinations.update", destinations.update, ["update-destination", decoder({ path: ["destination_id"], body: "json", fields: CONNECTOR_FIELDS })]],
+  ["destinations.delete", destinations.remove, ["delete-destination", decoder({ path: ["destination_id"] })]],
+
+  ["workflows.list", workflows.list, ["list-workflows", decoder({ query: WORKFLOW_QUERY, idem: false })]],
+  ["workflows.create", workflows.create, ["create-workflow", decoder({ body: "json", fields: WORKFLOW_FIELDS })]],
+  ["workflows.get", workflows.get, ["get-workflow", decoder({ path: ["workflow_id"], idem: false })]],
+  ["workflows.update", workflows.update, ["update-workflow", decoder({ path: ["workflow_id"], body: "json", fields: WORKFLOW_UPDATE_FIELDS })]],
+  ["workflows.delete", workflows.remove, ["delete-workflow", decoder({ path: ["workflow_id"] })]],
+  ["workflows.run", workflows.run, ["run-workflow", decodeRun]],
+
+  ["jobs.list", jobs.list, ["list-jobs", decoder({ query: { workflow_id: Q.str, status: Q.str, page: Q.int, page_size: Q.int }, idem: false })]],
+  ["jobs.get", jobs.get, ["get-job", decoder({ path: ["job_id"], idem: false })]],
+  ["jobs.cancel", jobs.cancel, ["cancel-job", decoder({ path: ["job_id"] })]],
+  ["jobs.get_details", jobs.details, ["get-job-details", decoder({ path: ["job_id"], idem: false })]],
+  ["jobs.get_failed_files", jobs.failedFiles, ["get-job-failed-files", decoder({ path: ["job_id"], idem: false })]],
+  ["jobs.download_output", jobs.downloadOutput, ["download-job-output", decoder({ path: ["job_id"], query: { file_id: Q.str, node_id: Q.str }, idem: false })]],
+];
+
+const operations = {};
+const http = {};
+for (const [operationId, handler, ...routes] of TABLE) {
+  operations[operationId] = handler;
+  for (const [routeId, decode, encode] of routes) http[routeId] = { decode, encode: encode ?? encoder };
+}
+
+export default { operations, http };

package/firedrill/tools/unstructured/lib/connectors.mjs

@@ -0,0 +1,68 @@
+// Connector vocabulary: the source and destination connector types of the Workflow Endpoint, the configuration key
+// each type needs for a successful connection check, and the secret keys masked in every response.
+
+export const SOURCE_TYPES = [
+  "azure", "confluence", "couchbase", "databricks_volumes", "dropbox", "elasticsearch", "gcs", "google_drive", "jira", "kafka_cloud",
+  "mongodb", "onedrive", "outlook", "postgres", "s3", "salesforce", "sharepoint", "slack", "snowflake", "zendesk", "box", "notion",
+];
+
+export const DESTINATION_TYPES = [
+  "astradb", "azure_ai_search", "couchbase", "databricks_volumes", "databricks_volume_delta_tables", "delta_table", "elasticsearch", "gcs",
+  "kafka_cloud", "milvus", "mongodb", "motherduck", "neo4j", "onedrive", "pinecone", "postgres", "redis", "qdrant_cloud", "s3", "snowflake",
+  "weaviate_cloud", "ibm_watsonx_s3", "duckdb",
+];
+
+/** First required configuration key per type (a non-empty string makes the connection check succeed). */
+const REQUIRED = new Map([
+  ["azure", ["remote_url"]], ["confluence", ["url"]], ["couchbase", ["connection_string", "bucket"]], ["databricks_volumes", ["host", "catalog", "volume"]],
+  ["dropbox", ["remote_url"]], ["elasticsearch", ["hosts", "index_name"]], ["gcs", ["remote_url"]], ["google_drive", ["drive_id"]], ["jira", ["url"]],
+  ["kafka_cloud", ["bootstrap_servers", "topic"]], ["mongodb", ["database", "collection"]], ["onedrive", ["client_id", "tenant", "user_pname"]],
+  ["outlook", ["client_id", "tenant", "user_email"]], ["postgres", ["host", "database", "table_name"]], ["s3", ["remote_url"]], ["salesforce", ["username"]],
+  ["sharepoint", ["site", "client_id", "tenant"]], ["slack", ["channels"]], ["snowflake", ["account", "database", "table_name"]], ["zendesk", ["subdomain"]],
+  ["box", ["remote_url"]], ["notion", []],
+  ["astradb", ["collection_name"]], ["azure_ai_search", ["endpoint", "index"]], ["databricks_volume_delta_tables", ["server_hostname", "catalog"]],
+  ["delta_table", ["table_uri"]], ["milvus", ["uri", "collection_name"]], ["motherduck", ["database"]], ["neo4j", ["uri", "database"]],
+  ["pinecone", ["index_name"]], ["redis", ["host"]], ["qdrant_cloud", ["url", "collection_name"]], ["weaviate_cloud", ["cluster_url", "collection"]],
+  ["ibm_watsonx_s3", ["iceberg_endpoint", "catalog"]], ["duckdb", ["database"]],
+]);
+
+export const requiredKeys = (type) => REQUIRED.get(type) ?? [];
+
+const SECRET_KEYS = new Set(["secret_access_key", "token", "password", "client_secret", "private_key", "api_key", "access_token", "client_cred", "aws_secret_access_key"]);
+export const MASK = "********";
+export const isSecretKey = (key) => SECRET_KEYS.has(key);
+
+/** Response shape of a connector row: `config` with secrets masked, internal fields removed. */
+export function connectorView(row) {
+  const config = {};
+  for (const key of Object.keys(row.config)) config[key] = isSecretKey(key) ? MASK : row.config[key];
+  return { id: row.id, name: row.name, type: row.type, config, created_at: row.created_at, updated_at: row.updated_at, key: row.key };
+}
+
+/** Validates a caller-supplied config object into a flat map of scalars; returns { config } or { issues: [[loc, msg]] }. */
+export function normaliseConfig(value, loc) {
+  const issues = [];
+  if (value === null || typeof value !== "object" || Array.isArray(value)) return { issues: [[loc, "Input should be a valid dictionary"]] };
+  const keys = Object.keys(value);
+  if (keys.length > 40) return { issues: [[loc, "Configuration has more than 40 keys"]] };
+  const config = {};
+  for (const key of keys) {
+    if (key === "__proto__" || key === "constructor" || key === "prototype") {
+      issues.push([`${loc}.${key}`, "Invalid configuration key"]);
+      continue;
+    }
+    if (key.length === 0 || key.length > 100 || !/^[A-Za-z0-9_.-]+$/.test(key)) {
+      issues.push([`${loc}.${key.slice(0, 60)}`, "Configuration keys must match ^[A-Za-z0-9_.-]{1,100}$"]);
+      continue;
+    }
+    const item = value[key];
+    if (item === null || typeof item === "boolean") config[key] = item;
+    else if (typeof item === "number" && Number.isFinite(item)) config[key] = item;
+    else if (typeof item === "string") {
+      if (item.length > 2000) issues.push([`${loc}.${key}`, "String should have at most 2000 characters"]);
+      else config[key] = item;
+    } else if (Array.isArray(item) && item.length <= 50 && item.every((x) => typeof x === "string" && x.length <= 500)) config[key] = item.join(",");
+    else issues.push([`${loc}.${key}`, "Configuration values must be strings, numbers, booleans or null"]);
+  }
+  return issues.length > 0 ? { issues } : { config };
+}

package/firedrill/tools/unstructured/lib/errors.mjs

@@ -0,0 +1,58 @@
+// Declared error codes, their HTTP statuses and the helpers every handler uses to raise them.
+import { clip } from "./util.mjs";
+
+/** code -> HTTP status (FastAPI-style `{ detail }` envelopes are rendered by wire.mjs). */
+export const STATUS = new Map([
+  ["UNAUTHORIZED", 401],
+  ["INVALID_REQUEST", 400],
+  ["UNSUPPORTED_FILE_TYPE", 400],
+  ["NOT_FOUND", 404],
+  ["RESPONSE_TOO_LARGE", 413],
+  ["INVALID_FILE", 422],
+  ["VALIDATION_ERROR", 422],
+  ["RATE_LIMITED", 429],
+  ["INTERNAL_ERROR", 500],
+  ["SERVICE_OVERLOADED", 503],
+]);
+
+export function fail(context, code, message) {
+  context.fail({ code, message: clip(message, 3000) });
+}
+
+/**
+ * Field-level validation failure. `loc` is a dotted location (`body.files`, `query.page_size`); several issues are
+ * joined by ` | ` and rendered as the FastAPI `detail` array by the codec.
+ */
+export function validation(context, loc, message) {
+  fail(context, "VALIDATION_ERROR", `loc=${loc}: ${clip(message, 200)}`);
+}
+
+export const notFound = (context, message) => fail(context, "NOT_FOUND", message);
+
+/** Fails when the decoder attached a request problem; the prefix chooses the code. */
+export function requestProblem(input, context) {
+  const problem = input.__request_error;
+  if (typeof problem !== "string" || problem.length === 0) return;
+  if (problem.startsWith("400:")) fail(context, "INVALID_REQUEST", problem.slice(4));
+  fail(context, "VALIDATION_ERROR", problem.startsWith("422:") ? problem.slice(4) : problem);
+}
+
+/** Collects `loc=…: message` issues and fails once with all of them (at most 20). */
+export class Issues {
+  constructor() {
+    this.items = [];
+  }
+  add(loc, message) {
+    if (this.items.length < 20) this.items.push(`loc=${loc}: ${clip(message, 200)}`);
+    return this;
+  }
+  get empty() {
+    return this.items.length === 0;
+  }
+  raise(context) {
+    if (!this.empty) fail(context, "VALIDATION_ERROR", this.items.join(" | "));
+  }
+}
+
+export const boundExceeded = (context, namespace, bound) =>
+  fail(context, "INTERNAL_ERROR", `state exceeds the supported bound of ${bound} ${namespace} rows`);

package/firedrill/tools/unstructured/lib/gzip.mjs

@@ -0,0 +1,38 @@
+// Real gzip stream with deflate *stored* blocks (RFC 1951 BTYPE=00, RFC 1952 header/trailer) for `orig_elements`.
+// No compression happens, but every gzip/zlib reader decodes it. Pure JS, no Node built-ins.
+import { base64Encode, utf8Bytes } from "./util.mjs";
+
+const CRC_TABLE = new Int32Array(256);
+for (let n = 0; n < 256; n += 1) {
+  let c = n;
+  for (let k = 0; k < 8; k += 1) c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1;
+  CRC_TABLE[n] = c;
+}
+
+export function crc32(bytes) {
+  let crc = -1;
+  for (let i = 0; i < bytes.length; i += 1) crc = CRC_TABLE[(crc ^ bytes[i]) & 255] ^ (crc >>> 8);
+  return (crc ^ -1) >>> 0;
+}
+
+/** gzip bytes (stored blocks of at most 65,535 bytes) for a byte array; mtime is fixed to 0 (deterministic). */
+export function gzipStored(bytes) {
+  const out = [0x1f, 0x8b, 0x08, 0x00, 0, 0, 0, 0, 0x00, 0x03];
+  const total = bytes.length;
+  if (total === 0) out.push(0x01, 0x00, 0x00, 0xff, 0xff);
+  for (let offset = 0; offset < total; offset += 65535) {
+    const len = Math.min(65535, total - offset);
+    const final = offset + len >= total ? 1 : 0;
+    out.push(final, len & 255, (len >> 8) & 255, ~len & 255, (~len >> 8) & 255);
+    for (let i = 0; i < len; i += 1) out.push(bytes[offset + i]);
+  }
+  const crc = crc32(bytes);
+  out.push(crc & 255, (crc >>> 8) & 255, (crc >>> 16) & 255, (crc >>> 24) & 255);
+  out.push(total & 255, (total >>> 8) & 255, (total >>> 16) & 255, (total >>> 24) & 255);
+  return out;
+}
+
+/** base64(gzip(JSON.stringify(value))) — the encoding the Unstructured library uses for `orig_elements`. */
+export function gzipBase64Json(value) {
+  return base64Encode(gzipStored(utf8Bytes(JSON.stringify(value))));
+}

package/firedrill/tools/unstructured/lib/identity.mjs

@@ -0,0 +1,19 @@
+// Workspace resolution. An API key belongs to one workspace; the actor attribute `unstructuredWorkspaceId` names it.
+// A fresh `firedrill tool add` actor has no attributes and falls back to the seeded default workspace.
+import { fail } from "./errors.mjs";
+
+export const DEFAULT_WORKSPACE = "ws_northgate";
+
+/** Returns the workspace row id or fails UNAUTHORIZED ("API key is invalid"). */
+export function workspaceOf(context) {
+  const claimed = context.actor.attributes.unstructuredWorkspaceId;
+  if (claimed === undefined || claimed === null) {
+    const fallback = context.state.get("workspaces", DEFAULT_WORKSPACE);
+    if (fallback === null) fail(context, "UNAUTHORIZED", "API key is invalid");
+    return DEFAULT_WORKSPACE;
+  }
+  if (typeof claimed !== "string" || claimed.length === 0 || claimed.length > 512 || context.state.get("workspaces", claimed) === null) {
+    fail(context, "UNAUTHORIZED", "API key is invalid");
+  }
+  return claimed;
+}

package/firedrill/tools/unstructured/lib/ids.mjs

@@ -0,0 +1,36 @@
+// Ids and counters: UUID v4 strings drawn from the seeded random source, `seq` from meta/counters.
+
+const HEX = "0123456789abcdef";
+
+/** Deterministic (per seed) UUID v4 from context.random. */
+export function uuid(context) {
+  let out = "";
+  for (let i = 0; i < 32; i += 1) {
+    if (i === 8 || i === 12 || i === 16 || i === 20) out += "-";
+    if (i === 12) out += "4";
+    else if (i === 16) out += HEX[8 + context.random.nextInteger(0, 4)];
+    else out += HEX[context.random.nextInteger(0, 16)];
+  }
+  return out;
+}
+
+export function nextSeq(context) {
+  const row = context.state.get("meta", "counters");
+  const current = row !== null && Number.isInteger(row.next_seq) ? row.next_seq : 1000;
+  context.state.put("meta", "counters", { next_seq: current + 1 });
+  return current;
+}
+
+const DEFAULT_LIMITS = { connectors: 2000, workflows: 5000, jobs: 5000, files_per_source: 200, source_file_bytes: 262144, response_bytes: 900 * 1024 };
+
+export function limits(context) {
+  const row = context.state.get("meta", "limits");
+  const out = { ...DEFAULT_LIMITS };
+  if (row !== null) for (const key of Object.keys(DEFAULT_LIMITS)) if (Number.isInteger(row[key]) && row[key] >= 0) out[key] = row[key];
+  return out;
+}
+
+export const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
+
+/** A caller-supplied row id is usable when it is a non-empty string short enough for the state store. */
+export const usableId = (value) => typeof value === "string" && value.length > 0 && value.length <= 512;

package/firedrill/tools/unstructured/lib/jobs-derive.mjs

@@ -0,0 +1,92 @@
+// Job status derived from virtual time: SCHEDULED for 5 s, IN_PROGRESS for 10 s per file plus 1 s per 4 KB, then
+// finished. Nothing time-dependent is stored on the job row, so the same world read at the same virtual time
+// always answers the same way.
+import { isoDuration, isoFromUs, usFromIso } from "./util.mjs";
+
+const SCHEDULE_US = 5000000;
+const PER_FILE_US = 10000000;
+const PER_4KB_US = 1000000;
+
+export function jobTiming(job) {
+  const createdUs = usFromIso(job.created_at) ?? 0;
+  const startUs = createdUs + SCHEDULE_US;
+  let bytes = 0;
+  for (const file of job.files) bytes += Number.isInteger(file.size_bytes) ? file.size_bytes : 0;
+  const finishUs = startUs + job.files.length * PER_FILE_US + Math.floor(bytes / 4096) * PER_4KB_US;
+  return { createdUs, startUs, finishUs };
+}
+
+/** "STOPPED" | "SCHEDULED" | "IN_PROGRESS" | "FINISHED" */
+export function jobPhase(job, nowUs) {
+  const { startUs, finishUs } = jobTiming(job);
+  if (typeof job.cancelled_at === "string") {
+    const cancelledUs = usFromIso(job.cancelled_at) ?? 0;
+    if (cancelledUs < finishUs) return "STOPPED";
+  }
+  if (nowUs < startUs) return "SCHEDULED";
+  if (nowUs < finishUs) return "IN_PROGRESS";
+  return "FINISHED";
+}
+
+export const isRunning = (phase) => phase === "SCHEDULED" || phase === "IN_PROGRESS";
+
+/** { status, processing_status, runtime, reason, phase } as the Workflow Endpoint reports them. */
+export function jobStatus(job, nowUs) {
+  const phase = jobPhase(job, nowUs);
+  if (phase === "STOPPED") return { phase, status: "STOPPED", processing_status: "STOPPED", runtime: null, reason: "Cancelled by user" };
+  if (phase === "SCHEDULED") return { phase, status: "SCHEDULED", processing_status: "SCHEDULED", runtime: null, reason: null };
+  if (phase === "IN_PROGRESS") return { phase, status: "IN_PROGRESS", processing_status: "IN_PROGRESS", runtime: null, reason: null };
+  const { startUs, finishUs } = jobTiming(job);
+  const runtime = isoDuration((finishUs - startUs) / 1000000);
+  const failed = job.files.filter((file) => file.error !== null).length;
+  if (job.files.length === 0) return { phase, status: "FAILED", processing_status: "FAILED", runtime, reason: "Source connector holds no files" };
+  if (failed === job.files.length) return { phase, status: "FAILED", processing_status: "FAILED", runtime, reason: "All files failed" };
+  if (failed > 0) return { phase, status: "COMPLETED", processing_status: "COMPLETED_WITH_ERRORS", runtime, reason: null };
+  return { phase, status: "COMPLETED", processing_status: "SUCCESS", runtime, reason: null };
+}
+
+/** The node whose output the destination receives: the last node of the snapshot. */
+export const outputNodeOf = (job) => (job.nodes.length > 0 ? job.nodes[job.nodes.length - 1] : null);
+
+/** `JobInformation` view of a job row at virtual `nowUs`. */
+export function jobView(job, nowUs) {
+  const derived = jobStatus(job, nowUs);
+  const node = outputNodeOf(job);
+  const outputs = [];
+  if (derived.phase === "FINISHED" && node !== null) {
+    for (const file of job.files) if (file.error === null) outputs.push({ node_id: node.id, file_id: file.file_id, node_type: node.type, node_subtype: node.subtype });
+  }
+  return {
+    id: job.id,
+    workflow_id: job.workflow_id,
+    workflow_name: job.workflow_name,
+    status: derived.status,
+    stop_requested: job.stop_requested,
+    created_at: job.created_at,
+    runtime: derived.runtime,
+    input_file_ids: job.files.map((file) => file.file_id),
+    output_node_files: outputs,
+    job_type: job.job_type,
+    created_by_id: job.created_by,
+    reason: derived.reason,
+  };
+}
+
+/** `node_stats` for `GET /jobs/{id}/details`. */
+export function nodeStats(job, phase) {
+  const total = job.files.length;
+  const failed = job.files.filter((file) => file.error !== null).length;
+  return job.nodes.map((node) => {
+    const stat = { node_name: node.name, node_type: node.type, node_subtype: node.subtype, ready: 0, in_progress: 0, success: 0, failure: 0 };
+    if (phase === "SCHEDULED") stat.ready = total;
+    else if (phase === "IN_PROGRESS") stat.in_progress = total;
+    else if (phase === "STOPPED") stat.ready = total;
+    else {
+      stat.success = total - failed;
+      stat.failure = failed;
+    }
+    return stat;
+  });
+}
+
+export const nowIso = (context) => isoFromUs(context.clock.nowUs());

package/firedrill/tools/unstructured/lib/multipart.mjs

@@ -0,0 +1,171 @@
+// Linear multipart/form-data parser over the UTF-8 text body the framework hands to `text` routes.
+// Returns { parts: [{ name, filename, contentType, transferEncoding, lastModified, body }] } or { error }.
+//
+// Cost model: the body is walked forward once. Delimiters are found with one `indexOf` cursor that only moves forward
+// (a candidate `--boundary` counts only at the start of a line and only when the line ends right after it or it is the
+// close delimiter); each part's header block is searched only inside a window of MAX_HEADER_BLOCK characters. No search
+// restarts from an earlier position, so the work is O(body length + parts x MAX_HEADER_BLOCK), whatever the part count.
+
+const MAX_PARTS = 64;
+const MAX_HEADER_BLOCK = 16384;
+const MAX_NAME = 200;
+// RFC 2046 bchars, plus ';' (only reachable inside a quoted boundary parameter).
+const BOUNDARY_RE = /^[0-9A-Za-z'()+_,\-./:=?; ]{1,70}$/;
+
+/**
+ * Splits a header value `type; key=value; key="quoted \"value\"; with ;"` in one scan.
+ * Returns { type, params: Map(lower-case key -> first value) }. Quoted values are unescaped (`\x` -> `x`).
+ */
+export function headerParams(value) {
+  const params = new Map();
+  const n = value.length;
+  let i = value.indexOf(";");
+  if (i < 0) i = n;
+  const type = value.slice(0, i).trim().toLowerCase();
+  while (i < n) {
+    i += 1; // past ';'
+    let keyEnd = i;
+    while (keyEnd < n && value[keyEnd] !== "=" && value[keyEnd] !== ";") keyEnd += 1;
+    const key = value.slice(i, keyEnd).trim().toLowerCase();
+    if (keyEnd >= n || value[keyEnd] === ";") {
+      i = keyEnd;
+      continue;
+    }
+    let j = keyEnd + 1;
+    while (j < n && (value[j] === " " || value[j] === "\t")) j += 1;
+    let parsed;
+    if (value[j] === '"') {
+      let out = "";
+      j += 1;
+      while (j < n && value[j] !== '"') {
+        if (value[j] === "\\" && j + 1 < n) j += 1;
+        out += value[j];
+        j += 1;
+      }
+      parsed = out;
+      j += 1; // past closing quote (or end)
+      while (j < n && value[j] !== ";") j += 1;
+    } else {
+      const start = j;
+      while (j < n && value[j] !== ";") j += 1;
+      parsed = value.slice(start, j).trim();
+    }
+    if (key.length > 0 && !params.has(key)) params.set(key, parsed);
+    i = j;
+  }
+  return { type, params };
+}
+
+/** RFC 5987 `charset'lang'pct-encoded` (UTF-8 or US-ASCII); null when it does not decode. */
+function extValue(value) {
+  const first = value.indexOf("'");
+  const second = first < 0 ? -1 : value.indexOf("'", first + 1);
+  if (second < 0) return null;
+  const charset = value.slice(0, first).toLowerCase();
+  if (charset !== "utf-8" && charset !== "us-ascii") return null;
+  try {
+    return decodeURIComponent(value.slice(second + 1));
+  } catch {
+    return null;
+  }
+}
+
+/** The boundary parameter of a multipart/form-data content type, or null. */
+export function boundaryOf(contentType) {
+  if (typeof contentType !== "string" || contentType.length > MAX_HEADER_BLOCK) return null;
+  const { type, params } = headerParams(contentType);
+  if (type !== "multipart/form-data" || !params.has("boundary")) return null;
+  const value = params.get("boundary");
+  return BOUNDARY_RE.test(value) && !value.endsWith(" ") ? value : null;
+}
+
+export const isMultipart = (contentType) => typeof contentType === "string" && contentType.split(";")[0].trim().toLowerCase() === "multipart/form-data";
+
+function dispositionParams(value) {
+  const { type, params } = headerParams(value);
+  if (type !== "form-data") return null;
+  if (params.has("filename*")) {
+    // RFC 6266: the extended value wins when it decodes.
+    const decoded = extValue(params.get("filename*"));
+    if (decoded !== null) params.set("filename", decoded);
+  }
+  return params;
+}
+
+export function parseMultipart(contentType, body) {
+  const boundary = boundaryOf(contentType);
+  if (boundary === null) return { error: "Content-Type must be multipart/form-data with a boundary" };
+  if (typeof body !== "string") return { error: "Request body is empty" };
+  const delimiter = `--${boundary}`;
+  let searchFrom = 0;
+  // Next delimiter at or after `from` whose line break before it starts at or after `floor`. Forward-only.
+  const nextDelimiter = (floor) => {
+    for (;;) {
+      const d = body.indexOf(delimiter, searchFrom);
+      if (d < 0) return -1;
+      searchFrom = d + 1;
+      if (d - 1 < floor || body[d - 1] !== "\n") continue;
+      const after = d + delimiter.length;
+      if (body.startsWith("--", after) || body.startsWith("\r\n", after) || body.startsWith("\n", after)) return d;
+    }
+  };
+  let cursor = body.indexOf(delimiter);
+  if (cursor < 0) return { error: "Multipart body has no boundary delimiter" };
+  searchFrom = cursor + 1;
+  const parts = [];
+  for (;;) {
+    cursor += delimiter.length;
+    if (body.startsWith("--", cursor)) break;
+    if (body.startsWith("\r\n", cursor)) cursor += 2;
+    else if (body.startsWith("\n", cursor)) cursor += 1;
+    else return { error: "Malformed multipart delimiter line" };
+    if (parts.length >= MAX_PARTS) return { error: `Multipart body has more than ${MAX_PARTS} parts` };
+    const windowEnd = Math.min(body.length, cursor + MAX_HEADER_BLOCK + 4);
+    const window = body.slice(cursor, windowEnd);
+    let headerEnd = window.indexOf("\r\n\r\n");
+    let sepLength = 4;
+    const lfEnd = window.indexOf("\n\n");
+    if (headerEnd < 0 || (lfEnd >= 0 && lfEnd < headerEnd)) {
+      headerEnd = lfEnd;
+      sepLength = 2;
+    }
+    if (headerEnd < 0) return { error: windowEnd === body.length ? "Multipart part headers are not terminated" : "Multipart part headers are too long" };
+    if (headerEnd > MAX_HEADER_BLOCK) return { error: "Multipart part headers are too long" };
+    const headerText = window.slice(0, headerEnd);
+    headerEnd += cursor;
+    let disposition = null;
+    let partType = null;
+    let transfer = null;
+    let lastModified = null;
+    for (const rawLine of headerText.split("\n")) {
+      const line = rawLine.endsWith("\r") ? rawLine.slice(0, -1) : rawLine;
+      const colon = line.indexOf(":");
+      if (colon <= 0) continue;
+      const name = line.slice(0, colon).trim().toLowerCase();
+      const value = line.slice(colon + 1).trim();
+      if (name === "content-disposition") disposition = dispositionParams(value);
+      else if (name === "content-type") partType = value.split(";")[0].trim().toLowerCase();
+      else if (name === "content-transfer-encoding") transfer = value.toLowerCase();
+      else if (name === "last-modified") lastModified = value;
+    }
+    if (disposition === null || !disposition.has("name")) return { error: "Multipart part is missing a Content-Disposition form-data name" };
+    const name = disposition.get("name");
+    if (name.length === 0 || name.length > MAX_NAME) return { error: "Multipart part name is empty or too long" };
+    const contentStart = headerEnd + sepLength;
+    if (searchFrom < contentStart) searchFrom = contentStart;
+    // The floor admits an empty part whose close delimiter shares the blank line ending its headers.
+    const d = nextDelimiter(contentStart - 1);
+    if (d < 0) return { error: "Multipart body is not terminated by the closing boundary" };
+    const contentEnd = d - 2 >= contentStart && body[d - 2] === "\r" ? d - 2 : Math.max(contentStart, d - 1);
+    parts.push({
+      name,
+      filename: disposition.has("filename") ? disposition.get("filename") : null,
+      contentType: partType,
+      transferEncoding: transfer,
+      lastModified,
+      body: body.slice(contentStart, contentEnd),
+    });
+    cursor = d;
+  }
+  return { parts };
+}

package/firedrill/tools/unstructured/lib/pages.mjs

@@ -0,0 +1,40 @@
+// Response size budgeting: every route response must stay under the framework's 1 MiB cap, measured in UTF-8 bytes.
+// The budget is `meta/limits.response_bytes` (default 900 KB). A response that would pass it fails RESPONSE_TOO_LARGE
+// (413) — a page is never shortened silently.
+import { fail } from "./errors.mjs";
+import { limits } from "./ids.mjs";
+import { utf8Length } from "./util.mjs";
+
+/** UTF-8 size of the JSON encoding of a value. */
+export const jsonBytes = (value) => utf8Length(JSON.stringify(value));
+
+/** The response byte budget of this world (`meta/limits.response_bytes`). */
+export const responseBudget = (context) => limits(context).response_bytes;
+
+/**
+ * Fails RESPONSE_TOO_LARGE when `bytes` passes the budget. `what` names the payload ("Partition output"); `hint` tells the
+ * caller how to shrink it.
+ */
+export function assertWithinBudget(context, bytes, what, hint) {
+  const budget = responseBudget(context);
+  if (bytes > budget) fail(context, "RESPONSE_TOO_LARGE", `${what} of ${bytes} bytes exceeds the ${budget} byte response limit${hint ? `; ${hint}` : ""}`);
+}
+
+/** Fails RESPONSE_TOO_LARGE when the JSON encoding of `value` passes the budget; otherwise returns `value`. */
+export function withinBudget(context, value, what, hint) {
+  assertWithinBudget(context, jsonBytes(value), what, hint);
+  return value;
+}
+
+/**
+ * The page-numbered slice of `rows` (already sorted): rows [start, start + count). Every requested row is returned or
+ * the call fails RESPONSE_TOO_LARGE telling the caller to lower `page_size` — a page never stops early at the budget,
+ * because the next page would start at (page - 1) * page_size and the rows in between would be unreachable.
+ */
+export function pageOf(context, rows, start, count) {
+  const items = rows.slice(start, start + count);
+  let bytes = 2;
+  for (const item of items) bytes += jsonBytes(item) + 1;
+  assertWithinBudget(context, bytes, `A page of ${items.length} rows`, items.length > 1 ? "lower page_size" : "the row itself exceeds the budget");
+  return items;
+}