@firedrill-tools/unstructured 0.1.4

package/test/conformance.mjs

@@ -0,0 +1,37 @@
+// Unstructured Tool conformance target: a scripted Tool test, not a model-driven agent. Node built-ins only.
+// Reads the drill task from stdin, picks the flow named in the instruction and runs it against the provider-shaped
+// routes (FIREDRILL_HTTP_URL with the unstructured-api-key header set to FIREDRILL_HTTP_TOKEN).
+import { archivistFlow, deniedFlow, freshFlow, revokedFlow } from "./flows/access.mjs";
+import { chunkingFlow } from "./flows/chunking.mjs";
+import { connectorsFlow } from "./flows/connectors.mjs";
+import { errorsFlow } from "./flows/errors.mjs";
+import { overloadedFlow, rateLimitedFlow, runLostFlow, smallResponsesFlow, tightLimitsFlow } from "./flows/faults.mjs";
+import { partitionFlow } from "./flows/partition.mjs";
+import { workflowsFlow } from "./flows/workflows.mjs";
+
+let task = "";
+for await (const chunk of process.stdin) task += chunk;
+const invocation = JSON.parse(task);
+const instruction = String(invocation.instruction ?? invocation.task?.instruction ?? "");
+
+const flows = {
+  partition: partitionFlow,
+  chunking: chunkingFlow,
+  errors: errorsFlow,
+  connectors: connectorsFlow,
+  workflows: workflowsFlow,
+  fresh: freshFlow,
+  archivist: archivistFlow,
+  revoked: revokedFlow,
+  denied: deniedFlow,
+  "rate-limited": rateLimitedFlow,
+  overloaded: overloadedFlow,
+  "run-lost": runLostFlow,
+  "tight-limits": tightLimitsFlow,
+  "small-responses": smallResponsesFlow,
+};
+
+const selected = /Run the ([a-z-]+) flow/.exec(instruction)?.[1];
+if (selected === undefined || !Object.hasOwn(flows, selected)) throw new Error(`Unknown drill instruction: ${instruction}`);
+await flows[selected]();
+process.stdout.write(JSON.stringify({ completed: true, flow: selected }));

package/test/flows/access.mjs

@@ -0,0 +1,54 @@
+// Access flow: fresh-install fallback, workspace scoping, an invalid key on every operation, framework denial.
+import assert from "node:assert/strict";
+import { ALL_OPERATIONS, ID, api, detail, get, op, partition } from "../lib.mjs";
+
+/** Arguments that would succeed for a valid key, used to prove UNAUTHORIZED fires before anything else. */
+const ARGS = {
+  "general.partition": { files: [{ filename: "a.txt", content: "Hello" }] },
+  "sources.list": {}, "sources.create": { name: "x", type: "s3", config: { remote_url: "s3://x/" } }, "sources.get": { source_id: ID.S1 }, "sources.update": { source_id: ID.S1, name: "x" },
+  "sources.delete": { source_id: ID.S1 }, "sources.check_connection": { source_id: ID.S1 }, "sources.get_connection_check": { source_id: ID.S1 },
+  "destinations.list": {}, "destinations.create": { name: "x", type: "s3", config: { remote_url: "s3://x/" } }, "destinations.get": { destination_id: ID.D1 }, "destinations.update": { destination_id: ID.D1, name: "x" }, "destinations.delete": { destination_id: ID.D1 },
+  "workflows.list": {}, "workflows.create": { name: "x", workflow_type: "auto" }, "workflows.get": { workflow_id: ID.W1 }, "workflows.update": { workflow_id: ID.W1, name: "x" }, "workflows.delete": { workflow_id: ID.W1 }, "workflows.run": { workflow_id: ID.W1 },
+  "jobs.list": {}, "jobs.get": { job_id: ID.J1 }, "jobs.cancel": { job_id: ID.J5 }, "jobs.get_details": { job_id: ID.J1 }, "jobs.get_failed_files": { job_id: ID.J1 }, "jobs.download_output": { job_id: ID.J2, file_id: `${ID.S1}:handbook-md`, node_id: ID.W1_EMBED_NODE },
+};
+
+/** A fresh `firedrill tool add` actor (no attributes) sees the seeded Northgate account. */
+export async function freshFlow() {
+  assert.equal((await get("/api/v1/sources")).json.length, 3);
+  assert.equal((await get("/api/v1/destinations")).json.length, 2);
+  assert.equal((await get("/api/v1/workflows")).json.length, 3);
+  assert.equal((await get("/api/v1/jobs")).json.length, 6);
+  const elements = (await partition([{ filename: "hello.txt", content: "Fresh Install\n\nThe fresh actor can partition documents without any identity attribute." }])).json;
+  assert.deepEqual(elements.map((e) => e.type), ["Title", "NarrativeText"]);
+  assert.equal((await get(`/api/v1/jobs/${ID.J1}`)).json.status, "COMPLETED");
+}
+
+/** The archive workspace key sees only S4 / D3 / W4; Northgate ids do not exist for it. */
+export async function archivistFlow() {
+  assert.deepEqual((await get("/api/v1/sources")).json.map((s) => s.id), [ID.S4]);
+  assert.deepEqual((await get("/api/v1/destinations")).json.map((d) => d.id), [ID.D3]);
+  assert.deepEqual((await get("/api/v1/workflows")).json.map((w) => w.id), [ID.W4]);
+  assert.deepEqual((await get("/api/v1/jobs")).json, []);
+  await detail("GET", `/api/v1/sources/${ID.S1}`, 404, "Source connector not found");
+  await detail("GET", `/api/v1/workflows/${ID.W1}`, 404, "Workflow not found");
+  await detail("GET", `/api/v1/jobs/${ID.J1}`, 404, "Job not found");
+  await detail("POST", "/api/v1/workflows", 422, "Source connector not found", { body: { name: "cross", workflow_type: "auto", source_id: ID.S1 } });
+  await detail("POST", `/api/v1/workflows/${ID.W4}/run`, 422, "holds no files");
+  assert.equal((await get("/api/v1/jobs")).json.length, 0);
+}
+
+/** A key whose workspace does not exist: 401 API key is invalid on every operation, wire and canonical. */
+export async function revokedFlow() {
+  await detail("POST", "/general/v0/general", 401, "API key is invalid", { raw: "--b\r\nContent-Disposition: form-data; name=\"files\"; filename=\"a.txt\"\r\n\r\nHi\r\n--b--\r\n", contentType: "multipart/form-data; boundary=b" });
+  await detail("GET", "/api/v1/sources", 401, "API key is invalid");
+  await detail("GET", `/api/v1/jobs/${ID.J1}`, 401, "API key is invalid");
+  for (const operationId of ALL_OPERATIONS) await op(operationId, ARGS[operationId], "UNAUTHORIZED");
+}
+
+/** An actor without grants: the framework denies before the Tool runs; the codec renders it in the detail envelope. */
+export async function deniedFlow() {
+  await detail("GET", "/api/v1/sources", 403, "not granted");
+  await detail("POST", "/general/v0/general", 403, "not granted", { raw: "--b\r\nContent-Disposition: form-data; name=\"files\"; filename=\"a.txt\"\r\n\r\nHi\r\n--b--\r\n", contentType: "multipart/form-data; boundary=b" });
+  await op("sources.list", {}, "denied");
+  await api("GET", "/api/v1/sources", { status: 401, token: "wrong-key" });
+}

package/test/flows/chunking.mjs

@@ -0,0 +1,95 @@
+// Chunking flow: basic, by_title, by_page, overlap, orig_elements (real gzip), TableChunk.
+import assert from "node:assert/strict";
+import { gunzipSync } from "node:zlib";
+import { fixtures, partition } from "../lib.mjs";
+
+const decodeOrig = (element) => {
+  const bytes = Buffer.from(element.metadata.orig_elements, "base64");
+  assert.equal(bytes[0], 0x1f);
+  assert.equal(bytes[1], 0x8b);
+  return JSON.parse(gunzipSync(bytes).toString("utf8"));
+};
+
+export async function chunkingFlow() {
+  const fx = fixtures();
+  const file = (name) => ({ filename: name, content: fx.get(name).content });
+
+  // 1. basic, max 500: every chunk fits, the long paragraph is split into continuation chunks, orig_elements decode.
+  const basic = (await partition([file("glossary.md")], { chunking_strategy: "basic", max_characters: "500" })).json;
+  assert.ok(basic.length >= 6, `chunks ${basic.length}`);
+  assert.ok(basic.every((c) => c.type === "CompositeElement" && c.text.length <= 500));
+  assert.ok(basic.some((c) => c.metadata.is_continuation === true), "continuation chunks");
+  for (const chunk of basic) {
+    const originals = decodeOrig(chunk);
+    assert.ok(Array.isArray(originals) && originals.length > 0);
+    assert.ok(originals.every((e) => typeof e.type === "string" && typeof e.text === "string" && e.metadata.filename === "glossary.md"));
+    if (chunk.metadata.is_continuation !== true && originals.length > 1) assert.equal(chunk.text, originals.map((e) => e.text).join("\n\n"));
+  }
+  assert.match(basic[1].text, /^A glossary entry explains/);
+
+  // 2. overlap=50: each continuation chunk starts with the last 50 characters of the previous chunk.
+  const overlapped = (await partition([file("glossary.md")], { chunking_strategy: "basic", max_characters: "500", overlap: "50" })).json;
+  let continuations = 0;
+  for (let i = 1; i < overlapped.length; i += 1) {
+    if (overlapped[i].metadata.is_continuation !== true) continue;
+    continuations += 1;
+    assert.ok(overlapped[i].text.startsWith(overlapped[i - 1].text.slice(-50)), `overlap prefix at ${i}`);
+    assert.ok(overlapped[i].text.length <= 500);
+  }
+  assert.ok(continuations >= 4, `continuations ${continuations}`);
+
+  // 3. overlap_all: every chunk after the first carries the prefix.
+  const all = (await partition([file("glossary.md")], { chunking_strategy: "basic", max_characters: "500", overlap: "40", overlap_all: "true" })).json;
+  for (let i = 1; i < all.length; i += 1) assert.ok(all[i].text.startsWith(all[i - 1].text.slice(-40)), `overlap_all prefix at ${i}`);
+
+  // 4. by_title with combine_under_n_chars: chunks start at titles; small sections are merged.
+  const byTitle = (await partition([file("handbook.md")], { chunking_strategy: "by_title", max_characters: "800", combine_under_n_chars: "200", include_orig_elements: "false" })).json;
+  assert.ok(byTitle[0].text.startsWith("Northgate Research Employee Handbook\n\nWelcome to Northgate Research."));
+  assert.ok(byTitle.some((c) => c.text.startsWith("Working Hours\n\n")));
+  assert.ok(byTitle.some((c) => c.type === "Table"), "table kept whole");
+  assert.ok(byTitle.every((c) => c.metadata.orig_elements === undefined), "orig_elements omitted");
+  assert.ok(byTitle.every((c) => c.text.length <= 800));
+
+  // 5. by_page: one chunk per page of the refund policy.
+  const byPage = (await partition([file("refund-policy.txt")], { chunking_strategy: "by_page", max_characters: "1500", new_after_n_chars: "1500" })).json;
+  assert.equal(byPage.length, 2);
+  assert.deepEqual(byPage.map((c) => c.metadata.page_number), [1, 2]);
+  assert.ok(byPage[1].text.startsWith("Exceptions"));
+  assert.ok(byPage.every((c) => c.type === "CompositeElement"));
+
+  // 6. include_orig_elements=false on basic; 7. new_after_n_chars larger than max_characters is lowered.
+  const noOrig = (await partition([file("2026-09-08-site-visit.txt")], { chunking_strategy: "basic", include_orig_elements: "false" })).json;
+  assert.ok(noOrig.every((c) => c.metadata.orig_elements === undefined));
+  const lowered = (await partition([file("2026-09-08-site-visit.txt")], { chunking_strategy: "basic", max_characters: "300", new_after_n_chars: "5000" })).json;
+  assert.ok(lowered.length >= 3 && lowered.every((c) => c.text.length <= 300));
+
+  // 8. A table longer than max_characters becomes TableChunk pieces split by rows.
+  const chunks = (await partition([file("vendors.csv")], { chunking_strategy: "basic", max_characters: "300" })).json;
+  assert.ok(chunks.length >= 2);
+  assert.ok(chunks.every((c) => c.type === "TableChunk" && c.text.length <= 300 && c.metadata.text_as_html.startsWith("<table><tr>")));
+  assert.equal(chunks[0].metadata.is_continuation, undefined);
+  assert.ok(chunks.slice(1).every((c) => c.metadata.is_continuation === true));
+  assert.equal(chunks.map((c) => c.text.split("\n").length).reduce((a, b) => a + b, 0), 13);
+
+  // 9. The largest chunk size (100,000, the element text bound) over two 90,000-character paragraphs: one chunk each,
+  // never a schema violation; 10. many tiny blocks accumulate into few chunks in linear time.
+  const wide = { filename: "wide.txt", content: `${"b".repeat(90000)}\n\n${"c".repeat(90000)}` };
+  const widest = (await partition([wide], { chunking_strategy: "basic", max_characters: "100000", new_after_n_chars: "100000", include_orig_elements: "false" })).json;
+  assert.equal(widest.length, 2);
+  assert.ok(widest.every((c) => c.type === "CompositeElement" && c.text.length === 90000));
+  const many = { filename: "many.txt", content: "a\n\n".repeat(20000) };
+  const started = Date.now();
+  const few = (await partition([many], { chunking_strategy: "basic", max_characters: "100000", new_after_n_chars: "100000", include_orig_elements: "false" })).json;
+  assert.ok(Date.now() - started < 5000, `chunking 20,000 blocks took ${Date.now() - started}ms`);
+  assert.equal(few.length, 1);
+  assert.equal(few[0].text.length, 20000 * 3 - 2);
+
+  // 11. Continuation pieces of one chunk group share the group's metadata: every piece of a 40-link paragraph split at
+  // max_characters=100 carries all 40 merged link_urls/link_texts (built once per group, never rebuilt per piece).
+  const linked = { filename: "linked.html", content: `<p>${'<a href="https://example.test/u">link text</a> '.repeat(40)}</p>` };
+  const shared = (await partition([linked], { chunking_strategy: "basic", max_characters: "100", include_orig_elements: "false" })).json;
+  assert.equal(shared.length, 4);
+  assert.ok(shared.every((c) => c.metadata.link_urls.length === 40 && c.metadata.link_texts.length === 40 && c.metadata.link_urls[39] === "https://example.test/u"));
+  assert.equal(shared[0].metadata.is_continuation, undefined);
+  assert.ok(shared.slice(1).every((c) => c.metadata.is_continuation === true));
+}

package/test/flows/connectors.mjs

@@ -0,0 +1,76 @@
+// Connector flow: sources and destinations CRUD, masking, connection checks, in-use protection.
+import assert from "node:assert/strict";
+import { ID, api, del, detail, get, post, put } from "../lib.mjs";
+
+export async function connectorsFlow() {
+  const sources = (await get("/api/v1/sources")).json;
+  assert.deepEqual(sources.map((s) => s.name), ["Policy archive", "Field notes", "Legacy scans"]);
+  assert.equal((await get("/api/v1/sources?source_type=s3")).json.length, 1);
+  await detail("GET", "/api/v1/sources?source_type=ftp", 422, "query.source_type");
+  await api("GET", "/api/v1/sources/", { status: 404 }); // trailing slash: framework 404 (route templates cannot end with /)
+  const s1 = (await get(`/api/v1/sources/${ID.S1}`)).json;
+  assert.equal(s1.config.secret_access_key, "********");
+  assert.equal(s1.config.remote_url, "s3://northgate-policies/");
+  assert.equal(s1.key, "policy-archive");
+  await detail("GET", `/api/v1/sources/${ID.S4}`, 404, "Source connector not found");
+  await detail("GET", `/api/v1/sources/${"x".repeat(600)}`, 400, "source_id"); // schema-invalid argument: framework status 400, detail array from the codec
+
+  const missing = await detail("POST", "/api/v1/sources", 422, "drive_id", { body: { name: "Drive", type: "google_drive", config: {} } });
+  assert.deepEqual(missing.json.detail[0].loc, ["body", "config", "drive_id"]);
+  await detail("POST", "/api/v1/sources", 422, "already exists", { body: { name: "Dup", type: "s3", config: { remote_url: "s3://x/" }, key: "policy-archive" } });
+  await detail("POST", "/api/v1/sources", 422, "Extra inputs are not permitted", { body: { name: "Dup", type: "s3", config: { remote_url: "s3://x/" }, colour: "red" } });
+  await detail("POST", "/api/v1/sources", 422, "at most 200 characters", { body: { name: "n".repeat(5000), type: "s3", config: { remote_url: "s3://x/" } } });
+  await detail("POST", "/api/v1/sources", 422, "Invalid key", { body: { name: "Bad", type: "s3", config: { remote_url: "s3://x/", constructor: 1 } } });
+  await detail("POST", "/api/v1/sources", 422, "Input should be a valid dictionary or object", { raw: "[1,2]" });
+  await api("POST", "/api/v1/sources", { raw: "", status: 400 }); // empty JSON body: framework HTTP_BODY_INVALID before any codec runs
+  await api("POST", "/api/v1/sources", { raw: "{not json", status: 400 });
+
+  const created = (await post("/api/v1/sources", { name: "Survey bucket", type: "s3", config: { remote_url: "s3://northgate-surveys/", access_key_id: "AKIAEXAMPLE0000009", secret_access_key: "not-a-real-secret" }, key: "survey-bucket" })).json;
+  assert.equal(created.config.secret_access_key, "********");
+  assert.equal(created.updated_at, null);
+  assert.equal((await get(`/api/v1/sources/${created.id}`)).json.config.secret_access_key, "********");
+  const renamed = (await put(`/api/v1/sources/${created.id}`, { name: "Survey bucket (EU)" })).json;
+  assert.equal(renamed.name, "Survey bucket (EU)");
+  assert.equal(renamed.updated_at, "2026-09-16T09:00:00.000Z");
+  await detail("PUT", `/api/v1/sources/${created.id}`, 422, "cannot be changed", { body: { type: "gcs" } });
+  await detail("PUT", `/api/v1/sources/${created.id}`, 422, "remote_url", { body: { config: { access_key_id: "x" } } });
+  await detail("PUT", `/api/v1/sources/${ID.NONE}`, 404, "Source connector not found", { body: { name: "x" } });
+  const masked = (await put(`/api/v1/sources/${created.id}`, { config: { remote_url: "s3://northgate-surveys-eu/", secret_access_key: "********" } })).json;
+  assert.equal(masked.config.remote_url, "s3://northgate-surveys-eu/");
+  assert.equal(masked.config.secret_access_key, "********");
+  assert.equal(masked.config.access_key_id, undefined, "config is replaced whole");
+
+  // Connection checks: computed from the required keys of the type.
+  const check = (await post(`/api/v1/sources/${ID.S1}/connection-check`, undefined)).json;
+  assert.deepEqual(check, { status: "SUCCESS", reason: null, created_at: "2026-09-16T09:00:00.000Z" });
+  assert.equal((await get(`/api/v1/sources/${ID.S3}/connection-check`)).json.status, "FAILURE");
+  await detail("GET", `/api/v1/sources/${created.id}/connection-check`, 404, "No connection check found");
+  const broken = (await post("/api/v1/sources", { name: "No drive", type: "google_drive", config: { drive_id: "1", service_account_key: "x" } })).json;
+  await put(`/api/v1/sources/${broken.id}`, { config: { drive_id: "", service_account_key: "x" } }, { status: 422 });
+  await detail("POST", `/api/v1/sources/${ID.NONE}/connection-check`, 404, "Source connector not found");
+  await detail("GET", `/api/v1/sources/${ID.NONE}/connection-check`, 404, "Source connector not found");
+
+  // Destinations mirror the same rules.
+  const destinations = (await get("/api/v1/destinations")).json;
+  assert.deepEqual(destinations.map((d) => d.name), ["Parsed output bucket", "Policy index"]);
+  assert.equal((await get("/api/v1/destinations?destination_type=pinecone")).json[0].config.api_key, "********");
+  await detail("GET", "/api/v1/destinations?destination_type=s3x", 422, "destination_type");
+  await detail("POST", "/api/v1/destinations", 422, "cluster_url", { body: { name: "Vectors", type: "weaviate_cloud", config: { collection: "docs" } } });
+  const weaviate = (await post("/api/v1/destinations", { name: "Vectors", type: "weaviate_cloud", config: { cluster_url: "https://vectors.example.test", collection: "docs", api_key: "x" } })).json;
+  assert.equal((await get(`/api/v1/destinations/${weaviate.id}`)).json.config.api_key, "********");
+  assert.equal((await put(`/api/v1/destinations/${weaviate.id}`, { name: "Vectors (prod)" })).json.name, "Vectors (prod)");
+  await detail("PUT", `/api/v1/destinations/${weaviate.id}`, 422, "cannot be changed", { body: { type: "s3" } });
+  await detail("PUT", `/api/v1/destinations/${ID.NONE}`, 404, "Destination connector not found", { body: { name: "x" } });
+  assert.deepEqual((await del(`/api/v1/destinations/${weaviate.id}`)).json, {});
+  await detail("GET", `/api/v1/destinations/${weaviate.id}`, 404, "Destination connector not found");
+  await detail("DELETE", `/api/v1/destinations/${weaviate.id}`, 404, "Destination connector not found");
+  await detail("GET", `/api/v1/destinations/${ID.D3}`, 404, "Destination connector not found");
+
+  // Connectors used by a running job cannot be deleted (J5 runs W2 = S2 -> D1; J6 runs W1 = S1 -> D2).
+  await detail("DELETE", `/api/v1/sources/${ID.S1}`, 422, "in use by a running job");
+  await detail("DELETE", `/api/v1/destinations/${ID.D1}`, 422, "in use by a running job");
+  await detail("DELETE", `/api/v1/sources/${ID.NONE}`, 404, "Source connector not found");
+  assert.deepEqual((await del(`/api/v1/sources/${broken.id}`)).json, {});
+  await detail("GET", `/api/v1/sources/${broken.id}`, 404);
+  assert.equal((await get("/api/v1/sources")).json.length, 4);
+}

package/test/flows/errors.mjs

@@ -0,0 +1,115 @@
+// Partition error flow: provider error envelopes for every rejected request; nothing is logged for failures.
+import assert from "node:assert/strict";
+import { api, detail, multipart, partition } from "../lib.mjs";
+
+export async function errorsFlow() {
+  const one = [{ filename: "note.txt", content: "Just a short note." }];
+  const bad = (fields, status, fragment, files = one) => partition(files, fields, { status }).then((r) => check(r, status, fragment));
+  const check = (r, status, fragment) => {
+    const d = r.json?.detail;
+    assert.ok(d !== undefined, `expected detail envelope, got ${r.text.slice(0, 200)}`);
+    const text = typeof d === "string" ? d : JSON.stringify(d);
+    assert.ok(text.includes(fragment), `expected ${JSON.stringify(fragment)} in ${text.slice(0, 300)}`);
+    return d;
+  };
+
+  const missing = await bad({ strategy: "fast" }, 422, "Field required", []);
+  assert.deepEqual(missing[0].loc, ["body", "files"]);
+  assert.equal(missing[0].type, "missing");
+  await bad({ strategy: "layout" }, 400, "Invalid strategy: layout. Must be one of ['fast', 'hi_res', 'auto', 'ocr_only', 'vlm']");
+  await bad({ chunking_strategy: "by_similarity" }, 400, "by_similarity is not supported");
+  await bad({ chunking_strategy: "by_word" }, 400, "Invalid chunking strategy: by_word");
+  await bad({ output_format: "text/xml" }, 400, "Invalid output format: text/xml");
+  await bad({ hi_res_model_name: "chipper" }, 400, "Unknown model type: chipper");
+  const vlm = await bad({ strategy: "vlm" }, 422, "vlm_model_provider");
+  assert.ok(vlm.some((i) => i.loc.join(".") === "body.vlm_model_provider" && i.type === "missing"));
+  const int = await bad({ max_characters: "abc", chunking_strategy: "basic" }, 422, "valid integer");
+  assert.equal(int[0].type, "int_parsing");
+  assert.deepEqual(int[0].loc, ["body", "max_characters"]);
+  await bad({ chunking_strategy: "basic", max_characters: "500", overlap: "600" }, 422, "less than max_characters");
+  const tooBig = await bad({ chunking_strategy: "basic", max_characters: "100001" }, 422, "less than or equal to 100000");
+  assert.equal(tooBig[0].type, "less_than_equal");
+  assert.deepEqual(tooBig[0].loc, ["body", "max_characters"]);
+  await bad({ include_page_breaks: "maybe" }, 422, "valid boolean");
+  await bad({ encoding: "latin-1" }, 400, "Unsupported encoding: latin-1");
+  await bad({}, 400, "application/pdf not currently supported", [{ filename: "scan.pdf", content: "%PDF-1.7 pretend" }]);
+  await bad({ content_type: "image/png" }, 400, "image/png not currently supported");
+  await bad({}, 400, "None not currently supported", [{ filename: "noext", content: "x" }]);
+  await bad({}, 422, "File is not a valid csv", [{ filename: "broken.csv", content: 'a,b\n1,"unterminated' }]);
+  await bad({}, 422, "File is not a valid xml", [{ filename: "broken.xml", content: "<a><b>x</a>" }]);
+  await bad({}, 400, "Json schema does not match the Unstructured schema", [{ filename: "notes.json", content: '{"a":1}' }]);
+  await bad({}, 422, "too many text blocks", [{ filename: "blocks.txt", content: "x\n\n".repeat(100001) }]);
+  await bad({ chunking_strategy: "basic", max_characters: "100" }, 413, "exceeds the 921600 byte response limit", [{ filename: "big.txt", content: Array.from({ length: 3000 }, (_, i) => `Entry ${i} posted.`).join("\n\n") }]);
+
+  // Transport-level problems the codec reports itself (never a framework runtime message).
+  await detail("POST", "/general/v0/general", 400, "boundary", { raw: "not multipart", contentType: "multipart/form-data" });
+  await detail("POST", "/general/v0/general", 400, "Content-Disposition", { raw: "--b\r\nContent-Type: text/plain\r\n\r\nx\r\n--b--\r\n", contentType: "multipart/form-data; boundary=b" });
+  await detail("POST", "/general/v0/general", 422, "Expected multipart/form-data", { raw: "files=x", contentType: "application/x-www-form-urlencoded" });
+  await detail("POST", "/general/v0/general", 422, "Expected multipart/form-data", { body: { files: [] } });
+  const b64bad = multipart([{ name: "files", filename: "a.txt", value: "!!!notbase64", headers: { "Content-Transfer-Encoding": "base64" } }]);
+  await detail("POST", "/general/v0/general", 422, "base64", { raw: b64bad.raw, contentType: b64bad.contentType });
+
+  // A valid base64 part is decoded and partitioned (the one success of this flow).
+  const b64 = multipart([{ name: "files", filename: "a.txt", value: Buffer.from("Encoded Title\n\nThis sentence arrived as base64 text and was decoded.").toString("base64"), headers: { "Content-Transfer-Encoding": "base64" } }]);
+  const ok = await api("POST", "/general/v0/general", { raw: b64.raw, contentType: b64.contentType });
+  assert.deepEqual(ok.json.map((e) => e.type), ["Title", "NarrativeText"]);
+
+  // Wrong key: the framework's own 401 (before any codec runs).
+  await api("POST", "/general/v0/general", { raw: b64.raw, contentType: b64.contentType, status: 401, token: "not-the-key" });
+
+  // Partitioner bounds fail with the declared 422, never truncate, and answer in linear time: 300 nested elements,
+  // a 12,000-row table, a 201-cell row and 300 nested XML elements.
+  const bounds = [
+    ["<div>a".repeat(300), "deep.html", "elements nested deeper than 256 levels"],
+    ["<table>" + "<tr><td>a</td><td>b</td></tr>".repeat(12000) + "</table>", "rows.html", "File has a table with more than 10000 rows"],
+    ["<table><tr>" + "<td>a</td>".repeat(201) + "</tr></table>", "cells.html", "File has a table row with more than 200 cells"],
+    ["<a>".repeat(300) + "t" + "</a>".repeat(300), "deep.xml", "File is not a valid xml: elements nested deeper than 256 levels"],
+    // Markdown pipe tables share the bound at any width: a 5,000-column table (separator row over 25,000 characters)
+    // and a one-column header over a 300-cell separator both fail instead of being read as text.
+    [`${"| a ".repeat(5000)}|\n${"| --- ".repeat(5000)}|\n${"| 1 ".repeat(5000)}|\n`, "wide.md", "File has a table row with more than 200 cells"],
+    [`| a |\n${"| --- ".repeat(300)}|\n| 1 |\n`, "narrow-header.md", "File has a table row with more than 200 cells"],
+  ];
+  for (const [content, filename, fragment] of bounds) {
+    const started = Date.now();
+    await bad({}, 422, fragment, [{ filename, content }]);
+    assert.ok(Date.now() - started < 3000, `bound ${filename} took ${Date.now() - started}ms`);
+  }
+
+  // Chunk output is sized arithmetically before any piece is built: overlap_all with overlap = max_characters - 1 would
+  // multiply two 20,000-character paragraphs into 400 MB of chunk text; the request answers 413 in milliseconds instead.
+  const paragraphs = `${"a".repeat(20000)}\n\n${"b".repeat(20000)}`;
+  for (const chunking_strategy of ["basic", "by_title"]) {
+    const started = Date.now();
+    await bad({ chunking_strategy, max_characters: "20000", overlap: "19999", overlap_all: "true" }, 413, "exceeds the 921600 byte response limit", [{ filename: "wide.txt", content: paragraphs }]);
+    assert.ok(Date.now() - started < 3000, `overlap_all ${chunking_strategy} took ${Date.now() - started}ms`);
+  }
+  await bad({ chunking_strategy: "basic", max_characters: "1" }, 413, "exceeds the 921600 byte response limit", [{ filename: "one.txt", content: "a".repeat(100000) }]);
+
+  // Every piece of a chunk group carries a copy of the group's metadata, so large merged link/emphasis arrays, e-mail recipient
+  // lists or orig_elements multiply by the piece count: 5,000 links at the default max_characters (8.9 MB of pieces), 20,000
+  // `<b>` marks at max_characters=19 (169 MB), a 100,000-character paragraph at 19 with orig_elements (700 MB), an e-mail with
+  // 50,000 recipients (2.4 GB) and a 5,000-row table at 19 with orig_elements (2.1 GB) each answer 413 before any piece exists.
+  const multiplied = [
+    [{ chunking_strategy: "basic" }, { filename: "links.html", content: `<p>${'<a href="https://example.test/u">x</a> '.repeat(5000)}</p>` }],
+    [{ chunking_strategy: "basic", max_characters: "19", include_orig_elements: "false" }, { filename: "marks.html", content: `<p>${"<b>x</b>".repeat(20000)}</p>` }],
+    [{ chunking_strategy: "basic", max_characters: "19" }, { filename: "orig.html", content: `<p>${"a".repeat(100000)}</p>` }],
+    [{ chunking_strategy: "basic", max_characters: "19", include_orig_elements: "false" }, { filename: "wide.eml", content: `From: a@example.test\nTo: ${"a@example.test,".repeat(50000)}\nSubject: s\n\n${"x".repeat(100000)}` }],
+    [{ chunking_strategy: "by_title", max_characters: "19" }, { filename: "table.html", content: `<table>${"<tr><td>abcdefghijklmnopqrst</td></tr>".repeat(5000)}</table>` }],
+  ];
+  for (const [fields, file] of multiplied) {
+    const started = Date.now();
+    await bad(fields, 413, "exceeds the 921600 byte response limit", [file]);
+    assert.ok(Date.now() - started < 3000, `${file.filename} took ${Date.now() - started}ms`);
+  }
+
+  // JSON metadata that cannot be returned as given (more than 100 keys, a string over 10,000 characters, an array over 200
+  // items, a nested object, a mistyped known key) fails the upload with the schema message; nothing is dropped on a 200.
+  const many = {};
+  for (let i = 0; i < 101; i += 1) many[`k${i}`] = i;
+  const metadataShapes = [many, { big: "x".repeat(10001) }, { arr: Array(201).fill(1) }, { nested: { a: 1 } }, { page_number: -1 }, { languages: Array(51).fill("x") }];
+  // The languages parameter is bounded (20 entries of at most 20 characters): over the bound is a 422, never a shortened list.
+  await bad({ languages: Array.from({ length: 25 }, (_, i) => `l${i}`) }, 422, "at most 20 items");
+  await bad({ languages: ["abcdefghijklmnopqrstuvwxyz0123"] }, 422, "at most 20 characters");
+  await bad({ languages: "eng,deu,fra,spa,ita,por,nld,swe,nor,dan,fin,pol,ces,slk,hun,ron,bul,ell,tur,rus,ukr" }, 422, "at most 20 characters");
+  for (const metadata of metadataShapes) await bad({}, 400, "Json schema does not match the Unstructured schema", [{ filename: "meta.json", content: JSON.stringify([{ type: "Title", text: "t", metadata }]) }]);
+}

package/test/flows/faults.mjs

@@ -0,0 +1,73 @@
+// Fault and bound flows: rate limit, overload, committed-but-lost run, lowered state bounds and a lowered response budget.
+import assert from "node:assert/strict";
+import { ID, api, detail, get, partition, post } from "../lib.mjs";
+
+const NOTE = [{ filename: "note.txt", content: "Fault Drill\n\nThis note is uploaded while a fault is active." }];
+
+export async function rateLimitedFlow() {
+  const first = await detail("POST", "/general/v0/general", 429, "Rate limit exceeded", { ...multipartNote() });
+  assert.equal(first.headers.get("retry-after"), "1");
+  await partition(NOTE, {}, { status: 429 });
+  assert.equal((await get("/api/v1/sources")).json.length, 3, "platform routes are unaffected");
+}
+
+export async function overloadedFlow() {
+  for (let i = 0; i < 2; i += 1) await detail("POST", "/general/v0/general", 503, "Server is under heavy load. Please try again later.", { ...multipartNote() });
+  assert.equal((await get("/api/v1/workflows")).json.length, 3);
+}
+
+export async function runLostFlow() {
+  const before = (await get(`/api/v1/jobs?workflow_id=${ID.W2}`)).json.length;
+  await detail("POST", `/api/v1/workflows/${ID.W2}/run`, 500, "Internal Server Error");
+  const after = (await get(`/api/v1/jobs?workflow_id=${ID.W2}`)).json;
+  assert.equal(after.length, before + 1, "the job was committed although the response was lost");
+  assert.equal(after[0].status, "SCHEDULED");
+  await detail("POST", `/api/v1/workflows/${ID.W2}/run`, 500, "Internal Server Error");
+  assert.equal((await get(`/api/v1/jobs?workflow_id=${ID.W2}`)).json.length, before + 2, "a naive retry creates a second job");
+}
+
+export async function tightLimitsFlow() {
+  await detail("GET", "/api/v1/sources", 500, "state exceeds the supported bound of 2 sources rows");
+  await detail("GET", "/api/v1/destinations", 500, "state exceeds the supported bound of 2 destinations rows");
+  await detail("GET", "/api/v1/workflows", 500, "state exceeds the supported bound of 2 workflows rows");
+  await detail("GET", "/api/v1/jobs", 500, "state exceeds the supported bound of 2 jobs rows");
+  await detail("POST", "/api/v1/sources", 500, "supported bound", { body: { name: "n", type: "s3", config: { remote_url: "s3://x/" }, key: "fresh-key" } });
+  await detail("POST", "/api/v1/destinations", 500, "supported bound", { body: { name: "n", type: "s3", config: { remote_url: "s3://x/" }, key: "fresh-key" } });
+  await detail("POST", "/api/v1/workflows", 500, "supported bound", { body: { name: "n", workflow_type: "auto", key: "fresh-key" } });
+  await detail("DELETE", `/api/v1/sources/${ID.S1}`, 500, "supported bound of 2 jobs rows");
+  await detail("DELETE", `/api/v1/destinations/${ID.D1}`, 500, "supported bound of 2 jobs rows");
+  await detail("DELETE", `/api/v1/workflows/${ID.W1}`, 500, "supported bound of 2 jobs rows");
+  await detail("POST", `/api/v1/workflows/${ID.W1}/run`, 500, "supported bound of 2 source-files rows");
+  assert.equal((await get(`/api/v1/sources/${ID.S1}`)).json.name, "Policy archive", "point reads still work");
+  assert.equal((await post("/api/v1/sources", { name: "No key", type: "s3", config: { remote_url: "s3://x/" } })).json.type, "s3", "a create without a key needs no scan");
+  await api("GET", `/api/v1/jobs/${ID.J1}`);
+}
+
+/** response_bytes = 520: every list or page over the budget answers 413 and is never shortened; small lists and point reads work. */
+export async function smallResponsesFlow() {
+  const tooLarge = (path, fragment) => detail("GET", path, 413, fragment);
+  await tooLarge("/api/v1/sources", "The list of 3 source connectors of ");
+  await tooLarge("/api/v1/sources", "exceeds the 520 byte response limit; filter by source_type or delete unused source connectors");
+  await tooLarge("/api/v1/destinations", "The list of 2 destination connectors of ");
+  await tooLarge("/api/v1/workflows", "A page of 3 rows of ");
+  await tooLarge("/api/v1/workflows", "exceeds the 520 byte response limit; lower page_size");
+  await tooLarge("/api/v1/workflows?page_size=2", "A page of 2 rows of ");
+  await tooLarge("/api/v1/workflows?page_size=2&page=2", "A page of 1 rows of ");
+  await tooLarge("/api/v1/workflows?page_size=1", "the row itself exceeds the budget");
+  await tooLarge("/api/v1/jobs", "A page of 6 rows of ");
+  await tooLarge("/api/v1/jobs?page_size=1", "A page of 1 rows of ");
+  assert.equal((await get("/api/v1/sources?source_type=s3")).json.length, 1, "a filtered list under the budget is returned whole");
+  assert.equal((await get("/api/v1/destinations?destination_type=s3")).json.length, 1);
+  assert.equal((await get("/api/v1/jobs?status=FAILED")).json.length, 1, "a page under the budget is returned whole");
+  assert.equal((await get("/api/v1/jobs?status=FAILED&page=2")).json.length, 0, "an empty page is fine");
+  assert.equal((await get("/api/v1/workflows?name=no-such-workflow")).json.length, 0, "an empty workflow page is fine");
+  assert.equal((await get(`/api/v1/workflows/${ID.W1}`)).json.name, "policy-ingest", "point reads are not budgeted");
+  const created = (await post("/api/v1/sources", { name: "Fourth", type: "s3", config: { remote_url: "s3://x/" } })).json;
+  assert.equal(created.type, "s3", "writes still work");
+  await tooLarge("/api/v1/sources", "The list of 4 source connectors of ");
+  assert.equal((await get(`/api/v1/sources/${created.id}`)).json.name, "Fourth", "the created row is reachable by id although the list is over budget");
+}
+
+function multipartNote() {
+  return { raw: `--b\r\nContent-Disposition: form-data; name="files"; filename="note.txt"\r\n\r\n${NOTE[0].content}\r\n--b--\r\n`, contentType: "multipart/form-data; boundary=b" };
+}