@firedrill-tools/unstructured 0.1.4

package/firedrill/tools/unstructured/lib/wire.mjs

@@ -0,0 +1,154 @@
+// Pure HTTP codecs. `decode` folds path, query and JSON body into snake_case arguments; a request that cannot be
+// mapped travels as `__request_error` ("400:…" or "422:…") so the handler answers the FastAPI `{ detail }` envelope.
+// `encode` renders success bodies and every failure as `{ "detail": "…" }` or `{ "detail": [ { loc, msg, type } ] }`.
+import { STATUS } from "./errors.mjs";
+import { clip } from "./util.mjs";
+
+const MAX_DEPTH = 512;
+const MAX_REPEATS = 50;
+const MAX_MEMBERS = 20000;
+const BANNED = new Set(["__proto__", "constructor", "prototype"]);
+export const RESERVED = "__request_error";
+
+/** Iterative shape check (explicit stack): nesting depth, member count and prototype-poisoning keys. */
+export function shapeProblem(value) {
+  const stack = [[value, 1]];
+  let visited = 0;
+  while (stack.length > 0) {
+    const [node, depth] = stack.pop();
+    if (depth > MAX_DEPTH) return "Request body is nested too deeply";
+    if (node === null || typeof node !== "object") continue;
+    visited += 1;
+    if (visited > MAX_MEMBERS) return "Request body has too many members";
+    if (Array.isArray(node)) {
+      if (node.length > MAX_MEMBERS) return "Request body has too many array entries";
+      for (const child of node) if (child !== null && typeof child === "object") stack.push([child, depth + 1]);
+    } else {
+      for (const key of Object.keys(node)) {
+        if (BANNED.has(key)) return `Invalid key "${clip(key, 60)}"`;
+        const child = node[key];
+        if (child !== null && typeof child === "object") stack.push([child, depth + 1]);
+      }
+    }
+  }
+  return null;
+}
+
+export const Q = {
+  str: (value) => value,
+  int: (value) => (typeof value === "string" && /^-?[0-9]{1,15}$/.test(value) ? Number(value) : value),
+};
+
+export function headerOf(request, name) {
+  const values = request.headers[name];
+  return Array.isArray(values) && values.length > 0 ? values[values.length - 1] : null;
+}
+
+/**
+ * Builds a decoder: `path` lists path parameters copied verbatim; `query` maps names to converters; `body` is "json"
+ * (object required; known `fields` copied, unknown keys reported as 422 extra inputs) or "none".
+ */
+export function decoder({ path = [], query = {}, body = "none", fields = [], idem = true }) {
+  const known = new Set(fields);
+  return (request) => {
+    const args = {};
+    for (const name of path) args[name] = typeof request.path[name] === "string" ? request.path[name] : "";
+    const refuse = (message) => ({ arguments: { ...args, [RESERVED]: clip(message, 300) } });
+    for (const [name, convert] of Object.entries(query)) {
+      const values = request.query[name];
+      if (!Array.isArray(values) || values.length === 0) continue;
+      if (values.length > MAX_REPEATS) return refuse(`422:loc=query.${name}: Parameter is repeated more than ${MAX_REPEATS} times`);
+      args[name] = convert(values[values.length - 1]);
+    }
+    if (body === "json") {
+      if (request.body.kind === "json") {
+        const value = request.body.value;
+        if (value === null || typeof value !== "object" || Array.isArray(value)) return refuse("422:loc=body: Input should be a valid dictionary or object");
+        const problem = shapeProblem(value);
+        if (problem !== null) return refuse(`422:loc=body: ${problem}`);
+        const extra = [];
+        for (const key of Object.keys(value)) {
+          if (key === RESERVED || Object.hasOwn(args, key)) continue;
+          if (known.has(key)) args[key] = value[key];
+          else extra.push(key);
+        }
+        if (extra.length > 0) return refuse(`422:${extra.slice(0, 20).map((key) => `loc=body.${clip(key, 100)}: Extra inputs are not permitted`).join(" | ")}`);
+      } else if (request.body.kind === "form" || (request.body.kind === "text" && request.body.value.trim().length > 0)) {
+        return refuse("422:loc=body: Input should be a valid dictionary or object");
+      } else if (request.body.kind === "none" || request.body.kind === "text") {
+        if (request.method === "POST" || request.method === "PUT") {
+          if (fields.length > 0) return refuse("422:loc=body: Field required");
+        }
+      }
+    }
+    const out = { arguments: args };
+    if (idem) {
+      const key = headerOf(request, "idempotency-key");
+      if (typeof key === "string" && key.length > 0 && key.length <= 255) out.idempotencyKey = key;
+    }
+    return out;
+  };
+}
+
+const TYPE_BY_TEXT = [
+  ["Field required", "missing"],
+  ["Input should be a valid integer", "int_parsing"],
+  ["Input should be a valid boolean", "bool_parsing"],
+  ["Input should be a valid dictionary", "dict_type"],
+  ["Input should be a valid list", "list_type"],
+  ["Input should be a valid string", "string_type"],
+  ["Extra inputs are not permitted", "extra_forbidden"],
+  ["Input should be less than max_characters", "value_error"],
+  ["Input should be greater than or equal to", "greater_than_equal"],
+  ["Input should be greater than", "greater_than"],
+  ["Input should be less than or equal to", "less_than_equal"],
+  ["Input should be less than", "less_than"],
+  ["String should have at least", "string_too_short"],
+  ["String should have at most", "string_too_long"],
+  ["String should match pattern", "string_pattern_mismatch"],
+  ["List should have at most", "too_long"],
+  ["Input should be ", "enum"],
+];
+
+/** "loc=body.files: Field required | loc=query.page: …" -> FastAPI detail array. */
+export function detailArray(message) {
+  const out = [];
+  for (const piece of message.split(" | ")) {
+    const match = /^loc=([^:]*): ?(.*)$/s.exec(piece);
+    const loc = match ? match[1].split(".").filter((s) => s.length > 0) : ["body"];
+    const msg = clip(match ? match[2] : piece, 300);
+    let type = "value_error";
+    for (const [prefix, name] of TYPE_BY_TEXT) if (msg.startsWith(prefix)) {
+      type = name;
+      break;
+    }
+    out.push({ loc, msg, type });
+  }
+  return out;
+}
+
+export function encodeError({ invocation, outcome }) {
+  const error = outcome.error ?? {};
+  const refusal = invocation?.arguments?.[RESERVED];
+  const json = (value, headers) => (headers ? { headers, body: { kind: "json", value } } : { body: { kind: "json", value } });
+  if (outcome.status === "invalid") {
+    if (typeof refusal === "string") return json({ detail: detailArray(refusal.replace(/^\d{3}:/, "")) });
+    const issues = Array.isArray(error.issues) ? error.issues.slice(0, 20) : [];
+    const detail = issues.map((issue) => ({
+      loc: ["body", ...(Array.isArray(issue.path) ? issue.path.slice(0, 8).map((p) => clip(String(p), 100)) : [])],
+      msg: clip(String(issue.message ?? "Input is invalid"), 300),
+      type: "value_error",
+    }));
+    return json({ detail: detail.length > 0 ? detail : [{ loc: ["body"], msg: "Input is invalid", type: "value_error" }] });
+  }
+  if (outcome.status === "denied") return json({ detail: "This API key is not granted this operation in the Firedrill world." });
+  if (outcome.status === "unsupported") return json({ detail: "Not Found" });
+  const code = String(error.code ?? "").replace(/^tool\./, "");
+  const message = typeof error.message === "string" && error.message.length > 0 ? error.message : "Internal Server Error";
+  if (code === "VALIDATION_ERROR" && message.startsWith("loc=")) return json({ detail: detailArray(message) });
+  if (code === "RATE_LIMITED") return json({ detail: message }, { "retry-after": "1" });
+  return json({ detail: STATUS.has(code) ? message : "Internal Server Error" });
+}
+
+/** Success -> the canonical value verbatim; failure -> the envelope. */
+export const encoder = (result) => (result.outcome.status === "ok" ? { body: { kind: "json", value: result.outcome.value } } : encodeError(result));

package/firedrill/tools/unstructured/ops/connectors.mjs

@@ -0,0 +1,129 @@
+// Source and destination connectors: list, create, get, update, delete, plus source connection checks.
+import { MASK, connectorView, isSecretKey, normaliseConfig, requiredKeys } from "../lib/connectors.mjs";
+import { Issues, notFound, requestProblem } from "../lib/errors.mjs";
+import { limits, nextSeq, usableId, uuid } from "../lib/ids.mjs";
+import { workspaceOf } from "../lib/identity.mjs";
+import { isRunning, jobPhase, nowIso } from "../lib/jobs-derive.mjs";
+import { scanAll, scanPrefix } from "../lib/store.mjs";
+import { withinBudget } from "../lib/pages.mjs";
+import { mangled, parseEnum, parseString } from "../lib/validate.mjs";
+
+const KEY_RE = /^[a-z0-9](-?[a-z0-9])*$/;
+
+/** The connector row of `namespace` with this id in the caller's workspace, or NOT_FOUND. */
+export function findConnector(context, namespace, id, workspace, label) {
+  if (!usableId(id) || mangled(id)) notFound(context, `${label} not found`);
+  const row = context.state.get(namespace, id);
+  if (row === null || row.workspace_id !== workspace) notFound(context, `${label} not found`);
+  return row;
+}
+
+/** Validates `config` for `type`; masked secrets keep the stored value. Adds issues; returns the config or null. */
+function validateConfig(value, type, loc, issues, stored) {
+  const result = normaliseConfig(value, loc);
+  if (result.issues !== undefined) {
+    for (const [where, message] of result.issues) issues.add(where, message);
+    return null;
+  }
+  const config = result.config;
+  if (stored !== null) for (const key of Object.keys(config)) if (config[key] === MASK && isSecretKey(key) && Object.hasOwn(stored, key)) config[key] = stored[key];
+  for (const key of requiredKeys(type)) if (typeof config[key] !== "string" || config[key].length === 0) issues.add(`${loc}.${key}`, "Field required");
+  return config;
+}
+
+/** True when a SCHEDULED or IN_PROGRESS job of the workspace runs a workflow that references this connector. */
+function inUseByRunningJob(context, workspace, idField, id) {
+  const bound = limits(context).jobs;
+  const now = context.clock.nowUs();
+  const workflows = new Map();
+  for (const job of scanAll(context, "jobs", bound)) {
+    if (job.workspace_id !== workspace || !isRunning(jobPhase(job, now))) continue;
+    if (!workflows.has(job.workflow_id)) workflows.set(job.workflow_id, context.state.get("workflows", job.workflow_id));
+    const workflow = workflows.get(job.workflow_id);
+    if (workflow !== null && workflow[idField] === id) return true;
+  }
+  return false;
+}
+
+export function connectorOps({ namespace, idField, types, typeFilter, label }) {
+  const find = (context, id, workspace) => findConnector(context, namespace, id, workspace, label);
+  const all = (context, workspace) => scanAll(context, namespace, limits(context).connectors).filter((row) => row.workspace_id === workspace);
+
+  const list = (input, context) => {
+    const workspace = workspaceOf(context);
+    requestProblem(input, context);
+    const issues = new Issues();
+    const type = parseEnum(input[typeFilter], `query.${typeFilter}`, issues, types, null);
+    issues.raise(context);
+    const rows = all(context, workspace)
+      .filter((row) => type === null || row.type === type)
+      .sort((a, b) => a.seq - b.seq)
+      .map(connectorView);
+    return withinBudget(context, rows, `The list of ${rows.length} ${label.toLowerCase()}s`, `filter by ${typeFilter} or delete unused ${label.toLowerCase()}s`);
+  };
+
+  const create = (input, context) => {
+    const workspace = workspaceOf(context);
+    requestProblem(input, context);
+    const issues = new Issues();
+    const name = parseString(input.name, "body.name", issues, { required: true, min: 1, max: 200 });
+    const type = input.type === undefined || input.type === null ? (issues.add("body.type", "Field required"), null) : parseEnum(input.type, "body.type", issues, types, null);
+    const key = parseString(input.key, "body.key", issues, { max: 63 });
+    if (key !== null && !KEY_RE.test(key)) issues.add("body.key", "String should match pattern '^[a-z0-9](-?[a-z0-9])*$'");
+    if (input.config === undefined || input.config === null) issues.add("body.config", "Field required");
+    const config = type !== null && input.config !== undefined && input.config !== null ? validateConfig(input.config, type, "body.config", issues, null) : null;
+    issues.raise(context);
+    if (key !== null && all(context, workspace).some((row) => row.key === key)) issues.add("body.key", "Connector key already exists in this workspace").raise(context);
+    const row = { id: uuid(context), workspace_id: workspace, seq: nextSeq(context), name, type, config, key, created_at: nowIso(context), updated_at: null };
+    context.state.put(namespace, row.id, row);
+    return connectorView(row);
+  };
+
+  const get = (input, context) => connectorView(find(context, input[idField], workspaceOf(context)));
+
+  const update = (input, context) => {
+    const workspace = workspaceOf(context);
+    requestProblem(input, context);
+    const row = find(context, input[idField], workspace);
+    const issues = new Issues();
+    const name = parseString(input.name, "body.name", issues, { min: 1, max: 200 });
+    if (input.type !== undefined && input.type !== null) {
+      const type = parseEnum(input.type, "body.type", issues, types, null);
+      if (type !== null && type !== row.type) issues.add("body.type", "Connector type cannot be changed");
+    }
+    const config = input.config !== undefined && input.config !== null ? validateConfig(input.config, row.type, "body.config", issues, row.config) : null;
+    issues.raise(context);
+    const next = { ...row, name: name ?? row.name, config: config ?? row.config, updated_at: nowIso(context) };
+    context.state.put(namespace, row.id, next);
+    return connectorView(next);
+  };
+
+  const remove = (input, context) => {
+    const workspace = workspaceOf(context);
+    const row = find(context, input[idField], workspace);
+    if (inUseByRunningJob(context, workspace, idField, row.id)) new Issues().add("body", `${label} is in use by a running job`).raise(context);
+    if (namespace === "sources") {
+      for (const file of scanPrefix(context, "source-files", `${row.id}:`, limits(context).files_per_source)) context.state.delete("source-files", `${row.id}:${file.file_id}`);
+      context.state.delete("connection-checks", row.id);
+    }
+    context.state.delete(namespace, row.id);
+    return {};
+  };
+
+  const checkConnection = (input, context) => {
+    const row = find(context, input[idField], workspaceOf(context));
+    const missing = requiredKeys(row.type).find((key) => typeof row.config[key] !== "string" || row.config[key].length === 0);
+    const check = { source_id: row.id, status: missing === undefined ? "SUCCESS" : "FAILURE", reason: missing === undefined ? null : `Missing required configuration key: ${missing}`, created_at: nowIso(context) };
+    context.state.put("connection-checks", row.id, check);
+    return { status: check.status, reason: check.reason, created_at: check.created_at };
+  };
+
+  const getConnectionCheck = (input, context) => {
+    const row = find(context, input[idField], workspaceOf(context));
+    const check = context.state.get("connection-checks", row.id);
+    if (check === null) notFound(context, "No connection check found");
+    return { status: check.status, reason: check.reason, created_at: check.created_at };
+  };
+
+  return { list, create, get, update, remove, checkConnection, getConnectionCheck };
+}

package/firedrill/tools/unstructured/ops/jobs.mjs

@@ -0,0 +1,83 @@
+// Jobs: list, get, cancel, details, failed files and output download (recomputed from the stored node snapshot).
+import { Issues, fail, notFound, requestProblem } from "../lib/errors.mjs";
+import { limits, usableId } from "../lib/ids.mjs";
+import { workspaceOf } from "../lib/identity.mjs";
+import { isRunning, jobPhase, jobStatus, jobView, nodeStats, nowIso, outputNodeOf } from "../lib/jobs-derive.mjs";
+import { pageOf, responseBudget, withinBudget } from "../lib/pages.mjs";
+import { partitionFile } from "../lib/partition/index.mjs";
+import { scanAll } from "../lib/store.mjs";
+import { mangled, parseEnum, parseInteger } from "../lib/validate.mjs";
+import { nodeOptions } from "./nodes.mjs";
+
+const JOB_STATUSES = ["SCHEDULED", "IN_PROGRESS", "COMPLETED", "STOPPED", "FAILED"];
+
+export function findJob(context, id, workspace) {
+  if (!usableId(id) || mangled(id)) notFound(context, "Job not found");
+  const row = context.state.get("jobs", id);
+  if (row === null || row.workspace_id !== workspace) notFound(context, "Job not found");
+  return row;
+}
+
+export const list = (input, context) => {
+  const workspace = workspaceOf(context);
+  requestProblem(input, context);
+  const issues = new Issues();
+  const status = parseEnum(input.status, "query.status", issues, JOB_STATUSES, null);
+  const page = parseInteger(input.page, "query.page", issues, 1, { min: 1, max: 1000000 });
+  const pageSize = parseInteger(input.page_size, "query.page_size", issues, 20, { min: 1, max: 100 });
+  if (input.workflow_id !== undefined && (typeof input.workflow_id !== "string" || mangled(input.workflow_id))) issues.add("query.workflow_id", "Input should be a valid string");
+  issues.raise(context);
+  const now = context.clock.nowUs();
+  const rows = scanAll(context, "jobs", limits(context).jobs)
+    .filter((job) => job.workspace_id === workspace && (input.workflow_id === undefined || job.workflow_id === input.workflow_id))
+    .map((job) => ({ job, view: jobView(job, now) }))
+    .filter((entry) => status === null || entry.view.status === status)
+    .sort((a, b) => (a.job.created_at < b.job.created_at ? 1 : a.job.created_at > b.job.created_at ? -1 : b.job.seq - a.job.seq));
+  return pageOf(context, rows.map((entry) => entry.view), (page - 1) * pageSize, pageSize);
+};
+
+export const get = (input, context) => jobView(findJob(context, input.job_id, workspaceOf(context)), context.clock.nowUs());
+
+export const cancel = (input, context) => {
+  const job = findJob(context, input.job_id, workspaceOf(context));
+  const phase = jobPhase(job, context.clock.nowUs());
+  if (!isRunning(phase)) new Issues().add("body", phase === "STOPPED" ? "Job is already stopped" : "Job is already finished").raise(context);
+  context.state.put("jobs", job.id, { ...job, cancelled_at: nowIso(context), stop_requested: true });
+  context.events.emit("job.cancelled", { job_id: job.id, workflow_id: job.workflow_id });
+  return { id: job.id, status: "STOPPED", message: "Job cancellation requested" };
+};
+
+export const details = (input, context) => {
+  const job = findJob(context, input.job_id, workspaceOf(context));
+  const derived = jobStatus(job, context.clock.nowUs());
+  const failed = job.files.filter((file) => file.error !== null).length;
+  const message = derived.phase === "FINISHED" ? `${job.files.length - failed} of ${job.files.length} files processed successfully` : derived.reason ?? `Job is ${derived.status.toLowerCase().replace("_", " ")}`;
+  return { id: job.id, processing_status: derived.processing_status, node_stats: nodeStats(job, derived.phase), message };
+};
+
+export const failedFiles = (input, context) => {
+  const job = findJob(context, input.job_id, workspaceOf(context));
+  return { failed_files: job.files.filter((file) => file.error !== null).map((file) => ({ document: file.path, error: file.error })) };
+};
+
+export const downloadOutput = (input, context) => {
+  const workspace = workspaceOf(context);
+  requestProblem(input, context);
+  const job = findJob(context, input.job_id, workspace);
+  const issues = new Issues();
+  if (typeof input.file_id !== "string" || input.file_id.length === 0) issues.add("query.file_id", "Field required");
+  if (typeof input.node_id !== "string" || input.node_id.length === 0) issues.add("query.node_id", "Field required");
+  issues.raise(context);
+  const phase = jobPhase(job, context.clock.nowUs());
+  if (phase !== "FINISHED") issues.add("body", "Job output is not available yet").raise(context);
+  const node = outputNodeOf(job);
+  const file = job.files.find((entry) => entry.file_id === input.file_id && entry.error === null);
+  if (node === null || node.id !== input.node_id || file === undefined) notFound(context, "Job output file not found");
+  const content = file.source_row !== null ? context.state.get("source-files", file.source_row) : context.state.get("job-files", `${job.id}:${file.file_id}`);
+  if (content === null) notFound(context, "Job output file not found");
+  const options = { ...nodeOptions(job.nodes, content.last_modified ?? job.created_at), responseBudget: responseBudget(context) };
+  const result = partitionFile({ filename: content.filename, content: content.content, content_type: content.content_type, last_modified: content.last_modified }, options, context);
+  if (!result.ok && result.code === "RESPONSE_TOO_LARGE") fail(context, "RESPONSE_TOO_LARGE", result.message.replace("Partition output", "Job output"));
+  if (!result.ok) notFound(context, "Job output file not found");
+  return withinBudget(context, result.elements, "Job output");
+};

package/firedrill/tools/unstructured/ops/nodes.mjs

@@ -0,0 +1,107 @@
+// Workflow node validation and the mapping from a node snapshot to partitioner options.
+import { normaliseConfig } from "../lib/connectors.mjs";
+import { uuid } from "../lib/ids.mjs";
+import { parseBool, parseInteger, parseLanguages, parseString } from "../lib/validate.mjs";
+import { Issues } from "../lib/errors.mjs";
+import { MAX_TEXT } from "../lib/partition/elements.mjs";
+
+export const NODE_TYPES = ["partition", "chunk", "embed", "prompter"];
+const PARTITION_SUBTYPES = ["auto", "fast", "hi_res", "vlm"];
+const CHUNK_SUBTYPES = ["chunk_by_title", "chunk_by_character", "chunk_by_page", "chunk_by_similarity"];
+const CHUNK_STRATEGY = new Map([["chunk_by_title", "by_title"], ["chunk_by_character", "basic"], ["chunk_by_page", "by_page"]]);
+
+/** Default nodes of an `auto` workflow. */
+export const defaultNodes = (context) => [
+  { id: uuid(context), name: "Partitioner", type: "partition", subtype: "auto", settings: {} },
+  { id: uuid(context), name: "Chunker", type: "chunk", subtype: "chunk_by_title", settings: { max_characters: 1500, new_after_n_chars: 1500, overlap: 0 } },
+];
+
+/** Validates `workflow_nodes` of a custom workflow; returns the normalised node array or null (issues added). */
+export function validateNodes(value, loc, issues, context) {
+  if (!Array.isArray(value) || value.length === 0) {
+    issues.add(loc, value === undefined || value === null ? "Field required" : "Input should be a valid list");
+    return null;
+  }
+  if (value.length > 16) {
+    issues.add(loc, "List should have at most 16 items");
+    return null;
+  }
+  const nodes = [];
+  let partitions = 0;
+  let chunks = 0;
+  for (const [index, node] of value.entries()) {
+    const at = `${loc}.${index}`;
+    if (node === null || typeof node !== "object" || Array.isArray(node)) {
+      issues.add(at, "Input should be a valid dictionary or object");
+      continue;
+    }
+    const name = parseString(node.name, `${at}.name`, issues, { required: true, min: 1, max: 200 });
+    const type = typeof node.type === "string" && NODE_TYPES.includes(node.type) ? node.type : (issues.add(`${at}.type`, "Input should be 'partition', 'chunk', 'embed' or 'prompter'"), null);
+    const subtype = parseString(node.subtype, `${at}.subtype`, issues, { required: true, min: 1, max: 100 });
+    const id = node.id === undefined || node.id === null ? uuid(context) : parseString(node.id, `${at}.id`, issues, { min: 1, max: 64 });
+    let settings = {};
+    if (node.settings !== undefined && node.settings !== null) {
+      const result = normaliseConfig(node.settings, `${at}.settings`);
+      if (result.issues !== undefined) for (const [where, message] of result.issues) issues.add(where, message);
+      else settings = result.config;
+    }
+    if (type === "partition") {
+      partitions += 1;
+      if (index !== 0) issues.add(`${at}.type`, "The partition node must be the first node");
+      if (subtype !== null && !PARTITION_SUBTYPES.includes(subtype)) issues.add(`${at}.subtype`, "Input should be 'auto', 'fast', 'hi_res' or 'vlm'");
+      parseLanguages(settings.languages, `${at}.settings.languages`, issues);
+    } else if (type === "chunk") {
+      chunks += 1;
+      if (subtype !== null && !CHUNK_SUBTYPES.includes(subtype)) issues.add(`${at}.subtype`, "Input should be 'chunk_by_title', 'chunk_by_character', 'chunk_by_page' or 'chunk_by_similarity'");
+      if (subtype === "chunk_by_similarity") issues.add(`${at}.subtype`, "Chunking strategy by_similarity is not supported by this Tool");
+      const check = new Issues();
+      const max = parseInteger(settings.max_characters, `${at}.settings.max_characters`, check, 500, { min: 1, max: MAX_TEXT });
+      parseInteger(settings.new_after_n_chars, `${at}.settings.new_after_n_chars`, check, max, { min: 0, max: MAX_TEXT });
+      const overlap = parseInteger(settings.overlap, `${at}.settings.overlap`, check, 0, { min: 0, max: MAX_TEXT });
+      parseInteger(settings.combine_under_n_chars, `${at}.settings.combine_under_n_chars`, check, max, { min: 0, max: MAX_TEXT });
+      for (const flag of ["overlap_all", "multipage_sections", "include_orig_elements"]) parseBool(settings[flag], `${at}.settings.${flag}`, check, false);
+      if (check.empty && overlap >= max) check.add(`${at}.settings.overlap`, `Input should be less than max_characters (${max})`);
+      for (const item of check.items) issues.add(item.slice(4, item.indexOf(": ")), item.slice(item.indexOf(": ") + 2));
+    }
+    if (name !== null && type !== null && subtype !== null && id !== null) nodes.push({ id, name, type, subtype, settings });
+  }
+  if (partitions !== 1) issues.add(loc, "A custom workflow needs exactly one partition node");
+  if (chunks > 1) issues.add(loc, "A custom workflow may have at most one chunk node");
+  return issues.empty ? nodes : null;
+}
+
+/** Partitioner options for a node snapshot (validated on create/update, so parsing cannot fail here). */
+export function nodeOptions(nodes, nowIso) {
+  const quiet = new Issues();
+  const chunkNode = nodes.find((node) => node.type === "chunk") ?? null;
+  let chunk = null;
+  if (chunkNode !== null && CHUNK_STRATEGY.has(chunkNode.subtype)) {
+    const s = chunkNode.settings;
+    // Clamped to MAX_TEXT so a chunk never exceeds the declared element text bound, whatever a stored node says.
+    const maxCharacters = Math.min(parseInteger(s.max_characters, "x", quiet, 500, { min: 1 }), MAX_TEXT);
+    const newAfter = parseInteger(s.new_after_n_chars, "x", quiet, maxCharacters, { min: 0 });
+    chunk = {
+      strategy: CHUNK_STRATEGY.get(chunkNode.subtype),
+      maxCharacters,
+      newAfterNChars: Math.min(newAfter, maxCharacters),
+      overlap: Math.min(parseInteger(s.overlap, "x", quiet, 0, { min: 0 }), maxCharacters - 1),
+      overlapAll: parseBool(s.overlap_all, "x", quiet, false),
+      combineUnderNChars: chunkNode.subtype === "chunk_by_title" ? parseInteger(s.combine_under_n_chars, "x", quiet, maxCharacters, { min: 0 }) : 0,
+      multipageSections: parseBool(s.multipage_sections, "x", quiet, true),
+      includeOrigElements: parseBool(s.include_orig_elements, "x", quiet, true),
+    };
+  }
+  const partitionNode = nodes.find((node) => node.type === "partition") ?? null;
+  const settings = partitionNode === null ? {} : partitionNode.settings;
+  return {
+    contentType: null,
+    // Validated on create/update (at most 20 entries of at most 20 characters), so the list is used whole.
+    languages: typeof settings.languages === "string" && settings.languages.length > 0 ? settings.languages.split(",").map((l) => l.trim()).filter((l) => l.length > 0) : ["eng"],
+    includePageBreaks: parseBool(settings.include_page_breaks, "x", quiet, false),
+    startingPage: 1,
+    xmlKeepTags: parseBool(settings.xml_keep_tags, "x", quiet, false),
+    uniqueIds: false,
+    nowIso,
+    chunk,
+  };
+}

package/firedrill/tools/unstructured/ops/partition.mjs

@@ -0,0 +1,112 @@
+// `general.partition`: the legacy Partition Endpoint over the deterministic text partitioner.
+import { Issues, fail, requestProblem } from "../lib/errors.mjs";
+import { nextSeq } from "../lib/ids.mjs";
+import { workspaceOf } from "../lib/identity.mjs";
+import { assertWithinBudget, jsonBytes, responseBudget } from "../lib/pages.mjs";
+import { elementsToCsv } from "../lib/partition/csvout.mjs";
+import { MAX_TEXT } from "../lib/partition/elements.mjs";
+import { partitionFile } from "../lib/partition/index.mjs";
+import { clip, isoFromUs, utf8Length } from "../lib/util.mjs";
+import { parseBool, parseEnum, parseInteger, parseLanguages, parseString } from "../lib/validate.mjs";
+
+const STRATEGIES = ["fast", "hi_res", "auto", "ocr_only", "vlm"];
+const CHUNKING = ["basic", "by_title", "by_page", "by_similarity"];
+const OUTPUTS = ["application/json", "text/csv"];
+const ENCODINGS = new Set(["utf-8", "utf8", "utf_8", "ascii", "us-ascii"]);
+const MAX_FILES = 32;
+const MAX_CONTENT = 1048576;
+
+
+/** Validates every partition/chunk parameter. Returns { options, strategy, chunkingStrategy, outputFormat }. */
+export function partitionOptions(input, context) {
+  const invalid = (message) => fail(context, "INVALID_REQUEST", message);
+  const strategyRaw = input.strategy === undefined || input.strategy === null || input.strategy === "" ? "auto" : input.strategy;
+  const strategy = typeof strategyRaw === "string" ? strategyRaw.toLowerCase() : "";
+  if (!STRATEGIES.includes(strategy)) invalid(`Invalid strategy: ${clip(strategyRaw)}. Must be one of ['fast', 'hi_res', 'auto', 'ocr_only', 'vlm']`);
+  if (typeof input.hi_res_model_name === "string" && input.hi_res_model_name.length > 0) invalid(`Unknown model type: ${clip(input.hi_res_model_name)}`);
+  const outputFormat = input.output_format === undefined || input.output_format === null || input.output_format === "" ? "application/json" : input.output_format;
+  if (!OUTPUTS.includes(outputFormat)) invalid(`Invalid output format: ${clip(outputFormat)}. Must be one of ['application/json', 'text/csv']`);
+  let chunkingStrategy = null;
+  if (input.chunking_strategy !== undefined && input.chunking_strategy !== null && input.chunking_strategy !== "") {
+    if (!CHUNKING.includes(input.chunking_strategy)) invalid(`Invalid chunking strategy: ${clip(input.chunking_strategy)}. Must be one of ['basic', 'by_title', 'by_page', 'by_similarity']`);
+    if (input.chunking_strategy === "by_similarity") invalid("Chunking strategy by_similarity is not supported by this Tool");
+    chunkingStrategy = input.chunking_strategy;
+  }
+  if (input.encoding !== undefined && input.encoding !== null && input.encoding !== "") {
+    if (typeof input.encoding !== "string" || !ENCODINGS.has(input.encoding.toLowerCase())) invalid(`Unsupported encoding: ${clip(input.encoding)}`);
+  }
+  const issues = new Issues();
+  const contentType = parseString(input.content_type, "body.content_type", issues, { max: 200 });
+  const languages = parseLanguages(input.languages, "body.languages", issues) ?? [];
+  const includePageBreaks = parseBool(input.include_page_breaks, "body.include_page_breaks", issues, false);
+  const startingPage = parseInteger(input.starting_page_number, "body.starting_page_number", issues, 1, { min: 0, max: 1000000 });
+  const uniqueIds = parseBool(input.unique_element_ids, "body.unique_element_ids", issues, false);
+  const xmlKeepTags = parseBool(input.xml_keep_tags, "body.xml_keep_tags", issues, false);
+  parseBool(input.coordinates, "body.coordinates", issues, false);
+  const maxCharacters = parseInteger(input.max_characters, "body.max_characters", issues, 500, { min: 1, max: MAX_TEXT });
+  let newAfterNChars = parseInteger(input.new_after_n_chars, "body.new_after_n_chars", issues, 1500, { min: 0, max: MAX_TEXT });
+  const overlap = parseInteger(input.overlap, "body.overlap", issues, 0, { min: 0, max: MAX_TEXT });
+  const overlapAll = parseBool(input.overlap_all, "body.overlap_all", issues, false);
+  const combineUnderNChars = parseInteger(input.combine_under_n_chars, "body.combine_under_n_chars", issues, maxCharacters, { min: 0, max: MAX_TEXT });
+  const multipageSections = parseBool(input.multipage_sections, "body.multipage_sections", issues, true);
+  const includeOrigElements = parseBool(input.include_orig_elements, "body.include_orig_elements", issues, true);
+  for (const name of ["split_pdf_page", "split_pdf_allow_failed", "include_slide_notes", "pdf_infer_table_structure"]) parseBool(input[name], `body.${name}`, issues, false);
+  parseInteger(input.split_pdf_concurrency_level, "body.split_pdf_concurrency_level", issues, 5, { min: 1, max: 50 });
+  parseInteger(input.split_pdf_page_range, "body.split_pdf_page_range", issues, 0, { min: 0 });
+  if (strategy === "vlm") {
+    if (typeof input.vlm_model !== "string" || input.vlm_model.length === 0) issues.add("body.vlm_model", "Field required");
+    if (typeof input.vlm_model_provider !== "string" || input.vlm_model_provider.length === 0) issues.add("body.vlm_model_provider", "Field required");
+  }
+  if (chunkingStrategy !== null && overlap >= maxCharacters) issues.add("body.overlap", `Input should be less than max_characters (${maxCharacters})`);
+  if (newAfterNChars > maxCharacters) newAfterNChars = maxCharacters;
+  issues.raise(context);
+  const chunk = chunkingStrategy === null ? null : { strategy: chunkingStrategy, maxCharacters, newAfterNChars, overlap, overlapAll, combineUnderNChars: chunkingStrategy === "by_title" ? combineUnderNChars : 0, multipageSections, includeOrigElements };
+  const options = { contentType, languages: languages.length > 0 ? languages : ["eng"], includePageBreaks, startingPage, xmlKeepTags, uniqueIds, nowIso: isoFromUs(context.clock.nowUs()), chunk, responseBudget: responseBudget(context), chunkSizing: { chars: 0, count: 0 }, elementSizing: { bytes: 0 } };
+  return { options, strategy, chunkingStrategy, outputFormat };
+}
+
+/** Validates the `files` array shape (a 422 in the FastAPI envelope). */
+export function checkFiles(files, context, loc = "body.files") {
+  const issues = new Issues();
+  if (!Array.isArray(files) || files.length === 0) issues.add(loc, "Field required");
+  else if (files.length > MAX_FILES) issues.add(loc, `At most ${MAX_FILES} files per request`);
+  else
+    for (const [index, file] of files.entries()) {
+      if (file === null || typeof file !== "object" || typeof file.content !== "string" || typeof file.filename !== "string") issues.add(`${loc}.${index}`, "Input should be a valid file upload");
+      else if (file.content.length > MAX_CONTENT) issues.add(`${loc}.${index}`, `File content exceeds ${MAX_CONTENT} characters`);
+      else if (file.filename.length > 500) issues.add(`${loc}.${index}.filename`, "String should have at most 500 characters");
+    }
+  issues.raise(context);
+}
+
+export function partition(input, context) {
+  workspaceOf(context);
+  requestProblem(input, context);
+  const { options, strategy, chunkingStrategy, outputFormat } = partitionOptions(input, context);
+  checkFiles(input.files, context);
+  const elements = [];
+  const filenames = [];
+  const filetypes = [];
+  let inputBytes = 0;
+  let pageCount = 0;
+  for (const file of input.files) {
+    const result = partitionFile(file, options, context);
+    if (!result.ok) fail(context, result.code, result.message);
+    inputBytes += utf8Length(file.content);
+    filenames.push(clip(file.filename, 200));
+    filetypes.push(result.filetype);
+    const pages = new Set();
+    for (const element of result.elements) pages.add(element.metadata.page_number);
+    pageCount += pages.size;
+    for (const element of result.elements) elements.push(element);
+  }
+  const body = outputFormat === "text/csv" ? elementsToCsv(elements) : null;
+  const bytes = body === null ? jsonBytes(elements) : utf8Length(body);
+  assertWithinBudget(context, bytes, "Partition output", "upload fewer or smaller files");
+  const seq = nextSeq(context);
+  const log = { seq, actor_id: clip(context.actor.id, 200), filenames, filetypes, strategy, chunking_strategy: chunkingStrategy, output_format: outputFormat, element_count: elements.length, page_count: pageCount, input_bytes: inputBytes, created_at: options.nowIso };
+  const logId = `p_${String(seq).padStart(10, "0")}`;
+  context.state.put("partition-log", logId, log);
+  context.events.emit("partition.completed", { log_id: logId, filenames, filetypes, strategy, chunking_strategy: chunkingStrategy, element_count: elements.length, page_count: pageCount, input_bytes: inputBytes });
+  return body === null ? elements : { csv: body };
+}