@sembl/core 0.3.0 → 0.5.0

package/dist/index.cjs

@@ -26,6 +26,7 @@ __export(index_exports, {
   Constrain: () => Constrain,
   Describe: () => Describe,
   EnumResolutionError: () => EnumResolutionError,
+  FIELD_FORMATS: () => FIELD_FORMATS,
   PROVENANCE_INSTRUCTIONS: () => PROVENANCE_INSTRUCTIONS,
   SOURCE_INSTRUCTIONS: () => SOURCE_INSTRUCTIONS,
   Schema: () => Schema,
@@ -35,19 +36,25 @@ __export(index_exports, {
   ValuesFrom: () => ValuesFrom,
   budgetSources: () => budgetSources,
   buildPrompt: () => buildPrompt,
+  buildRepairCorrection: () => buildRepairCorrection,
   buildRepairInput: () => buildRepairInput,
   bundleOf: () => bundleOf,
   coerce: () => coerce,
+  coerceDetailed: () => coerceDetailed,
   coerceMany: () => coerceMany,
   coerceWithProvenance: () => coerceWithProvenance,
   collectEnumSources: () => collectEnumSources,
   defineSchema: () => defineSchema,
+  describeFormat: () => describeFormat,
   field: () => field,
+  formatToJsonSchema: () => formatToJsonSchema,
   isCoerceInput: () => isCoerceInput,
   isSource: () => isSource,
   normalizeInstructions: () => normalizeInstructions,
   partialCoerce: () => partialCoerce,
+  partialCoerceDetailed: () => partialCoerceDetailed,
   partialCoerceWithProvenance: () => partialCoerceWithProvenance,
+  primeCache: () => primeCache,
   provenanceInstructions: () => provenanceInstructions,
   renderSources: () => renderSources,
   resolveEnumSources: () => resolveEnumSources,
@@ -58,11 +65,164 @@ __export(index_exports, {
   toOpenAIJsonSchema: () => toOpenAIJsonSchema,
   toProvenanceSchema: () => toProvenanceSchema,
   toSources: () => toSources,
+  validateFormat: () => validateFormat,
   validatePartial: () => validatePartial,
   validateStrict: () => validateStrict
 });
 module.exports = __toCommonJS(index_exports);
 
+// src/schema/formats.ts
+var FIELD_FORMATS = [
+  "url",
+  "email",
+  "date",
+  "datetime",
+  "iso-country",
+  "us-state",
+  "us-state-name",
+  "currency"
+];
+var ISO_COUNTRIES = new Set(
+  "AD AE AF AG AI AL AM AO AQ AR AS AT AU AW AX AZ BA BB BD BE BF BG BH BI BJ BL BM BN BO BQ BR BS BT BV BW BY BZ CA CC CD CF CG CH CI CK CL CM CN CO CR CU CV CW CX CY CZ DE DJ DK DM DO DZ EC EE EG EH ER ES ET FI FJ FK FM FO FR GA GB GD GE GF GG GH GI GL GM GN GP GQ GR GS GT GU GW GY HK HM HN HR HT HU ID IE IL IM IN IO IQ IR IS IT JE JM JO JP KE KG KH KI KM KN KP KR KW KY KZ LA LB LC LI LK LR LS LT LU LV LY MA MC MD ME MF MG MH MK ML MM MN MO MP MQ MR MS MT MU MV MW MX MY MZ NA NC NE NF NG NI NL NO NP NR NU NZ OM PA PE PF PG PH PK PL PM PN PR PS PT PW PY QA RE RO RS RU RW SA SB SC SD SE SG SH SI SJ SK SL SM SN SO SR SS ST SV SX SY SZ TC TD TF TG TH TJ TK TL TM TN TO TR TT TV TW TZ UA UG UM US UY UZ VA VC VE VG VI VN VU WF WS YE YT ZA ZM ZW".split(" ")
+);
+var US_STATES = {
+  AL: "Alabama",
+  AK: "Alaska",
+  AZ: "Arizona",
+  AR: "Arkansas",
+  CA: "California",
+  CO: "Colorado",
+  CT: "Connecticut",
+  DE: "Delaware",
+  FL: "Florida",
+  GA: "Georgia",
+  HI: "Hawaii",
+  ID: "Idaho",
+  IL: "Illinois",
+  IN: "Indiana",
+  IA: "Iowa",
+  KS: "Kansas",
+  KY: "Kentucky",
+  LA: "Louisiana",
+  ME: "Maine",
+  MD: "Maryland",
+  MA: "Massachusetts",
+  MI: "Michigan",
+  MN: "Minnesota",
+  MS: "Mississippi",
+  MO: "Missouri",
+  MT: "Montana",
+  NE: "Nebraska",
+  NV: "Nevada",
+  NH: "New Hampshire",
+  NJ: "New Jersey",
+  NM: "New Mexico",
+  NY: "New York",
+  NC: "North Carolina",
+  ND: "North Dakota",
+  OH: "Ohio",
+  OK: "Oklahoma",
+  OR: "Oregon",
+  PA: "Pennsylvania",
+  RI: "Rhode Island",
+  SC: "South Carolina",
+  SD: "South Dakota",
+  TN: "Tennessee",
+  TX: "Texas",
+  UT: "Utah",
+  VT: "Vermont",
+  VA: "Virginia",
+  WA: "Washington",
+  WV: "West Virginia",
+  WI: "Wisconsin",
+  WY: "Wyoming",
+  DC: "District of Columbia",
+  PR: "Puerto Rico",
+  GU: "Guam",
+  VI: "U.S. Virgin Islands",
+  AS: "American Samoa",
+  MP: "Northern Mariana Islands"
+};
+var US_STATE_NAMES = new Set(Object.values(US_STATES));
+var CURRENCIES = new Set(
+  "AED AFN ALL AMD ANG AOA ARS AUD AWG AZN BAM BBD BDT BGN BHD BIF BMD BND BOB BRL BSD BTN BWP BYN BZD CAD CDF CHF CLP CNY COP CRC CUP CVE CZK DJF DKK DOP DZD EGP ERN ETB EUR FJD FKP GBP GEL GHS GIP GMD GNF GTQ GYD HKD HNL HTG HUF IDR ILS INR IQD IRR ISK JMD JOD JPY KES KGS KHR KMF KPW KRW KWD KYD KZT LAK LBP LKR LRD LSL LYD MAD MDL MGA MKD MMK MNT MOP MRU MUR MVR MWK MXN MYR MZN NAD NGN NIO NOK NPR NZD OMR PAB PEN PGK PHP PKR PLN PYG QAR RON RSD RUB RWF SAR SBD SCR SDG SEK SGD SHP SLE SOS SRD SSP STN SVC SYP SZL THB TJS TMT TND TOP TRY TTD TWD TZS UAH UGX USD UYU UZS VES VND VUV WST XAF XCD XOF XPF YER ZAR ZMW ZWG".split(" ")
+);
+var EMAIL = /^[^\s@]+@[^\s@]+\.[^\s@]+$/;
+var DATE = /^(\d{4})-(\d{2})-(\d{2})$/;
+var DATETIME = /^\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?(?:Z|[+-]\d{2}:?\d{2})?$/;
+function isCalendarDate(y, m, d) {
+  const date = new Date(Date.UTC(y, m - 1, d));
+  return date.getUTCFullYear() === y && date.getUTCMonth() === m - 1 && date.getUTCDate() === d;
+}
+function validateFormat(value, format) {
+  switch (format) {
+    case "url": {
+      try {
+        const url = new URL(value);
+        if (url.protocol === "http:" || url.protocol === "https:") return void 0;
+      } catch {
+      }
+      return `Expected an absolute http(s) URL, got ${JSON.stringify(value)}`;
+    }
+    case "email":
+      return EMAIL.test(value) ? void 0 : `Expected an email address, got ${JSON.stringify(value)}`;
+    case "date": {
+      const m = DATE.exec(value);
+      if (m && isCalendarDate(Number(m[1]), Number(m[2]), Number(m[3]))) return void 0;
+      return `Expected a calendar date as YYYY-MM-DD, got ${JSON.stringify(value)}`;
+    }
+    case "datetime":
+      return DATETIME.test(value) && !Number.isNaN(Date.parse(value)) ? void 0 : `Expected an ISO 8601 timestamp, got ${JSON.stringify(value)}`;
+    case "iso-country":
+      return ISO_COUNTRIES.has(value) ? void 0 : `Expected an ISO 3166-1 alpha-2 country code such as US or DE, got ${JSON.stringify(value)}`;
+    case "us-state":
+      return value in US_STATES ? void 0 : `Expected a two-letter USPS state code such as CA or NY, got ${JSON.stringify(value)}`;
+    case "us-state-name":
+      return US_STATE_NAMES.has(value) ? void 0 : `Expected a US state's full name such as California, got ${JSON.stringify(value)}`;
+    case "currency":
+      return CURRENCIES.has(value) ? void 0 : `Expected an ISO 4217 currency code such as USD or EUR, got ${JSON.stringify(value)}`;
+  }
+}
+function describeFormat(format) {
+  switch (format) {
+    case "url":
+      return "an absolute http(s) URL";
+    case "email":
+      return "an email address";
+    case "date":
+      return "a calendar date as YYYY-MM-DD";
+    case "datetime":
+      return "an ISO 8601 timestamp (e.g. 2026-09-05T14:30:00Z)";
+    case "iso-country":
+      return "an ISO 3166-1 alpha-2 country code (e.g. US, DE, PT), never a country name";
+    case "us-state":
+      return "a two-letter USPS state code (e.g. CA, NY), never the state's name";
+    case "us-state-name":
+      return "a US state's full name (e.g. California, New York), never its abbreviation";
+    case "currency":
+      return "an ISO 4217 currency code (e.g. USD, EUR, GBP), never a symbol or a word";
+  }
+}
+function formatToJsonSchema(format) {
+  switch (format) {
+    case "url":
+      return { format: "uri" };
+    case "email":
+      return { format: "email" };
+    case "date":
+      return { format: "date" };
+    case "datetime":
+      return { format: "date-time" };
+    case "iso-country":
+    case "us-state":
+      return { pattern: "^[A-Z]{2}$" };
+    case "currency":
+      return { pattern: "^[A-Z]{3}$" };
+    case "us-state-name":
+      return {};
+  }
+}
+
 // src/schema/json-schema.ts
 var CONSTRAINT_KEYWORDS = [
   "maxLength",
@@ -79,6 +239,9 @@ function constraintsToJsonSchema(constraints, dialect) {
     return {};
   }
   const out = {};
+  if (constraints.format !== void 0) {
+    Object.assign(out, formatToJsonSchema(constraints.format));
+  }
   for (const keyword of CONSTRAINT_KEYWORDS) {
     const value = constraints[keyword];
     if (value !== void 0) {
@@ -228,7 +391,7 @@ async function resolveEnumSources(schema, resolver, bundle) {
   await Promise.all(
     [...usages].map(async ([sourceId, usage]) => {
       try {
-        const values = await resolver(sourceId);
+        const values = await resolver(sourceId, { sourceId, schema, ...usage });
         if (!values || values.length === 0) {
           failures.push({ sourceId, reason: "empty", ...usage });
           return;
@@ -463,7 +626,7 @@ ${summary}`);
 // src/coerce/sources.ts
 var SOURCE_TAG = "source";
 function isSource(value) {
-  return typeof value === "object" && value !== null && !Array.isArray(value) && typeof value.text === "string" && (value.label === void 0 || typeof value.label === "string");
+  return typeof value === "object" && value !== null && !Array.isArray(value) && typeof value.text === "string" && (value.label === void 0 || typeof value.label === "string") && (value.maxChars === void 0 || typeof value.maxChars === "number");
 }
 function isCoerceInput(value) {
   return typeof value === "string" || isSource(value) || Array.isArray(value) && value.every(isSource);
@@ -483,7 +646,9 @@ function toSources(input) {
 }
 function cleanLabel(source) {
   const label = source.label?.trim();
-  return label ? { label, text: source.text } : { text: source.text };
+  const cleaned = label ? { label, text: source.text } : { text: source.text };
+  if (source.maxChars !== void 0) cleaned.maxChars = source.maxChars;
+  return cleaned;
 }
 function escapeText(text) {
   return text.replace(new RegExp(`</(\\s*${SOURCE_TAG}\\b)`, "gi"), "<\\/$1");
@@ -521,7 +686,10 @@ function describeConstraints(constraints) {
     return [];
   }
   const phrases = [];
-  const { minLength, maxLength, minimum, maximum, minItems, maxItems, pattern } = constraints;
+  const { minLength, maxLength, minimum, maximum, minItems, maxItems, pattern, format } = constraints;
+  if (format !== void 0) {
+    phrases.push(describeFormat(format));
+  }
   if (minLength !== void 0 && maxLength !== void 0) {
     phrases.push(`between ${minLength} and ${maxLength} characters`);
   } else if (maxLength !== void 0) {
@@ -634,6 +802,8 @@ function buildPrompt(schema, bundle, options = {}) {
   lines.push("Instructions:");
   lines.push("- Extract values from the sources that match the schema fields.");
   lines.push("- Use null for optional fields that cannot be determined from the input.");
+  lines.push("- A value the input states is never omitted because it looks like a default: return 1, 0, false or an empty list when that is what the input says.");
+  lines.push("- Never return an empty object when the input states values for any field.");
   lines.push("- Required fields must always have a valid, non-null value.");
   lines.push("- Interpret the user's input semantically \u2014 infer meaning, don't just pattern match.");
   lines.push("- Respect every stated limit exactly; truncate or drop lower-priority content to stay within it.");
@@ -659,7 +829,7 @@ function renderReceived(received) {
   return text.length > MAX_RECEIVED_LENGTH ? `${text.slice(0, MAX_RECEIVED_LENGTH)}\u2026 (truncated)` : text;
 }
 function buildRepairInput(originalInput, rejected, issues) {
-  const lines = [
+  return [
     originalInput,
     "",
     "---",
@@ -668,9 +838,11 @@ function buildRepairInput(originalInput, rejected, issues) {
     "",
     JSON.stringify(rejected, null, 2),
     "",
-    "It was rejected because:",
-    ""
-  ];
+    buildRepairCorrection(issues)
+  ].join("\n");
+}
+function buildRepairCorrection(issues) {
+  const lines = ["The output was rejected because:", ""];
   for (const issue of issues) {
     lines.push(`- ${issue.path}: ${issue.message} (received: ${renderReceived(issue.received)})`);
   }
@@ -699,11 +871,31 @@ var PROVENANCE_INSTRUCTIONS = [
   "- Judge each field on its own. A confident value next to a guessed one is",
   "  normal, and marking the guess honestly is more useful than looking sure."
 ].join("\n");
+function provenanceFieldNames(schema, fields) {
+  if (fields === void 0) return new Set(schema.fields.map((f) => f.name));
+  const known = new Set(schema.fields.map((f) => f.name));
+  for (const name of fields) {
+    if (!known.has(name)) {
+      throw new RangeError(
+        `provenance field "${name}" is not a field of schema "${schema.id}" (fields: ${[...known].join(", ")})`
+      );
+    }
+  }
+  return new Set(fields);
+}
 function provenanceInstructions(options = {}) {
   const labels = options.sourceLabels ?? [];
-  if (labels.length < 2) return PROVENANCE_INSTRUCTIONS;
-  return `${PROVENANCE_INSTRUCTIONS}
-- Set \`source\` to the label of the source the value was read from.`;
+  let text = PROVENANCE_INSTRUCTIONS;
+  if (options.fields !== void 0) {
+    text = text.replace(
+      "- Every field is wrapped as an object: put the extracted value in `value`.",
+      `- Only these fields are wrapped as objects, with the extracted value in \`value\`: ${options.fields.join(", ")}. Every other field is a plain value.`
+    );
+  }
+  if (labels.length >= 2) {
+    text += "\n- Always set `source` to the label of the source the value was read from. When several agree, name the one quoted in `evidence`.";
+  }
+  return text;
 }
 function annotationSchema(parentId, field2, sourceLabels) {
   const valueField = {
@@ -735,7 +927,7 @@ function annotationSchema(parentId, field2, sourceLabels) {
           name: "source",
           description: "The label of the source this value was read from.",
           type: { kind: "enum", values: [...sourceLabels] },
-          required: false
+          required: true
         }
       ] : []
     ]
@@ -745,7 +937,12 @@ function toProvenanceSchema(schema, bundle, options = {}) {
   const schemas = { ...bundle?.schemas ?? {} };
   const fields = [];
   const sourceLabels = options.sourceLabels ?? [];
+  const wrapped = provenanceFieldNames(schema, options.fields);
   for (const field2 of schema.fields) {
+    if (!wrapped.has(field2.name)) {
+      fields.push(field2);
+      continue;
+    }
     const annotation = annotationSchema(schema.id, field2, sourceLabels);
     schemas[annotation.id] = annotation;
     fields.push({
@@ -821,10 +1018,30 @@ ${tailRoom > 0 ? text.slice(text.length - tailRoom) : ""}`;
   }
 }
 function budgetSources(sources, maxChars, policy = "tail") {
-  const total = sources.reduce((sum, s) => sum + s.text.length, 0);
-  if (total <= maxChars) {
-    return { sources: [...sources], truncated: [] };
+  const records = /* @__PURE__ */ new Map();
+  const record = (index, source, text) => {
+    const existing = records.get(index);
+    if (existing) {
+      existing.keptLength = text.length;
+    } else {
+      records.set(index, {
+        ...source.label !== void 0 ? { label: source.label } : {},
+        originalLength: source.text.length,
+        keptLength: text.length
+      });
+    }
+  };
+  const capped = sources.map((source, index) => {
+    if (source.maxChars === void 0 || source.text.length <= source.maxChars) return source;
+    const text = truncateText(source.text, source.maxChars, policy);
+    record(index, source, text);
+    return { ...source, text };
+  });
+  const total = capped.reduce((sum, s) => sum + s.text.length, 0);
+  if (maxChars === void 0 || total <= maxChars) {
+    return { sources: capped, truncated: [...records.values()] };
   }
+  sources = capped;
   const allowance = /* @__PURE__ */ new Map();
   const order = sources.map((s, i) => i).sort((a, b) => sources[a].text.length - sources[b].text.length);
   let remaining = maxChars;
@@ -834,19 +1051,14 @@ function budgetSources(sources, maxChars, policy = "tail") {
     allowance.set(index, granted);
     remaining -= granted;
   });
-  const truncated = [];
   const budgeted = sources.map((source, index) => {
     const limit = allowance.get(index) ?? 0;
     if (source.text.length <= limit) return source;
     const text = truncateText(source.text, limit, policy);
-    truncated.push({
-      ...source.label !== void 0 ? { label: source.label } : {},
-      originalLength: source.text.length,
-      keptLength: text.length
-    });
+    record(index, source, text);
     return { ...source, text };
   });
-  return { sources: budgeted, truncated };
+  return { sources: budgeted, truncated: [...records.values()] };
 }
 
 // src/coerce/validator.ts
@@ -862,7 +1074,7 @@ function entries(count) {
   return `${count} ${count === 1 ? "entry" : "entries"}`;
 }
 function validateConstraints(value, constraints, path, issues) {
-  const { minLength, maxLength, minimum, maximum, minItems, maxItems, pattern } = constraints;
+  const { minLength, maxLength, minimum, maximum, minItems, maxItems, pattern, format } = constraints;
   if (Array.isArray(value)) {
     if (minItems !== void 0 && value.length < minItems) {
       issues.push({
@@ -906,6 +1118,10 @@ function validateConstraints(value, constraints, path, issues) {
         received: value
       });
     }
+    if (format !== void 0) {
+      const message = validateFormat(value, format);
+      if (message) issues.push({ path, message, received: value });
+    }
     return;
   }
   if (typeof value === "number") {
@@ -1247,8 +1463,15 @@ function generateSpanId() {
 }
 var Tracer = class {
   sinks;
-  constructor(sinks) {
+  baseAttributes;
+  /**
+   * `baseAttributes` are merged into every span this tracer opens — how a
+   * batch stamps `itemIndex` on the spans of each item, so a sink can tell
+   * whose `llmCall` it is looking at under concurrency.
+   */
+  constructor(sinks, baseAttributes) {
     this.sinks = sinks ?? [];
+    this.baseAttributes = baseAttributes;
   }
   startSpan(name, attributes, parent) {
     return {
@@ -1256,7 +1479,7 @@ var Tracer = class {
       name,
       startTime: Date.now(),
       events: [],
-      attributes,
+      attributes: this.baseAttributes ? { ...this.baseAttributes, ...attributes } : attributes,
       parentId: parent?.id
     };
   }
@@ -1311,15 +1534,31 @@ async function resolveEnums(schema, bundle, enumResolver, tracer, parent) {
     tracer.endSpan(span);
   }
 }
-async function runCoercion(input, options, { mode, provenance }) {
-  const { provider, schema, enumResolver, traceSinks } = options;
-  const bundle = options.bundle ?? bundleOf(schema);
+function emptyUsage() {
+  return { calls: 0, promptTokens: 0, completionTokens: 0, totalTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0 };
+}
+function addUsage(into, usage) {
+  into.calls += 1;
+  if (!usage) return;
+  into.promptTokens += usage.promptTokens;
+  into.completionTokens += usage.completionTokens;
+  into.totalTokens += usage.totalTokens;
+  into.cacheReadTokens += usage.cacheReadTokens ?? 0;
+  into.cacheWriteTokens += usage.cacheWriteTokens ?? 0;
+}
+function checkOptions(options) {
   const maxRepairAttempts = options.maxRepairAttempts ?? 0;
   if (!Number.isInteger(maxRepairAttempts) || maxRepairAttempts < 0) {
     throw new RangeError(
       `maxRepairAttempts must be a non-negative integer, got ${String(options.maxRepairAttempts)}`
     );
   }
+  const retryOnEmpty = options.retryOnEmpty ?? 0;
+  if (!Number.isInteger(retryOnEmpty) || retryOnEmpty < 0) {
+    throw new RangeError(
+      `retryOnEmpty must be a non-negative integer, got ${String(options.retryOnEmpty)}`
+    );
+  }
   const instructions = normalizeInstructions(options.instructions);
   const onInvalidField = options.onInvalidField ?? "throw";
   if (!INVALID_FIELD_POLICIES.includes(onInvalidField)) {
@@ -1332,61 +1571,106 @@ async function runCoercion(input, options, { mode, provenance }) {
       `maxInputChars must be a positive integer, got ${String(options.maxInputChars)}`
     );
   }
+  return { maxRepairAttempts, retryOnEmpty, onInvalidField, instructions };
+}
+async function prepareRequest(options, { mode, provenance }, instructions, sourceLabels, tracer, rootSpan) {
+  const { schema, enumResolver } = options;
+  const bundle = options.bundle ?? bundleOf(schema);
+  const resolvedEnums = await resolveEnums(schema, bundle, enumResolver, tracer, rootSpan);
+  const promptSpan = tracer.startSpan("buildPrompt", {}, rootSpan);
+  const basePrompt = buildPrompt(schema, bundle, { resolvedEnums, instructions });
+  const provenanceOptions = { sourceLabels, fields: options.provenanceFields };
+  const systemPrompt = provenance ? `${basePrompt}
+${provenanceInstructions(provenanceOptions)}` : basePrompt;
+  tracer.addEvent(promptSpan, "promptBuilt", {
+    promptLength: systemPrompt.length,
+    instructionCount: instructions.length,
+    mode
+  });
+  tracer.endSpan(promptSpan);
+  const request = provenance ? toProvenanceSchema(schema, bundle, provenanceOptions) : { schema, bundle };
+  const schemaSpan = tracer.startSpan("buildJsonSchema", {}, rootSpan);
+  const jsonSchema = runtimeSchemaToJsonSchema(request.schema, request.bundle, {
+    resolvedEnums
+  });
+  tracer.endSpan(schemaSpan);
+  return { systemPrompt, jsonSchema, schema: request.schema, bundle: request.bundle, resolvedEnums };
+}
+function isEmptyResult(data) {
+  return Object.values(data).every((value) => value === null || value === void 0);
+}
+var EMPTY_RETRY_NOTE = "A previous attempt at this extraction returned no fields. The sources above do state values for at least some fields; read them again and return every value that is stated, leaving out only what the sources genuinely do not say.";
+async function runCoercion(input, options, { mode, provenance, traceAttributes }) {
+  const { provider, schema, traceSinks } = options;
+  const bundle = options.bundle ?? bundleOf(schema);
+  const { maxRepairAttempts, retryOnEmpty, onInvalidField, instructions } = checkOptions(options);
   const rawSources = toSources(input);
-  const tracer = new Tracer(traceSinks);
+  const tracer = new Tracer(traceSinks, traceAttributes);
   const rootSpan = tracer.startSpan(mode, {
     schemaId: schema.id,
     provenance,
     onInvalidField,
     sourceCount: rawSources.length
   });
+  const usage = emptyUsage();
   try {
     const sources = await prepareSources(rawSources, options, tracer, rootSpan);
     const sourceLabels = sources.length > 1 ? sources.map((s) => s.label ?? "") : [];
-    const resolvedEnums = await resolveEnums(
-      schema,
-      bundle,
-      enumResolver,
+    const prepared = await prepareRequest(
+      options,
+      { mode, provenance },
+      instructions,
+      sourceLabels,
       tracer,
       rootSpan
     );
-    const promptSpan = tracer.startSpan("buildPrompt", {}, rootSpan);
-    const basePrompt = buildPrompt(schema, bundle, { resolvedEnums, instructions });
-    const systemPrompt = provenance ? `${basePrompt}
-${provenanceInstructions({ sourceLabels })}` : basePrompt;
-    tracer.addEvent(promptSpan, "promptBuilt", {
-      promptLength: systemPrompt.length,
-      instructionCount: instructions.length
-    });
-    tracer.endSpan(promptSpan);
-    const request = provenance ? toProvenanceSchema(schema, bundle, { sourceLabels }) : { schema, bundle };
-    const schemaSpan = tracer.startSpan("buildJsonSchema", {}, rootSpan);
-    const jsonSchema = runtimeSchemaToJsonSchema(request.schema, request.bundle, {
-      resolvedEnums
-    });
-    tracer.endSpan(schemaSpan);
+    const { systemPrompt, jsonSchema, resolvedEnums } = prepared;
     const validate = mode === "coerce" ? validateStrict : validatePartial;
     const renderedInput = renderSources(sources);
+    const hasInput = sources.some((s) => s.text.trim().length > 0);
     tracer.addEvent(rootSpan, "inputRendered", {
       sourceCount: sources.length,
       inputLength: renderedInput.length
     });
     let userInput = renderedInput;
     let issues = [];
-    let run = { data: {}, provenance: {}, issues: [] };
+    let run = { data: {}, provenance: {}, issues: [], usage };
+    let emptyRetries = 0;
+    const multiTurn = provider.supportsHistory === true;
+    const history = [];
+    const followUp = (rejected, text, folded) => {
+      if (multiTurn) {
+        history.push({ role: "assistant", data: rejected }, { role: "user", text });
+      } else {
+        userInput = folded;
+      }
+    };
     for (let attempt = 0; attempt <= maxRepairAttempts; attempt++) {
-      const llmSpan = tracer.startSpan("llmCall", { attempt }, rootSpan);
+      const llmSpan = tracer.startSpan("llmCall", { attempt, turns: history.length }, rootSpan);
       const response = await provider.complete({
         systemPrompt,
         userInput,
+        ...history.length > 0 ? { history: [...history] } : {},
         jsonSchema,
-        schema: request.schema,
-        bundle: request.bundle,
+        schema: prepared.schema,
+        bundle: prepared.bundle,
         resolvedEnums
       });
+      addUsage(usage, response.usage);
       tracer.addEvent(llmSpan, "responseReceived", { usage: response.usage });
       tracer.endSpan(llmSpan);
-      run = provenance ? { ...splitProvenance(response.data, schema), issues: [] } : { data: response.data, provenance: {}, issues: [] };
+      run = provenance ? { ...splitProvenance(response.data, schema), issues: [], usage } : { data: response.data, provenance: {}, issues: [], usage };
+      if (hasInput && emptyRetries < retryOnEmpty && isEmptyResult(run.data)) {
+        emptyRetries += 1;
+        tracer.addEvent(rootSpan, "emptyRetry", { retry: emptyRetries });
+        followUp(response.data, EMPTY_RETRY_NOTE, `${renderedInput}
+
+---
+
+${EMPTY_RETRY_NOTE}`);
+        attempt -= 1;
+        continue;
+      }
       const validationSpan = tracer.startSpan("validate", { attempt }, rootSpan);
       issues = validate(run.data, schema, bundle, { resolvedEnums });
       tracer.addEvent(validationSpan, "validated", { issueCount: issues.length });
@@ -1412,7 +1696,8 @@ ${provenanceInstructions({ sourceLabels })}` : basePrompt;
           return {
             data: outcome.data,
             provenance: pruneProvenance(run.provenance, outcome.resolved),
-            issues: outcome.resolved
+            issues: outcome.resolved,
+            usage
           };
         }
       }
@@ -1423,7 +1708,11 @@ ${provenanceInstructions({ sourceLabels })}` : basePrompt;
           issueCount: issues.length,
           paths: issues.map((issue) => issue.path)
         });
-        userInput = buildRepairInput(renderedInput, run.data, issues);
+        followUp(
+          response.data,
+          buildRepairCorrection(issues),
+          buildRepairInput(renderedInput, run.data, issues)
+        );
       }
     }
     throw new CoerceError(issues);
@@ -1433,7 +1722,8 @@ ${provenanceInstructions({ sourceLabels })}` : basePrompt;
 }
 async function prepareSources(sources, options, tracer, parent) {
   const { preprocess, maxInputChars, truncate } = options;
-  if (!preprocess && maxInputChars === void 0) {
+  const anyCapped = sources.some((s) => s.maxChars !== void 0);
+  if (!preprocess && maxInputChars === void 0 && !anyCapped) {
     return [...sources];
   }
   const span = tracer.startSpan("prepareInput", {}, parent);
@@ -1450,7 +1740,7 @@ async function prepareSources(sources, options, tracer, parent) {
         lengths: prepared.map((s) => s.text.length)
       });
     }
-    if (maxInputChars !== void 0) {
+    if (maxInputChars !== void 0 || prepared.some((s) => s.maxChars !== void 0)) {
       const budgeted = budgetSources(prepared, maxInputChars, truncate);
       prepared = budgeted.sources;
       if (budgeted.truncated.length > 0) {
@@ -1498,19 +1788,59 @@ async function partialCoerce(input, options) {
   });
   return stripNulls(data);
 }
+async function coerceDetailed(input, options) {
+  const { data, issues, usage } = await runCoercion(input, options, {
+    mode: "coerce",
+    provenance: false
+  });
+  return { data, issues, usage };
+}
+async function partialCoerceDetailed(input, options) {
+  const { data, issues, usage } = await runCoercion(input, options, {
+    mode: "partialCoerce",
+    provenance: false
+  });
+  return { data: stripNulls(data), issues, usage };
+}
+var PRIME_INPUT = "Cache warm-up. There is no input to extract from; return an object with every field null.";
+async function primeCache(options) {
+  const { mode = "coerce", provenance = false, ...coerceOptions } = options;
+  const { instructions } = checkOptions(coerceOptions);
+  const tracer = new Tracer(coerceOptions.traceSinks);
+  const rootSpan = tracer.startSpan("primeCache", { schemaId: coerceOptions.schema.id, mode, provenance });
+  const usage = emptyUsage();
+  try {
+    const prepared = await prepareRequest(coerceOptions, { mode, provenance }, instructions, [], tracer, rootSpan);
+    const llmSpan = tracer.startSpan("llmCall", { attempt: 0, warmup: true }, rootSpan);
+    const response = await coerceOptions.provider.complete({
+      systemPrompt: prepared.systemPrompt,
+      userInput: renderSources(toSources(PRIME_INPUT)),
+      jsonSchema: prepared.jsonSchema,
+      schema: prepared.schema,
+      bundle: prepared.bundle,
+      resolvedEnums: prepared.resolvedEnums
+    });
+    addUsage(usage, response.usage);
+    tracer.addEvent(llmSpan, "responseReceived", { usage: response.usage });
+    tracer.endSpan(llmSpan);
+    return { schemaId: coerceOptions.schema.id, mode, provenance, usage, primedAt: (/* @__PURE__ */ new Date()).toISOString() };
+  } finally {
+    tracer.endSpan(rootSpan);
+  }
+}
 async function coerceWithProvenance(input, options) {
-  const { data, provenance, issues } = await runCoercion(input, options, {
+  const { data, provenance, issues, usage } = await runCoercion(input, options, {
     mode: "coerce",
     provenance: true
   });
-  return { data, provenance, issues };
+  return { data, provenance, issues, usage };
 }
 async function partialCoerceWithProvenance(input, options) {
-  const { data, provenance, issues } = await runCoercion(input, options, {
+  const { data, provenance, issues, usage } = await runCoercion(input, options, {
     mode: "partialCoerce",
     provenance: true
   });
-  return { data: stripNulls(data), provenance, issues };
+  return { data: stripNulls(data), provenance, issues, usage };
 }
 
 // src/coerce/coerce-many.ts
@@ -1551,60 +1881,107 @@ var BackoffGate = class {
     this.streak = 0;
   }
 };
+var InputQueue = class {
+  iterator;
+  pulling = Promise.resolve();
+  index = 0;
+  constructor(inputs) {
+    this.iterator = Symbol.asyncIterator in inputs ? inputs[Symbol.asyncIterator]() : inputs[Symbol.iterator]();
+  }
+  next() {
+    const pull = this.pulling.then(async () => {
+      const result = await this.iterator.next();
+      if (result.done) return void 0;
+      return { index: this.index++, input: result.value };
+    });
+    this.pulling = pull.catch(() => void 0);
+    return pull;
+  }
+};
+function labelOf(input) {
+  if (typeof input === "string") return void 0;
+  if (isSource(input)) return input.label;
+  return input[0]?.label;
+}
 async function coerceMany(inputs, options) {
   const {
     concurrency = DEFAULT_CONCURRENCY,
     mode = "coerce",
-    provenance = false,
-    primeCache = true,
+    provenance: provenanceOption = false,
+    primeCache: primeCache2 = true,
+    primed,
     onItem,
     signal,
+    retry: retryOptions,
     ...coerceOptions
   } = options;
-  const retry = { ...DEFAULT_RETRY, ...options.retry };
+  const retry = { ...DEFAULT_RETRY, ...retryOptions };
+  const provenance = provenanceOption !== false;
+  if (Array.isArray(provenanceOption)) {
+    coerceOptions.provenanceFields = provenanceOption;
+  }
   if (!Number.isInteger(concurrency) || concurrency < 1) {
     throw new RangeError(`concurrency must be a positive integer, got ${String(concurrency)}`);
   }
   if (!Number.isInteger(retry.attempts) || retry.attempts < 0) {
     throw new RangeError(`retry.attempts must be a non-negative integer, got ${String(retry.attempts)}`);
   }
-  const results = new Array(inputs.length);
+  const results = [];
   const gate = new BackoffGate(retry);
-  async function runOne(index) {
+  const queue = new InputQueue(inputs);
+  async function runOne(index, input) {
     let attempts = 0;
     let result;
+    let usage = emptyUsage();
+    const label = labelOf(input);
+    const traceAttributes = { itemIndex: index, ...label !== void 0 ? { itemLabel: label } : {} };
     for (; ; ) {
       if (signal?.aborted) {
-        result = { ok: false, index, error: signal.reason ?? new Error("Batch aborted"), attempts };
+        result = { ok: false, index, error: signal.reason ?? new Error("Batch aborted"), usage, attempts };
         break;
       }
       await gate.wait();
       attempts += 1;
       try {
-        const run = await runCoercion(inputs[index], coerceOptions, { mode, provenance });
+        const run = await runCoercion(input, coerceOptions, { mode, provenance, traceAttributes });
         const data = mode === "partialCoerce" ? stripNulls(run.data) : run.data;
         gate.succeeded();
-        result = { ok: true, index, data, provenance: run.provenance, issues: run.issues, attempts };
+        result = { ok: true, index, data, provenance: run.provenance, issues: run.issues, usage: run.usage, attempts };
         break;
       } catch (error) {
+        usage = emptyUsage();
         if (isRetryable(error) && attempts <= retry.attempts) {
           gate.failed();
           continue;
         }
-        result = { ok: false, index, error, attempts };
+        result = { ok: false, index, error, usage, attempts };
         break;
       }
     }
     results[index] = result;
     onItem?.(result);
   }
-  let next = 0;
-  if (primeCache && inputs.length > 1) {
-    await runOne(next++);
+  let pending;
+  const warmup = primed ?? (primeCache2 === "eager" ? primeCache({ ...coerceOptions, mode, provenance }) : void 0);
+  if (warmup) {
+    await Promise.resolve(warmup).catch(() => void 0);
+  } else if (primeCache2 === true) {
+    const first = await queue.next();
+    if (first === void 0) return results;
+    const second = await queue.next();
+    if (second === void 0) {
+      await runOne(first.index, first.input);
+      return results;
+    }
+    await runOne(first.index, first.input);
+    pending = second;
   }
-  const workers = Array.from({ length: Math.min(concurrency, inputs.length) }, async () => {
-    while (next < inputs.length) {
-      await runOne(next++);
+  const workers = Array.from({ length: concurrency }, async () => {
+    for (; ; ) {
+      const item = pending ?? await queue.next();
+      pending = void 0;
+      if (item === void 0) return;
+      await runOne(item.index, item.input);
     }
   });
   await Promise.all(workers);
@@ -1643,6 +2020,7 @@ function resolveConfig(callConfig) {
     maxRepairAttempts: callConfig?.maxRepairAttempts ?? global.maxRepairAttempts,
     onInvalidField: callConfig?.onInvalidField ?? global.onInvalidField,
     instructions: callConfig?.instructions ?? global.instructions,
+    retryOnEmpty: callConfig?.retryOnEmpty ?? global.retryOnEmpty,
     maxInputChars: callConfig?.maxInputChars ?? global.maxInputChars,
     truncate: callConfig?.truncate ?? global.truncate,
     preprocess: callConfig?.preprocess ?? global.preprocess
@@ -1677,6 +2055,7 @@ var Coercible = class _Coercible {
       maxRepairAttempts: this._config.maxRepairAttempts,
       onInvalidField: this._config.onInvalidField,
       instructions: this._config.instructions,
+      retryOnEmpty: this._config.retryOnEmpty,
       maxInputChars: this._config.maxInputChars,
       truncate: this._config.truncate,
       preprocess: this._config.preprocess
@@ -1743,6 +2122,7 @@ var ConsoleSink = class {
   Constrain,
   Describe,
   EnumResolutionError,
+  FIELD_FORMATS,
   PROVENANCE_INSTRUCTIONS,
   SOURCE_INSTRUCTIONS,
   Schema,
@@ -1752,19 +2132,25 @@ var ConsoleSink = class {
   ValuesFrom,
   budgetSources,
   buildPrompt,
+  buildRepairCorrection,
   buildRepairInput,
   bundleOf,
   coerce,
+  coerceDetailed,
   coerceMany,
   coerceWithProvenance,
   collectEnumSources,
   defineSchema,
+  describeFormat,
   field,
+  formatToJsonSchema,
   isCoerceInput,
   isSource,
   normalizeInstructions,
   partialCoerce,
+  partialCoerceDetailed,
   partialCoerceWithProvenance,
+  primeCache,
   provenanceInstructions,
   renderSources,
   resolveEnumSources,
@@ -1775,6 +2161,7 @@ var ConsoleSink = class {
   toOpenAIJsonSchema,
   toProvenanceSchema,
   toSources,
+  validateFormat,
   validatePartial,
   validateStrict
 });