@wildo-ai/external-connectors-models 1.1.8 → 1.1.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -23,7 +23,7 @@ import { ExternalProvider_ExchangeProtocol_Kind, } from "../core/commons.externa
|
|
|
23
23
|
// executor never dispatches it, the capability slot never fills), whereas a new
|
|
24
24
|
// `runtimeContract.kind` fails COMPILATION at every switch over the union. That
|
|
25
25
|
// compile-enforcement is the whole argument, and it is verified rather than
|
|
26
|
-
// asserted — deleting an arm from `
|
|
26
|
+
// asserted — deleting an arm from `describeProviderProtocolSecretRef` reddens the
|
|
27
27
|
// `never` check with the offending contract named in the diagnostic.
|
|
28
28
|
//
|
|
29
29
|
// It also fills its own `EngineCapability.AI_DOCUMENT_EXTRACTION` slot, so an
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"document-extraction.external-connectors-models.schemas.js","sourceRoot":"","sources":["../../../../src/protocols/document-extraction.external-connectors-models.schemas.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AACvB,OAAO,EACL,sCAAsC,GACrC,MAAM,oDAAoD,CAAA;AAE7D,iEAAiE;AACjE,+BAA+B;AAC/B,WAAW;AACX,iEAAiE;AACjE,6EAA6E;AAC7E,mDAAmD;AACnD,EAAE;AACF,kEAAkE;AAClE,EAAE;AACF,iFAAiF;AACjF,iFAAiF;AACjF,gFAAgF;AAChF,+EAA+E;AAC/E,iFAAiF;AACjF,EAAE;AACF,+DAA+D;AAC/D,EAAE;AACF,kFAAkF;AAClF,iFAAiF;AACjF,8EAA8E;AAC9E,gFAAgF;AAChF,gFAAgF;AAChF,4EAA4E;AAC5E,4EAA4E;AAC5E,qEAAqE;AACrE,EAAE;AACF,8EAA8E;AAC9E,gFAAgF;AAChF,gFAAgF;AAChF,iFAAiF;AACjF,8EAA8E;AAC9E,iEAAiE;AAEjE;;;;GAIG;AACH,MAAM,CAAN,IAAY,gCASX;AATD,WAAY,gCAAgC;IAC1C;;;;;;OAMG;IACH,6EAAyC,CAAA;AAC3C,CAAC,EATW,gCAAgC,KAAhC,gCAAgC,QAS3C;AAED;;;;;;;GAOG;AACH,MAAM,CAAN,IAAY,uCAGX;AAHD,WAAY,uCAAuC;IACjD,qDAAqD;IACrD,2GAAgE,CAAA;AAClE,CAAC,EAHW,uCAAuC,KAAvC,uCAAuC,QAGlD;AAED;;;;;;;;;;;GAWG;AACH,MAAM,CAAN,IAAY,gCAGX;AAHD,WAAY,gCAAgC;IAC1C,+FAA+F;IAC/F,mFAA+C,CAAA;AACjD,CAAC,EAHW,gCAAgC,KAAhC,gCAAgC,QAG3C;AAsBD;;;;GAIG;AACH,MAAM,CAAC,MAAM,iCAAiC,GAG1C;IACF,CAAC,gCAAgC,CAAC,kBAAkB,CAAC,EAAE;QACrD,WAAW,EAAE,SAAS;QACtB;;;;;;;;;;;WAWG;QACH,kBAAkB,EAAE,CAAC,iBAAiB,EAAE,WAAW,EAAE,YAAY,EAAE,YAAY,EAAE,YAAY,CAAC;QAC9F,eAAe,EAAE,CAAC,gCAAgC,CAAC,qBAAqB,CAAC;KAC1E;CACF,CAAC;AAEF;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,kDAAkD,GAG3D;IACF,CAAC,gCAAgC,CAAC,qBAAqB,CAAC,EAAE;QACxD,gCAAgC,CAAC,kBAAkB;KACpD;CACF,CAAC;AAEF,8HAA8H;AAC9H,MAAM,CAAN,IAAY,yCAGX;AAHD,WAAY,yCAAyC;IACnD,mEAAmE;IACnE,oGAAuD,CAAA;AACzD,CAAC,EAHW,yCAAyC,KAAzC,yCAAyC,QAGpD;AAUD;;;;;GAKG;AACH,MAAM,CAAC,MAAM,8CAA8C,GAAG,CAC5D,KAA+E,EAC/E,UAA0F,iCAAiC,EACvF,EAAE;IACtC,MAAM,UAAU,GAAG,kDAAkD,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;IAC3F,MAAM,WAAW,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE;QAC9C,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;QAC7B,OAAO,KAAK,CAAC,WAAW,KAAK,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,eAAe,CAAC,QAAQ,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;IACxG,CAAC,CAAC,CAAC;IACH,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7B,OAAO;YACL,QAAQ,EAAE,KAAK;YACf,OAAO,EAAE,yCAAyC,CAAC,yBAAyB;YAC5E,gBAAgB,EAAE,UAAU;SAC7B,CAAC;IACJ,CAAC;IACD,MAAM,KAAK,GAAG,WAAW,CAAC,CAAC,CAAE,CAAC;IAC9B,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;AAC1D,CAAC,CAAC;AAEF,kEAAkE;AAClE,yCAAyC;AACzC,kEAAkE;AAElE;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,sCAAsC,GAAG,CAAC,CAAC,MAAM,CAAC;IAC7D;;;;;;OAMG;IACH,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACrC,8FAA8F;IAC9F,iBAAiB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CACzC,CAAC,CAAC;AAGH,MAAM,CAAC,MAAM,2DAA2D,GAAG,CAAC,CAAC,MAAM,CAAC;IAClF,YAAY,EAAE,CAAC,CAAC,OAAO,CAAC,sCAAsC,CAAC,4BAA4B,CAAC;IAC5F,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE;IACvB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CAC7B,CAAC,CAAC;AAKH,kEAAkE;AAClE,kCAAkC;AAClC,kEAAkE;AAElE;;;;;;;;;;;;;;;GAeG;AACH,MAAM,CAAC,MAAM,yDAAyD,GAAG,CAAC,CAAC,MAAM,CAAC;IAChF,4FAA4F;IAC5F,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE;IACpB,oFAAoF;IACpF,kBAAkB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE;CACnD,CAAC,CAAC;AAKH;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,0DAA0D,GAAG,CAAC,CAAC,MAAM,CAAC;IACjF,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,kGAAkG;IAClG,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE;IACzC,iDAAiD;IACjD,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,KAAK,EAAE,sCAAsC;CAC9C,CAAC,CAAC;AAKH,kEAAkE;AAClE,+BAA+B;AAC/B,WAAW;AACX,kGAAkG;AAClG,mGAAmG;AACnG,0FAA0F;AAC1F,4FAA4F;AAC5F,+FAA+F;AAC/F,gGAAgG;AAChG,EAAE;AACF,gGAAgG;AAChG,iGAAiG;AACjG,+FAA+F;AAC/F,+FAA+F;AAC/F,8CAA8C;AAC9C,kEAAkE;AAElE;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAN,IAAY,+BAaX;AAbD,WAAY,+BAA+B;IACzC;;;;OAIG;IACH,8EAA2C,CAAA;IAC3C;;;;OAIG;IACH,gFAA6C,CAAA;AAC/C,CAAC,EAbW,+BAA+B,KAA/B,+BAA+B,QAa1C;AAiCD;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAN,IAAY,8BAGX;AAHD,WAAY,8BAA8B;IACxC,mEAAiC,CAAA;IACjC,qEAAmC,CAAA;AACrC,CAAC,EAHW,8BAA8B,KAA9B,8BAA8B,QAGzC;AAoBD,iGAAiG;AACjG,MAAM,CAAN,IAAY,2BAGX;AAHD,WAAY,2BAA2B;IACrC,kDAAmB,CAAA;IACnB,oDAAqB,CAAA;AACvB,CAAC,EAHW,2BAA2B,KAA3B,2BAA2B,QAGtC;AAgBD;;;;;;;;;;;;;;;GAeG;AACH,MAAM,CAAC,MAAM,oCAAoC,GAAG,CAAC,MAAc,EAAmC,EAAE;IACtG,MAAM,iBAAiB,GAAG,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG,CAAC;IAC3D,MAAM,mBAAmB,GAAG,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG,CAAC;IAC7D,MAAM,aAAa,GAAG,MAAM,IAAI,GAAG,CAAC;IACpC,OAAO,iBAAiB,IAAI,mBAAmB,IAAI,aAAa;QAC9D,CAAC,CAAC,+BAA+B,CAAC,oBAAoB;QACtD,CAAC,CAAC,+BAA+B,CAAC,mBAAmB,CAAC;AAC1D,CAAC,CAAC","sourcesContent":["import { z } from \"zod\"\nimport {\n ExternalProvider_ExchangeProtocol_Kind,\n } from \"../core/commons.external-connectors-models.schemas\"\n\n//===============================================================\n// Document Extraction Provider\n// --------\n// Document bytes → text. Serves the file-side TEXT PROVIDER that\n// `FileTextExtractionBackendService` owns, whose largest consumer is the RAG\n// ingestion path (`.ragSource()` corpus building).\n//\n// ## Why this is a provider at all, and not just a better library\n//\n// The local `pdfjs-dist` path reads a PDF's TEXT LAYER. A scanned document — the\n// common case for uploaded contracts, invoices and forms — is an IMAGE of a page\n// and carries no text layer, so it extracts to the empty string, contributes no\n// chunks, and becomes silently unanswerable. That is not a defect in the local\n// library; it is its nature, and it is the entire reason a provider exists here.\n//\n// ## Deliberately a SEPARATE protocol kind from `LLM_PROVIDER`\n//\n// Same reasoning as `EMBEDDINGS_PROVIDER` (RAG plan DL2) and the `IMAGE_PROVIDER`\n// precedent before it: an unmapped LLM operation key fails SILENTLY across every\n// consumer surface (the registry's semantic-operation map filters it out, the\n// executor never dispatches it, the capability slot never fills), whereas a new\n// `runtimeContract.kind` fails COMPILATION at every switch over the union. That\n// compile-enforcement is the whole argument, and it is verified rather than\n// asserted — deleting an arm from `secretRefForRuntimeContract` reddens the\n// `never` check with the offending contract named in the diagnostic.\n//\n// It also fills its own `EngineCapability.AI_DOCUMENT_EXTRACTION` slot, so an\n// application selects its OCR provider INDEPENDENTLY of its chat and embeddings\n// providers. Those are three unrelated procurement decisions — a deployment may\n// well want Anthropic for chat, OpenAI for embeddings, and Mistral for OCR — and\n// fusing any two of them into one `selection` entry makes that unexpressible.\n//===============================================================\n\n/**\n * Concrete document-extraction model ids the framework knows.\n *\n * Import path: `@wildo-ai/external-connectors-models`.\n */\nexport enum DocumentExtractionProvider_Model {\n /**\n * Mistral's OCR model alias. An ALIAS deliberately, not a dated snapshot: unlike an embedding\n * model — where the concrete id is persisted as a stamp on every chunk and a silent change\n * would fracture the vector space — extraction output is consumed immediately and re-derived\n * on every ingestion pass, so tracking the vendor's current model is a benefit rather than a\n * hazard.\n */\n MISTRAL_OCR_LATEST = \"mistral-ocr-latest\",\n}\n\n/**\n * The document-extraction operation vocabulary — deliberately DISJOINT from\n * `LLMProvider_OperationKey`, `ImageProvider_OperationKey` and\n * `EmbeddingsProvider_OperationKey` (each protocol kind owns its own operation enum, so a key's\n * membership decides its protocol without inspection).\n *\n * Import path: `@wildo-ai/external-connectors-models`.\n */\nexport enum DocumentExtractionProvider_OperationKey {\n /** One document's bytes → its text, page by page. */\n EXTRACT_DOCUMENT_TEXT = \"documentExtraction.extractDocumentText\",\n}\n\n/**\n * @wildo_source:part:start saas.external-providers.document-extraction.applicability facet:kind:api facet:audience:app-developer\n *\n * WHAT a document is being extracted FOR — the durable axis of extraction-model selection\n * (model-selection-by-applicability rule: an author names a CLASS, never a concrete model id;\n * the framework resolves the id from the catalog + preference below).\n *\n * One class today, and that is a statement rather than a placeholder: every extraction in the\n * engine feeds `FileTextExtractionBackendService`, and a second class would only be earned by a\n * surface with genuinely different requirements (a layout-preserving export, say, or a\n * cheap-and-approximate preview tier).\n */\nexport enum DocumentExtraction_Applicability {\n /** The default document-text surface: file text extraction, and the RAG corpus built on it. */\n DOCUMENT_TEXT_DEFAULT = \"document-text-default\",\n}\n/** @wildo_source:part:end saas.external-providers.document-extraction.applicability */\n\n/**\n * One row per document-extraction model: everything the framework knows about a concrete id.\n * Mirrors `Embeddings_ModelCatalogEntry`'s shape at the size this axis needs today.\n */\nexport type DocumentExtraction_ModelCatalogEntry = {\n /** The `ref` of the provider module that serves this id. */\n providerRef: string;\n /**\n * MIME types this model can process, as declared by the provider's documentation.\n *\n * Consulted BEFORE bytes leave the deployment: sending a document the model cannot read wastes\n * an egress of application content, which the data-minimization posture on the execution seam\n * exists to avoid.\n */\n supportedMimeTypes: readonly string[];\n /** FRAMEWORK OPINION — which applicability classes this model may be resolved for. */\n applicabilities: readonly DocumentExtraction_Applicability[];\n};\n\n/**\n * THE document-extraction model catalog — the single per-model fact table (the\n * `EMBEDDINGS_MODEL_CATALOG` discipline: an EXHAUSTIVE `Record`, so a new\n * `DocumentExtractionProvider_Model` member without a row is a compile error).\n */\nexport const DOCUMENT_EXTRACTION_MODEL_CATALOG: Record<\n DocumentExtractionProvider_Model,\n DocumentExtraction_ModelCatalogEntry\n> = {\n [DocumentExtractionProvider_Model.MISTRAL_OCR_LATEST]: {\n providerRef: \"mistral\",\n /**\n * Mistral's published OCR format list (docs.mistral.ai \"Known limitations\": \"Supported\n * formats for OCR: PDF, PNG, JPG, JPEG, TIFF, BMP, GIF, WEBP\"), narrowed to the MIME types\n * the engine's own extraction gate admits today.\n *\n * `application/pdf` is the only entry that is currently REACHABLE: image MIME types are\n * refused by `FileTextExtractionBackendService.isExtractionSupported`, which is a separate,\n * deliberate decision (an uploaded photo is not a document). The image types are declared\n * here anyway because this row states what the MODEL can do, not what the engine currently\n * asks it to do — and admitting images to extraction later must be an engine decision, not\n * an accidental consequence of a catalog edit.\n */\n supportedMimeTypes: [\"application/pdf\", \"image/png\", \"image/jpeg\", \"image/tiff\", \"image/webp\"],\n applicabilities: [DocumentExtraction_Applicability.DOCUMENT_TEXT_DEFAULT],\n },\n};\n\n/**\n * The framework's DEFAULT answer to \"which extraction model serves this class\", ordered by\n * preference, highest first.\n *\n * Unlike the embeddings preference — which is a MIGRATION KEY, because editing it re-binds a\n * class and forces every stamped chunk to be re-embedded — this one carries no durable\n * consequence: extraction output is re-derived on the next ingestion pass and nothing persists\n * the model that produced it. Editing this list changes what the next extraction uses and\n * nothing else.\n */\nexport const DOCUMENT_EXTRACTION_MODEL_APPLICABILITY_PREFERENCE: Record<\n DocumentExtraction_Applicability,\n readonly DocumentExtractionProvider_Model[]\n> = {\n [DocumentExtraction_Applicability.DOCUMENT_TEXT_DEFAULT]: [\n DocumentExtractionProvider_Model.MISTRAL_OCR_LATEST,\n ],\n};\n\n/** Why {@link resolveDocumentExtractionModelForApplicability} could not answer — carried into a named CONFIGURATION error. */\nexport enum DocumentExtraction_ModelResolutionFailure {\n /** The applicability has no candidate at all for this provider. */\n NO_CANDIDATE_FOR_PROVIDER = \"no_candidate_for_provider\",\n}\n\nexport type DocumentExtraction_ModelResolution =\n | { resolved: true; model: DocumentExtractionProvider_Model; entry: DocumentExtraction_ModelCatalogEntry }\n | {\n resolved: false;\n failure: DocumentExtraction_ModelResolutionFailure;\n consideredModels: readonly DocumentExtractionProvider_Model[];\n };\n\n/**\n * Resolve an applicability class to ONE concrete extraction model for a given provider.\n *\n * Fails CLOSED and explains itself (the `resolveModelForApplicability` contract): a silent\n * fallback to some other model would send application documents to a model nobody chose.\n */\nexport const resolveDocumentExtractionModelForApplicability = (\n input: { applicability: DocumentExtraction_Applicability; providerRef: string },\n catalog: Record<DocumentExtractionProvider_Model, DocumentExtraction_ModelCatalogEntry> = DOCUMENT_EXTRACTION_MODEL_CATALOG,\n): DocumentExtraction_ModelResolution => {\n const candidates = DOCUMENT_EXTRACTION_MODEL_APPLICABILITY_PREFERENCE[input.applicability];\n const forProvider = candidates.filter((model) => {\n const entry = catalog[model];\n return entry.providerRef === input.providerRef && entry.applicabilities.includes(input.applicability);\n });\n if (forProvider.length === 0) {\n return {\n resolved: false,\n failure: DocumentExtraction_ModelResolutionFailure.NO_CANDIDATE_FOR_PROVIDER,\n consideredModels: candidates,\n };\n }\n const model = forProvider[0]!;\n return { resolved: true, model, entry: catalog[model] };\n};\n\n// ===============================================================\n// Document Extraction Operations Context\n// ===============================================================\n\n/**\n * Usage of one extraction call.\n *\n * There is NO token dimension, and its absence is structural rather than an omission: an OCR\n * call is billed and reported per PAGE, not per token. Any consumer judging usage completeness\n * for an extraction must do so on `pagesProcessed` alone — the operation-aware discipline the\n * embeddings arm records for its own missing completion-token dimension.\n */\nexport const DocumentExtractionProvider_UsageSchema = z.object({\n /**\n * Pages the provider reports having processed.\n *\n * The FRAMEWORK's name for the dimension. Which vendor field it is projected from is the\n * provider's `parseExtractResponse` to know — the wired one reads Mistral's\n * `usage_info.pages_processed`, and a second provider will read something else.\n */\n pagesProcessed: z.number().optional(),\n /** Document size the provider reports having received. Same ownership as `pagesProcessed`. */\n documentSizeBytes: z.number().optional(),\n});\nexport type DocumentExtractionProvider_Usage = z.infer<typeof DocumentExtractionProvider_UsageSchema>;\n\nexport const DocumentExtractionProvider_ExecutionTransportMetadataSchema = z.object({\n protocolKind: z.literal(ExternalProvider_ExchangeProtocol_Kind.DOCUMENT_EXTRACTION_PROVIDER),\n providerRef: z.string(),\n model: z.string().optional(),\n});\nexport type DocumentExtractionProvider_ExecutionTransportMetadata = z.infer<\n typeof DocumentExtractionProvider_ExecutionTransportMetadataSchema\n>;\n\n// ===============================================================\n// Document Extraction I/O Schemas\n// ===============================================================\n\n/**\n * Input DESCRIPTOR of the {@link DocumentExtractionProvider_OperationKey.EXTRACT_DOCUMENT_TEXT}\n * operation — the metadata about the document, deliberately NOT the document.\n *\n * ⚠️ THE BYTES ARE ABSENT ON PURPOSE, and this is the design decision most likely to be\n * \"corrected\" by a later reader. A `documentBase64: z.string()` field here would be structurally\n * correct and operationally dangerous: a Zod-shaped input object is the thing every diagnostic\n * path in this codebase reaches for when something goes wrong — error `context`, debug logs,\n * agent-call traces — and a multi-megabyte base64 blob of an application's private contract is\n * the single worst payload to make casually loggable. The embeddings arm reached the same\n * conclusion from the other direction (its header forbids raw texts and vectors from ever\n * entering a trace); here the prohibition is enforced by the SHAPE rather than by discipline.\n *\n * The bytes travel as a `Buffer` on the execution service's own request type, which no\n * serializer walks.\n */\nexport const DocumentExtractionProvider_ExtractDocumentTextInputSchema = z.object({\n /** The document's MIME type — checked against the model's catalog row before any egress. */\n mimeType: z.string(),\n /** Byte length of the document being sent. Diagnostic and ceiling-checking only. */\n documentByteLength: z.number().int().nonnegative(),\n});\nexport type DocumentExtractionProvider_ExtractDocumentTextInput = z.infer<\n typeof DocumentExtractionProvider_ExtractDocumentTextInputSchema\n>;\n\n/**\n * Output of {@link DocumentExtractionProvider_OperationKey.EXTRACT_DOCUMENT_TEXT}.\n *\n * `text` is the concatenation of the provider's per-page output in document order. Providers in\n * this family return MARKDOWN per page (Mistral's `pages[].markdown`), which is kept as-is:\n * headings and table structure are signal for a retrieval corpus, and stripping them to \"plain\"\n * text would discard information the provider was paid to recover.\n */\nexport const DocumentExtractionProvider_ExtractDocumentTextOutputSchema = z.object({\n text: z.string(),\n /** Pages the provider actually returned — the authoritative count, not the request's estimate. */\n pageCount: z.number().int().nonnegative(),\n /** The concrete model that produced the text. */\n model: z.string(),\n usage: DocumentExtractionProvider_UsageSchema,\n});\nexport type DocumentExtractionProvider_ExtractDocumentTextOutput = z.infer<\n typeof DocumentExtractionProvider_ExtractDocumentTextOutputSchema\n>;\n\n// ===============================================================\n// The PROVIDER-OWNED WIRE SEAM\n// --------\n// Until 2026-09-09 this arm declared `baseUrl`, `endpointPath`, `apiKey` and `models` — where the\n// call goes and how it authenticates — and NOTHING about the body or the response. So the engine's\n// execution seam composed Mistral's `/v1/ocr` request (`document: { type: 'document_url',\n// document_url }`, `include_image_base64`) and read Mistral's response (`pages[].markdown`,\n// `usage_info.pages_processed`) with the vendor named nowhere, and a second provider declaring\n// DOCUMENT_EXTRACTION_PROVIDER would have been sent through Mistral's API carrying its own key.\n//\n// The remedy is the one the EMAIL and SMS arms already converged on: the provider declares PURE\n// FUNCTIONS and the engine owns transport and credentials. The types those functions are written\n// against live HERE rather than beside the contract, because the failure vocabulary is read by\n// BOTH sides — the provider produces it, the engine's caller branches on it — and a vocabulary\n// with two homes is a vocabulary that drifts.\n// ===============================================================\n\n/**\n * WHY an extraction failed, in the only two senses a caller can act on differently.\n *\n * This is the arm's most load-bearing output after the text itself, and the consequence of\n * confusing the two is CORPUS DELETION rather than a misleading log line:\n * `RagIngestionBackendService` deletes a field's entire chunk set when an ingestion pass yields no\n * text, so reporting a provider OUTAGE as an unreadable document silently destroys a live,\n * already-embedded corpus and reports success.\n *\n * It lived in `saas-backend-lib`'s execution seam until 2026-09-09, which is exactly why the wire\n * shape lived there too: a vocabulary a provider cannot reach is one a provider cannot answer in,\n * so classification had to be done by whoever could. Moving it here is what makes\n * {@link DocumentExtraction_ProviderExtraction} expressible at all.\n */\nexport enum DocumentExtraction_FailureClass {\n /**\n * The provider read the request and judged the DOCUMENT unprocessable — corrupt bytes, an\n * unsupported format, a password-protected file. PERMANENT: re-sending the same bytes cannot\n * succeed, so a sweep may safely stop retrying and skip the document.\n */\n UNREADABLE_DOCUMENT = \"unreadable_document\",\n /**\n * The provider could not be reached, refused the credential, throttled, or failed internally —\n * a network error, a 401/403, a 429, any 5xx, or a timeout. TRANSIENT and, decisively, says\n * NOTHING about the document. A caller must not conclude anything about the bytes from this.\n */\n PROVIDER_UNAVAILABLE = \"provider_unavailable\",\n}\n\n/**\n * What a provider's `parseExtractResponse` returns, and what a COMPLETE poll carries (#2020): the extraction, or a\n * CLASSIFIED refusal.\n *\n * A discriminated union rather than \"return the output or throw\", for the reason the failure class\n * exists: a thrown `Error` carries no class, and the engine would have to guess one — which is the\n * guess that deletes a corpus. A provider that cannot classify a response must still say which of\n * the two it is, and the arm's JSDoc tells it to prefer `PROVIDER_UNAVAILABLE` when unsure,\n * because that is the answer with no destructive consequence.\n */\nexport type DocumentExtraction_ProviderExtraction =\n | {\n extracted: true;\n /** Per-page text joined in document order. Markdown is kept as-is; see the output schema. */\n text: string;\n /** Pages the provider actually returned — the authoritative count, not the request estimate. */\n pageCount: number;\n /** The model the provider says produced the text. Absent means \"the resolved model\". */\n model?: string;\n usage?: DocumentExtractionProvider_Usage;\n }\n | {\n extracted: false;\n failureClass: DocumentExtraction_FailureClass;\n /**\n * Operator-facing reason. MUST NOT contain the provider's response body or any part of the\n * document: this message reaches logs, and a provider error body can echo request content.\n */\n message: string;\n };\n\n/**\n * HOW a document-extraction provider exchanges with its vendor (#2020) — the discriminator of the runtime arm's\n * `exchange`.\n *\n * Two shapes exist in the field and they cannot be folded into one:\n * - SINGLE_REQUEST: one request carries the document and the answer carries the text (Mistral OCR).\n * - SUBMIT_AND_POLL: the vendor accepts the document, answers with a job, and the text is fetched later by polling\n * that job until it is done (Azure Document Intelligence's `Operation-Location`, VLM Run). Asynchronous Textract is\n * a job-based vendor this form does NOT serve (SigV4, S3 input, POST polls); see the private exchange type.\n * Folding this into one request is what #2001 showed failing: the first request answers a job id, not text.\n *\n * A closed vocabulary with one home, so the engine's switch over it is exhaustive: a new form, or a deleted one, is a\n * compile error at the one place that executes the exchange.\n */\nexport enum DocumentExtractionExchangeForm {\n SINGLE_REQUEST = \"single-request\",\n SUBMIT_AND_POLL = \"submit-and-poll\",\n}\n\n/**\n * What a SUBMIT_AND_POLL provider's `parseSubmitResponse` returns: the vendor's job id, or a CLASSIFIED refusal. The\n * refusal carries the same two classes as an extraction, for the same reason: a vendor refusing THIS document at\n * submission is permanent, an outage is not, and confusing them deletes a corpus.\n */\nexport type DocumentExtraction_JobSubmission =\n | {\n submitted: true;\n /** The vendor's identifier for the job. Opaque to the engine; handed back to `buildPollRequest`. */\n jobId: string;\n }\n | {\n submitted: false;\n failureClass: DocumentExtraction_FailureClass;\n /** Operator-facing; MUST NOT contain the response body or the document (see the extraction's `message`). */\n message: string;\n };\n\n/** Where a polled job stands. PENDING keeps the engine polling until the provider's deadline. */\nexport enum DocumentExtraction_JobState {\n PENDING = \"pending\",\n COMPLETE = \"complete\",\n}\n\n/**\n * What a SUBMIT_AND_POLL provider's `parsePollResponse` returns for one poll: still running, or done — with the\n * extraction or its classified refusal, exactly what a single-request parser returns.\n *\n * COMPLETE means the VENDOR says the job has ended. A job the vendor reports failed is COMPLETE with a refusal, and\n * only then may the engine submit the document again, because only then is nothing still running. A poll the vendor\n * answered with a transient error (a 5xx or 429 on the poll route) says nothing about the job, so it is PENDING: the\n * engine polls again until the deadline. Reading it as COMPLETE would stop the engine while the job may still run,\n * and a retry would then pay the vendor twice for one document.\n */\nexport type DocumentExtraction_JobPoll =\n | { state: DocumentExtraction_JobState.PENDING }\n | { state: DocumentExtraction_JobState.COMPLETE; extraction: DocumentExtraction_ProviderExtraction };\n\n/**\n * The framework's DEFAULT reading of an extraction response's HTTP status.\n *\n * Offered to every provider's `parseExtractResponse` rather than applied by the engine, because\n * only the provider knows whether its vendor also signals a per-document verdict inside a 2xx\n * body (Twilio does exactly that on the SMS arm). What it prevents is each provider re-deriving a\n * classification whose worst case is corpus deletion.\n *\n * Every boundary is deliberate:\n * - 4xx generally means the provider READ the request and refused THIS DOCUMENT → permanent.\n * - 401/403 is a CREDENTIAL fault, not a document fault. Classifying it as unreadable would make\n * a rotated key look like a corpus of corrupt files and delete chunk sets across every\n * document for as long as the key was wrong.\n * - 408/429 are explicitly retryable by their own definition.\n * - 5xx is the provider's fault, never the document's.\n */\nexport const classifyDocumentExtractionHttpStatus = (status: number): DocumentExtraction_FailureClass => {\n const isCredentialFault = status === 401 || status === 403;\n const isThrottleOrTimeout = status === 408 || status === 429;\n const isServerFault = status >= 500;\n return isCredentialFault || isThrottleOrTimeout || isServerFault\n ? DocumentExtraction_FailureClass.PROVIDER_UNAVAILABLE\n : DocumentExtraction_FailureClass.UNREADABLE_DOCUMENT;\n};\n\n"]}
|
|
1
|
+
{"version":3,"file":"document-extraction.external-connectors-models.schemas.js","sourceRoot":"","sources":["../../../../src/protocols/document-extraction.external-connectors-models.schemas.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAA;AACvB,OAAO,EACL,sCAAsC,GACrC,MAAM,oDAAoD,CAAA;AAE7D,iEAAiE;AACjE,+BAA+B;AAC/B,WAAW;AACX,iEAAiE;AACjE,6EAA6E;AAC7E,mDAAmD;AACnD,EAAE;AACF,kEAAkE;AAClE,EAAE;AACF,iFAAiF;AACjF,iFAAiF;AACjF,gFAAgF;AAChF,+EAA+E;AAC/E,iFAAiF;AACjF,EAAE;AACF,+DAA+D;AAC/D,EAAE;AACF,kFAAkF;AAClF,iFAAiF;AACjF,8EAA8E;AAC9E,gFAAgF;AAChF,gFAAgF;AAChF,4EAA4E;AAC5E,kFAAkF;AAClF,qEAAqE;AACrE,EAAE;AACF,8EAA8E;AAC9E,gFAAgF;AAChF,gFAAgF;AAChF,iFAAiF;AACjF,8EAA8E;AAC9E,iEAAiE;AAEjE;;;;GAIG;AACH,MAAM,CAAN,IAAY,gCASX;AATD,WAAY,gCAAgC;IAC1C;;;;;;OAMG;IACH,6EAAyC,CAAA;AAC3C,CAAC,EATW,gCAAgC,KAAhC,gCAAgC,QAS3C;AAED;;;;;;;GAOG;AACH,MAAM,CAAN,IAAY,uCAGX;AAHD,WAAY,uCAAuC;IACjD,qDAAqD;IACrD,2GAAgE,CAAA;AAClE,CAAC,EAHW,uCAAuC,KAAvC,uCAAuC,QAGlD;AAED;;;;;;;;;;;GAWG;AACH,MAAM,CAAN,IAAY,gCAGX;AAHD,WAAY,gCAAgC;IAC1C,+FAA+F;IAC/F,mFAA+C,CAAA;AACjD,CAAC,EAHW,gCAAgC,KAAhC,gCAAgC,QAG3C;AAsBD;;;;GAIG;AACH,MAAM,CAAC,MAAM,iCAAiC,GAG1C;IACF,CAAC,gCAAgC,CAAC,kBAAkB,CAAC,EAAE;QACrD,WAAW,EAAE,SAAS;QACtB;;;;;;;;;;;WAWG;QACH,kBAAkB,EAAE,CAAC,iBAAiB,EAAE,WAAW,EAAE,YAAY,EAAE,YAAY,EAAE,YAAY,CAAC;QAC9F,eAAe,EAAE,CAAC,gCAAgC,CAAC,qBAAqB,CAAC;KAC1E;CACF,CAAC;AAEF;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,kDAAkD,GAG3D;IACF,CAAC,gCAAgC,CAAC,qBAAqB,CAAC,EAAE;QACxD,gCAAgC,CAAC,kBAAkB;KACpD;CACF,CAAC;AAEF,8HAA8H;AAC9H,MAAM,CAAN,IAAY,yCAGX;AAHD,WAAY,yCAAyC;IACnD,mEAAmE;IACnE,oGAAuD,CAAA;AACzD,CAAC,EAHW,yCAAyC,KAAzC,yCAAyC,QAGpD;AAUD;;;;;GAKG;AACH,MAAM,CAAC,MAAM,8CAA8C,GAAG,CAC5D,KAA+E,EAC/E,UAA0F,iCAAiC,EACvF,EAAE;IACtC,MAAM,UAAU,GAAG,kDAAkD,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;IAC3F,MAAM,WAAW,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE;QAC9C,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;QAC7B,OAAO,KAAK,CAAC,WAAW,KAAK,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,eAAe,CAAC,QAAQ,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;IACxG,CAAC,CAAC,CAAC;IACH,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7B,OAAO;YACL,QAAQ,EAAE,KAAK;YACf,OAAO,EAAE,yCAAyC,CAAC,yBAAyB;YAC5E,gBAAgB,EAAE,UAAU;SAC7B,CAAC;IACJ,CAAC;IACD,MAAM,KAAK,GAAG,WAAW,CAAC,CAAC,CAAE,CAAC;IAC9B,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;AAC1D,CAAC,CAAC;AAEF,kEAAkE;AAClE,yCAAyC;AACzC,kEAAkE;AAElE;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,sCAAsC,GAAG,CAAC,CAAC,MAAM,CAAC;IAC7D;;;;;;OAMG;IACH,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACrC,8FAA8F;IAC9F,iBAAiB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CACzC,CAAC,CAAC;AAGH,MAAM,CAAC,MAAM,2DAA2D,GAAG,CAAC,CAAC,MAAM,CAAC;IAClF,YAAY,EAAE,CAAC,CAAC,OAAO,CAAC,sCAAsC,CAAC,4BAA4B,CAAC;IAC5F,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE;IACvB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CAC7B,CAAC,CAAC;AAKH,kEAAkE;AAClE,kCAAkC;AAClC,kEAAkE;AAElE;;;;;;;;;;;;;;;GAeG;AACH,MAAM,CAAC,MAAM,yDAAyD,GAAG,CAAC,CAAC,MAAM,CAAC;IAChF,4FAA4F;IAC5F,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE;IACpB,oFAAoF;IACpF,kBAAkB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE;CACnD,CAAC,CAAC;AAKH;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,0DAA0D,GAAG,CAAC,CAAC,MAAM,CAAC;IACjF,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,kGAAkG;IAClG,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE;IACzC,iDAAiD;IACjD,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,KAAK,EAAE,sCAAsC;CAC9C,CAAC,CAAC;AAKH,kEAAkE;AAClE,+BAA+B;AAC/B,WAAW;AACX,kGAAkG;AAClG,mGAAmG;AACnG,0FAA0F;AAC1F,4FAA4F;AAC5F,+FAA+F;AAC/F,gGAAgG;AAChG,EAAE;AACF,gGAAgG;AAChG,iGAAiG;AACjG,+FAA+F;AAC/F,+FAA+F;AAC/F,8CAA8C;AAC9C,kEAAkE;AAElE;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAN,IAAY,+BAaX;AAbD,WAAY,+BAA+B;IACzC;;;;OAIG;IACH,8EAA2C,CAAA;IAC3C;;;;OAIG;IACH,gFAA6C,CAAA;AAC/C,CAAC,EAbW,+BAA+B,KAA/B,+BAA+B,QAa1C;AAiCD;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAN,IAAY,8BAGX;AAHD,WAAY,8BAA8B;IACxC,mEAAiC,CAAA;IACjC,qEAAmC,CAAA;AACrC,CAAC,EAHW,8BAA8B,KAA9B,8BAA8B,QAGzC;AAoBD,iGAAiG;AACjG,MAAM,CAAN,IAAY,2BAGX;AAHD,WAAY,2BAA2B;IACrC,kDAAmB,CAAA;IACnB,oDAAqB,CAAA;AACvB,CAAC,EAHW,2BAA2B,KAA3B,2BAA2B,QAGtC;AAgBD;;;;;;;;;;;;;;;GAeG;AACH,MAAM,CAAC,MAAM,oCAAoC,GAAG,CAAC,MAAc,EAAmC,EAAE;IACtG,MAAM,iBAAiB,GAAG,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG,CAAC;IAC3D,MAAM,mBAAmB,GAAG,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG,CAAC;IAC7D,MAAM,aAAa,GAAG,MAAM,IAAI,GAAG,CAAC;IACpC,OAAO,iBAAiB,IAAI,mBAAmB,IAAI,aAAa;QAC9D,CAAC,CAAC,+BAA+B,CAAC,oBAAoB;QACtD,CAAC,CAAC,+BAA+B,CAAC,mBAAmB,CAAC;AAC1D,CAAC,CAAC","sourcesContent":["import { z } from \"zod\"\nimport {\n ExternalProvider_ExchangeProtocol_Kind,\n } from \"../core/commons.external-connectors-models.schemas\"\n\n//===============================================================\n// Document Extraction Provider\n// --------\n// Document bytes → text. Serves the file-side TEXT PROVIDER that\n// `FileTextExtractionBackendService` owns, whose largest consumer is the RAG\n// ingestion path (`.ragSource()` corpus building).\n//\n// ## Why this is a provider at all, and not just a better library\n//\n// The local `pdfjs-dist` path reads a PDF's TEXT LAYER. A scanned document — the\n// common case for uploaded contracts, invoices and forms — is an IMAGE of a page\n// and carries no text layer, so it extracts to the empty string, contributes no\n// chunks, and becomes silently unanswerable. That is not a defect in the local\n// library; it is its nature, and it is the entire reason a provider exists here.\n//\n// ## Deliberately a SEPARATE protocol kind from `LLM_PROVIDER`\n//\n// Same reasoning as `EMBEDDINGS_PROVIDER` (RAG plan DL2) and the `IMAGE_PROVIDER`\n// precedent before it: an unmapped LLM operation key fails SILENTLY across every\n// consumer surface (the registry's semantic-operation map filters it out, the\n// executor never dispatches it, the capability slot never fills), whereas a new\n// `runtimeContract.kind` fails COMPILATION at every switch over the union. That\n// compile-enforcement is the whole argument, and it is verified rather than\n// asserted — deleting an arm from `describeProviderProtocolSecretRef` reddens the\n// `never` check with the offending contract named in the diagnostic.\n//\n// It also fills its own `EngineCapability.AI_DOCUMENT_EXTRACTION` slot, so an\n// application selects its OCR provider INDEPENDENTLY of its chat and embeddings\n// providers. Those are three unrelated procurement decisions — a deployment may\n// well want Anthropic for chat, OpenAI for embeddings, and Mistral for OCR — and\n// fusing any two of them into one `selection` entry makes that unexpressible.\n//===============================================================\n\n/**\n * Concrete document-extraction model ids the framework knows.\n *\n * Import path: `@wildo-ai/external-connectors-models`.\n */\nexport enum DocumentExtractionProvider_Model {\n /**\n * Mistral's OCR model alias. An ALIAS deliberately, not a dated snapshot: unlike an embedding\n * model — where the concrete id is persisted as a stamp on every chunk and a silent change\n * would fracture the vector space — extraction output is consumed immediately and re-derived\n * on every ingestion pass, so tracking the vendor's current model is a benefit rather than a\n * hazard.\n */\n MISTRAL_OCR_LATEST = \"mistral-ocr-latest\",\n}\n\n/**\n * The document-extraction operation vocabulary — deliberately DISJOINT from\n * `LLMProvider_OperationKey`, `ImageProvider_OperationKey` and\n * `EmbeddingsProvider_OperationKey` (each protocol kind owns its own operation enum, so a key's\n * membership decides its protocol without inspection).\n *\n * Import path: `@wildo-ai/external-connectors-models`.\n */\nexport enum DocumentExtractionProvider_OperationKey {\n /** One document's bytes → its text, page by page. */\n EXTRACT_DOCUMENT_TEXT = \"documentExtraction.extractDocumentText\",\n}\n\n/**\n * @wildo_source:part:start saas.external-providers.document-extraction.applicability facet:kind:api facet:audience:app-developer\n *\n * WHAT a document is being extracted FOR — the durable axis of extraction-model selection\n * (model-selection-by-applicability rule: an author names a CLASS, never a concrete model id;\n * the framework resolves the id from the catalog + preference below).\n *\n * One class today, and that is a statement rather than a placeholder: every extraction in the\n * engine feeds `FileTextExtractionBackendService`, and a second class would only be earned by a\n * surface with genuinely different requirements (a layout-preserving export, say, or a\n * cheap-and-approximate preview tier).\n */\nexport enum DocumentExtraction_Applicability {\n /** The default document-text surface: file text extraction, and the RAG corpus built on it. */\n DOCUMENT_TEXT_DEFAULT = \"document-text-default\",\n}\n/** @wildo_source:part:end saas.external-providers.document-extraction.applicability */\n\n/**\n * One row per document-extraction model: everything the framework knows about a concrete id.\n * Mirrors `Embeddings_ModelCatalogEntry`'s shape at the size this axis needs today.\n */\nexport type DocumentExtraction_ModelCatalogEntry = {\n /** The `ref` of the provider module that serves this id. */\n providerRef: string;\n /**\n * MIME types this model can process, as declared by the provider's documentation.\n *\n * Consulted BEFORE bytes leave the deployment: sending a document the model cannot read wastes\n * an egress of application content, which the data-minimization posture on the execution seam\n * exists to avoid.\n */\n supportedMimeTypes: readonly string[];\n /** FRAMEWORK OPINION — which applicability classes this model may be resolved for. */\n applicabilities: readonly DocumentExtraction_Applicability[];\n};\n\n/**\n * THE document-extraction model catalog — the single per-model fact table (the\n * `EMBEDDINGS_MODEL_CATALOG` discipline: an EXHAUSTIVE `Record`, so a new\n * `DocumentExtractionProvider_Model` member without a row is a compile error).\n */\nexport const DOCUMENT_EXTRACTION_MODEL_CATALOG: Record<\n DocumentExtractionProvider_Model,\n DocumentExtraction_ModelCatalogEntry\n> = {\n [DocumentExtractionProvider_Model.MISTRAL_OCR_LATEST]: {\n providerRef: \"mistral\",\n /**\n * Mistral's published OCR format list (docs.mistral.ai \"Known limitations\": \"Supported\n * formats for OCR: PDF, PNG, JPG, JPEG, TIFF, BMP, GIF, WEBP\"), narrowed to the MIME types\n * the engine's own extraction gate admits today.\n *\n * `application/pdf` is the only entry that is currently REACHABLE: image MIME types are\n * refused by `FileTextExtractionBackendService.isExtractionSupported`, which is a separate,\n * deliberate decision (an uploaded photo is not a document). The image types are declared\n * here anyway because this row states what the MODEL can do, not what the engine currently\n * asks it to do — and admitting images to extraction later must be an engine decision, not\n * an accidental consequence of a catalog edit.\n */\n supportedMimeTypes: [\"application/pdf\", \"image/png\", \"image/jpeg\", \"image/tiff\", \"image/webp\"],\n applicabilities: [DocumentExtraction_Applicability.DOCUMENT_TEXT_DEFAULT],\n },\n};\n\n/**\n * The framework's DEFAULT answer to \"which extraction model serves this class\", ordered by\n * preference, highest first.\n *\n * Unlike the embeddings preference — which is a MIGRATION KEY, because editing it re-binds a\n * class and forces every stamped chunk to be re-embedded — this one carries no durable\n * consequence: extraction output is re-derived on the next ingestion pass and nothing persists\n * the model that produced it. Editing this list changes what the next extraction uses and\n * nothing else.\n */\nexport const DOCUMENT_EXTRACTION_MODEL_APPLICABILITY_PREFERENCE: Record<\n DocumentExtraction_Applicability,\n readonly DocumentExtractionProvider_Model[]\n> = {\n [DocumentExtraction_Applicability.DOCUMENT_TEXT_DEFAULT]: [\n DocumentExtractionProvider_Model.MISTRAL_OCR_LATEST,\n ],\n};\n\n/** Why {@link resolveDocumentExtractionModelForApplicability} could not answer — carried into a named CONFIGURATION error. */\nexport enum DocumentExtraction_ModelResolutionFailure {\n /** The applicability has no candidate at all for this provider. */\n NO_CANDIDATE_FOR_PROVIDER = \"no_candidate_for_provider\",\n}\n\nexport type DocumentExtraction_ModelResolution =\n | { resolved: true; model: DocumentExtractionProvider_Model; entry: DocumentExtraction_ModelCatalogEntry }\n | {\n resolved: false;\n failure: DocumentExtraction_ModelResolutionFailure;\n consideredModels: readonly DocumentExtractionProvider_Model[];\n };\n\n/**\n * Resolve an applicability class to ONE concrete extraction model for a given provider.\n *\n * Fails CLOSED and explains itself (the `resolveModelForApplicability` contract): a silent\n * fallback to some other model would send application documents to a model nobody chose.\n */\nexport const resolveDocumentExtractionModelForApplicability = (\n input: { applicability: DocumentExtraction_Applicability; providerRef: string },\n catalog: Record<DocumentExtractionProvider_Model, DocumentExtraction_ModelCatalogEntry> = DOCUMENT_EXTRACTION_MODEL_CATALOG,\n): DocumentExtraction_ModelResolution => {\n const candidates = DOCUMENT_EXTRACTION_MODEL_APPLICABILITY_PREFERENCE[input.applicability];\n const forProvider = candidates.filter((model) => {\n const entry = catalog[model];\n return entry.providerRef === input.providerRef && entry.applicabilities.includes(input.applicability);\n });\n if (forProvider.length === 0) {\n return {\n resolved: false,\n failure: DocumentExtraction_ModelResolutionFailure.NO_CANDIDATE_FOR_PROVIDER,\n consideredModels: candidates,\n };\n }\n const model = forProvider[0]!;\n return { resolved: true, model, entry: catalog[model] };\n};\n\n// ===============================================================\n// Document Extraction Operations Context\n// ===============================================================\n\n/**\n * Usage of one extraction call.\n *\n * There is NO token dimension, and its absence is structural rather than an omission: an OCR\n * call is billed and reported per PAGE, not per token. Any consumer judging usage completeness\n * for an extraction must do so on `pagesProcessed` alone — the operation-aware discipline the\n * embeddings arm records for its own missing completion-token dimension.\n */\nexport const DocumentExtractionProvider_UsageSchema = z.object({\n /**\n * Pages the provider reports having processed.\n *\n * The FRAMEWORK's name for the dimension. Which vendor field it is projected from is the\n * provider's `parseExtractResponse` to know — the wired one reads Mistral's\n * `usage_info.pages_processed`, and a second provider will read something else.\n */\n pagesProcessed: z.number().optional(),\n /** Document size the provider reports having received. Same ownership as `pagesProcessed`. */\n documentSizeBytes: z.number().optional(),\n});\nexport type DocumentExtractionProvider_Usage = z.infer<typeof DocumentExtractionProvider_UsageSchema>;\n\nexport const DocumentExtractionProvider_ExecutionTransportMetadataSchema = z.object({\n protocolKind: z.literal(ExternalProvider_ExchangeProtocol_Kind.DOCUMENT_EXTRACTION_PROVIDER),\n providerRef: z.string(),\n model: z.string().optional(),\n});\nexport type DocumentExtractionProvider_ExecutionTransportMetadata = z.infer<\n typeof DocumentExtractionProvider_ExecutionTransportMetadataSchema\n>;\n\n// ===============================================================\n// Document Extraction I/O Schemas\n// ===============================================================\n\n/**\n * Input DESCRIPTOR of the {@link DocumentExtractionProvider_OperationKey.EXTRACT_DOCUMENT_TEXT}\n * operation — the metadata about the document, deliberately NOT the document.\n *\n * ⚠️ THE BYTES ARE ABSENT ON PURPOSE, and this is the design decision most likely to be\n * \"corrected\" by a later reader. A `documentBase64: z.string()` field here would be structurally\n * correct and operationally dangerous: a Zod-shaped input object is the thing every diagnostic\n * path in this codebase reaches for when something goes wrong — error `context`, debug logs,\n * agent-call traces — and a multi-megabyte base64 blob of an application's private contract is\n * the single worst payload to make casually loggable. The embeddings arm reached the same\n * conclusion from the other direction (its header forbids raw texts and vectors from ever\n * entering a trace); here the prohibition is enforced by the SHAPE rather than by discipline.\n *\n * The bytes travel as a `Buffer` on the execution service's own request type, which no\n * serializer walks.\n */\nexport const DocumentExtractionProvider_ExtractDocumentTextInputSchema = z.object({\n /** The document's MIME type — checked against the model's catalog row before any egress. */\n mimeType: z.string(),\n /** Byte length of the document being sent. Diagnostic and ceiling-checking only. */\n documentByteLength: z.number().int().nonnegative(),\n});\nexport type DocumentExtractionProvider_ExtractDocumentTextInput = z.infer<\n typeof DocumentExtractionProvider_ExtractDocumentTextInputSchema\n>;\n\n/**\n * Output of {@link DocumentExtractionProvider_OperationKey.EXTRACT_DOCUMENT_TEXT}.\n *\n * `text` is the concatenation of the provider's per-page output in document order. Providers in\n * this family return MARKDOWN per page (Mistral's `pages[].markdown`), which is kept as-is:\n * headings and table structure are signal for a retrieval corpus, and stripping them to \"plain\"\n * text would discard information the provider was paid to recover.\n */\nexport const DocumentExtractionProvider_ExtractDocumentTextOutputSchema = z.object({\n text: z.string(),\n /** Pages the provider actually returned — the authoritative count, not the request's estimate. */\n pageCount: z.number().int().nonnegative(),\n /** The concrete model that produced the text. */\n model: z.string(),\n usage: DocumentExtractionProvider_UsageSchema,\n});\nexport type DocumentExtractionProvider_ExtractDocumentTextOutput = z.infer<\n typeof DocumentExtractionProvider_ExtractDocumentTextOutputSchema\n>;\n\n// ===============================================================\n// The PROVIDER-OWNED WIRE SEAM\n// --------\n// Until 2026-09-09 this arm declared `baseUrl`, `endpointPath`, `apiKey` and `models` — where the\n// call goes and how it authenticates — and NOTHING about the body or the response. So the engine's\n// execution seam composed Mistral's `/v1/ocr` request (`document: { type: 'document_url',\n// document_url }`, `include_image_base64`) and read Mistral's response (`pages[].markdown`,\n// `usage_info.pages_processed`) with the vendor named nowhere, and a second provider declaring\n// DOCUMENT_EXTRACTION_PROVIDER would have been sent through Mistral's API carrying its own key.\n//\n// The remedy is the one the EMAIL and SMS arms already converged on: the provider declares PURE\n// FUNCTIONS and the engine owns transport and credentials. The types those functions are written\n// against live HERE rather than beside the contract, because the failure vocabulary is read by\n// BOTH sides — the provider produces it, the engine's caller branches on it — and a vocabulary\n// with two homes is a vocabulary that drifts.\n// ===============================================================\n\n/**\n * WHY an extraction failed, in the only two senses a caller can act on differently.\n *\n * This is the arm's most load-bearing output after the text itself, and the consequence of\n * confusing the two is CORPUS DELETION rather than a misleading log line:\n * `RagIngestionBackendService` deletes a field's entire chunk set when an ingestion pass yields no\n * text, so reporting a provider OUTAGE as an unreadable document silently destroys a live,\n * already-embedded corpus and reports success.\n *\n * It lived in `saas-backend-lib`'s execution seam until 2026-09-09, which is exactly why the wire\n * shape lived there too: a vocabulary a provider cannot reach is one a provider cannot answer in,\n * so classification had to be done by whoever could. Moving it here is what makes\n * {@link DocumentExtraction_ProviderExtraction} expressible at all.\n */\nexport enum DocumentExtraction_FailureClass {\n /**\n * The provider read the request and judged the DOCUMENT unprocessable — corrupt bytes, an\n * unsupported format, a password-protected file. PERMANENT: re-sending the same bytes cannot\n * succeed, so a sweep may safely stop retrying and skip the document.\n */\n UNREADABLE_DOCUMENT = \"unreadable_document\",\n /**\n * The provider could not be reached, refused the credential, throttled, or failed internally —\n * a network error, a 401/403, a 429, any 5xx, or a timeout. TRANSIENT and, decisively, says\n * NOTHING about the document. A caller must not conclude anything about the bytes from this.\n */\n PROVIDER_UNAVAILABLE = \"provider_unavailable\",\n}\n\n/**\n * What a provider's `parseExtractResponse` returns, and what a COMPLETE poll carries (#2020): the extraction, or a\n * CLASSIFIED refusal.\n *\n * A discriminated union rather than \"return the output or throw\", for the reason the failure class\n * exists: a thrown `Error` carries no class, and the engine would have to guess one — which is the\n * guess that deletes a corpus. A provider that cannot classify a response must still say which of\n * the two it is, and the arm's JSDoc tells it to prefer `PROVIDER_UNAVAILABLE` when unsure,\n * because that is the answer with no destructive consequence.\n */\nexport type DocumentExtraction_ProviderExtraction =\n | {\n extracted: true;\n /** Per-page text joined in document order. Markdown is kept as-is; see the output schema. */\n text: string;\n /** Pages the provider actually returned — the authoritative count, not the request estimate. */\n pageCount: number;\n /** The model the provider says produced the text. Absent means \"the resolved model\". */\n model?: string;\n usage?: DocumentExtractionProvider_Usage;\n }\n | {\n extracted: false;\n failureClass: DocumentExtraction_FailureClass;\n /**\n * Operator-facing reason. MUST NOT contain the provider's response body or any part of the\n * document: this message reaches logs, and a provider error body can echo request content.\n */\n message: string;\n };\n\n/**\n * HOW a document-extraction provider exchanges with its vendor (#2020) — the discriminator of the runtime arm's\n * `exchange`.\n *\n * Two shapes exist in the field and they cannot be folded into one:\n * - SINGLE_REQUEST: one request carries the document and the answer carries the text (Mistral OCR).\n * - SUBMIT_AND_POLL: the vendor accepts the document, answers with a job, and the text is fetched later by polling\n * that job until it is done (Azure Document Intelligence's `Operation-Location`, VLM Run). Asynchronous Textract is\n * a job-based vendor this form does NOT serve (SigV4, S3 input, POST polls); see the private exchange type.\n * Folding this into one request is what #2001 showed failing: the first request answers a job id, not text.\n *\n * A closed vocabulary with one home, so the engine's switch over it is exhaustive: a new form, or a deleted one, is a\n * compile error at the one place that executes the exchange.\n */\nexport enum DocumentExtractionExchangeForm {\n SINGLE_REQUEST = \"single-request\",\n SUBMIT_AND_POLL = \"submit-and-poll\",\n}\n\n/**\n * What a SUBMIT_AND_POLL provider's `parseSubmitResponse` returns: the vendor's job id, or a CLASSIFIED refusal. The\n * refusal carries the same two classes as an extraction, for the same reason: a vendor refusing THIS document at\n * submission is permanent, an outage is not, and confusing them deletes a corpus.\n */\nexport type DocumentExtraction_JobSubmission =\n | {\n submitted: true;\n /** The vendor's identifier for the job. Opaque to the engine; handed back to `buildPollRequest`. */\n jobId: string;\n }\n | {\n submitted: false;\n failureClass: DocumentExtraction_FailureClass;\n /** Operator-facing; MUST NOT contain the response body or the document (see the extraction's `message`). */\n message: string;\n };\n\n/** Where a polled job stands. PENDING keeps the engine polling until the provider's deadline. */\nexport enum DocumentExtraction_JobState {\n PENDING = \"pending\",\n COMPLETE = \"complete\",\n}\n\n/**\n * What a SUBMIT_AND_POLL provider's `parsePollResponse` returns for one poll: still running, or done — with the\n * extraction or its classified refusal, exactly what a single-request parser returns.\n *\n * COMPLETE means the VENDOR says the job has ended. A job the vendor reports failed is COMPLETE with a refusal, and\n * only then may the engine submit the document again, because only then is nothing still running. A poll the vendor\n * answered with a transient error (a 5xx or 429 on the poll route) says nothing about the job, so it is PENDING: the\n * engine polls again until the deadline. Reading it as COMPLETE would stop the engine while the job may still run,\n * and a retry would then pay the vendor twice for one document.\n */\nexport type DocumentExtraction_JobPoll =\n | { state: DocumentExtraction_JobState.PENDING }\n | { state: DocumentExtraction_JobState.COMPLETE; extraction: DocumentExtraction_ProviderExtraction };\n\n/**\n * The framework's DEFAULT reading of an extraction response's HTTP status.\n *\n * Offered to every provider's `parseExtractResponse` rather than applied by the engine, because\n * only the provider knows whether its vendor also signals a per-document verdict inside a 2xx\n * body (Twilio does exactly that on the SMS arm). What it prevents is each provider re-deriving a\n * classification whose worst case is corpus deletion.\n *\n * Every boundary is deliberate:\n * - 4xx generally means the provider READ the request and refused THIS DOCUMENT → permanent.\n * - 401/403 is a CREDENTIAL fault, not a document fault. Classifying it as unreadable would make\n * a rotated key look like a corpus of corrupt files and delete chunk sets across every\n * document for as long as the key was wrong.\n * - 408/429 are explicitly retryable by their own definition.\n * - 5xx is the provider's fault, never the document's.\n */\nexport const classifyDocumentExtractionHttpStatus = (status: number): DocumentExtraction_FailureClass => {\n const isCredentialFault = status === 401 || status === 403;\n const isThrottleOrTimeout = status === 408 || status === 429;\n const isServerFault = status >= 500;\n return isCredentialFault || isThrottleOrTimeout || isServerFault\n ? DocumentExtraction_FailureClass.PROVIDER_UNAVAILABLE\n : DocumentExtraction_FailureClass.UNREADABLE_DOCUMENT;\n};\n\n"]}
|