@wildo-ai/saas-backend-lib 1.1.3 → 1.1.4

package/dist/esm/external-providers/services/document-extraction.backend.service.js

@@ -20,11 +20,41 @@
  * Every sibling provider seam in this package hands an API key to an `@ai-sdk/*` client factory.
  * This one cannot: **`@ai-sdk/mistral@4.0.12` has ZERO OCR support** — the package exports no
  * `ocr` symbol and no client method reaching `POST /v1/ocr`, because the AI SDK models language,
- * embedding and image models and document extraction is none of them. So the request is built
- * here from the provider's declared `baseUrl`, `endpointPath` and `apiKey` header shape.
+ * embedding and image models and document extraction is none of them. So this service is the
+ * transport, issuing the request itself.
  *
- * That is a cost, and it is worth naming what it buys back: the contract declares the route and
- * the auth header, so a second provider in this family is a module edit rather than a change here.
+ * ## The line between engine and provider
+ *
+ * Until 2026-09-09 being the transport had quietly become being the CLIENT. This service composed
+ * MISTRAL's `/v1/ocr` request — `document: { type: 'document_url', document_url }` as an inline
+ * data URI, `include_image_base64: false` — and read MISTRAL's response — `pages[].markdown`,
+ * `usage_info.pages_processed` — while the arm it narrowed to (`kind: DOCUMENT_EXTRACTION`) is a
+ * PROTOCOL any provider may declare, and the vendor was named nowhere. A second provider selected
+ * for `AI_DOCUMENT_EXTRACTION` would have been sent through Mistral's API carrying its own key,
+ * and nothing would have refused it or logged it.
+ *
+ * It now mirrors `EmailBackendService` and `SMSBackendService`, the two arms that already have a
+ * solved shape for this: the provider declares PURE FUNCTIONS on its contract, and the engine owns
+ * transport.
+ *
+ *   ENGINE (here)  — resolve the provider, the model and the secret; compose the URL from the
+ *                    declared `baseUrl` + `endpointPath` and the API-key header from the declared
+ *                    `apiKey`; apply the engine's own byte ceiling and timeout; issue the request;
+ *                    validate what the provider's parser returns; retry per the failure class.
+ *   PROVIDER       — the request body and its `content-type`, how the bytes travel, and the
+ *                    projection of the vendor's response onto the framework's extraction or onto
+ *                    a {@link DocumentExtraction_FailureClass}.
+ *
+ * There is deliberately no consumer-side adapter registry of the kind billing uses. That shape
+ * exists only because a billing adapter is a CLASS in this package, which the provider package
+ * cannot reach without a reverse dependency edge; a document-extraction provider owes two
+ * functions over `@wildo-ai/external-connectors-models` types, which it supplies directly.
+ *
+ * Three floors stay HERE and a provider cannot lower them, because each one's failure mode is
+ * corpus deletion rather than a bad log line: a transport failure or timeout never reaches the
+ * parser and is `PROVIDER_UNAVAILABLE`; a parser that THROWS is `PROVIDER_UNAVAILABLE` rather
+ * than an unclassified crash; and a parser result that violates the standard output schema is
+ * `PROVIDER_UNAVAILABLE`, never "this document has no text".
  *
  * ## Failure posture — THREE outcomes, not two, and the third is the whole point
  *
@@ -32,7 +62,7 @@
  * not ask". Collapsing them is not a tidiness issue: `RagIngestionBackendService` deletes a
  * field's entire chunk set when an ingestion pass yields no text, so reporting a provider OUTAGE
  * as an unreadable document would silently destroy a live, already-embedded corpus and report
- * success. {@link DocumentExtractionFailureClass} carries that distinction, and
+ * success. {@link DocumentExtraction_FailureClass} carries that distinction, and
  * `FileTextExtractionBackendService` branches on it.
  *
  * ## Observability
@@ -58,9 +88,10 @@ var __param = (this && this.__param) || function (paramIndex, decorator) {
 };
 import { injectable, inject, optional } from 'inversify';
 import { EngineCapability, ErrorType, ErrorCustomMessageReference, AgentCallOutcome, AgentCallTraceKind, AgentTraceabilityLevel, } from '@wildo-ai/saas-models';
-import { DocumentExtractionProvider_OperationKey, ExternalProvider_ExchangeProtocol_Kind, resolveDocumentExtractionModelForApplicability, ProviderProtocolRuntimeKind, } from '@wildo-ai/external-connectors-models';
+import { DocumentExtraction_FailureClass, DocumentExtractionProvider_ExtractDocumentTextOutputSchema, DocumentExtractionProvider_OperationKey, ExternalProvider_ExchangeProtocol_Kind, resolveDocumentExtractionModelForApplicability, ProviderProtocolRuntimeKind, } from '@wildo-ai/external-connectors-models';
 import { SAAS_SERVICE_TYPES } from '../../inject/models/service-types.js';
 import { ErrorBuilderBackendService } from '../../errors/services/error-builder.backend.service.js';
+import { withEngineOwnedHeaders } from '../../security/outbound/outbound-request-headers.backend.utils.js';
 /**
  * Engine-side ceiling on the document bytes sent in one extraction request, applied as
  * `min(provider-declared, engine)` — and UNCONDITIONAL, so a provider that declares no limit
@@ -95,27 +126,15 @@ const DOCUMENT_EXTRACTION_DEFAULT_MAX_ATTEMPTS = 2;
  */
 export const DOCUMENT_EXTRACTION_TRACE_AGENT_REF = 'engine:document-extraction';
 /**
- * WHY an extraction failed, in the only two senses a caller can act on differently.
+ * A failed extraction, classified.
  *
- * This is the service's most load-bearing output after the text itself. See the file header: the
- * consequence of confusing the two is corpus deletion, not a misleading log line.
+ * The CLASS lives in `@wildo-ai/external-connectors-models` rather than here, and the move is what
+ * made the provider seam possible: a vocabulary a provider module cannot reach is one a provider
+ * cannot answer in, so before 2026-09-09 the classification had to be performed by whoever could
+ * reach it — this service — which is precisely how one vendor's status semantics became the
+ * framework's. The ERROR stays here because it is an engine construct: `FileTextExtractionBackendService`
+ * catches it, and a provider never throws.
  */
-export var DocumentExtractionFailureClass;
-(function (DocumentExtractionFailureClass) {
-    /**
-     * The provider read the request and judged the DOCUMENT unprocessable — corrupt bytes, an
-     * unsupported format, a password-protected file. PERMANENT: re-sending the same bytes cannot
-     * succeed, so a sweep may safely stop retrying and skip the document.
-     */
-    DocumentExtractionFailureClass["UNREADABLE_DOCUMENT"] = "unreadable_document";
-    /**
-     * The provider could not be reached, refused the credential, throttled, or failed internally —
-     * a network error, a 401/403, a 429, any 5xx, or a timeout. TRANSIENT and, decisively, says
-     * NOTHING about the document. A caller must not conclude anything about the bytes from this.
-     */
-    DocumentExtractionFailureClass["PROVIDER_UNAVAILABLE"] = "provider_unavailable";
-})(DocumentExtractionFailureClass || (DocumentExtractionFailureClass = {}));
-/** A failed extraction, classified. Thrown as {@link DocumentExtractionError}. */
 export class DocumentExtractionError extends Error {
     failureClass;
     httpStatus;
@@ -181,6 +200,40 @@ let DocumentExtractionBackendService = class DocumentExtractionBackendService {
                 },
             });
         }
+        /*
+         * A contract that reached the registry without an executable wire shape.
+         *
+         * This is NOT belt-and-braces on top of the type system, and it must not be deleted as such —
+         * the SMS arm carries the identical guard for the identical reason. A provider reaches a
+         * running application through a COMPANION CONTRIBUTION, and that wire record is deliberately
+         * permissive at both ends: `ProviderCompanionContributionSchema`
+         * (`@wildo-ai/saas-models`, `src/companion/provider-discovery.shared.schemas.ts`) is a
+         * `z.looseObject` so a record from a package version this repository never compiled rides
+         * through instead of failing the parse, it announces its `protocols` as bare strings rather
+         * than the typed enum, and its `runtimeImportPath` is resolved and imported at runtime. So the
+         * module supplying a document-extraction contract can be one no compilation here ever checked.
+         *
+         * Without this refusal that case is a `TypeError` from inside transport naming no provider —
+         * or, worse in this arm, a silent send through whatever shape the engine last knew. With it,
+         * the compile-time requirement on the arm and this refusal together are what make "never
+         * silently handed another vendor's shape" true for every declarer, not only the ones built in
+         * this repository. The message NAMES what the engine serves, so an operator learns the arm's
+         * requirement rather than a stack trace.
+         */
+        if (typeof runtimeContract.buildExtractRequest !== 'function'
+            || typeof runtimeContract.parseExtractResponse !== 'function') {
+            throw this.errorBuilder.buildError(ErrorType.CONFIGURATION, undefined, {
+                customMessageReference: ErrorCustomMessageReference.CONFIGURATION,
+                context: {
+                    providerRef,
+                    message: `Provider "${providerRef}" declares no document-extraction wire shape and cannot be used. `
+                        + 'Its DOCUMENT_EXTRACTION_PROVIDER runtime contract must declare buildExtractRequest and '
+                        + 'parseExtractResponse: the engine serves the transport (URL from baseUrl + endpointPath, the '
+                        + 'declared apiKey header, the byte ceiling, the timeout and the retry) and the provider owns the '
+                        + 'request body and the response projection.',
+                },
+            });
+        }
         const resolution = resolveDocumentExtractionModelForApplicability({ applicability, providerRef });
         // `=== false` rather than `!…`: this compiler configuration narrows the literal-discriminant
         // union on the explicit comparison but not on the truthiness form (the embeddings seam
@@ -210,6 +263,8 @@ let DocumentExtractionBackendService = class DocumentExtractionBackendService {
                 endpointPath: runtimeContract.endpointPath,
                 apiKeyHeaderName: runtimeContract.apiKey.headerName,
                 apiKeyPrefix: runtimeContract.apiKey.prefix ?? '',
+                buildExtractRequest: runtimeContract.buildExtractRequest,
+                parseExtractResponse: runtimeContract.parseExtractResponse,
             },
         };
     }
@@ -229,17 +284,21 @@ let DocumentExtractionBackendService = class DocumentExtractionBackendService {
      * physical call emits an agent-call trace (provider, model, pages, outcome) through the standard
      * sink chain, so a reviewer can enumerate what was sent and when.
      * assurance-control: WILDO.PRIVACY.DATA_MINIMIZATION — only the document bytes reach the
-     * provider: no filename, no row context, no identifiers, no organization or user id. The
-     * document is sent as an inline data URI rather than uploaded to the provider's file store,
-     * which means the provider retains no object of ours to delete (its Files API states a 30-day
-     * retention that an inline request never enters). Observability surfaces receive SUMMARIES only
-     * — the bytes and the extracted text never enter a trace, a log, or an error context.
+     * provider: no filename, no row context, no identifiers, no organization or user id. Since
+     * 2026-09-09 that is STRUCTURAL rather than a property of a body this service composes: the
+     * provider's `buildExtractRequest` receives `DocumentExtractionProviderRequestInput`, which
+     * carries the bytes, the MIME type and the resolved model and nothing else, so no provider can
+     * include an identifier it was never given. What FORM the bytes take on the wire is now the
+     * provider's decision and is recorded and pinned in its own module — the wired one inlines them
+     * as a `data:` URI precisely so no object of ours enters the vendor's file-store retention
+     * window. Observability surfaces receive SUMMARIES only — the bytes and the extracted text never
+     * enter a trace, a log, or an error context.
      * assurance-control: WILDO.PRIVACY.THIRD_PARTY_PROCESSING — routing documents here makes the
      * provider a PROCESSOR of application content. It is opt-in by construction: no bytes leave
      * unless an application selects a provider for `AI_DOCUMENT_EXTRACTION`, and with no selection
      * the local library serves every extraction with no egress at all.
      *
-     * Throws {@link DocumentExtractionError} with a {@link DocumentExtractionFailureClass}. See the
+     * Throws {@link DocumentExtractionError} with a {@link DocumentExtraction_FailureClass}. See the
      * file header for why the caller MUST branch on that class rather than treating any failure as
      * an unreadable document.
      */
@@ -320,7 +379,7 @@ let DocumentExtractionBackendService = class DocumentExtractionBackendService {
     }
     /**
      * Attempt the call up to `binding.maxAttempts` times, retrying ONLY
-     * {@link DocumentExtractionFailureClass.PROVIDER_UNAVAILABLE}.
+     * {@link DocumentExtraction_FailureClass.PROVIDER_UNAVAILABLE}.
      *
      * An `UNREADABLE_DOCUMENT` is never retried, and that asymmetry is the point: the same bytes
      * will be judged the same way, so a retry spends a page of billed quota to receive the identical
@@ -334,7 +393,7 @@ let DocumentExtractionBackendService = class DocumentExtractionBackendService {
             }
             catch (error) {
                 if (!(error instanceof DocumentExtractionError)
-                    || error.failureClass !== DocumentExtractionFailureClass.PROVIDER_UNAVAILABLE) {
+                    || error.failureClass !== DocumentExtraction_FailureClass.PROVIDER_UNAVAILABLE) {
                     throw error;
                 }
                 lastError = error;
@@ -347,95 +406,117 @@ let DocumentExtractionBackendService = class DocumentExtractionBackendService {
             }
         }
         // Unreachable with `maxAttempts >= 1` unless every attempt threw, in which case `lastError` is set.
-        throw lastError ?? new DocumentExtractionError(DocumentExtractionFailureClass.PROVIDER_UNAVAILABLE, 'Document extraction exhausted its attempts without producing a result.');
+        throw lastError ?? new DocumentExtractionError(DocumentExtraction_FailureClass.PROVIDER_UNAVAILABLE, 'Document extraction exhausted its attempts without producing a result.');
     }
-    /** One physical HTTPS request against the provider's declared route. */
+    /**
+     * One physical HTTPS request against the provider's declared route, with the provider's own body
+     * and the provider's own reading of the answer.
+     *
+     * Everything vendor-shaped comes off the contract. The engine contributes the URL, the API-key
+     * header and the timeout — note the header is applied AFTER the provider's transport headers, so
+     * a provider cannot take authentication over by naming that header.
+     */
     async invokeOnce(binding, apiKey, request) {
         const url = `${binding.baseUrl.replace(/\/+$/, '')}/${binding.endpointPath.replace(/^\/+/, '')}`;
-        // The document travels as an inline data URI. The provider's document chunk accepts a URL, and
-        // a `data:` URI is the form that keeps the bytes inside THIS request — the alternative (upload
-        // to the provider's file store, then reference the id) would leave an object of ours sitting in
-        // their retention window, and the alternative-alternative (a public URL to our object store)
-        // would require exposing application documents to the internet. Verified working against the
-        // live API on 2026-08-27 for a 129-page document.
-        const documentUrl = `data:${request.mimeType};base64,${request.document.toString('base64')}`;
+        // Only the bytes, the MIME type and the resolved model — the input type carries no identifier
+        // for a provider to include, which is what makes the data-minimization claim structural.
+        const providerRequest = binding.buildExtractRequest({
+            document: request.document,
+            mimeType: request.mimeType,
+            model: binding.model,
+        });
         const abortController = new AbortController();
         const timeout = setTimeout(() => abortController.abort(), DOCUMENT_EXTRACTION_REQUEST_TIMEOUT_MS);
         let response;
+        let responseText;
         try {
             response = await fetch(url, {
                 method: 'POST',
-                headers: {
-                    'Content-Type': 'application/json',
+                headers: withEngineOwnedHeaders(providerRequest.transportHeaders, {
                     [binding.apiKeyHeaderName]: `${binding.apiKeyPrefix}${apiKey}`,
-                },
-                body: JSON.stringify({
-                    model: binding.model,
-                    document: { type: 'document_url', document_url: documentUrl },
-                    // Images are not requested: this seam produces TEXT, and asking for base64 images would
-                    // multiply the response size by an order of magnitude for bytes nothing here consumes.
-                    include_image_base64: false,
                 }),
+                body: providerRequest.body,
                 signal: abortController.signal,
             });
+            responseText = await response.text();
         }
         catch (transportError) {
-            // A transport failure — DNS, TCP, TLS, or our own abort. Says nothing about the document.
-            throw new DocumentExtractionError(DocumentExtractionFailureClass.PROVIDER_UNAVAILABLE, `Document-extraction provider "${binding.providerRef}" was unreachable: `
+            // A transport failure — DNS, TCP, TLS, our own abort, or a body that never finished
+            // arriving. The parser never runs, so the engine classifies: this says nothing about the
+            // document. FLOOR ONE, and a provider cannot lower it.
+            throw new DocumentExtractionError(DocumentExtraction_FailureClass.PROVIDER_UNAVAILABLE, `Document-extraction provider "${binding.providerRef}" was unreachable: `
                 + (transportError instanceof Error ? transportError.message : String(transportError)));
         }
         finally {
             clearTimeout(timeout);
         }
-        if (!response.ok) {
-            // The status→class mapping, and every boundary in it is deliberate:
-            //   - 4xx generally means the provider READ our request and refused THIS DOCUMENT → permanent.
-            //   - 401/403 is a CREDENTIAL fault, not a document fault. Classifying it as unreadable would
-            //     make a rotated key look like a corpus of corrupt files, and would silently delete
-            //     chunk sets across every document while the key was wrong.
-            //   - 408/429 are explicitly retryable by their own definition.
-            //   - 5xx is the provider's fault, never the document's.
-            const isCredentialFault = response.status === 401 || response.status === 403;
-            const isThrottleOrTimeout = response.status === 408 || response.status === 429;
-            const isServerFault = response.status >= 500;
-            const failureClass = isCredentialFault || isThrottleOrTimeout || isServerFault
-                ? DocumentExtractionFailureClass.PROVIDER_UNAVAILABLE
-                : DocumentExtractionFailureClass.UNREADABLE_DOCUMENT;
-            // The response BODY is deliberately not carried into the message: a provider error body can
-            // echo request content, and this message reaches logs.
-            throw new DocumentExtractionError(failureClass, `Document-extraction provider "${binding.providerRef}" answered HTTP ${response.status}.`, response.status);
-        }
-        let parsed;
+        const parserInput = {
+            status: response.status,
+            headers: this.responseHeadersToRecord(response.headers),
+            body: this.bestEffortParseResponseBody(responseText),
+        };
+        let extraction;
         try {
-            parsed = (await response.json());
+            extraction = binding.parseExtractResponse(parserInput);
         }
-        catch (parseError) {
-            // A 200 whose body is not JSON is the provider misbehaving, not the document being bad.
-            throw new DocumentExtractionError(DocumentExtractionFailureClass.PROVIDER_UNAVAILABLE, `Document-extraction provider "${binding.providerRef}" returned a 200 with an unreadable body: `
-                + (parseError instanceof Error ? parseError.message : String(parseError)));
+        catch (parserError) {
+            // FLOOR TWO. A provider function that throws has produced no class, and the engine must not
+            // invent one — `PROVIDER_UNAVAILABLE` is the member with no destructive consequence, so a
+            // broken parser costs a retry and never a corpus.
+            throw new DocumentExtractionError(DocumentExtraction_FailureClass.PROVIDER_UNAVAILABLE, `Document-extraction provider "${binding.providerRef}" response parser threw: `
+                + (parserError instanceof Error ? parserError.message : String(parserError)), response.status);
         }
-        if (!Array.isArray(parsed.pages)) {
-            // Validate the ONE field this engine depends on. A response with no `pages` array is not a
-            // document with no text — it is a contract violation, and treating it as "this document is
-            // empty" is precisely how an upstream change would silently empty a corpus.
-            throw new DocumentExtractionError(DocumentExtractionFailureClass.PROVIDER_UNAVAILABLE, `Document-extraction provider "${binding.providerRef}" returned a response with no pages array.`);
+        if (extraction.extracted === false) {
+            // The provider's own verdict. `httpStatus` is carried for diagnostics only; the CLASS is
+            // what the caller branches on, and the message is the provider's (which the arm's JSDoc
+            // requires to be free of response-body and document content).
+            throw new DocumentExtractionError(extraction.failureClass, extraction.message, response.status);
+        }
+        /*
+         * FLOOR THREE, and the one most easily mistaken for belt-and-braces.
+         *
+         * A parser that returns `{ extracted: true }` with a missing `text` or a non-numeric
+         * `pageCount` produces, after coercion, exactly the shape of a document that legitimately
+         * contained no text — and THAT answer deletes a field's chunk set. So a result the standard
+         * output schema rejects is a contract violation, reported as `PROVIDER_UNAVAILABLE` rather
+         * than allowed to read as an empty document. The validation error is carried; the parser's
+         * output is not, because it holds the extracted text.
+         */
+        const validated = DocumentExtractionProvider_ExtractDocumentTextOutputSchema.safeParse({
+            text: extraction.text,
+            pageCount: extraction.pageCount,
+            model: extraction.model ?? binding.model,
+            usage: extraction.usage ?? {},
+        });
+        if (!validated.success) {
+            throw new DocumentExtractionError(DocumentExtraction_FailureClass.PROVIDER_UNAVAILABLE, `Document-extraction provider "${binding.providerRef}" response parser violated `
+                + `DocumentExtractionProvider_ExtractDocumentTextOutput: ${validated.error.message}`, response.status);
+        }
+        return validated.data;
+    }
+    /** Lower-cased header projection, so a provider's parser can index headers by a stable name. */
+    responseHeadersToRecord(headers) {
+        const result = {};
+        headers.forEach((value, key) => {
+            result[key.toLowerCase()] = value;
+        });
+        return result;
+    }
+    /**
+     * Decode a response body without assuming it is JSON. A vendor error page is frequently HTML or
+     * plain text, and handing the raw string to the provider's parser is more useful than throwing
+     * before it gets a chance to classify — the parser is the half that knows what its vendor emits.
+     */
+    bestEffortParseResponseBody(responseText) {
+        if (responseText.length === 0) {
+            return undefined;
+        }
+        try {
+            return JSON.parse(responseText);
+        }
+        catch {
+            return responseText;
         }
-        // Per-page markdown, joined in document order. Non-string page content is skipped rather than
-        // stringified — `String(undefined)` in a retrieval corpus is worse than a shorter document.
-        const pageTexts = parsed.pages.map((page) => (typeof page?.markdown === 'string' ? page.markdown : ''));
-        return {
-            text: pageTexts.join('\n'),
-            pageCount: parsed.pages.length,
-            model: typeof parsed.model === 'string' ? parsed.model : binding.model,
-            usage: {
-                ...(typeof parsed.usage_info?.pages_processed === 'number'
-                    ? { pagesProcessed: parsed.usage_info.pages_processed }
-                    : {}),
-                ...(typeof parsed.usage_info?.doc_size_bytes === 'number'
-                    ? { documentSizeBytes: parsed.usage_info.doc_size_bytes }
-                    : {}),
-            },
-        };
     }
     /**
      * One trace per physical extraction call — summaries only (file header: the bytes and the