@isomorph.ai/app-sdk 1.34.1 → 1.35.1

package/README.md

@@ -36,6 +36,21 @@ Replace the connection/resource with the app's existing approved integration. To
 
 For images call `generateImage({model: gateway.image(process.env.ISOMORPH_AI_IMAGE_MODEL_ALIAS!), prompt: 'A blue notebook'})`. Editing uses `prompt: {text: 'Make it red', images: [inputBytes]}` only when `isomorph ai models` advertises edit. Reuse app files upload/signed reads for authorized input and output storage; respect existing transfer/action limits and never return credentials.
 
+To read a PDF or an image (a scan, a photo of a form) call `isomorph.ai.ocr(file, options?)` from an action or a job. It sends the document to the governed gateway's OCR route under the company's document model and answers with the text of each page as markdown; `layout: true` also returns each page's blocks (kind, text, box normalised to 0..1 of the page, confidence), and `pages` narrows the read (`'0-2'` or `[0, 3]`). The file is a `Blob`, bytes (`Uint8Array` or `ArrayBuffer`, taken as a PDF) or `{ mediaType, data }`. It is a server call: a page that needs a document read invokes the action. When IT has not routed an OCR model the call fails with `aiErrorCode(error) === 'AI_NOT_ENABLED'`.
+
+```ts
+// actions/read-invoice.ts
+import { createClient, aiErrorCode } from '@isomorph.ai/app-sdk';
+
+export default async function readInvoice(input: { path: string }) {
+  const isomorph = createClient();
+  const { signedUrl } = await isomorph.files.createSignedUrl(input.path);
+  const file = await (await fetch(signedUrl)).blob();
+  const read = await isomorph.ai.ocr(file, { layout: true });
+  return { text: read.markdown, pages: read.pages.length, tables: read.pages.flatMap(page => page.blocks!.filter(block => block.kind === 'table').map(block => block.text)) };
+}
+```
+
 ## Company integrations
 
 `isomorph.integrations.execute(connection, request)` is one method over a closed

package/dist/index.d.ts

@@ -1815,6 +1815,55 @@ export interface AiEmbedResult {
     };
     traceId?: string;
 }
+export type AiOcrFile = Blob | Uint8Array | ArrayBuffer | {
+    /** The document's media type: application/pdf, image/png, image/jpeg, … */
+    mediaType: string;
+    /** The bytes, or their base64 text. */
+    data: Uint8Array | ArrayBuffer | string;
+    /** For the caller's bookkeeping only; it is not sent to the gateway. */
+    filename?: string;
+};
+export interface AiOcrOptions {
+    /** The pages to read: a range such as '0-2' or a list of zero-based page indexes. Omitted reads every page. */
+    pages?: string | number[];
+    /** true also returns each page's layout blocks, with their boxes and confidence. */
+    layout?: boolean;
+}
+export interface AiOcrBlock {
+    /** The block's kind as the provider names it: text, table, figure, … */
+    kind: string;
+    text: string;
+    /** The block's box on its page, normalised to 0..1 of the page's width and height. */
+    box: {
+        x0: number;
+        y0: number;
+        x1: number;
+        y1: number;
+    };
+    confidence?: number;
+}
+export interface AiOcrPage {
+    /** The zero-based page index in the document. */
+    index: number;
+    /** The page's text as markdown. */
+    text: string;
+    /** The page size in the provider's rendering units (pixels at `dpi`). */
+    width: number;
+    height: number;
+    confidence?: number;
+    /** Present when the call asked for layout. */
+    blocks?: AiOcrBlock[];
+}
+export interface AiOcrResult {
+    /** Every page's text, in order, separated by blank lines. */
+    markdown: string;
+    pages: AiOcrPage[];
+    /** The model that read the document, as the gateway reports it. */
+    model: string;
+    usage: {
+        pages: number;
+    };
+}
 /** Stable reason codes governed AI answers with in IsomorphError.details.code. */
 export type AiErrorCode = 'AI_NOT_ENABLED' | 'AI_POLICY_DENIED' | 'AI_QUOTA_EXCEEDED' | 'AI_PROVIDER_ERROR' | 'AI_UNAVAILABLE' | 'REQUEST_TOO_LARGE' | 'VALIDATION_FAILED';
 /** The AI reason code carried by an error, when it is a governed-AI refusal. */
@@ -2008,6 +2057,7 @@ export declare function createClient(options?: ClientOptions): {
     ai: {
         chat: (request: AiChatRequest) => Promise<AiChatResult>;
         embed: (request: AiEmbedRequest) => Promise<AiEmbedResult>;
+        ocr: (file: AiOcrFile, ocrOptions?: AiOcrOptions) => Promise<AiOcrResult>;
     };
 };
 export type IsomorphClient = ReturnType<typeof createClient>;

package/dist/index.js

@@ -212,7 +212,11 @@ class Transport {
         const action = currentAction();
         // (headers with an undefined value are removed so a multipart body can
         // clear the JSON default and let the browser set its own boundary)
-        const response = await this.fetch(`${action?.baseUrl ?? this.baseUrl}${path}`, {
+        // A trusted action uses its local gateway for database calls, including
+        // transactions. The local gateway deliberately has only the data
+        // capability; every other declared surface remains on the shared gateway.
+        const baseUrl = action?.baseUrl && path.startsWith('/_harbour/data/') ? action.baseUrl : this.baseUrl;
+        const response = await this.fetch(`${baseUrl}${path}`, {
             credentials: 'same-origin',
             ...init,
             headers: mergeHeaders(
@@ -840,9 +844,180 @@ export function createClient(options = {}) {
             embed: (request) => transport.json('/_harbour/ai/embed', {
                 method: 'POST', body: JSON.stringify(request),
             }),
+            ocr: (file, ocrOptions = {}) => ocrDocument(options.fetch ?? globalThis.fetch.bind(globalThis), file, ocrOptions),
         },
     };
 }
+const OCR_NOT_ROUTED = 'No OCR model is routed for this app. Ask IT to connect an OCR provider under AI governance.';
+const OCR_ROUTING_CODES = new Set(['model_capability_unassigned', 'model_route_not_approved', 'route_targets_unavailable', 'file_input_unsupported']);
+// The OCR call: the document goes to the governed gateway as a data URL in
+// Mistral's OCR wire shape and the answer comes back as pages of markdown,
+// with boxes normalised to the page. Refusals are IsomorphErrors whose
+// details.code is an AiErrorCode, like the rest of governed AI.
+async function ocrDocument(fetchImpl, file, options) {
+    if (typeof window !== 'undefined' && typeof window.document !== 'undefined') {
+        throw new IsomorphError('VALIDATION_FAILED', 'isomorph.ai.ocr runs in actions or jobs', undefined, { code: 'VALIDATION_FAILED' });
+    }
+    const env = typeof process === 'undefined' ? {} : process.env;
+    const baseUrl = env.OPENAI_BASE_URL;
+    const key = env.OPENAI_API_KEY;
+    if (!baseUrl || !key) {
+        throw new IsomorphError('FORBIDDEN', 'Governed AI is not connected for this app version yet.', undefined, { code: 'AI_NOT_ENABLED' });
+    }
+    const model = env.ISOMORPH_AI_DOCUMENT_MODEL_ALIAS;
+    if (!model)
+        throw new IsomorphError('FORBIDDEN', OCR_NOT_ROUTED, undefined, { code: 'AI_NOT_ENABLED' });
+    const { mediaType, base64 } = await ocrDocumentBytes(file);
+    if (options.pages !== undefined && typeof options.pages !== 'string' && !(Array.isArray(options.pages) && options.pages.every(page => Number.isInteger(page) && page >= 0))) {
+        throw new IsomorphError('VALIDATION_FAILED', 'pages must be a range string or a list of zero-based page indexes', undefined, { code: 'VALIDATION_FAILED' });
+    }
+    const dataUrl = `data:${mediaType};base64,${base64}`;
+    const body = {
+        model,
+        document: mediaType.startsWith('image/') ? { type: 'image_url', image_url: dataUrl } : { type: 'document_url', document_url: dataUrl },
+        ...(options.pages !== undefined ? { pages: options.pages } : {}),
+        include_blocks: options.layout === true,
+        ...(options.layout === true ? { confidence_scores_granularity: 'block' } : {}),
+    };
+    let response;
+    try {
+        response = await fetchImpl(`${baseUrl.replace(/\/$/, '')}/ocr`, {
+            method: 'POST',
+            headers: { authorization: `Bearer ${key}`, 'content-type': 'application/json' },
+            body: JSON.stringify(body),
+        });
+    }
+    catch (error) {
+        throw new IsomorphError('UNAVAILABLE', `The AI gateway could not be reached: ${error instanceof Error ? error.message : String(error)}`, undefined, { code: 'AI_UNAVAILABLE' });
+    }
+    const requestId = response.headers.get('x-request-id') ?? undefined;
+    const raw = await response.json().catch(() => undefined);
+    if (!response.ok) {
+        const gatewayCode = response.headers.get('x-harbour-error-code') ?? raw?.error?.code ?? undefined;
+        const message = raw?.error?.message;
+        const [category, code, fallback] = ocrRefusal(response.status, gatewayCode);
+        throw new IsomorphError(category, code === 'AI_NOT_ENABLED' ? OCR_NOT_ROUTED : message ?? fallback, requestId, { code, gatewayCode }, response.status);
+    }
+    const wire = raw;
+    if (!wire || !Array.isArray(wire.pages)) {
+        throw new IsomorphError('UNAVAILABLE', 'The AI gateway answered without pages.', requestId, { code: 'AI_PROVIDER_ERROR' }, response.status);
+    }
+    const pages = wire.pages.map((page, position) => {
+        const width = finiteNumber(page.dimensions?.width) ?? 0;
+        const height = finiteNumber(page.dimensions?.height) ?? 0;
+        const confidence = confidenceOf(page.confidence_scores);
+        const blocks = options.layout === true
+            ? (page.blocks ?? []).map((block) => {
+                const blockConfidence = confidenceOf(block.confidence_scores);
+                return {
+                    kind: typeof block.type === 'string' ? block.type : 'text',
+                    text: typeof block.content === 'string' ? block.content : '',
+                    box: {
+                        x0: normalise(block.top_left_x, width), y0: normalise(block.top_left_y, height),
+                        x1: normalise(block.bottom_right_x, width), y1: normalise(block.bottom_right_y, height),
+                    },
+                    ...(blockConfidence !== undefined ? { confidence: blockConfidence } : {}),
+                };
+            })
+            : undefined;
+        return {
+            index: finiteNumber(page.index) ?? position,
+            text: typeof page.markdown === 'string' ? page.markdown : '',
+            width, height,
+            ...(confidence !== undefined ? { confidence } : {}),
+            ...(blocks !== undefined ? { blocks } : {}),
+        };
+    });
+    return {
+        markdown: pages.map(page => page.text).join('\n\n'),
+        pages,
+        model: typeof wire.model === 'string' ? wire.model : model,
+        usage: { pages: finiteNumber(wire.usage_info?.pages_processed) ?? pages.length },
+    };
+}
+function ocrRefusal(status, gatewayCode) {
+    if (gatewayCode !== undefined && OCR_ROUTING_CODES.has(gatewayCode))
+        return ['FORBIDDEN', 'AI_NOT_ENABLED', OCR_NOT_ROUTED];
+    if (status === 401 || status === 403)
+        return ['FORBIDDEN', 'AI_POLICY_DENIED', 'AI governance refused this document.'];
+    if (status === 429)
+        return ['RATE_LIMITED', 'AI_QUOTA_EXCEEDED', 'The AI provider is rate limited; try again shortly.'];
+    if (status === 413)
+        return ['VALIDATION_FAILED', 'REQUEST_TOO_LARGE', 'The document is larger than the AI gateway accepts.'];
+    if (status >= 400 && status < 500)
+        return ['VALIDATION_FAILED', 'VALIDATION_FAILED', `The AI gateway refused the OCR request (${status}).`];
+    return ['UNAVAILABLE', 'AI_PROVIDER_ERROR', `The OCR provider failed (${status}).`];
+}
+async function ocrDocumentBytes(file) {
+    const invalid = (message) => new IsomorphError('VALIDATION_FAILED', message, undefined, { code: 'VALIDATION_FAILED' });
+    if (typeof Blob !== 'undefined' && file instanceof Blob) {
+        const bytes = new Uint8Array(await file.arrayBuffer());
+        if (bytes.byteLength === 0)
+            throw invalid('The document is empty.');
+        return { mediaType: file.type || 'application/pdf', base64: toBase64(bytes) };
+    }
+    if (file instanceof Uint8Array || file instanceof ArrayBuffer) {
+        const bytes = file instanceof Uint8Array ? file : new Uint8Array(file);
+        if (bytes.byteLength === 0)
+            throw invalid('The document is empty.');
+        return { mediaType: 'application/pdf', base64: toBase64(bytes) };
+    }
+    if (file && typeof file === 'object' && 'mediaType' in file && typeof file.mediaType === 'string' && /^[a-z]+\/[a-z0-9.+-]+$/i.test(file.mediaType)) {
+        const { mediaType, data } = file;
+        if (typeof data === 'string') {
+            if (data.length === 0)
+                throw invalid('The document is empty.');
+            return { mediaType, base64: data };
+        }
+        if (data instanceof Uint8Array || data instanceof ArrayBuffer) {
+            const bytes = data instanceof Uint8Array ? data : new Uint8Array(data);
+            if (bytes.byteLength === 0)
+                throw invalid('The document is empty.');
+            return { mediaType, base64: toBase64(bytes) };
+        }
+    }
+    throw invalid('file must be a Blob, bytes, or { mediaType, data }');
+}
+function toBase64(bytes) {
+    if (typeof Buffer !== 'undefined')
+        return Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength).toString('base64');
+    let binary = '';
+    for (let at = 0; at < bytes.length; at += 0x8000)
+        binary += String.fromCharCode(...bytes.subarray(at, at + 0x8000));
+    return btoa(binary);
+}
+function finiteNumber(value) {
+    return typeof value === 'number' && Number.isFinite(value) ? value : undefined;
+}
+function normalise(value, extent) {
+    const at = finiteNumber(value);
+    if (at === undefined || !(extent > 0))
+        return 0;
+    return Math.min(1, Math.max(0, at / extent));
+}
+// A confidence is a number, or the provider's object of numbers (the
+// granularity's score among them): the first finite number is the score.
+// Mistral's OCR shape names the averages (`average_page_confidence_score`,
+// `average_content_confidence_score`); another vendor mapped onto the shape
+// may give a bare number. The first finite number is the last resort.
+function confidenceOf(value) {
+    if (typeof value === 'number')
+        return finiteNumber(value);
+    if (value && typeof value === 'object') {
+        const scores = value;
+        for (const key of ['average_page_confidence_score', 'average_content_confidence_score', 'average', 'confidence']) {
+            const score = finiteNumber(scores[key]);
+            if (score !== undefined)
+                return score;
+        }
+        for (const inner of Object.values(scores)) {
+            const score = finiteNumber(inner);
+            if (score !== undefined)
+                return score;
+        }
+    }
+    return undefined;
+}
 // The contract's integration input rules (vendored verbatim from
 // @fourier-labs/harbour-contracts gen-ts/integration-execute-input-rules.ts;
 // test/integration-input-rules.test.mjs pins the copy's digest), so an app or

package/dist/runtime.js

@@ -486,6 +486,8 @@ export class Runtime {
                 process.env.ISOMORPH_AI_GENERATION_MODEL_ALIAS = binding.modelAliases.generation;
             if (binding.modelAliases?.image)
                 process.env.ISOMORPH_AI_IMAGE_MODEL_ALIAS = binding.modelAliases.image;
+            if (binding.modelAliases?.document)
+                process.env.ISOMORPH_AI_DOCUMENT_MODEL_ALIAS = binding.modelAliases.document;
         })().catch(error => { this.aiBinding = undefined; throw error; });
         await this.aiBinding;
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@isomorph.ai/app-sdk",
-  "version": "1.34.1",
+  "version": "1.35.1",
   "type": "module",
   "repository": {
     "type": "git",