@vertesia/common 1.5.0-dev.20260714.072725Z → 1.5.0-dev.20260722.120446Z

package/src/schema-for-extraction.ts

@@ -0,0 +1,231 @@
+/**
+ * Extraction schema filtering.
+ *
+ * Content-type object_schema properties may set `"x-extract": false` to mark fields
+ * that must not be filled by extraction models (match scores, ERP ids filled later,
+ * internal bookkeeping). The full schema remains the object model for UI/storage;
+ * only the filtered copy is used as result_schema for constrained decoding.
+ */
+
+export const X_EXTRACT = 'x-extract';
+
+function unescapeJsonPointerSegment(segment: string): string {
+    return segment.replace(/~1/g, '/').replace(/~0/g, '~');
+}
+
+function resolveLocalRef(ref: string, root: unknown): unknown {
+    if (!ref.startsWith('#/')) {
+        return undefined;
+    }
+    let current = root;
+    for (const segment of ref.slice(2).split('/').map(unescapeJsonPointerSegment)) {
+        if (!current || typeof current !== 'object' || Array.isArray(current)) {
+            return undefined;
+        }
+        current = (current as Record<string, unknown>)[segment];
+    }
+    return current;
+}
+
+function isExtractableSchemaNodeInternal(schema: unknown, root: unknown, seenRefs: Set<string>): boolean {
+    if (!schema || typeof schema !== 'object' || Array.isArray(schema)) {
+        return true;
+    }
+    const schemaObj = schema as Record<string, unknown>;
+    if (schemaObj[X_EXTRACT] === false) {
+        return false;
+    }
+    if (typeof schemaObj.$ref === 'string' && !seenRefs.has(schemaObj.$ref)) {
+        seenRefs.add(schemaObj.$ref);
+        const resolved = resolveLocalRef(schemaObj.$ref, root);
+        if (resolved !== undefined) {
+            return isExtractableSchemaNodeInternal(resolved, root, seenRefs);
+        }
+    }
+    return true;
+}
+
+export function isExtractableSchemaNode(schema: unknown, root: unknown = schema): boolean {
+    return isExtractableSchemaNodeInternal(schema, root, new Set());
+}
+
+function filterSchemaArray(value: unknown, root: unknown): unknown[] | undefined {
+    if (!Array.isArray(value)) {
+        return undefined;
+    }
+    const filtered = value
+        .map((item) => schemaForExtractionNode(item, root))
+        .filter((item): item is unknown => item !== undefined);
+    return filtered.length > 0 ? filtered : undefined;
+}
+
+function filterSchemaMap(value: unknown, root: unknown): Record<string, unknown> | undefined {
+    if (!value || typeof value !== 'object' || Array.isArray(value)) {
+        return undefined;
+    }
+    const filtered: Record<string, unknown> = {};
+    for (const [key, child] of Object.entries(value)) {
+        const childFiltered = schemaForExtractionNode(child, root);
+        if (childFiltered !== undefined) {
+            filtered[key] = childFiltered;
+        }
+    }
+    return Object.keys(filtered).length > 0 ? filtered : undefined;
+}
+
+function schemaForExtractionNode(schema: unknown, root: unknown): unknown {
+    if (!schema || typeof schema !== 'object') {
+        return schema;
+    }
+    if (Array.isArray(schema)) {
+        return schema.map((item) => schemaForExtractionNode(item, root));
+    }
+    if (!isExtractableSchemaNode(schema, root)) {
+        return undefined;
+    }
+
+    const node = { ...(schema as Record<string, unknown>) };
+
+    for (const defsKey of ['$defs', 'definitions']) {
+        const defs = filterSchemaMap(node[defsKey], root);
+        if (defs) {
+            node[defsKey] = defs;
+        } else {
+            delete node[defsKey];
+        }
+    }
+
+    if (node.items && typeof node.items === 'object') {
+        const items = schemaForExtractionNode(node.items, root);
+        if (items !== undefined) {
+            node.items = items;
+        } else {
+            delete node.items;
+        }
+    }
+    for (const unionKey of ['anyOf', 'oneOf', 'allOf']) {
+        if (Array.isArray(node[unionKey])) {
+            const filtered = filterSchemaArray(node[unionKey], root);
+            if (!filtered) {
+                return undefined;
+            }
+            node[unionKey] = filtered;
+        }
+    }
+    if (node.additionalProperties && typeof node.additionalProperties === 'object') {
+        const additionalProperties = schemaForExtractionNode(node.additionalProperties, root);
+        node.additionalProperties = additionalProperties ?? false;
+    }
+
+    if (node.properties && typeof node.properties === 'object' && !Array.isArray(node.properties)) {
+        const props: Record<string, unknown> = {};
+        for (const [key, child] of Object.entries(node.properties as Record<string, unknown>)) {
+            const childFiltered = schemaForExtractionNode(child, root);
+            if (childFiltered !== undefined) {
+                props[key] = childFiltered;
+            }
+        }
+        node.properties = props;
+
+        if (Array.isArray(node.required)) {
+            const kept = node.required.filter((r) => typeof r === 'string' && r in props);
+            if (kept.length > 0) {
+                node.required = kept;
+            } else {
+                delete node.required;
+            }
+        }
+    }
+
+    // Drop the flag from the extraction copy (not needed by providers)
+    delete node[X_EXTRACT];
+
+    return node;
+}
+
+/**
+ * Deep-clones a JSON schema and removes properties (recursively) marked
+ * `x-extract: false`, cleaning `required` arrays to match.
+ */
+export function schemaForExtraction<T>(schema: T): T {
+    return (schemaForExtractionNode(schema, schema) ?? {}) as T;
+}
+
+function resolveSchemaNode(schema: unknown, root: unknown): unknown {
+    if (!schema || typeof schema !== 'object' || Array.isArray(schema)) {
+        return schema;
+    }
+    const schemaObj = schema as Record<string, unknown>;
+    if (typeof schemaObj.$ref !== 'string') {
+        return schema;
+    }
+    return resolveLocalRef(schemaObj.$ref, root) ?? schema;
+}
+
+function mergePreservingNonExtractableNode(
+    existing: unknown,
+    extracted: unknown,
+    schema: unknown,
+    root: unknown,
+): unknown {
+    if (!schema || typeof schema !== 'object' || Array.isArray(schema)) {
+        return extracted;
+    }
+    const resolved = resolveSchemaNode(schema, root);
+    if (!resolved || typeof resolved !== 'object' || Array.isArray(resolved)) {
+        return extracted;
+    }
+    const schemaObj = resolved as Record<string, unknown>;
+
+    // Arrays are new extraction output. Without a declared stable item key, index
+    // merging can attach preserved ERP/match fields to the wrong row.
+    if (Array.isArray(extracted)) {
+        const itemSchema = schemaObj.items;
+        if (!itemSchema || typeof itemSchema !== 'object') {
+            return extracted;
+        }
+        return extracted.map((item) => mergePreservingNonExtractableNode(undefined, item, itemSchema, root));
+    }
+
+    if (!extracted || typeof extracted !== 'object' || Array.isArray(extracted)) {
+        return extracted;
+    }
+
+    const props = schemaObj.properties;
+    if (!props || typeof props !== 'object' || Array.isArray(props)) {
+        return extracted;
+    }
+
+    const existingObj =
+        existing && typeof existing === 'object' && !Array.isArray(existing)
+            ? (existing as Record<string, unknown>)
+            : {};
+    const extractedObj = extracted as Record<string, unknown>;
+    const propSchemas = props as Record<string, unknown>;
+    const result: Record<string, unknown> = { ...extractedObj };
+
+    for (const [key, propSchema] of Object.entries(propSchemas)) {
+        if (!isExtractableSchemaNode(propSchema, root)) {
+            // Model should not have filled this; restore prior value if any.
+            if (key in existingObj) {
+                result[key] = existingObj[key];
+            } else {
+                delete result[key];
+            }
+            continue;
+        }
+        if (key in extractedObj) {
+            result[key] = mergePreservingNonExtractableNode(existingObj[key], extractedObj[key], propSchema, root);
+        }
+    }
+
+    return result;
+}
+
+/**
+ * Merge model output over existing properties while preserving values at paths
+ * marked `x-extract: false` on the full (unfiltered) schema.
+ */
+export function mergePreservingNonExtractable(existing: unknown, extracted: unknown, schema: unknown): unknown {
+    return mergePreservingNonExtractableNode(existing, extracted, schema, schema);
+}

package/src/store/agent-run.ts

@@ -453,6 +453,8 @@ export interface UpdateAgentRunStatusPayload {
     title?: string;
     topic?: string;
     lessons_learned?: string[];
+    /** Shallow-merged into the run's existing properties. */
+    properties?: Record<string, unknown>;
     /** ES-only: conversation content text (not stored in MongoDB) */
     content?: string;
     /**

package/src/store/conversation-state.ts

@@ -14,6 +14,34 @@ export interface ToolReference {
     stored_at: string;
 }
 
+/** Reference to text content externalized to agent artifact storage. */
+export interface TextArtifactReference {
+    storage_id: string;
+    artifact_path: string;
+    display_ref: string;
+    sha256: string;
+    size_bytes: number;
+    content_type: string;
+}
+
+/**
+ * Sidecar metadata for generated tool input fields that were stored outside
+ * model-visible tool_input. Keyed by tool_use.id on ConversationState.
+ */
+export interface ExternalizedToolInputRef {
+    tool_name: string;
+    input_path: ['content'];
+    ref: TextArtifactReference;
+}
+
+export interface ExternalizedToolInputRefs {
+    [toolUseId: string]: ExternalizedToolInputRef[];
+}
+
+export function toolInputRefsArtifactPath(storageId: string): string {
+    return `agents/${storageId}/tool-input-refs.json`;
+}
+
 /**
  * Conversation state passed between workflow activities.
  * Contains all context needed to continue a multi-turn agent conversation.
@@ -51,6 +79,15 @@ export interface ConversationState {
     /** Compact, redacted latest user intent for reviewer-style system interactions. */
     latest_user_message?: string;
 
+    /**
+     * Transport sidecar for large generated tool input fields.
+     *
+     * These refs are intentionally kept out of tool_use.tool_input so they are
+     * not shown to the model. Tool execution hydrates them from artifact storage
+     * immediately before activity validation.
+     */
+    tool_input_refs?: ExternalizedToolInputRefs;
+
     /**
      * The output of the this conversation step
      */
@@ -204,6 +241,16 @@ export interface ConversationState {
      * to consolidate all artifacts under the parent agent run.
      */
     launch_id?: string;
+
+    /**
+     * The exact app version this run is pinned to, derived from the `@version` on the
+     * started interaction ref / the `x-vertesia-app-version` header at start. Persisted on the state
+     * so it survives resume, and applied to the activity client (`withAppVersion`) so every app-owned
+     * ref the run resolves — interactions, types, processes, tools — targets this version instead of
+     * the current/promoted one. Undefined → current/promoted. Resolution-time only; never a stored
+     * capability-ref version.
+     */
+    app_version?: string;
 }
 
 /**

package/src/store/doc-analyzer.ts

@@ -1,104 +1,42 @@
-import type { JSONObject } from '../json.js';
 import type { WorkflowExecutionPayload, WorkflowRunStatus } from './workflow.js';
 
-export interface PdfToRichtextOptions {
+export interface DocumentPrepOptions {
     features?: string[];
     debug?: boolean;
+    output_format?: DocProcessorOutputFormat;
     [key: string]: unknown;
 }
 
-export interface PdfToRichTextWorkflowPayload extends Omit<WorkflowExecutionPayload, 'vars'> {
-    vars: PdfToRichtextOptions;
-}
-
-export interface TransformTablesWorkflowPayload extends Omit<WorkflowExecutionPayload, 'vars'> {
-    vars: AdaptTablesParams;
-    environment?: string;
+export interface DocumentPrepWorkflowPayload extends Omit<WorkflowExecutionPayload, 'vars'> {
+    vars: DocumentPrepOptions;
 }
 
-export interface TransformTablesWorkflowResult {
-    result_path: string;
-    status: string;
-    table_count: number;
-    item_count: number;
-}
-
-/**
- * Represents a image in a document that has been analyzed
- */
-export interface DocImage {
-    id?: string;
-    page_number?: number;
-    description?: string;
-    is_meaningful?: boolean;
-    width?: number;
-    height?: number;
-}
-
-/**
- * The export type formats for tables.
- */
-export type ExportTableFormats = 'json' | 'csv' | 'xml';
-
-/**
- * Represents a table in a document that has been analyzed
- */
-export interface DocTable {
-    page_number?: number;
-    table_number?: number;
-    title?: string;
-    format: 'application/csv' | 'application/json';
-}
-
-/**
- * Represents a table in a document that has been analyzed in CSV format
- */
-export interface DocTableCsv extends DocTable {
-    format: 'application/csv';
-    title?: string;
-    data: string;
-}
-
-/**
- * Represents a table in a document that has been analyzed in JSON format
- */
-export interface DocTableJson extends DocTable {
-    format: 'application/json';
-    title?: string;
-    data: JSONObject[];
-}
-
-export type DocTableResponse = DocTableCsv | DocTableJson;
+export type DocumentProcessingPhase = 'markdown' | 'grounded_extraction';
 
 /**
  * Output format for document processing workflows
  */
-export type DocProcessorOutputFormat = 'xml' | 'markdown';
+export type DocProcessorOutputFormat = 'markdown';
 
 /**
  * Represents a document analysis run status
  */
 export interface DocAnalyzeRunStatusResponse extends WorkflowRunStatus {
+    phase?: DocumentProcessingPhase;
     progress?: DocAnalyzerProgress;
-    /** The output format being used for processing (markdown or xml) */
+    /** The output format being used for processing. */
     output_format?: DocProcessorOutputFormat;
 }
 
-export interface DocAnalyzerResultResponse {
-    document?: string;
-    tables?: DocTableResponse[];
-    images?: DocImage[];
-    annotated?: string | null;
-}
-
 export interface DocAnalyzerProgress {
+    phase?: DocumentProcessingPhase;
     pages: DocAnalyzerProgressStatus;
     images: DocAnalyzerProgressStatus;
     tables: DocAnalyzerProgressStatus;
     visuals: DocAnalyzerProgressStatus;
     started_at?: number;
     percent: number;
-    /** The output format being used for processing (markdown or xml) */
+    /** The output format being used for processing. */
     output_format?: DocProcessorOutputFormat;
 }
 
@@ -176,7 +114,3 @@ export interface AdaptedTable {
 }
 
 export type AdaptedTableResponse = Record<string, AdaptedTable>;
-
-export interface AnnotatedPdfResponse {
-    url: string | null;
-}

package/src/store/dsl-workflow.ts

@@ -48,6 +48,7 @@ export interface DSLWorkflowExecutionPayload extends WorkflowExecutionPayload<Re
  */
 export interface DSLActivityOptions {
     startToCloseTimeout?: DurationValue;
+    heartbeatTimeout?: DurationValue;
     scheduleToStartTimeout?: DurationValue;
     scheduleToCloseTimeout?: DurationValue;
     retry?: DSLRetryPolicy;

package/src/store/grounded-extraction.ts

@@ -0,0 +1,154 @@
+import type { InteractionExecutionConfiguration } from '../interaction.js';
+import type { WorkflowRunStatus } from './workflow.js';
+
+/**
+ * Document-level trust verdict for a grounded extraction. `good_to_go` means the
+ * extracted content (after any review corrections) can be used without a human
+ * check; `needs_review` means a human should verify it. This reflects content
+ * correctness, not how many citation boxes rendered.
+ */
+export type GroundedExtractionVerdict = 'good_to_go' | 'needs_review';
+
+/**
+ * Canonical workflow id for object-scoped grounded extraction. Every entry point
+ * must use this id so Temporal prevents overlapping extraction runs per object.
+ */
+export function getGroundedExtractionWorkflowId(accountId: string, objectId: string): string {
+    return `${accountId.slice(0, 6)}:workflow_execution_request:${objectId}:grounded`;
+}
+
+/**
+ * Request body to start a grounded extraction on a content object. All fields are
+ * optional: with none set, the object's own content-type schema drives the
+ * extraction with default models and settings.
+ */
+export interface GroundedExtractionRequest {
+    /** JSON schema describing the data to extract. Takes precedence over type_ref. */
+    schema?: Record<string, unknown>;
+    /** Content type id or catalog ref whose object_schema drives the extraction. */
+    type_ref?: string;
+    /** Interaction to use. Defaults to sys:ExtractInformationGrounded. */
+    interaction_name?: string;
+    /** Maximum number of pages to process. */
+    max_pages?: number;
+    /** Run OCR on every page even when a text layer exists. */
+    force_ocr?: boolean;
+    /** Re-run OCR on pages that need it instead of restoring the stored OCR result. */
+    refresh_ocr?: boolean;
+    /** Attach clean page images for layout/semantic context; direct-vision pages also receive checkerboards. */
+    use_vision?: boolean;
+    /**
+     * A1 locate-grid cell size in PDF points for vision pages (drives both the drawn
+     * grid and cell→box resolution). Smaller = finer grid / more cells. Default: 14.
+     */
+    grid_cell_pt?: number;
+    /**
+     * How to read pages that have no digital text layer (scans / image-only pages).
+     * 'vision' (default): read them off the page image with the extraction model,
+     * skipping OCR entirely. 'ocr': legacy path — OCR those pages and block-ground on
+     * the (lossy) OCR text. Set to 'ocr' to revert to the pre-vision behavior.
+     */
+    raster_mode?: 'vision' | 'ocr';
+    /** Maximum pages per extraction call; larger documents are split into sequential windows. */
+    window_pages?: number;
+    /**
+     * Extract with an autonomous agent (views the whole document at once) instead of
+     * the deterministic windowed pipeline. Sidesteps window-boundary splits on long
+     * documents. The workflow stages the artifacts into an agent space, runs a
+     * conversation agent that writes the extraction, then folds it back.
+     */
+    agentic_extraction?: boolean;
+    /** Agent interaction for agentic_extraction. Defaults to sys:GeneralAgent. */
+    extract_agent?: string;
+    /** Update the object's properties with the extracted data. Default: true. */
+    update_properties?: boolean;
+    /** LLM execution configuration (model, environment, ...) for the main pass. */
+    config?: InteractionExecutionConfiguration;
+    /** Execution configuration used instead of `config` on hard content (scans, handwriting). */
+    hard_config?: InteractionExecutionConfiguration;
+    /** Hardness score (0..1) at or above which `hard_config` is used. Default: 0.5. */
+    hardness_threshold?: number;
+    /** Execution configuration for the post-extraction review pass. No review runs when absent. */
+    review_config?: InteractionExecutionConfiguration;
+    /** Hardness score (0..1) at or above which the review runs. Defaults to hardness_threshold. */
+    review_threshold?: number;
+    /** Review triggers when any page's citation coverage falls below this floor. Default: 0.2. */
+    coverage_review_threshold?: number;
+    /** Run the model review even when every citation was digitally verified. Requires review_config. */
+    force_review?: boolean;
+    /**
+     * Free-text operator guidance folded into the extraction prompt to steer a
+     * (re-)extraction, e.g. "part numbers are in the third column; some line items
+     * wrap onto the next row".
+     */
+    operator_instructions?: string;
+    /** Interactive assistant only: the operator's opening message for the assistant conversation. */
+    user_prompt?: string;
+}
+
+/**
+ * Response from starting the interactive grounded extraction assistant. The agent
+ * run + conversation workflow are launched server-side (recordRun -> stage the
+ * document into the agent space -> launch the interactive conversation); the
+ * client renders the conversation with `agent_run_id`.
+ */
+export interface GroundedAssistantResponse {
+    /** The AgentRun id to stream/render the conversation. */
+    agent_run_id: string;
+    /** The conversation workflow id backing the run. */
+    workflow_id: string;
+    /** The object the assistant is scoped to. */
+    object_id: string;
+}
+
+/**
+ * Status of a grounded extraction workflow. Carries the doc-level verdict once the
+ * run has completed and written its result.
+ */
+export interface GroundedExtractionRunStatusResponse extends WorkflowRunStatus {
+    /** The trust verdict, present once the run has completed. */
+    verdict?: GroundedExtractionVerdict;
+}
+
+/**
+ * How each extracted value was verified. Two kinds, both trustworthy:
+ * digitally verified (matched the document's text — digital layer or OCR) and
+ * AI verified (the reviewer confirmed it against the page image — the primary
+ * signal for scanned or handwritten content, which has no text layer to match).
+ */
+export interface GroundedVerificationBreakdown {
+    /** Total number of cited values. */
+    total: number;
+    /** Values matched verbatim against the document's text (digital layer or OCR). */
+    digitally_verified: number;
+    /** Values the reviewer model confirmed against the page image. */
+    ai_verified: number;
+    /** Values read from the image but neither text-matched nor reviewer-confirmed. */
+    unverified: number;
+}
+
+/**
+ * Completed grounded extraction result: the extracted data with its trust verdict
+ * and verification breakdown, plus a download URL for the full citations artifact.
+ */
+export interface GroundedExtractionResultResponse {
+    object_id: string;
+    /** The extracted data, shaped by the requested schema. */
+    data: Record<string, unknown>;
+    /** Document-level trust verdict. */
+    verdict?: GroundedExtractionVerdict;
+    /** One-sentence rationale for the verdict. */
+    verdict_reason?: string;
+    /** Mean citation confidence in [0,1]. */
+    confidence?: number;
+    /** Per-value verification breakdown. */
+    verification: GroundedVerificationBreakdown;
+    /** Review outcome, when a review pass ran. */
+    review?: {
+        assessment: 'complete' | 'issues_found';
+        summary?: string;
+        corrections_applied?: number;
+    };
+    /** Signed download URL for the full grounded-extraction.json (data + citations + boxes). */
+    result_url?: string | null;
+}

package/src/store/index.ts

@@ -6,6 +6,7 @@ export * from './common.js';
 export * from './conversation-state.js';
 export * from './doc-analyzer.js';
 export * from './dsl-workflow.js';
+export * from './grounded-extraction.js';
 export * from './hive-memory.js';
 export * from './object-types.js';
 export * from './process.js';