@vertesia/common 1.5.0-dev.20260717.131047Z → 1.5.0-dev.20260725.083715Z

package/src/schema-for-extraction.test.ts

@@ -0,0 +1,191 @@
+import { describe, expect, it } from 'vitest';
+import {
+    isExtractableSchemaNode,
+    mergePreservingNonExtractable,
+    schemaForExtraction,
+} from './schema-for-extraction.js';
+
+describe('schemaForExtraction', () => {
+    it('drops top-level properties marked x-extract: false and cleans required', () => {
+        const schema = {
+            type: 'object',
+            properties: {
+                po_number: { type: 'string' },
+                match: { type: 'string', 'x-extract': false },
+                match_reason: { type: 'string', 'x-extract': false },
+            },
+            required: ['po_number', 'match'],
+        };
+        const filtered = schemaForExtraction(schema);
+        expect(Object.keys(filtered.properties as object).sort()).toEqual(['po_number']);
+        expect(filtered.required).toEqual(['po_number']);
+        expect((filtered.properties as Record<string, unknown>).po_number).toEqual({ type: 'string' });
+    });
+
+    it('drops nested and array-item properties with x-extract: false', () => {
+        const schema = {
+            type: 'object',
+            properties: {
+                line_items: {
+                    type: 'array',
+                    items: {
+                        type: 'object',
+                        properties: {
+                            part_number: { type: 'string' },
+                            match: { type: 'string', 'x-extract': false },
+                            quantity: { type: 'string' },
+                        },
+                        required: ['part_number', 'match'],
+                    },
+                },
+            },
+        };
+        const filtered = schemaForExtraction(schema);
+        const items = (filtered.properties as Record<string, Record<string, unknown>>).line_items.items as Record<
+            string,
+            unknown
+        >;
+        expect(Object.keys(items.properties as object).sort()).toEqual(['part_number', 'quantity']);
+        expect(items.required).toEqual(['part_number']);
+    });
+
+    it('treats missing x-extract as extractable', () => {
+        expect(isExtractableSchemaNode({ type: 'string' })).toBe(true);
+        expect(isExtractableSchemaNode({ type: 'string', 'x-extract': true })).toBe(true);
+        expect(isExtractableSchemaNode({ type: 'string', 'x-extract': false })).toBe(false);
+    });
+
+    it('filters $defs and drops properties that reference non-extractable definitions', () => {
+        const schema = {
+            type: 'object',
+            $defs: {
+                hidden: { type: 'string', 'x-extract': false },
+                visible: {
+                    type: 'object',
+                    properties: {
+                        value: { type: 'string' },
+                        match: { type: 'string', 'x-extract': false },
+                    },
+                    required: ['value', 'match'],
+                },
+            },
+            properties: {
+                hidden: { $ref: '#/$defs/hidden' },
+                visible: { $ref: '#/$defs/visible' },
+            },
+            required: ['hidden', 'visible'],
+        };
+
+        const filtered = schemaForExtraction(schema);
+
+        expect(filtered.$defs.hidden).toBeUndefined();
+        expect(filtered.$defs.visible).toEqual({
+            type: 'object',
+            properties: { value: { type: 'string' } },
+            required: ['value'],
+        });
+        expect(filtered.properties).toEqual({ visible: { $ref: '#/$defs/visible' } });
+        expect(filtered.required).toEqual(['visible']);
+    });
+
+    it('drops non-extractable anyOf branches', () => {
+        const schema = {
+            type: 'object',
+            properties: {
+                value: {
+                    anyOf: [{ type: 'string', 'x-extract': false }, { type: 'number' }],
+                },
+            },
+        };
+
+        const filtered = schemaForExtraction(schema);
+
+        expect((filtered.properties.value as { anyOf: unknown[] }).anyOf).toEqual([{ type: 'number' }]);
+    });
+});
+
+describe('mergePreservingNonExtractable', () => {
+    it('preserves non-extractable top-level values over model output', () => {
+        const schema = {
+            type: 'object',
+            properties: {
+                po_number: { type: 'string' },
+                match: { type: 'string', 'x-extract': false },
+            },
+        };
+        const existing = { po_number: 'OLD', match: 'erp-hit' };
+        const extracted = { po_number: 'NEW', match: 'model-invented' };
+        expect(mergePreservingNonExtractable(existing, extracted, schema)).toEqual({
+            po_number: 'NEW',
+            match: 'erp-hit',
+        });
+    });
+
+    it('clears extractable fields omitted by the latest extraction', () => {
+        const schema = {
+            type: 'object',
+            properties: {
+                po_number: { type: 'string' },
+                total: { type: 'number' },
+                match: { type: 'string', 'x-extract': false },
+            },
+        };
+        const existing = { po_number: 'STALE', total: 42, match: 'erp-hit' };
+        const extracted = { total: 50 };
+
+        expect(mergePreservingNonExtractable(existing, extracted, schema)).toEqual({
+            total: 50,
+            match: 'erp-hit',
+        });
+    });
+
+    it('does not preserve non-extractable array item fields by index', () => {
+        const schema = {
+            type: 'object',
+            properties: {
+                line_items: {
+                    type: 'array',
+                    items: {
+                        type: 'object',
+                        properties: {
+                            part: { type: 'string' },
+                            match: { type: 'string', 'x-extract': false },
+                        },
+                    },
+                },
+            },
+        };
+        const existing = {
+            line_items: [
+                { part: 'A', match: 'hit-a' },
+                { part: 'B', match: 'hit-b' },
+            ],
+        };
+        const extracted = {
+            line_items: [{ part: 'B2' }],
+        };
+        expect(mergePreservingNonExtractable(existing, extracted, schema)).toEqual({
+            line_items: [{ part: 'B2' }],
+        });
+    });
+
+    it('preserves top-level values referenced through non-extractable $defs', () => {
+        const schema = {
+            type: 'object',
+            $defs: {
+                match: { type: 'string', 'x-extract': false },
+            },
+            properties: {
+                po_number: { type: 'string' },
+                match: { $ref: '#/$defs/match' },
+            },
+        };
+        const existing = { po_number: 'OLD', match: 'erp-hit' };
+        const extracted = { po_number: 'NEW', match: 'model-invented' };
+
+        expect(mergePreservingNonExtractable(existing, extracted, schema)).toEqual({
+            po_number: 'NEW',
+            match: 'erp-hit',
+        });
+    });
+});

package/src/schema-for-extraction.ts

@@ -0,0 +1,231 @@
+/**
+ * Extraction schema filtering.
+ *
+ * Content-type object_schema properties may set `"x-extract": false` to mark fields
+ * that must not be filled by extraction models (match scores, ERP ids filled later,
+ * internal bookkeeping). The full schema remains the object model for UI/storage;
+ * only the filtered copy is used as result_schema for constrained decoding.
+ */
+
+export const X_EXTRACT = 'x-extract';
+
+function unescapeJsonPointerSegment(segment: string): string {
+    return segment.replace(/~1/g, '/').replace(/~0/g, '~');
+}
+
+function resolveLocalRef(ref: string, root: unknown): unknown {
+    if (!ref.startsWith('#/')) {
+        return undefined;
+    }
+    let current = root;
+    for (const segment of ref.slice(2).split('/').map(unescapeJsonPointerSegment)) {
+        if (!current || typeof current !== 'object' || Array.isArray(current)) {
+            return undefined;
+        }
+        current = (current as Record<string, unknown>)[segment];
+    }
+    return current;
+}
+
+function isExtractableSchemaNodeInternal(schema: unknown, root: unknown, seenRefs: Set<string>): boolean {
+    if (!schema || typeof schema !== 'object' || Array.isArray(schema)) {
+        return true;
+    }
+    const schemaObj = schema as Record<string, unknown>;
+    if (schemaObj[X_EXTRACT] === false) {
+        return false;
+    }
+    if (typeof schemaObj.$ref === 'string' && !seenRefs.has(schemaObj.$ref)) {
+        seenRefs.add(schemaObj.$ref);
+        const resolved = resolveLocalRef(schemaObj.$ref, root);
+        if (resolved !== undefined) {
+            return isExtractableSchemaNodeInternal(resolved, root, seenRefs);
+        }
+    }
+    return true;
+}
+
+export function isExtractableSchemaNode(schema: unknown, root: unknown = schema): boolean {
+    return isExtractableSchemaNodeInternal(schema, root, new Set());
+}
+
+function filterSchemaArray(value: unknown, root: unknown): unknown[] | undefined {
+    if (!Array.isArray(value)) {
+        return undefined;
+    }
+    const filtered = value
+        .map((item) => schemaForExtractionNode(item, root))
+        .filter((item): item is unknown => item !== undefined);
+    return filtered.length > 0 ? filtered : undefined;
+}
+
+function filterSchemaMap(value: unknown, root: unknown): Record<string, unknown> | undefined {
+    if (!value || typeof value !== 'object' || Array.isArray(value)) {
+        return undefined;
+    }
+    const filtered: Record<string, unknown> = {};
+    for (const [key, child] of Object.entries(value)) {
+        const childFiltered = schemaForExtractionNode(child, root);
+        if (childFiltered !== undefined) {
+            filtered[key] = childFiltered;
+        }
+    }
+    return Object.keys(filtered).length > 0 ? filtered : undefined;
+}
+
+function schemaForExtractionNode(schema: unknown, root: unknown): unknown {
+    if (!schema || typeof schema !== 'object') {
+        return schema;
+    }
+    if (Array.isArray(schema)) {
+        return schema.map((item) => schemaForExtractionNode(item, root));
+    }
+    if (!isExtractableSchemaNode(schema, root)) {
+        return undefined;
+    }
+
+    const node = { ...(schema as Record<string, unknown>) };
+
+    for (const defsKey of ['$defs', 'definitions']) {
+        const defs = filterSchemaMap(node[defsKey], root);
+        if (defs) {
+            node[defsKey] = defs;
+        } else {
+            delete node[defsKey];
+        }
+    }
+
+    if (node.items && typeof node.items === 'object') {
+        const items = schemaForExtractionNode(node.items, root);
+        if (items !== undefined) {
+            node.items = items;
+        } else {
+            delete node.items;
+        }
+    }
+    for (const unionKey of ['anyOf', 'oneOf', 'allOf']) {
+        if (Array.isArray(node[unionKey])) {
+            const filtered = filterSchemaArray(node[unionKey], root);
+            if (!filtered) {
+                return undefined;
+            }
+            node[unionKey] = filtered;
+        }
+    }
+    if (node.additionalProperties && typeof node.additionalProperties === 'object') {
+        const additionalProperties = schemaForExtractionNode(node.additionalProperties, root);
+        node.additionalProperties = additionalProperties ?? false;
+    }
+
+    if (node.properties && typeof node.properties === 'object' && !Array.isArray(node.properties)) {
+        const props: Record<string, unknown> = {};
+        for (const [key, child] of Object.entries(node.properties as Record<string, unknown>)) {
+            const childFiltered = schemaForExtractionNode(child, root);
+            if (childFiltered !== undefined) {
+                props[key] = childFiltered;
+            }
+        }
+        node.properties = props;
+
+        if (Array.isArray(node.required)) {
+            const kept = node.required.filter((r) => typeof r === 'string' && r in props);
+            if (kept.length > 0) {
+                node.required = kept;
+            } else {
+                delete node.required;
+            }
+        }
+    }
+
+    // Drop the flag from the extraction copy (not needed by providers)
+    delete node[X_EXTRACT];
+
+    return node;
+}
+
+/**
+ * Deep-clones a JSON schema and removes properties (recursively) marked
+ * `x-extract: false`, cleaning `required` arrays to match.
+ */
+export function schemaForExtraction<T>(schema: T): T {
+    return (schemaForExtractionNode(schema, schema) ?? {}) as T;
+}
+
+function resolveSchemaNode(schema: unknown, root: unknown): unknown {
+    if (!schema || typeof schema !== 'object' || Array.isArray(schema)) {
+        return schema;
+    }
+    const schemaObj = schema as Record<string, unknown>;
+    if (typeof schemaObj.$ref !== 'string') {
+        return schema;
+    }
+    return resolveLocalRef(schemaObj.$ref, root) ?? schema;
+}
+
+function mergePreservingNonExtractableNode(
+    existing: unknown,
+    extracted: unknown,
+    schema: unknown,
+    root: unknown,
+): unknown {
+    if (!schema || typeof schema !== 'object' || Array.isArray(schema)) {
+        return extracted;
+    }
+    const resolved = resolveSchemaNode(schema, root);
+    if (!resolved || typeof resolved !== 'object' || Array.isArray(resolved)) {
+        return extracted;
+    }
+    const schemaObj = resolved as Record<string, unknown>;
+
+    // Arrays are new extraction output. Without a declared stable item key, index
+    // merging can attach preserved ERP/match fields to the wrong row.
+    if (Array.isArray(extracted)) {
+        const itemSchema = schemaObj.items;
+        if (!itemSchema || typeof itemSchema !== 'object') {
+            return extracted;
+        }
+        return extracted.map((item) => mergePreservingNonExtractableNode(undefined, item, itemSchema, root));
+    }
+
+    if (!extracted || typeof extracted !== 'object' || Array.isArray(extracted)) {
+        return extracted;
+    }
+
+    const props = schemaObj.properties;
+    if (!props || typeof props !== 'object' || Array.isArray(props)) {
+        return extracted;
+    }
+
+    const existingObj =
+        existing && typeof existing === 'object' && !Array.isArray(existing)
+            ? (existing as Record<string, unknown>)
+            : {};
+    const extractedObj = extracted as Record<string, unknown>;
+    const propSchemas = props as Record<string, unknown>;
+    const result: Record<string, unknown> = { ...extractedObj };
+
+    for (const [key, propSchema] of Object.entries(propSchemas)) {
+        if (!isExtractableSchemaNode(propSchema, root)) {
+            // Model should not have filled this; restore prior value if any.
+            if (key in existingObj) {
+                result[key] = existingObj[key];
+            } else {
+                delete result[key];
+            }
+            continue;
+        }
+        if (key in extractedObj) {
+            result[key] = mergePreservingNonExtractableNode(existingObj[key], extractedObj[key], propSchema, root);
+        }
+    }
+
+    return result;
+}
+
+/**
+ * Merge model output over existing properties while preserving values at paths
+ * marked `x-extract: false` on the full (unfiltered) schema.
+ */
+export function mergePreservingNonExtractable(existing: unknown, extracted: unknown, schema: unknown): unknown {
+    return mergePreservingNonExtractableNode(existing, extracted, schema, schema);
+}

package/src/store/agent-run.ts

@@ -16,6 +16,7 @@ import type { UserChannel } from '../email.js';
 import type {
     AgentSearchScope,
     ConversationVisibility,
+    InitialToolCall,
     InteractionExecutionConfiguration,
     InteractionRef,
     RunSource,
@@ -173,6 +174,15 @@ export interface AgentRunBase<TData = Record<string, unknown>, TProperties = Rec
     /** Tools configured for this run (+/- syntax supported) */
     tool_names?: string[];
 
+    /** Builtin system skills activated before the first model turn. */
+    initial_skills?: string[];
+
+    /** Ordered, bounded hydration/read calls executed before the first model turn. */
+    initial_tool_calls?: InitialToolCall[];
+
+    /** Hard denylist of tool names: never exposed to the model, refused at execution time. */
+    excluded_tools?: string[];
+
     /** Scoped collection (if any) */
     collection_id?: string;
 
@@ -547,6 +557,25 @@ export interface AgentArtifactUrlResponse {
     path: string;
 }
 
+/** Text content and concurrency token for an agent artifact. */
+export interface AgentArtifactContentResponse {
+    path: string;
+    content: string;
+    generation: string;
+}
+
+/** Conditional text update for an agent artifact. */
+export interface UpdateAgentArtifactContentPayload {
+    content: string;
+    generation: string;
+}
+
+/** Result of a conditional agent artifact update. */
+export interface UpdateAgentArtifactContentResponse {
+    path: string;
+    generation: string;
+}
+
 /**
  * Telemetry ingestion payload for an agent run.
  */

package/src/store/content-type-editing.test.ts

@@ -0,0 +1,17 @@
+import Ajv from 'ajv';
+import { describe, expect, it } from 'vitest';
+import { ContentTypeEditingPolicySchema } from './store.js';
+
+const validate = new Ajv.default({ allErrors: true, strict: false }).compile(ContentTypeEditingPolicySchema);
+
+describe('ContentTypeEditingPolicySchema', () => {
+    it('accepts an omitted or configured interaction', () => {
+        expect(validate({})).toBe(true);
+        expect(validate({ interaction: 'sys:GeneralAgent' })).toBe(true);
+    });
+
+    it('rejects unknown fields and non-string interactions', () => {
+        expect(validate({ interaction: 42 })).toBe(false);
+        expect(validate({ interaction: 'sys:GeneralAgent', tools: [] })).toBe(false);
+    });
+});

package/src/store/conversation-state.ts

@@ -206,6 +206,9 @@ export interface ConversationState {
      */
     skill_instructions_delivered?: string[];
 
+    /** Stable ids of initialization tool calls completed before the first model turn. */
+    initialization_call_ids?: string[];
+
     /**
      * Denylist of MCP tool-collection ids deactivated for this conversation.
      * `undefined`/empty means all installed/connected MCP collections are active.
@@ -243,7 +246,7 @@ export interface ConversationState {
     launch_id?: string;
 
     /**
-     * The app version this run is pinned to (candidate testing), derived from the `@version` on the
+     * The exact app version this run is pinned to, derived from the `@version` on the
      * started interaction ref / the `x-vertesia-app-version` header at start. Persisted on the state
      * so it survives resume, and applied to the activity client (`withAppVersion`) so every app-owned
      * ref the run resolves — interactions, types, processes, tools — targets this version instead of

package/src/store/doc-analyzer.ts

@@ -1,104 +1,42 @@
-import type { JSONObject } from '../json.js';
 import type { WorkflowExecutionPayload, WorkflowRunStatus } from './workflow.js';
 
-export interface PdfToRichtextOptions {
+export interface DocumentPrepOptions {
     features?: string[];
     debug?: boolean;
+    output_format?: DocProcessorOutputFormat;
     [key: string]: unknown;
 }
 
-export interface PdfToRichTextWorkflowPayload extends Omit<WorkflowExecutionPayload, 'vars'> {
-    vars: PdfToRichtextOptions;
-}
-
-export interface TransformTablesWorkflowPayload extends Omit<WorkflowExecutionPayload, 'vars'> {
-    vars: AdaptTablesParams;
-    environment?: string;
+export interface DocumentPrepWorkflowPayload extends Omit<WorkflowExecutionPayload, 'vars'> {
+    vars: DocumentPrepOptions;
 }
 
-export interface TransformTablesWorkflowResult {
-    result_path: string;
-    status: string;
-    table_count: number;
-    item_count: number;
-}
-
-/**
- * Represents a image in a document that has been analyzed
- */
-export interface DocImage {
-    id?: string;
-    page_number?: number;
-    description?: string;
-    is_meaningful?: boolean;
-    width?: number;
-    height?: number;
-}
-
-/**
- * The export type formats for tables.
- */
-export type ExportTableFormats = 'json' | 'csv' | 'xml';
-
-/**
- * Represents a table in a document that has been analyzed
- */
-export interface DocTable {
-    page_number?: number;
-    table_number?: number;
-    title?: string;
-    format: 'application/csv' | 'application/json';
-}
-
-/**
- * Represents a table in a document that has been analyzed in CSV format
- */
-export interface DocTableCsv extends DocTable {
-    format: 'application/csv';
-    title?: string;
-    data: string;
-}
-
-/**
- * Represents a table in a document that has been analyzed in JSON format
- */
-export interface DocTableJson extends DocTable {
-    format: 'application/json';
-    title?: string;
-    data: JSONObject[];
-}
-
-export type DocTableResponse = DocTableCsv | DocTableJson;
+export type DocumentProcessingPhase = 'markdown' | 'grounded_extraction';
 
 /**
  * Output format for document processing workflows
  */
-export type DocProcessorOutputFormat = 'xml' | 'markdown';
+export type DocProcessorOutputFormat = 'markdown';
 
 /**
  * Represents a document analysis run status
  */
 export interface DocAnalyzeRunStatusResponse extends WorkflowRunStatus {
+    phase?: DocumentProcessingPhase;
     progress?: DocAnalyzerProgress;
-    /** The output format being used for processing (markdown or xml) */
+    /** The output format being used for processing. */
     output_format?: DocProcessorOutputFormat;
 }
 
-export interface DocAnalyzerResultResponse {
-    document?: string;
-    tables?: DocTableResponse[];
-    images?: DocImage[];
-    annotated?: string | null;
-}
-
 export interface DocAnalyzerProgress {
+    phase?: DocumentProcessingPhase;
     pages: DocAnalyzerProgressStatus;
     images: DocAnalyzerProgressStatus;
     tables: DocAnalyzerProgressStatus;
     visuals: DocAnalyzerProgressStatus;
     started_at?: number;
     percent: number;
-    /** The output format being used for processing (markdown or xml) */
+    /** The output format being used for processing. */
     output_format?: DocProcessorOutputFormat;
 }
 
@@ -176,7 +114,3 @@ export interface AdaptedTable {
 }
 
 export type AdaptedTableResponse = Record<string, AdaptedTable>;
-
-export interface AnnotatedPdfResponse {
-    url: string | null;
-}

package/src/store/dsl-workflow.ts

@@ -48,6 +48,7 @@ export interface DSLWorkflowExecutionPayload extends WorkflowExecutionPayload<Re
  */
 export interface DSLActivityOptions {
     startToCloseTimeout?: DurationValue;
+    heartbeatTimeout?: DurationValue;
     scheduleToStartTimeout?: DurationValue;
     scheduleToCloseTimeout?: DurationValue;
     retry?: DSLRetryPolicy;