@vertesia/workflow 1.5.0-dev.20260714.072725Z → 1.5.0-dev.20260722.120446Z

package/src/activities/generateDocumentProperties.ts

@@ -1,20 +1,28 @@
 import { ApplicationFailure, log } from '@temporalio/activity';
-import type { DSLActivityExecutionPayload, DSLActivitySpec, JSONObject } from '@vertesia/common';
+import {
+    type ContentMetadata,
+    type ContentSource,
+    type DSLActivityExecutionPayload,
+    type DSLActivitySpec,
+    type JSONObject,
+    mergePreservingNonExtractable,
+    PDF_RENDITION_NAME,
+    type Rendition,
+    schemaForExtraction,
+} from '@vertesia/common';
 import { setupActivity } from '../dsl/setup/ActivityContext.js';
+import { md5 } from '../utils/blobs.js';
 import { type TruncateSpec, truncByMaxTokens } from '../utils/tokens.js';
 import { executeInteractionFromActivity, type InteractionExecutionParams } from './executeInteraction.js';
 
 const INT_EXTRACT_INFORMATION = 'sys:ExtractInformation';
 
+export type GenerateDocumentPropertiesSource = 'auto' | 'text' | 'vision' | 'mixed';
+
 interface RetryableError extends Error {
     retryable?: boolean;
 }
 
-interface GeneratedPropertiesSummary {
-    title?: unknown;
-    description?: unknown;
-}
-
 function toRetryableError(error: unknown): RetryableError {
     if (error instanceof Error) {
         return error as RetryableError;
@@ -31,57 +39,209 @@ export interface GenerateDocumentPropertiesParams extends InteractionExecutionPa
 
     interactionName?: string;
 
+    /**
+     * @deprecated Use source: 'mixed'. Accepted for backward compatibility with existing DSL callers.
+     */
     use_vision?: boolean;
+
+    source?: GenerateDocumentPropertiesSource;
+
+    instructions?: string;
+
+    /**
+     * Scoped page-image evidence prepared by intake (`prepareVisionEvidence` scratch refs).
+     * When the param is PRESENT (even as an empty array) and the source calls for visual
+     * evidence, these refs are the ONLY visual evidence used — never the whole-object
+     * `store:{id}` ref, which resolves to ALL pages near-full-res through the rendition
+     * generate-and-poll path. An empty array means intake prepared no usable evidence
+     * (budget-impossible or no renderable source): extraction then runs text-only or fails
+     * with no-source instead of shipping an over-budget payload.
+     * Absent = legacy behavior for existing DSL callers.
+     */
+    evidence_images?: ContentSource[];
+
+    /**
+     * Opt-in freshness guard. When true, the activity computes a fingerprint of the extraction
+     * inputs (content etag, type + its object schema, source, instructions, interaction name)
+     * and returns `{ status: 'skipped' }` without executing the interaction when the fingerprint
+     * stored by a previous successful extraction matches and the object already has non-empty
+     * properties. Defaults to false so shared DSL callers keep the always-extract behavior.
+     * `payload.vars.forceGeneration` bypasses the skip.
+     */
+    skip_if_fresh?: boolean;
 }
 export interface GenerateDocumentProperties extends DSLActivitySpec<GenerateDocumentPropertiesParams> {
     name: 'generateDocumentProperties';
 }
+export type GenerateDocumentPropertiesResult =
+    | { status: 'completed' }
+    | { status: 'failed'; error: string }
+    | { document: string; status: 'skipped'; message: string };
+
+/**
+ * Stable fingerprint of the inputs that determine the extraction output. Key order is fixed by
+ * the object literal so the hash is deterministic.
+ */
+export function computeExtractionFingerprint(input: {
+    content_etag?: string;
+    type_id?: string;
+    source: GenerateDocumentPropertiesSource;
+    instructions?: string;
+    interactionName: string;
+    /** The type's object_schema: a schema edit under the same type id must invalidate the fingerprint. */
+    object_schema?: unknown;
+    /** Scoped vision evidence refs: a different page/profile selection must invalidate the fingerprint.
+     *  Undefined (legacy callers) keeps previously stored hashes valid. */
+    evidence?: string[];
+    /** The object's text etag: text can change under the same content etag (re-render,
+     *  re-conversion, manual edit) and text-consuming extractions must invalidate.
+     *  Undefined (vision-only callers) keeps previously stored hashes valid. */
+    text_etag?: string | null;
+}): string {
+    return md5(
+        JSON.stringify({
+            content_etag: input.content_etag,
+            type_id: input.type_id,
+            source: input.source,
+            instructions: input.instructions,
+            interactionName: input.interactionName,
+            object_schema: input.object_schema,
+            ...(input.evidence ? { evidence: input.evidence } : {}),
+            ...(input.text_etag !== undefined ? { text_etag: input.text_etag } : {}),
+        }),
+    );
+}
+
+/**
+ * The fingerprint is persisted inside the generation run info of the last successful extraction
+ * (the server appends it to `metadata.generation_runs`). Other writers append runs without a
+ * fingerprint, so scan backwards for the most recent extraction entry.
+ */
+function getStoredExtractionFingerprint(metadata: ContentMetadata | undefined): string | undefined {
+    const runs = metadata?.generation_runs;
+    if (!Array.isArray(runs)) {
+        return undefined;
+    }
+    for (let i = runs.length - 1; i >= 0; i--) {
+        if (runs[i]?.extraction_fingerprint) {
+            return runs[i].extraction_fingerprint;
+        }
+    }
+    return undefined;
+}
+
+function getPdfRenditionContent(doc: { metadata?: { renditions?: unknown } }): ContentSource | undefined {
+    const renditions = doc.metadata?.renditions;
+    if (!Array.isArray(renditions)) {
+        return undefined;
+    }
+    const pdfRendition = renditions.find(
+        (rendition): rendition is Rendition =>
+            typeof rendition === 'object' &&
+            rendition !== null &&
+            (rendition as { name?: unknown }).name === PDF_RENDITION_NAME &&
+            typeof (rendition as { content?: { source?: unknown } }).content?.source === 'string',
+    );
+    return pdfRendition?.content;
+}
 
 export async function generateDocumentProperties(
     payload: DSLActivityExecutionPayload<GenerateDocumentPropertiesParams>,
-) {
+): Promise<GenerateDocumentPropertiesResult> {
     const context = await setupActivity<GenerateDocumentPropertiesParams>(payload);
     const { params, client, objectId } = context;
     const interactionName = params.interactionName ?? INT_EXTRACT_INFORMATION;
 
     const project = await context.fetchProject();
 
-    const doc = await client.objects.retrieve(objectId, '+text');
+    const doc = await client.objects.retrieve(objectId, '+text +metadata +content');
     const type = doc.type ? await client.types.catalog.resolve(doc.type) : undefined;
 
-    if (!doc?.text && !params.use_vision && !doc?.content?.type?.startsWith('image/')) {
-        log.warn(`Object ${objectId} not found or text is empty`);
-        return { status: 'failed', error: 'no-text' };
-    }
-
     if (!type?.object_schema) {
         log.info(`Object ${objectId} has no schema`);
         return { document: objectId, status: 'skipped', message: 'no schema defined on type' };
     }
 
-    const getImageRef = () => {
+    const source = params.source ?? (params.use_vision ? 'mixed' : 'auto');
+    const evidenceProvided = Array.isArray(params.evidence_images);
+    const evidenceImages = params.evidence_images?.length ? params.evidence_images : undefined;
+
+    const extractionFingerprint = computeExtractionFingerprint({
+        content_etag: doc.content?.etag,
+        type_id: type.id,
+        source,
+        instructions: params.instructions,
+        interactionName,
+        object_schema: type.object_schema,
+        evidence: evidenceImages?.map((image) => image.source ?? image.name ?? ''),
+        // text may change under the same content etag; only vision-only extraction ignores it
+        text_etag: source === 'vision' ? undefined : (doc.text_etag ?? null),
+    });
+
+    if (params.skip_if_fresh && !payload.vars?.forceGeneration) {
+        const storedFingerprint = getStoredExtractionFingerprint(doc.metadata);
+        const hasProperties = !!doc.properties && Object.keys(doc.properties).length > 0;
+        if (storedFingerprint === extractionFingerprint && hasProperties) {
+            log.info(`Skipping property extraction for ${objectId}: extraction inputs unchanged`, {
+                extractionFingerprint,
+            });
+            return {
+                document: objectId,
+                status: 'skipped',
+                message: 'properties are fresh (extraction fingerprint matches)',
+            };
+        }
+    }
+
+    const getImageRef = (): string | ContentSource | undefined => {
         if (doc.content?.type?.startsWith('image/')) {
             return `store:${doc.id}`;
         }
 
-        if (params.use_vision && doc.content?.type?.startsWith('application/pdf')) {
+        if (doc.content?.type?.startsWith('application/pdf')) {
             return `store:${doc.id}`;
         }
 
+        const pdfRendition = getPdfRenditionContent(doc);
+        if (pdfRendition?.type?.startsWith('application/pdf')) {
+            return pdfRendition;
+        }
+
         log.info(`Object ${objectId} is not an image or pdf`);
         return undefined;
     };
 
-    const content = doc.text ? truncByMaxTokens(doc.text, params.truncate || 30000) : undefined;
+    const hasRealText = !!doc.text?.trim();
+    const content =
+        hasRealText && (source === 'auto' || source === 'text' || source === 'mixed')
+            ? truncByMaxTokens(doc.text ?? '', params.truncate || 30000)
+            : undefined;
+    const needsVisualEvidence = source === 'vision' || source === 'mixed' || (source === 'auto' && !content);
+    // Scoped scratch page refs win over the whole-object store: ref (which resolves to ALL
+    // pages near-full-res through the rendition generate-and-poll path). A PRESENT param —
+    // even empty — disables the store-ref fallback entirely: intake decided what evidence
+    // (if any) may be sent.
+    const scopedImages = needsVisualEvidence ? evidenceImages : undefined;
+    const imageRef = needsVisualEvidence && !evidenceProvided ? getImageRef() : undefined;
+
+    if (!content && !imageRef && !scopedImages) {
+        log.warn(`Object ${objectId} has no text or supported vision source`);
+        return { status: 'failed', error: 'no-source' };
+    }
 
     const promptData = {
         content: content,
-        image: getImageRef() ?? undefined,
+        image: imageRef,
+        images: scopedImages,
+        extraction_instructions: params.instructions,
         human_context: project?.configuration?.human_context ?? undefined,
     };
 
+    // Strip properties marked x-extract:false so constrained decoding cannot invent them.
+    const extractionSchema = schemaForExtraction(type.object_schema);
+
     log.info(
-        ` Extracting information from object ${objectId} with type ${type.name}`,
+        `Extracting information from object ${objectId} with type ${type.name}`,
         payload.debug_mode ? { params } : undefined,
     );
 
@@ -93,7 +253,7 @@ export async function generateDocumentProperties(
             {
                 ...params,
                 include_previous_error: true,
-                result_schema: type.object_schema,
+                result_schema: extractionSchema,
                 validate_result: type.strict_mode,
             },
             promptData,
@@ -125,38 +285,22 @@ export async function generateDocumentProperties(
         throw error;
     }
 
-    const getText = () => {
-        if (doc.text) {
-            return undefined;
-        }
-        let text = '';
-        const jsonResult = infoRes.result.object<GeneratedPropertiesSummary>();
-        if (typeof jsonResult.title === 'string') {
-            text += `${jsonResult.title}\n`;
-        }
-        if (typeof jsonResult.description === 'string') {
-            text += jsonResult.description;
-        }
-        if (text) {
-            return text;
-        } else {
-            return undefined;
-        }
-    };
-
     log.info(`Extracted information from object ${objectId} with type ${type.name}`, { runId: infoRes.id });
+    const extracted = infoRes.result.object<JSONObject>() ?? {};
+    const existing =
+        doc.properties && typeof doc.properties === 'object' && !Array.isArray(doc.properties)
+            ? (doc.properties as Record<string, unknown>)
+            : {};
+    const properties = mergePreservingNonExtractable(existing, extracted, type.object_schema) as JSONObject;
     await client.objects.update(
         doc.id,
         {
-            properties: {
-                ...infoRes.result.object<JSONObject>(),
-                ...(doc.text_etag !== undefined ? { etag: doc.text_etag } : {}),
-            },
-            text: getText(),
+            properties,
             generation_run_info: {
                 id: infoRes.id,
                 date: new Date().toISOString(),
                 model: infoRes.modelId ?? '',
+                extraction_fingerprint: extractionFingerprint,
             },
         },
         { suppressWorkflows: true },

package/src/activities/generateEmbeddings.test.ts

@@ -0,0 +1,114 @@
+import { MockActivityEnvironment } from '@temporalio/testing';
+import type { VertesiaClient } from '@vertesia/client';
+import {
+    ContentEventName,
+    type ContentObject,
+    type DSLActivityExecutionPayload,
+    SupportedEmbeddingTypes,
+} from '@vertesia/common';
+import { beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
+import type { ActivityContext } from '../dsl/setup/ActivityContext.js';
+import { type GenerateEmbeddingsParams, generateEmbeddings } from './generateEmbeddings.js';
+
+vi.mock('../dsl/setup/ActivityContext.js', async (importOriginal) => {
+    const actual = await importOriginal<typeof import('../dsl/setup/ActivityContext.js')>();
+    return { ...actual, setupActivity: vi.fn() };
+});
+
+let testEnv: MockActivityEnvironment;
+
+beforeAll(() => {
+    testEnv = new MockActivityEnvironment();
+});
+
+beforeEach(() => {
+    vi.clearAllMocks();
+});
+
+const createPayload = (params: GenerateEmbeddingsParams): DSLActivityExecutionPayload<GenerateEmbeddingsParams> => {
+    const activityParams: GenerateEmbeddingsParams & Record<string, unknown> = { ...params };
+    return {
+        auth_token: 'mock-token',
+        account_id: 'test-account',
+        project_id: 'test-project',
+        params,
+        config: { studio_url: 'http://mock-studio', store_url: 'http://mock-store' },
+        workflow_name: 'StandardDocumentIntake',
+        event: ContentEventName.create,
+        objectIds: ['properties-only-object'],
+        input: { inputType: 'objectIds', objectIds: ['properties-only-object'] },
+        vars: {},
+        activity: { name: 'generateEmbeddings', params: activityParams },
+    };
+};
+
+describe('generateEmbeddings', () => {
+    it('should generate property embeddings for an object without content', async () => {
+        const { setupActivity } = await import('../dsl/setup/ActivityContext.js');
+        const document = {
+            id: 'properties-only-object',
+            properties: {
+                title: 'Properties-only object',
+                category: 'metadata',
+            },
+        } satisfies Partial<ContentObject>;
+        const embeddingResponse = {
+            model: 'embedding-model',
+            results: [{ outputs: [{ values: [0.1, 0.2, 0.3] }] }],
+        };
+        const client = {
+            objects: {
+                retrieve: vi.fn().mockResolvedValue(document),
+                setEmbedding: vi.fn().mockResolvedValue(undefined),
+            },
+            environments: {
+                embeddings: vi.fn().mockResolvedValue(embeddingResponse),
+            },
+        } as unknown as VertesiaClient;
+        const params = {
+            type: SupportedEmbeddingTypes.properties,
+            force: false,
+        } satisfies GenerateEmbeddingsParams;
+
+        vi.mocked(setupActivity).mockResolvedValue({
+            client,
+            objectId: document.id,
+            params,
+            fetchProject: vi.fn().mockResolvedValue({
+                name: 'Test Project',
+                namespace: 'test-project',
+                configuration: {
+                    embeddings: {
+                        [SupportedEmbeddingTypes.properties]: {
+                            enabled: true,
+                            environment: 'test-environment',
+                            max_tokens: 8000,
+                        },
+                    },
+                },
+            }),
+        } as unknown as ActivityContext<GenerateEmbeddingsParams>);
+
+        const result = await testEnv.run(generateEmbeddings, createPayload(params));
+
+        expect(result).toEqual({
+            id: document.id,
+            type: SupportedEmbeddingTypes.properties,
+            status: 'completed',
+            len: 3,
+        });
+        expect(client.objects.retrieve).toHaveBeenCalledWith(
+            document.id,
+            '+text +parts +embeddings +tokens +properties',
+        );
+        expect(client.environments.embeddings).toHaveBeenCalledWith('test-environment', {
+            inputs: [{ type: 'text', text: JSON.stringify(document.properties) }],
+            model: undefined,
+        });
+        expect(client.objects.setEmbedding).toHaveBeenCalledWith(document.id, SupportedEmbeddingTypes.properties, {
+            values: embeddingResponse.results[0].outputs[0].values,
+            model: embeddingResponse.model,
+            etag: expect.any(String),
+        });
+    });
+});

package/src/activities/generateEmbeddings.ts

@@ -113,10 +113,6 @@ export async function generateEmbeddings(payload: DSLActivityExecutionPayload<Ge
         throw new DocumentNotFoundError('Document not found', [objectId]);
     }
 
-    if (!document.content) {
-        throw new DocumentNotFoundError('Document content not found', [objectId]);
-    }
-
     let res:
         | Awaited<ReturnType<typeof generateTextEmbeddings>>
         | Awaited<ReturnType<typeof generateImageEmbeddings>>
@@ -174,14 +170,19 @@ async function generateTextEmbeddings({ document, client, type, config, force }:
         };
     }
 
-    if (type === SupportedEmbeddingTypes.text && !document.text) {
-        return { id: document.id, status: 'failed', message: 'no text found' };
-    }
-    if (type === SupportedEmbeddingTypes.properties && !document?.properties) {
+    const sourceText =
+        type === SupportedEmbeddingTypes.text
+            ? document.text
+            : document.properties
+              ? JSON.stringify(document.properties)
+              : undefined;
+
+    if (!sourceText) {
         return {
             id: document.id,
-            status: 'failed',
-            message: 'no properties found',
+            type,
+            status: 'skipped',
+            message: type === SupportedEmbeddingTypes.text ? 'no text found' : 'no properties found',
         };
     }
 
@@ -192,12 +193,12 @@ async function generateTextEmbeddings({ document, client, type, config, force }:
         );
     }
 
-    // Compute text etag for comparison
-    const textEtag = document.text_etag ?? (document.text ? md5(document.text) : undefined);
+    const sourceEtag =
+        type === SupportedEmbeddingTypes.text ? (document.text_etag ?? md5(sourceText)) : md5(sourceText);
 
     // Skip if embeddings already exist with matching etag (unless force=true)
     const existingEmbedding = document.embeddings?.[type];
-    if (!force && existingEmbedding?.etag && textEtag && existingEmbedding.etag === textEtag) {
+    if (!force && existingEmbedding?.etag && existingEmbedding.etag === sourceEtag) {
         log.debug(`Skipping ${type} embeddings for document ${document.id} - etag unchanged`);
         return {
             id: document.id,
@@ -208,15 +209,7 @@ async function generateTextEmbeddings({ document, client, type, config, force }:
     }
 
     // Count tokens if needed, do not rely on existing token count
-    let tokenCount: number | undefined;
-    if (type === SupportedEmbeddingTypes.text && document.text) {
-        tokenCount = countTokens(document.text).count;
-    }
-
-    if (type === SupportedEmbeddingTypes.properties && document.properties) {
-        const propertiesText = JSON.stringify(document.properties);
-        tokenCount = countTokens(propertiesText).count;
-    }
+    const tokenCount = countTokens(sourceText).count;
 
     const maxTokens = config.max_tokens ?? 8000;
 
@@ -233,7 +226,7 @@ async function generateTextEmbeddings({ document, client, type, config, force }:
     } else {
         log.debug(`Generating ${type} embeddings for document`);
 
-        const res = await generateEmbeddingsFromStudio(JSON.stringify(document[type]), environment, client);
+        const res = await generateEmbeddingsFromStudio(sourceText, environment, client);
         const values = res?.results?.[0]?.outputs?.[0]?.values;
         if (!values) {
             return {
@@ -249,7 +242,7 @@ async function generateTextEmbeddings({ document, client, type, config, force }:
         await client.objects.setEmbedding(document.id, type, {
             values,
             model: res.model,
-            etag: textEtag,
+            etag: sourceEtag,
         });
 
         return {