@vertesia/workflow 0.24.0-dev.202601221707

package/src/activities/generateDocumentProperties.ts

@@ -0,0 +1,120 @@
+import { log } from "@temporalio/activity";
+import { DSLActivityExecutionPayload, DSLActivitySpec } from "@vertesia/common";
+import { setupActivity } from "../dsl/setup/ActivityContext.js";
+import { TruncateSpec, truncByMaxTokens } from "../utils/tokens.js";
+import { InteractionExecutionParams, executeInteractionFromActivity } from "./executeInteraction.js";
+
+const INT_EXTRACT_INFORMATION = "sys:ExtractInformation";
+export interface GenerateDocumentPropertiesParams extends InteractionExecutionParams {
+    typesHint?: string[];
+    /**
+     * truncate the input doc text to the specified max_tokens
+     */
+    truncate?: TruncateSpec;
+
+    interactionName?: string;
+
+    use_vision?: boolean;
+}
+export interface GenerateDocumentProperties extends DSLActivitySpec<GenerateDocumentPropertiesParams> {
+    name: "generateDocumentProperties";
+}
+
+export async function generateDocumentProperties(
+    payload: DSLActivityExecutionPayload<GenerateDocumentPropertiesParams>,
+) {
+    const context = await setupActivity<GenerateDocumentPropertiesParams>(payload);
+    const { params, client, objectId } = context;
+    const interactionName = params.interactionName ?? INT_EXTRACT_INFORMATION;
+
+    const project = await context.fetchProject();
+
+    const doc = await client.objects.retrieve(objectId, "+text");
+    const type = doc.type ? await client.types.retrieve(doc.type.id) : undefined;
+
+    if (!doc?.text && !params.use_vision && !doc?.content?.type?.startsWith("image/")) {
+        log.warn(`Object ${objectId} not found or text is empty`);
+        return { status: "failed", error: "no-text" };
+    }
+
+    if (!type || !type.object_schema) {
+        log.info(`Object ${objectId} has no schema`);
+        return { document: objectId, status: "skipped", message: "no schema defined on type" };
+    }
+
+    const getImageRef = () => {
+        if (doc.content?.type?.startsWith("image/")) {
+            return "store:" + doc.id;
+        }
+
+        if (params.use_vision && doc.content?.type?.startsWith("application/pdf")) {
+            return "store:" + doc.id;
+        }
+
+        log.info(`Object ${objectId} is not an image or pdf`);
+        return undefined;
+    };
+
+    const content = doc.text
+        ? truncByMaxTokens(doc.text, params.truncate || 30000)
+        : undefined;
+
+    const promptData = {
+        content: content,
+        image: getImageRef() ?? undefined,
+        human_context: project?.configuration?.human_context ?? undefined,
+    };
+
+    log.info(
+        ` Extracting information from object ${objectId} with type ${type.name}`,
+        payload.debug_mode ? { params } : undefined,
+    );
+
+    const infoRes = await executeInteractionFromActivity(
+        client,
+        interactionName,
+        {
+            ...params,
+            include_previous_error: true,
+            result_schema: type.object_schema,
+            validate_result: type.strict_mode,
+        },
+        promptData,
+        payload.debug_mode ?? false,
+    );
+
+    const getText = () => {
+        if (doc.text) {
+            return undefined;
+        }
+        let text = "";
+        const jsonResult = infoRes.result.object();
+        if (jsonResult.title) {
+            text += jsonResult.title + "\n";
+        }
+        if (jsonResult.description) {
+            text += jsonResult.description;
+        }
+        if (text) {
+            return text;
+        } else {
+            return undefined;
+        }
+    };
+
+    log.info(`Extracted information from object ${objectId} with type ${type.name}`, { runId: infoRes.id });
+    await client.objects.update(doc.id, {
+        properties: {
+            ...infoRes.result.object(),
+            etag: doc.text_etag,
+        },
+        text: getText(),
+        generation_run_info: {
+            id: infoRes.id,
+            date: new Date().toISOString(),
+            model: infoRes.modelId,
+        },
+    }, { suppressWorkflows: true });
+
+    return { status: "completed" };
+}

package/src/activities/generateEmbeddings.ts

@@ -0,0 +1,387 @@
+import { EmbeddingsResult } from "@llumiverse/common";
+import { log } from "@temporalio/activity";
+import { VertesiaClient } from "@vertesia/client";
+import {
+    ContentObject,
+    DSLActivityExecutionPayload,
+    DSLActivitySpec,
+    ImageRenditionFormat,
+    ProjectConfigurationEmbeddings,
+    SupportedEmbeddingTypes,
+} from "@vertesia/common";
+import { setupActivity } from "../dsl/setup/ActivityContext.js";
+import { DocumentNotFoundError } from "../errors.js";
+import { fetchBlobAsBase64, md5 } from "../utils/blobs.js";
+import { DocPart } from "../utils/chunks.js";
+import { countTokens } from "../utils/tokens.js";
+
+export interface GenerateEmbeddingsParams {
+    /**
+     * The model to use for embedding generation
+     * If not set, the default model for the project will be used
+     */
+    model?: string;
+
+    /**
+     * The environment to use for embedding generation
+     * If not set, the default environment for the project will be used
+     */
+    environment?: string;
+
+    /**
+     * If true, force embedding generation even if the document already has embeddings
+     */
+    force?: boolean;
+
+    /**
+     * The embedding type to generate
+     */
+    type: SupportedEmbeddingTypes;
+
+    /**
+     * The DocParts to use for long documents
+     */
+    parts?: DocPart[];
+}
+
+export interface GenerateEmbeddings
+    extends DSLActivitySpec<GenerateEmbeddingsParams> {
+    name: "generateEmbeddings";
+}
+
+export async function generateEmbeddings(
+    payload: DSLActivityExecutionPayload<GenerateEmbeddingsParams>,
+) {
+    const { params, client, objectId, fetchProject } =
+        await setupActivity<GenerateEmbeddingsParams>(payload);
+    const { force, type } = params;
+
+    const projectData = await fetchProject();
+    const config = projectData?.configuration.embeddings[type];
+    if (!projectData) {
+        throw new DocumentNotFoundError("Project not found", [payload.project_id]);
+    }
+    if (!config) {
+        throw new DocumentNotFoundError("Embeddings configuration not found", [
+            objectId,
+        ]);
+    }
+
+    if (!projectData) {
+        throw new DocumentNotFoundError("Project not found", [payload.project_id]);
+    }
+
+    if (!projectData?.configuration.embeddings[type]?.enabled) {
+        log.info(
+            `Embeddings generation disabled for type ${type} on project: ${projectData.name} (${projectData.namespace})`,
+            { config },
+        );
+        return {
+            id: objectId,
+            status: "skipped",
+            message: `Embeddings generation disabled for type ${type}`,
+        };
+    }
+
+    log.info(`${type} embedding generation starting for object ${objectId}`, {
+        force,
+        config,
+    });
+
+    if (!config.environment) {
+        throw new Error(
+            "No environment found in project configuration. Set environment in project configuration to generate embeddings.",
+        );
+    }
+
+    const document = await client.objects.retrieve(
+        objectId,
+        "+text +parts +embeddings +tokens +properties",
+    );
+
+    if (!document) {
+        throw new DocumentNotFoundError("Document not found", [objectId]);
+    }
+
+    if (!document.content) {
+        throw new DocumentNotFoundError("Document content not found", [objectId]);
+    }
+
+    let res;
+
+    switch (type) {
+        case SupportedEmbeddingTypes.text:
+            res = await generateTextEmbeddings({
+                client,
+                config,
+                document,
+                type,
+                force,
+            });
+            break;
+        case SupportedEmbeddingTypes.properties:
+            res = await generateTextEmbeddings({
+                client,
+                config,
+                document,
+                type,
+                force,
+            });
+            break;
+        case SupportedEmbeddingTypes.image:
+            res = await generateImageEmbeddings({
+                client,
+                config,
+                document,
+                type,
+                force,
+            });
+            break;
+        default:
+            res = {
+                id: objectId,
+                status: "failed",
+                message: `unsupported embedding type: ${type}`,
+            };
+    }
+
+    return res;
+}
+
+interface ExecuteGenerateEmbeddingsParams {
+    document: ContentObject;
+    client: VertesiaClient;
+    type: SupportedEmbeddingTypes;
+    config: ProjectConfigurationEmbeddings;
+    property?: string;
+    force?: boolean;
+}
+
+async function generateTextEmbeddings(
+    { document, client, type, config, force }: ExecuteGenerateEmbeddingsParams
+) {
+
+    if (!document) {
+        return { status: "error", message: "document is null or undefined" };
+    }
+
+    if (
+        type !== SupportedEmbeddingTypes.text &&
+        type !== SupportedEmbeddingTypes.properties
+    ) {
+        return {
+            id: document.id,
+            status: "failed",
+            message: `unsupported embedding type: ${type}`,
+        };
+    }
+
+    if (type === SupportedEmbeddingTypes.text && !document.text) {
+        return { id: document.id, status: "failed", message: "no text found" };
+    }
+    if (type === SupportedEmbeddingTypes.properties && !document?.properties) {
+        return {
+            id: document.id,
+            status: "failed",
+            message: "no properties found",
+        };
+    }
+
+    const { environment } = config;
+
+    // Compute text etag for comparison
+    const textEtag = document.text_etag ?? (document.text ? md5(document.text) : undefined);
+
+    // Skip if embeddings already exist with matching etag (unless force=true)
+    const existingEmbedding = document.embeddings?.[type];
+    if (!force && existingEmbedding?.etag && textEtag && existingEmbedding.etag === textEtag) {
+        log.info(`Skipping ${type} embeddings for document ${document.id} - etag unchanged`);
+        return {
+            id: document.id,
+            type,
+            status: "skipped",
+            message: "embeddings already exist with matching etag",
+        };
+    }
+
+    // Count tokens if needed, do not rely on existing token count
+    let tokenCount : number | undefined = undefined;
+    if (type === SupportedEmbeddingTypes.text && document.text) {
+        tokenCount = countTokens(document.text).count;
+    }
+
+    const maxTokens = config.max_tokens ?? 8000;
+
+    //generate embeddings for the main doc if document isn't too large
+    log.info(`Generating ${type} embeddings for document ${document.id}`);
+    if (
+        type === SupportedEmbeddingTypes.text &&
+        tokenCount !== undefined &&
+        tokenCount > maxTokens
+    ) {
+        //TODO: Review strategy for large documents
+        log.warn(
+            `Document too large for ${type} embeddings generation, skipping (${tokenCount} tokens)`,
+        );
+        return {
+            id: document.id,
+            status: "skipped",
+            message: `${type} embeddings generation, skipped for large document (${tokenCount} tokens)`,
+        }
+    } else {
+        log.info(`Generating ${type} embeddings for document`);
+
+        const res = await generateEmbeddingsFromStudio(
+            JSON.stringify(document[type]),
+            environment,
+            client,
+        );
+        if (!res || !res.values) {
+            return {
+                id: document.id,
+                status: "failed",
+                message: "no embeddings generated",
+            };
+        }
+
+        log.info(`${type} embeddings generated for document ${document.id}`, {
+            len: res.values.length,
+        });
+        await client.objects.setEmbedding(document.id, type, {
+            values: res.values,
+            model: res.model,
+            etag: textEtag,
+        });
+
+        return {
+            id: document.id,
+            type,
+            status: "completed",
+            len: res.values.length,
+        };
+    }
+}
+
+async function generateImageEmbeddings({
+    document,
+    client,
+    type,
+    config,
+    force,
+}: ExecuteGenerateEmbeddingsParams) {
+    log.info("Generating image embeddings for document " + document.id, {
+        content: document.content,
+    });
+    if (
+        !document.content?.type?.startsWith("image/") &&
+        !document.content?.type?.includes("pdf")
+    ) {
+        return {
+            id: document.id,
+            type,
+            status: "failed",
+            message: "content is not an image",
+        };
+    }
+
+    // Use content etag for image change detection
+    const contentEtag = document.content?.etag;
+
+    // Skip if embeddings already exist with matching etag (unless force=true)
+    const existingEmbedding = document.embeddings?.[type];
+    if (!force && existingEmbedding?.etag && contentEtag && existingEmbedding.etag === contentEtag) {
+        log.info(`Skipping ${type} embeddings for document ${document.id} - content etag unchanged`);
+        return {
+            id: document.id,
+            type,
+            status: "skipped",
+            message: "embeddings already exist with matching etag",
+        };
+    }
+
+    const { environment, model } = config;
+
+    const resRnd = await client.store.objects.getRendition(document.id, {
+        format: ImageRenditionFormat.jpeg,
+        generate_if_missing: true,
+        sign_url: true,
+    });
+
+    if (resRnd.status === "generating") {
+        throw new Error("Rendition is generating, will retry later");
+    } else if (
+        resRnd.status === "failed" ||
+        !resRnd.renditions ||
+        !resRnd.renditions.length
+    ) {
+        throw new DocumentNotFoundError("Rendition retrieval failed", [document.id]);
+    }
+
+    const renditions = resRnd.renditions;
+    if (!renditions?.length) {
+        throw new DocumentNotFoundError("No source found in rendition", [
+            document.id,
+        ]);
+    }
+
+    const rendition = renditions[0];
+    const image = await fetchBlobAsBase64(client, rendition);
+
+    const res = await client.environments
+        .embeddings(environment, {
+            image,
+            model,
+        })
+        .then((res) => res)
+        .catch((e) => {
+            log.error("Error generating embeddings for image", { error: e });
+            throw e;
+        });
+
+    if (!res || !res.values) {
+        return {
+            id: document.id,
+            status: "failed",
+            message: "no embeddings generated",
+        };
+    }
+
+    await client.objects.setEmbedding(
+        document.id,
+        SupportedEmbeddingTypes.image,
+        {
+            values: res.values,
+            model: res.model,
+            etag: contentEtag,
+        },
+    );
+
+    return {
+        id: document.id,
+        type,
+        status: "completed",
+        len: res.values.length,
+    };
+}
+
+async function generateEmbeddingsFromStudio(
+    text: string,
+    env: string,
+    client: VertesiaClient,
+    model?: string,
+): Promise<EmbeddingsResult> {
+    log.info(
+        `Generating embeddings for text of ${text.length} chars with environment ${env}`,
+    );
+
+    return client.environments
+        .embeddings(env, {
+            text,
+            model,
+        })
+        .then((res) => res)
+        .catch((e) => {
+            log.error("Error generating embeddings for text", { error: e });
+            throw e;
+        });
+}