@vertesia/workflow 0.24.0-dev.202601221707

package/src/activities/notifyWebhook.ts

@@ -0,0 +1,199 @@
+import { log } from "@temporalio/activity";
+import { VertesiaClient } from "@vertesia/client";
+import { ApiVersions, DSLActivityExecutionPayload, DSLActivitySpec, WebHookSpec, WorkflowExecutionBaseParams } from "@vertesia/common";
+import { setupActivity } from "../dsl/setup/ActivityContext.js";
+import { WorkflowParamNotFoundError } from "../errors.js";
+import { getVertesiaClientOptions } from "../utils/client.js";
+
+export interface NotifyWebhookParams {
+    webhook: string | WebHookSpec; //URL to send the notification to
+    workflow_id: string; //The ID of the workflow sending the notification
+    workflow_type: string; //The type of workflow sending the notification (the wf function name)
+    workflow_run_id: string; //The ID of the specific workflow run sending the notification
+    event_name: string; //The event that triggered the notification (e.g. "completed", "failed", etc.)
+    detail?: Record<string, any>; // additional data about the event if any. It will be send to the webhook when using POST 
+    //target_url: string; //URL to send the notification to
+    method: 'GET' | 'POST'; //HTTP method to use
+    headers?: Record<string, string>; // additional headers to send
+}
+
+export interface WebhookNotificationPayload {
+    workflow_id: string,
+    workflow_name: string,
+    workflow_run_id: string,
+    event_name: string,
+    detail?: Record<string, any>,
+}
+
+export interface NotifyWebhook extends DSLActivitySpec<NotifyWebhookParams> {
+    name: 'notifyWebhook';
+}
+
+
+export async function notifyWebhook(payload: DSLActivityExecutionPayload<NotifyWebhookParams>) {
+
+    const { params } = await setupActivity<NotifyWebhookParams>(payload);
+    const { webhook, method, headers: defaultHeaders } = params
+    // resolve the url and the api version of the webhook
+    let target_url: string, version: number | undefined;
+    if (typeof webhook === 'string') {
+        target_url = webhook;
+    } else {
+        target_url = webhook.url;
+        version = webhook.version;
+    }
+
+    if (!target_url) throw new WorkflowParamNotFoundError('target_url');
+
+    const hasBody = params.detail && method === 'POST'; //body is sent only for POST
+
+    const headers = {
+        ...defaultHeaders,
+    };
+    if (hasBody) {
+        headers['Content-Type'] = 'application/json';
+    }
+    const body = hasBody ? await createRequestBody(payload, params, version) : undefined
+
+    log.info(`Notifying webhook at ${target_url}`);
+    const res = await fetch(target_url, {
+        method,
+        body,
+        headers,
+    }).catch(err => {
+        log.error(`An error occurred while notifying webhook at ${target_url}`, { err });
+        throw err;
+    });
+
+    if (!res.ok) {
+        log.warn(`Webhook endpoint ${target_url} returned an error - ${res.status} ${res.statusText}`, { fetchResponse: res });
+
+        // Try to get response payload for error message
+        let errorMessage = `Webhook Notification to ${target_url} failed with status: ${res.status} ${res.statusText}`;
+        try {
+            const responseText = await res.text();
+            if (responseText) {
+                errorMessage += ` - Response: ${responseText}`;
+            }
+        } catch (readError) {
+            // If we can't read the response, just use the basic error message
+            log.debug('Could not read response body for error', { readError });
+        }
+
+        throw new Error(errorMessage);
+    }
+
+    return { status: res.status, message: res.statusText, url: res.url }
+
+}
+
+
+
+// --------------------------------------
+// Data provider for webhooks
+// this allows to customize the payload sent to the webhook depending on the 
+// type of workflow and the api version of the webhook
+// --------------------------------------
+
+
+function getWorkflowName(workflowType: string): string {
+    // remove trailing Workflow or _Workflow case insensitive from the workflow type
+    return workflowType.replace(/_?workflow$/i, '');
+}
+
+async function createRequestBody(payload: WorkflowExecutionBaseParams, params: NotifyWebhookParams, api_version: number | undefined): Promise<string> {
+    if (api_version === undefined || Number(api_version) < ApiVersions.COMPLETION_RESULT_V1) {
+        return createOldRequestBody(payload, params);
+    } else {
+        return createLatestRequestBody(payload, params, api_version);
+    }
+}
+
+async function createLatestRequestBody(payload: WorkflowExecutionBaseParams, params: NotifyWebhookParams, api_version: number | undefined): Promise<string> {
+    const data = await createEventData(payload, params, api_version);
+    return JSON.stringify({
+        workflow_id: params.workflow_id,
+        workflow_name: getWorkflowName(params.workflow_type),
+        workflow_run_id: params.workflow_run_id,
+        event_name: params.event_name,
+        detail: data,
+    } satisfies WebhookNotificationPayload);
+}
+
+async function createEventData(payload: WorkflowExecutionBaseParams, params: NotifyWebhookParams, api_version: number | undefined): Promise<any> {
+    const data = params.detail;
+    if (data && data.run_id && params.event_name === "workflow_completed" && params.workflow_type === 'ExecuteInteractionWorkflow') {
+        const client = getVersionedVertesiaClient(payload, api_version); //ensure client is initialized
+        // we replace the result property with the full execution run object
+        return await client.runs.retrieve(data.run_id);
+    }
+    return data;
+}
+
+
+function getVersionedVertesiaClient(payload: WorkflowExecutionBaseParams, version: string | number | undefined | null) {
+    // set the api version header
+    return new VertesiaClient(getVertesiaClientOptions(payload)).withApiVersion(version ? String(version) : null);
+}
+
+
+// ----------------- Compatibility code -----------------
+/* Before 2025-10-08 the notifyWebhook POST body was in the format:
+
+{
+    "workflowId": "generation:ExecuteInteractionWorkflow:WhatColor:ox8wu6t4",
+    "runId": "0199c2c6-818f-77eb-b931-c1ba8b9e5184",
+    "status": "completed",
+    "result": {
+        "run_id": "68e616274b0e9bb510462378",
+        "status": "completed",
+        "result": {"Color": "white"}
+    }
+}
+
+After Versions.COMPLETION_RESULT_V1  (20250925) when the completion result interface changed we improved the
+payload to the current format:
+
+{
+    "workflow_id": "generation:ExecuteInteractionWorkflow:WhatColor:bdedqjqj6",
+    "workflow_name": "ExecuteInteraction",
+    "workflow_run_id": "0199c2d4-6b1d-7cf2-a1e5-4cac6778091e",
+    "event_name": "workflow_completed",
+    "detail": ExecutionRun
+}
+
+where ExecutionRun contains a result property with the new completion result format.
+
+"result": [
+    {
+        "type": "json",
+        "value": {
+            "Color": "white"
+        }
+    }
+],
+
+*/
+
+//@ts-ignore
+async function createOldRequestBody(payload: WorkflowExecutionBaseParams, params: NotifyWebhookParams): Promise<string> {
+    let data = params.detail;
+    if (data && data.run_id && params.event_name === "workflow_completed" && params.workflow_type === 'ExecuteInteractionWorkflow') {
+        const client = getVersionedVertesiaClient(payload, null); //ensure client is using no specific version
+        // Important Note: we cannot use client.runs.retrieve since it will transform the run result to the new format because of InteractionOutput
+        const run = await client.runs.get(data.run_id);
+        // since we use an unversioned client the run will be in old format so we don't need to tranform the result
+        const result = run.result;
+        data = {
+            workflowId: params.workflow_id,
+            runId: params.workflow_run_id,
+            status: params.event_name === 'workflow_completed' ? 'completed' : params.event_name,
+            result: {
+                run_id: run.id,
+                status: run.status,
+                result: result || null
+            }
+        };
+    }
+    return JSON.stringify(data || {});
+}

package/src/activities/rateLimiter.ts

@@ -0,0 +1,41 @@
+import { DSLActivityExecutionPayload, RateLimitRequestPayload } from "@vertesia/common";
+import { activityInfo, log } from "@temporalio/activity";
+import { setupActivity } from "../dsl/setup/ActivityContext.js";
+
+export interface RateLimitParams {
+  interactionIdOrEndpoint: string;
+  environmentId?: string;
+  modelId?: string;
+}
+
+export interface RateLimitResult {
+  delayMs: number;
+}
+
+export async function checkRateLimit(payload: DSLActivityExecutionPayload<RateLimitParams>): Promise<RateLimitResult> {
+  const { client, params } = await setupActivity<RateLimitParams>(payload);
+  const { environmentId, modelId } = params;
+  
+  const result: RateLimitResult = {
+    delayMs: 0,
+  }
+
+  try {
+    // Call the studio-server endpoint to get rate limit delay using the Vertesia client
+    const info = activityInfo();
+    const requestPayload: RateLimitRequestPayload = {
+      interaction: params.interactionIdOrEndpoint,
+      workflow_run_id: info.workflowExecution.runId,
+      environment_id: environmentId,
+      model_id: modelId,
+    };
+    
+    const response = await client.interactions.requestSlot(requestPayload);
+    result.delayMs = response.delay_ms;
+  } catch (error) {
+    log.warn('Failed to call rate limit API:', {error});
+    throw error;
+  }
+
+  return result;
+}

package/src/activities/renditions/generateImageRendition.ts

@@ -0,0 +1,111 @@
+import { log } from "@temporalio/activity";
+import { DSLActivityExecutionPayload, DSLActivitySpec } from "@vertesia/common";
+import { setupActivity } from "../../dsl/setup/ActivityContext.js";
+import { DocumentNotFoundError, WorkflowParamNotFoundError } from "../../errors.js";
+import { saveBlobToTempFile } from "../../utils/blobs.js";
+import {
+    ImageRenditionParams,
+    uploadRenditionPages,
+} from "../../utils/renditions.js";
+
+interface GenerateImageRenditionParams extends ImageRenditionParams { }
+
+export interface GenerateImageRendition
+    extends DSLActivitySpec<GenerateImageRenditionParams> {
+    name: "generateImageRendition";
+}
+
+export async function generateImageRendition(
+    payload: DSLActivityExecutionPayload<GenerateImageRenditionParams>,
+) {
+    const {
+        client,
+        objectId,
+        params: originParams,
+    } = await setupActivity<GenerateImageRenditionParams>(payload);
+
+    // Fix: Use maxHeightWidth if max_hw is not provided
+    const params = {
+        ...originParams,
+        max_hw: originParams.max_hw || (originParams as any).maxHeightWidth || 1596, // Default to 1596 if both are missing
+        format: originParams.format || (originParams as any).format_output || "png", // Default to png if format is missing
+    };
+
+    log.info(`Generating image rendition for ${objectId}`, {
+        originParams,
+        params,
+    });
+
+    const inputObject = await client.objects.retrieve(objectId).catch((err) => {
+        log.error(`Failed to retrieve document ${objectId}`, { err });
+        if (err.message.includes("not found")) {
+            throw new DocumentNotFoundError(`Document ${objectId} not found`, [objectId]);
+        }
+        throw err;
+    });
+
+    if (!inputObject) {
+        log.error(`Document ${objectId} not found`);
+        throw new DocumentNotFoundError(`Document ${objectId} not found`, [objectId]);
+    }
+
+    if (!params.format) {
+        log.error(`Format not found`);
+        throw new WorkflowParamNotFoundError(`format`);
+    }
+
+    if (!inputObject.content?.source) {
+        log.error(`Document ${objectId} has no source`);
+        throw new DocumentNotFoundError(`Document ${objectId} has no source`, [objectId]);
+    }
+
+    if (
+        !inputObject.content.type ||
+        !inputObject.content.type?.startsWith("image/")
+    ) {
+        log.error(
+            `Document ${objectId} is not an image or a video: ${inputObject.content.type}`,
+        );
+        throw new DocumentNotFoundError(
+            `Document ${objectId} is not an image or a video: ${inputObject.content.type}`,
+            [objectId],
+        );
+    }
+
+    //array of rendition files to upload
+    let renditionPages: string[] = [];
+
+    const imageFile = await saveBlobToTempFile(
+        client,
+        inputObject.content.source,
+    );
+    log.info(`Image ${objectId} copied to ${imageFile}`);
+    renditionPages.push(imageFile);
+
+
+    //IF no etag, log and use use object id as etag
+    if (!inputObject.content.etag) {
+        log.warn(`Document ${objectId} has no etag, using object id as etag`);
+    }
+    const contentEtag = inputObject.content.etag ?? inputObject.id;
+
+    const uploaded = await uploadRenditionPages(
+        client,
+        contentEtag,
+        [imageFile],
+        params,
+    );
+
+    if (!uploaded || !uploaded.length || !uploaded[0]) {
+        log.error(`Failed to upload rendition for ${objectId}`, { uploaded });
+        throw new Error(
+            `Failed to upload rendition for ${objectId} - upload object is empty`,
+        );
+    }
+
+    return {
+        uploads: uploaded.map((u) => u),
+        format: params.format,
+        status: "success",
+    };
+}

package/src/activities/renditions/generateVideoRendition.ts

@@ -0,0 +1,293 @@
+import { log } from "@temporalio/activity";
+import { DSLActivityExecutionPayload, DSLActivitySpec } from "@vertesia/common";
+import { exec } from "child_process";
+import fs from "fs";
+import os from "os";
+import path from "path";
+import { promisify } from "util";
+import { setupActivity } from "../../dsl/setup/ActivityContext.js";
+import { DocumentNotFoundError, WorkflowParamNotFoundError } from "../../errors.js";
+import { saveBlobToTempFile } from "../../utils/blobs.js";
+import {
+    ImageRenditionParams,
+    uploadRenditionPages,
+} from "../../utils/renditions.js";
+
+const execAsync = promisify(exec);
+
+interface GenerateVideoRenditionParams extends ImageRenditionParams { }
+
+export interface GenerateVideoRendition
+    extends DSLActivitySpec<GenerateVideoRenditionParams> {
+    name: "generateImageRendition";
+}
+
+interface VideoMetadata {
+    duration: number;
+    width: number;
+    height: number;
+}
+
+async function getVideoMetadata(videoPath: string): Promise<VideoMetadata> {
+    try {
+        const command = `ffprobe -v quiet -print_format json -show_format -show_streams "${videoPath}"`;
+        const { stdout } = await execAsync(command);
+        const metadata = JSON.parse(stdout);
+
+        const videoStream = metadata.streams.find(
+            (stream: any) => stream.codec_type === "video",
+        );
+        const duration = parseFloat(metadata.format.duration) || 0;
+        const width = videoStream?.width || 0;
+        const height = videoStream?.height || 0;
+
+        return { duration, width, height };
+    } catch (error) {
+        log.error(
+            `Failed to get video metadata: ${error instanceof Error ? error.message : "Unknown error"}`,
+        );
+        throw new Error(
+            `Failed to probe video metadata: ${error instanceof Error ? error.message : "Unknown error"}`,
+        );
+    }
+}
+
+async function generateThumbnail(
+    videoPath: string,
+    outputDir: string,
+    timestamp: number,
+    maxSize: number,
+): Promise<string | undefined> {
+    //pad timestamp to 5 digits as filename
+    const outputFile = path.join(
+        outputDir,
+        `thumb-${timestamp.toString().padStart(5, "0")}.jpg`,
+    );
+
+    // FFmpeg command to extract thumbnail at specific timestamp
+    // Use proper scale filter syntax: scale=w:h:force_original_aspect_ratio=decrease
+    const scaleFilter = `scale=${maxSize}:${maxSize}:force_original_aspect_ratio=decrease`;
+
+    const command = [
+        "ffmpeg",
+        "-y", // Overwrite output files
+        "-ss",
+        timestamp.toString(), // Seek to timestamp
+        "-i",
+        `"${videoPath}"`, // Input file
+        "-vframes",
+        "1", // Extract only 1 frame
+        "-vf",
+        `"${scaleFilter}"`, // Scale maintaining aspect ratio
+        "-q:v",
+        "2", // High quality
+        `"${outputFile}"`,
+    ].join(" ");
+    log.info(`Generating thumbnail at ${timestamp}s`), { command };
+    try {
+        const { stderr } = await execAsync(command);
+
+        // Log any warnings from ffmpeg
+        if (stderr && !stderr.includes("frame=")) {
+            log.debug(
+                `FFmpeg stderr for thumbnail at ${timestamp}s: ${stderr}`,
+            );
+        }
+
+        // Verify the file was created
+        if (fs.existsSync(outputFile)) {
+            log.debug(`Generated thumbnail at ${timestamp}s`);
+            return outputFile;
+        } else {
+            log.warn(`Thumbnail not generated for timestamp ${timestamp}s`);
+            return undefined;
+        }
+    } catch (error) {
+        log.error(
+            `Failed to generate thumbnail at ${timestamp}s: ${error instanceof Error ? error.message : "Unknown error"}`,
+        );
+        return undefined;
+    }
+}
+
+export async function generateVideoRendition(
+    payload: DSLActivityExecutionPayload<GenerateVideoRenditionParams>,
+) {
+    const {
+        client,
+        objectId,
+        params: originParams,
+    } = await setupActivity<GenerateVideoRenditionParams>(payload);
+
+    // Fix: Use maxHeightWidth if max_hw is not provided
+    const params = {
+        ...originParams,
+        max_hw:
+            originParams.max_hw || (originParams as any).maxHeightWidth || 1024, // Default to 1024 if both are missing
+        format:
+            originParams.format || (originParams as any).format_output || "png", // Default to png if format is missing
+    };
+
+    log.info(`Generating video rendition for ${objectId}`, {
+        originParams,
+        params,
+    });
+
+    const inputObject = await client.objects.retrieve(objectId).catch((err) => {
+        log.error(`Failed to retrieve document ${objectId}`, { err });
+        if (err.message.includes("not found")) {
+            throw new DocumentNotFoundError(`Document ${objectId} not found`, [
+                objectId,
+            ]);
+        }
+        throw err;
+    });
+
+    if (!params.format) {
+        log.error(`Format not found`);
+        throw new WorkflowParamNotFoundError(`format`);
+    }
+
+    if (!inputObject.content?.source) {
+        log.error(`Document ${objectId} has no source`);
+        throw new DocumentNotFoundError(`Document ${objectId} has no source`, [
+            objectId,
+        ]);
+    }
+
+    if (
+        !inputObject.content.type ||
+        !inputObject.content.type?.startsWith("video/")
+    ) {
+        log.error(
+            `Document ${objectId} is not a video: ${inputObject.content.type}`,
+        );
+        throw new DocumentNotFoundError(
+            `Document ${objectId} is not a video: ${inputObject.content.type}`,
+            [objectId],
+        );
+    }
+
+    //array of rendition files to upload
+    let renditionPages: string[] = [];
+
+    const videoFile = await saveBlobToTempFile(
+        client,
+        inputObject.content.source,
+    );
+    const tempOutputDir = fs.mkdtempSync(
+        path.join(os.tmpdir(), "video-rendition-"),
+    );
+
+    try {
+        // Get video metadata using command line ffprobe
+        const metadata = await getVideoMetadata(videoFile);
+        const duration = metadata.duration;
+
+        // Calculate optimal number of thumbnails based on video length
+        const calculateThumbnailCount = (videoDuration: number): number => {
+            if (videoDuration <= 60) return 3; // Short videos: 3 thumbnails
+            if (videoDuration <= 300) return 5; // 5min videos: 5 thumbnails
+            if (videoDuration <= 600) return 8; // 10min videos: 8 thumbnails
+            if (videoDuration <= 1800) return 12; // 30min videos: 12 thumbnails
+            if (videoDuration <= 3600) return 16; // 1hr videos: 16 thumbnails
+            return 20; // Longer videos: max 20 thumbnails
+        };
+
+        const thumbnailCount = calculateThumbnailCount(duration);
+
+        // Generate evenly spaced timestamps, avoiding very beginning and end
+        const timestamps: number[] = [];
+        const startOffset = Math.min(duration * 0.05, 5); // Skip first 5% or 5 seconds
+        const endOffset = Math.min(duration * 0.05, 5); // Skip last 5% or 5 seconds
+        const usableDuration = duration - startOffset - endOffset;
+
+        for (let i = 0; i < thumbnailCount; i++) {
+            const progress = (i + 1) / (thumbnailCount + 1); // Evenly distribute
+            const timestamp = startOffset + usableDuration * progress;
+            timestamps.push(Math.max(timestamp, 1));
+        }
+
+        log.info(
+            `Generating ${thumbnailCount} thumbnails for ${duration}s video`,
+            {
+                objectId,
+                duration,
+                thumbnailCount,
+                timestamps: timestamps.map((t) => Math.round(t)),
+                tempOutputDir,
+            },
+        );
+
+        // Generate thumbnails using command line ffmpeg
+        const generatedThumbnails = await Promise.all(
+            timestamps.map(async (timestamp) => {
+                return await generateThumbnail(
+                    videoFile,
+                    tempOutputDir,
+                    timestamp,
+                    params.max_hw,
+                );
+            }),
+        );
+
+        if (generatedThumbnails.length === 0) {
+            log.info(`No thumbnails were generated for video ${objectId}`, {
+                objectId,
+                thumbnailCount,
+                tempOutputDir,
+            });
+            throw new Error(
+                `No thumbnails were generated for video ${objectId}`,
+            );
+        }
+
+        renditionPages.push(
+            ...generatedThumbnails.filter(
+                (thumbnail) => thumbnail !== undefined,
+            ),
+        );
+        log.info(
+            `Successfully generated ${generatedThumbnails.length} thumbnails for ${objectId}`,
+            {
+                objectId,
+                generatedCount: generatedThumbnails.length,
+                requestedCount: thumbnailCount,
+            },
+        );
+    } catch (error) {
+        log.error(
+            `Error generating thumbnails for video: ${error instanceof Error ? error.message : "Unknown error"}`,
+        );
+        throw new Error(`Failed to generate thumbnails for video: ${objectId}`);
+    } finally {
+        // Clean up temporary video file
+        try {
+            if (fs.existsSync(videoFile)) {
+                fs.unlinkSync(videoFile);
+            }
+        } catch (cleanupError) {
+            log.warn(`Failed to cleanup temporary video file: ${videoFile}`);
+        }
+    }
+
+    if (!inputObject.content?.etag) {
+        log.warn(`Document ${objectId} has no etag, using object id as etag`);
+    }
+    const etag = inputObject.content.etag ?? inputObject.id;
+
+    // Update the final upload call to handle multiple thumbnails
+    const uploaded = await uploadRenditionPages(
+        client,
+        etag,
+        renditionPages,
+        params,
+    );
+
+    return {
+        uploads: uploaded.map((u) => u),
+        format: params.format,
+        thumbnailCount: renditionPages.length,
+        status: "success",
+    };
+}

package/src/activities/setDocumentStatus.ts

@@ -0,0 +1,25 @@
+import { ContentObjectStatus, DSLActivityExecutionPayload, DSLActivitySpec } from "@vertesia/common";
+import { setupActivity } from "../dsl/setup/ActivityContext.js";
+
+export interface SetDocumentStatusParams {
+    status: ContentObjectStatus;
+}
+
+export interface SetDocumentStatus extends DSLActivitySpec<SetDocumentStatusParams> {
+    name: 'setDocumentStatus';
+    projection?: never;
+}
+
+/**
+ * We are using a union type for the status parameter since typescript enums breaks the workflow code generation
+ * @param objectId
+ * @param status
+ */
+export async function setDocumentStatus(payload: DSLActivityExecutionPayload<SetDocumentStatusParams>) {
+    const { client, params, objectId } = await setupActivity<SetDocumentStatusParams>(payload);
+
+    const res = await client.objects.update(objectId, { status: params.status });
+
+    return res.status;
+
+}