@vertesia/workflow 0.79.0 → 0.80.0-dev-20251118

package/src/activities/media/transcribeMediaWithGladia.ts

@@ -1,6 +1,6 @@
-import { DSLActivityExecutionPayload, DSLActivitySpec, GladiaConfiguration, SupportedIntegrations } from "@vertesia/common";
+import { DSLActivityExecutionPayload, DSLActivitySpec, GladiaConfiguration, SupportedIntegrations, AUDIO_RENDITION_NAME, VideoMetadata, ContentNature } from "@vertesia/common";
 import { activityInfo, CompleteAsyncError, log } from "@temporalio/activity";
-import { FetchClient } from "@vertesia/api-fetch-client";
+import { FetchClient, RequestError } from "@vertesia/api-fetch-client";
 import { setupActivity } from "../../dsl/setup/ActivityContext.js";
 import { DocumentNotFoundError } from "../../errors.js";
 import { TextExtractionResult, TextExtractionStatus } from "../../index.js";
@@ -27,7 +27,12 @@ export async function transcribeMedia(payload: DSLActivityExecutionPayload<Trans
 
     const gladiaConfig = await client.projects.integrations.retrieve(payload.project_id, SupportedIntegrations.gladia) as GladiaConfiguration | undefined;
     if (!gladiaConfig || !gladiaConfig.enabled) {
-        throw new DocumentNotFoundError("Gladia integration not enabled");
+        return {
+            hasText: false,
+            objectId,
+            status: TextExtractionStatus.error,
+            error: "Gladia integration not enabled",
+        }
     }
 
     const object = await client.objects.retrieve(objectId, "+text");
@@ -42,37 +47,63 @@ export async function transcribeMedia(payload: DSLActivityExecutionPayload<Trans
         throw new DocumentNotFoundError(`No source found for object ${objectId}`);
     }
 
-    const mediaUrl = await client.store.objects.getContentSource(objectId).then(res => res.source);
+    // Check for audio rendition in video metadata (preferred for videos)
+    let mediaSource: string = object.content.source;
+    if (object.metadata?.type === ContentNature.Video) {
+        const videoMetadata = object.metadata as VideoMetadata;
+        const audioRendition = videoMetadata.renditions?.find(r => r.name === AUDIO_RENDITION_NAME);
+        if (audioRendition?.content?.source) {
+            mediaSource = audioRendition.content.source;
+            log.info(`Found audio rendition for video object ${objectId}`, { mediaSource });
+        }
+    }
+
+    // Get download URL for the media source
+    const { url: mediaUrl } = await client.files.getDownloadUrl(mediaSource);
 
     if (!mediaUrl) {
-        throw new DocumentNotFoundError(`Error fetching source ${object.content.source}`);
+        throw new DocumentNotFoundError(`Error fetching media URL for ${mediaSource}`);
     }
 
+    log.info(`Using media URL for transcription`, { objectId, mediaUrl: mediaSource });
+
     const taskToken = Buffer.from(activityInfo().taskToken).toString('base64url');
     const callbackUrl = generateCallbackUrlForGladia(client.store.baseUrl, payload.auth_token, taskToken, objectId);
 
     log.info(`Transcribing media ${mediaUrl} with Gladia`, { objectId, callbackUrl });
 
-    const res = await gladiaClient.post("/transcription", {
-        payload: {
-            audio_url: mediaUrl,
-            callback_url: callbackUrl,
-            diarization_enhanced: true,
-            enable_code_switching: true,
-            subtitles: true,
-            subtitles_config: {
-                formats: ["vtt"],
+    try {
+        const res = await gladiaClient.post("/transcription", {
+            payload: {
+                audio_url: mediaUrl,
+                callback_url: callbackUrl,
+                diarization_enhanced: true,
+                enable_code_switching: true,
+                subtitles: true,
+                subtitles_config: {
+                    formats: ["vtt"],
+                }
+            }
+        }) as GladiaTranscriptRequestResponse;
+
+        log.info(`Transcription request sent to Gladia`, { objectId, res });
+        throw new CompleteAsyncError();
+
+    } catch (error: any) {
+        if (error instanceof RequestError && error.status === 422) {
+            return {
+                hasText: false,
+                objectId,
+                status: TextExtractionStatus.error,
+                error: `Gladia transcription error: ${error.message}`,
             }
+        } else {
+            log.error(`Error sending transcription request to Gladia for object ${objectId}`, { error });
+            throw error;
         }
-    }) as GladiaTranscriptRequestResponse;
-
-    log.info(`Transcription request sent to Gladia`, { objectId, res });
-
-    throw new CompleteAsyncError();
-
+    }
 }
 
-
 function generateCallbackUrlForGladia(baseUrl: string, authToken: string, taskToken: string, objectId: string) {
     return `${baseUrl}/api/v1/webhooks/gladia/${objectId}?access_token=${authToken}&task_token=${taskToken}`;
 }

package/src/activities/notifyWebhook.test.ts

@@ -33,7 +33,7 @@ const defaultParams = {
 const createTestPayload = (params: Partial<NotifyWebhookParams> = {}): DSLActivityExecutionPayload<NotifyWebhookParams> => {
   const mergedParams = { ...defaultParams, ...params };
   return {
-    auth_token: "unset",
+    auth_token: process.env.VERTESIA_KEY || "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJodHRwOi8vbW9jay10b2tlbi1zZXJ2ZXIiLCJzdWIiOiJ0ZXN0In0.signature",
     account_id: "unset",
     project_id: "unset",
     params: mergedParams,

package/src/activities/notifyWebhook.ts

@@ -180,7 +180,8 @@ async function createOldRequestBody(payload: WorkflowExecutionBaseParams, params
     let data = params.detail;
     if (data && data.run_id && params.event_name === "workflow_completed" && params.workflow_type === 'ExecuteInteractionWorkflow') {
         const client = getVersionedVertesiaClient(payload, null); //ensure client is using no specific version
-        const run = await client.runs.retrieve(data.run_id);
+        // Important Note: we cannot use client.runs.retrieve since it will transform the run result to the new format because of InteractionOutput
+        const run = await client.runs.get(data.run_id);
         // since we use an unversioned client the run will be in old format so we don't need to tranform the result
         const result = run.result;
         data = {

package/src/dsl/dsl-workflow.ts

@@ -225,7 +225,13 @@ async function executeChildWorkflow(step: DSLChildWorkflowStep, payload: DSLWork
 }
 
 function buildRateLimitParams(activity: DSLActivitySpec, executionPayload: DSLActivityExecutionPayload<any>): RateLimitParams {
-    const params = executionPayload.params;
+    // resolve payload params
+    const vars = new Vars({
+        ...executionPayload.params, // imported params (doesn't contain expressions)
+        ...executionPayload.activity.params, // activity params (may contain expressions)
+    });
+    const params = vars.resolve();
+
     let interactionId: string;
 
     switch (activity.name) {

package/src/dsl/setup/ActivityContext.ts

@@ -6,16 +6,32 @@ import {
     Project,
     WorkflowExecutionPayload,
 } from "@vertesia/common";
-import { DocumentNotFoundError, WorkflowParamNotFoundError } from "../../errors.js";
+import {
+    DocumentNotFoundError,
+    WorkflowParamNotFoundError,
+} from "../../errors.js";
 import { getProjectFromToken } from "../../utils/auth.js";
 import { getVertesiaClient } from "../../utils/client.js";
 import { Vars } from "../vars.js";
-import { getFetchProvider, registerFetchProviderFactory } from "./fetch/index.js";
-import { DocumentProvider, DocumentTypeProvider, InteractionRunProvider } from "./fetch/providers.js";
+import {
+    getFetchProvider,
+    registerFetchProviderFactory,
+} from "./fetch/index.js";
+import {
+    DocumentProvider,
+    DocumentTypeProvider,
+    InteractionRunProvider,
+} from "./fetch/providers.js";
 
 registerFetchProviderFactory(DocumentProvider.ID, DocumentProvider.factory);
-registerFetchProviderFactory(DocumentTypeProvider.ID, DocumentTypeProvider.factory);
-registerFetchProviderFactory(InteractionRunProvider.ID, InteractionRunProvider.factory);
+registerFetchProviderFactory(
+    DocumentTypeProvider.ID,
+    DocumentTypeProvider.factory,
+);
+registerFetchProviderFactory(
+    InteractionRunProvider.ID,
+    InteractionRunProvider.factory,
+);
 
 export class ActivityContext<ParamsT extends Record<string, any>> {
     client: VertesiaClient;
@@ -40,7 +56,10 @@ export class ActivityContext<ParamsT extends Record<string, any>> {
             log.error("No objectId found in payload");
             throw new WorkflowParamNotFoundError(
                 "objectIds[0]",
-                (this.payload as WorkflowExecutionPayload as DSLWorkflowExecutionPayload).workflow,
+                (
+                    this
+                        .payload as WorkflowExecutionPayload as DSLWorkflowExecutionPayload
+                ).workflow,
             );
         }
         return objectId;
@@ -56,7 +75,10 @@ export class ActivityContext<ParamsT extends Record<string, any>> {
             log.error("No runId found in activityInfo");
             throw new WorkflowParamNotFoundError(
                 "runId",
-                (this.payload as WorkflowExecutionPayload as DSLWorkflowExecutionPayload).workflow,
+                (
+                    this
+                        .payload as WorkflowExecutionPayload as DSLWorkflowExecutionPayload
+                ).workflow,
             );
         }
         return runId;
@@ -68,7 +90,10 @@ export class ActivityContext<ParamsT extends Record<string, any>> {
             log.error("No workflowId found in activityInfo");
             throw new WorkflowParamNotFoundError(
                 "workflowId",
-                (this.payload as WorkflowExecutionPayload as DSLWorkflowExecutionPayload).workflow,
+                (
+                    this
+                        .payload as WorkflowExecutionPayload as DSLWorkflowExecutionPayload
+                ).workflow,
             );
         }
         return workflowId;
@@ -102,7 +127,7 @@ export async function setupActivity<ParamsT extends Record<string, any>>(
         });
     }
 
-    const client = getVertesiaClient(payload);
+    const client = await getVertesiaClient(payload);
     const fetchSpecs = payload.activity.fetch;
     if (fetchSpecs) {
         const keys = Object.keys(fetchSpecs);
@@ -118,7 +143,10 @@ export async function setupActivity<ParamsT extends Record<string, any>>(
 
                 const provider = getFetchProvider(client, fetchSpec);
 
-                log.info(`Fetching data for ${key} with provider ${provider.name}`, { fetchSpec });
+                log.info(
+                    `Fetching data for ${key} with provider ${provider.name}`,
+                    { fetchSpec },
+                );
                 const result = await provider.fetch(fetchSpec);
                 if (result && result.length > 0) {
                     if (fetchSpec.limit === 1) {
@@ -127,7 +155,9 @@ export async function setupActivity<ParamsT extends Record<string, any>>(
                         vars.setValue(key, result);
                     }
                 } else if (fetchSpec.on_not_found === "throw") {
-                    throw new DocumentNotFoundError("No documents found for: " + JSON.stringify(fetchSpec));
+                    throw new DocumentNotFoundError(
+                        "No documents found for: " + JSON.stringify(fetchSpec),
+                    );
                 } else {
                     vars.setValue(key, null);
                 }
@@ -141,7 +171,10 @@ export async function setupActivity<ParamsT extends Record<string, any>>(
     return new ActivityContext<ParamsT>(payload, client, params);
 }
 
-async function _fetchProject(client: VertesiaClient, payload: WorkflowExecutionPayload) {
+async function _fetchProject(
+    client: VertesiaClient,
+    payload: WorkflowExecutionPayload,
+) {
     const project = await getProjectFromToken(payload.auth_token);
     return project ? await client.projects.retrieve(project.id) : undefined;
 }

package/src/dsl/workflow-exec-child.test.ts

@@ -159,7 +159,7 @@ describe('DSL Workflow with child workflows', () => {
             account_id: '123',
             project_id: '123',
             wf_rule_name: 'test',
-            auth_token: 'test',
+            auth_token: process.env.VERTESIA_KEY || 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJodHRwOi8vbW9jay10b2tlbi1zZXJ2ZXIiLCJzdWIiOiJ0ZXN0In0.signature',
             config: {
                 studio_url: process.env.CP_STUDIO_URL || "http://localhost:8081",
                 store_url: process.env.CP_STORE_URL || "http://localhost:8082",
@@ -203,7 +203,7 @@ describe('DSL Workflow with child workflows', () => {
             account_id: '123',
             project_id: '123',
             wf_rule_name: 'test',
-            auth_token: 'test',
+            auth_token: process.env.VERTESIA_KEY || 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJodHRwOi8vbW9jay10b2tlbi1zZXJ2ZXIiLCJzdWIiOiJ0ZXN0In0.signature',
             config: {
                 studio_url: process.env.CP_STUDIO_URL || "http://localhost:8081",
                 store_url: process.env.CP_STORE_URL || "http://localhost:8082",
@@ -247,7 +247,7 @@ describe('DSL Workflow with child workflows', () => {
             account_id: '123',
             project_id: '123',
             wf_rule_name: 'test',
-            auth_token: 'test',
+            auth_token: process.env.VERTESIA_KEY || 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJodHRwOi8vbW9jay10b2tlbi1zZXJ2ZXIiLCJzdWIiOiJ0ZXN0In0.signature',
             config: {
                 studio_url: process.env.CP_STUDIO_URL || "http://localhost:8081",
                 store_url: process.env.CP_STORE_URL || "http://localhost:8082",

package/src/dsl/workflow-fetch.test.ts

@@ -107,7 +107,7 @@ describe('DSL Workflow', () => {
             project_id: '123',
             timestamp: Date.now(),
             wf_rule_name: 'test',
-            auth_token: 'test',
+            auth_token: process.env.VERTESIA_KEY || 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJodHRwOi8vbW9jay10b2tlbi1zZXJ2ZXIiLCJzdWIiOiJ0ZXN0In0.signature',
             config: {
                 studio_url: process.env.CP_STUDIO_URL || "http://localhost:8081",
                 store_url: process.env.CP_STORE_URL || "http://localhost:8082",

package/src/dsl/workflow-import.test.ts

@@ -61,7 +61,7 @@ describe('DSL Workflow import vars', () => {
             project_id: '123',
             timestamp: Date.now(),
             wf_rule_name: 'test',
-            auth_token: 'test',
+            auth_token: process.env.VERTESIA_KEY || 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJodHRwOi8vbW9jay10b2tlbi1zZXJ2ZXIiLCJzdWIiOiJ0ZXN0In0.signature',
             config: {
                 studio_url: process.env.CP_STUDIO_URL || "http://localhost:8081",
                 store_url: process.env.CP_STORE_URL || "http://localhost:8082",

package/src/dsl/workflow.test.ts

@@ -91,7 +91,7 @@ describe('DSL Workflow', () => {
             project_id: '123',
             timestamp: Date.now(),
             wf_rule_name: 'test',
-            auth_token: 'test',
+            auth_token: process.env.VERTESIA_KEY || 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJodHRwOi8vbW9jay10b2tlbi1zZXJ2ZXIiLCJzdWIiOiJ0ZXN0In0.signature',
             config: {
                 studio_url: process.env.CP_STUDIO_URL || "http://localhost:8081",
                 store_url: process.env.CP_STORE_URL || "http://localhost:8082",

package/src/errors.ts

@@ -64,9 +64,24 @@ export class ResourceExhaustedError extends ApplicationFailure {
     }
 }
 
+export class InvalidContentTypeError extends ApplicationFailure {
+    constructor(
+        public objectId: string,
+        public expectedType: string,
+        public actualType: string,
+    ) {
+        super(
+            `Document ${objectId} has invalid content type. Expected ${expectedType}, got ${actualType}`,
+            "InvalidContentTypeError",
+            true, // non-retryable
+        );
+    }
+}
+
 export const WF_NON_RETRYABLE_ERRORS = [
     "DocumentNotFoundError",
     "ActivityParamInvalidError",
     "ActivityParamNotFoundError",
     "WorkflowParamNotFoundError",
+    "InvalidContentTypeError",
 ];

package/src/index.ts

@@ -25,6 +25,7 @@ export * from "./activities/notifyWebhook.js";
 export * from "./activities/rateLimiter.js";
 export * from "./activities/setDocumentStatus.js";
 export * from "./iterative-generation/activities/index.js";
+export * from "./activities/media/prepareVideo.js";
 
 export * from "./dsl/setup/ActivityContext.js";
 export * from "./errors.js";

package/src/iterative-generation/activities/extractToc.ts

@@ -2,8 +2,16 @@ import { log } from "@temporalio/activity";
 import { WorkflowExecutionPayload } from "@vertesia/common";
 import { parse as parseYaml } from "yaml";
 import { getVertesiaClient } from "../../utils/client.js";
-import { buildAndPublishMemoryPack, loadMemoryPack } from "../../utils/memory.js";
-import { IterativeGenerationPayload, OutputMemoryMeta, Toc, TocIndex } from "../types.js";
+import {
+    buildAndPublishMemoryPack,
+    loadMemoryPack,
+} from "../../utils/memory.js";
+import {
+    IterativeGenerationPayload,
+    OutputMemoryMeta,
+    Toc,
+    TocIndex,
+} from "../types.js";
 import { tocIndex } from "../utils.js";
 
 /**
@@ -12,10 +20,12 @@ import { tocIndex } from "../utils.js";
  *
  * @param payload
  */
-export async function it_gen_extractToc(payload: WorkflowExecutionPayload): Promise<TocIndex | null> {
+export async function it_gen_extractToc(
+    payload: WorkflowExecutionPayload,
+): Promise<TocIndex | null> {
     const vars = payload.vars as IterativeGenerationPayload;
     const memory = vars.memory;
-    const client = getVertesiaClient(payload);
+    const client = await getVertesiaClient(payload);
 
     const inMemory = await loadMemoryPack(client, `${memory}/input`);
     let tocJson: string | null = null;
@@ -37,13 +47,17 @@ export async function it_gen_extractToc(payload: WorkflowExecutionPayload): Prom
 
     log.info(`Found a TOC in the input memory pack`);
 
-    await buildAndPublishMemoryPack(client, `${vars.memory}/output`, async () => {
-        return {
-            toc,
-            lastProcessedPart: undefined, // the part index (a number array)
-            previouslyGenerated: ""
-        } as OutputMemoryMeta
-    });
+    await buildAndPublishMemoryPack(
+        client,
+        `${vars.memory}/output`,
+        async () => {
+            return {
+                toc,
+                lastProcessedPart: undefined, // the part index (a number array)
+                previouslyGenerated: "",
+            } as OutputMemoryMeta;
+        },
+    );
 
     return tocIndex(toc);
-}
+}

package/src/iterative-generation/activities/finalizeOutput.ts

@@ -2,14 +2,24 @@ import { log } from "@temporalio/activity";
 import { WorkflowExecutionPayload } from "@vertesia/common";
 import { getVertesiaClient } from "../../utils/client.js";
 import { expandVars } from "../../utils/expand-vars.js";
-import { buildAndPublishMemoryPack, loadMemoryPack } from "../../utils/memory.js";
-import { IterativeGenerationPayload, Section, SECTION_ID_PLACEHOLDER, TocSection } from "../types.js";
+import {
+    buildAndPublishMemoryPack,
+    loadMemoryPack,
+} from "../../utils/memory.js";
+import {
+    IterativeGenerationPayload,
+    Section,
+    SECTION_ID_PLACEHOLDER,
+    TocSection,
+} from "../types.js";
 
-export async function it_gen_finalizeOutput(payload: WorkflowExecutionPayload): Promise<string> {
+export async function it_gen_finalizeOutput(
+    payload: WorkflowExecutionPayload,
+): Promise<string> {
     const vars = payload.vars as IterativeGenerationPayload;
 
     const memory = vars.memory;
-    const client = getVertesiaClient(payload);
+    const client = await getVertesiaClient(payload);
     const inMemory = await loadMemoryPack(client, `${memory}/input`);
     const outMemory = await loadMemoryPack(client, `${memory}/output`);
 
@@ -35,43 +45,56 @@ export async function it_gen_finalizeOutput(payload: WorkflowExecutionPayload):
     }
     const sections = JSON.parse(content) as Section[];
 
-    await buildAndPublishMemoryPack(client, `${memory}/output`, async ({ copyText }) => {
-        // copy the input toc file if any
-        if (toc) {
-            copyText(toc, tocName);
-        }
-        // copy the raw JSON content
-        copyText(content, "content.json");
-        if (vars.section_file_pattern) {
-            log.info(`Saving sections to files using pattern: ${vars.section_file_pattern}`);
-            // save sections to files
-            for (const section of sections) {
-                let content = section.content;
-                if (vars.section_file_header) {
-                    content = getSectionFileHeader(section, vars.section_file_header) + '\n\n' + content;
+    await buildAndPublishMemoryPack(
+        client,
+        `${memory}/output`,
+        async ({ copyText }) => {
+            // copy the input toc file if any
+            if (toc) {
+                copyText(toc, tocName);
+            }
+            // copy the raw JSON content
+            copyText(content, "content.json");
+            if (vars.section_file_pattern) {
+                log.info(
+                    `Saving sections to files using pattern: ${vars.section_file_pattern}`,
+                );
+                // save sections to files
+                for (const section of sections) {
+                    let content = section.content;
+                    if (vars.section_file_header) {
+                        content =
+                            getSectionFileHeader(
+                                section,
+                                vars.section_file_header,
+                            ) +
+                            "\n\n" +
+                            content;
+                    }
+                    copyText(
+                        content,
+                        getSectionFileName(section, vars.section_file_pattern),
+                    );
                 }
-                copyText(content, getSectionFileName(section, vars.section_file_pattern));
             }
-        }
-        return {
-            ...inMeta,
-            vars
-        };
-    });
+            return {
+                ...inMeta,
+                vars,
+            };
+        },
+    );
 
     return `Processing done. Extracted files to: ${vars.section_file_pattern}`;
 }
 
-
-
 function getSectionFileHeader(section: TocSection, header: string): string {
     const date = new Date().toISOString();
     return expandVars(header, {
         section,
-        date
+        date,
     });
 }
 
 function getSectionFileName(section: TocSection, pattern: string): string {
     return pattern.replace(SECTION_ID_PLACEHOLDER, section.id);
-}
+}