@vertesia/workflow 0.79.0 → 0.79.1

package/src/activities/media/transcribeMediaWithGladia.ts

@@ -1,6 +1,6 @@
-import { DSLActivityExecutionPayload, DSLActivitySpec, GladiaConfiguration, SupportedIntegrations } from "@vertesia/common";
+import { DSLActivityExecutionPayload, DSLActivitySpec, GladiaConfiguration, SupportedIntegrations, AUDIO_RENDITION_NAME, VideoMetadata, ContentNature } from "@vertesia/common";
 import { activityInfo, CompleteAsyncError, log } from "@temporalio/activity";
-import { FetchClient } from "@vertesia/api-fetch-client";
+import { FetchClient, RequestError } from "@vertesia/api-fetch-client";
 import { setupActivity } from "../../dsl/setup/ActivityContext.js";
 import { DocumentNotFoundError } from "../../errors.js";
 import { TextExtractionResult, TextExtractionStatus } from "../../index.js";
@@ -27,7 +27,12 @@ export async function transcribeMedia(payload: DSLActivityExecutionPayload<Trans
 
     const gladiaConfig = await client.projects.integrations.retrieve(payload.project_id, SupportedIntegrations.gladia) as GladiaConfiguration | undefined;
     if (!gladiaConfig || !gladiaConfig.enabled) {
-        throw new DocumentNotFoundError("Gladia integration not enabled");
+        return {
+            hasText: false,
+            objectId,
+            status: TextExtractionStatus.error,
+            error: "Gladia integration not enabled",
+        }
     }
 
     const object = await client.objects.retrieve(objectId, "+text");
@@ -42,37 +47,63 @@ export async function transcribeMedia(payload: DSLActivityExecutionPayload<Trans
         throw new DocumentNotFoundError(`No source found for object ${objectId}`);
     }
 
-    const mediaUrl = await client.store.objects.getContentSource(objectId).then(res => res.source);
+    // Check for audio rendition in video metadata (preferred for videos)
+    let mediaSource: string = object.content.source;
+    if (object.metadata?.type === ContentNature.Video) {
+        const videoMetadata = object.metadata as VideoMetadata;
+        const audioRendition = videoMetadata.renditions?.find(r => r.name === AUDIO_RENDITION_NAME);
+        if (audioRendition?.content?.source) {
+            mediaSource = audioRendition.content.source;
+            log.info(`Found audio rendition for video object ${objectId}`, { mediaSource });
+        }
+    }
+
+    // Get download URL for the media source
+    const { url: mediaUrl } = await client.files.getDownloadUrl(mediaSource);
 
     if (!mediaUrl) {
-        throw new DocumentNotFoundError(`Error fetching source ${object.content.source}`);
+        throw new DocumentNotFoundError(`Error fetching media URL for ${mediaSource}`);
     }
 
+    log.info(`Using media URL for transcription`, { objectId, mediaUrl: mediaSource });
+
     const taskToken = Buffer.from(activityInfo().taskToken).toString('base64url');
     const callbackUrl = generateCallbackUrlForGladia(client.store.baseUrl, payload.auth_token, taskToken, objectId);
 
     log.info(`Transcribing media ${mediaUrl} with Gladia`, { objectId, callbackUrl });
 
-    const res = await gladiaClient.post("/transcription", {
-        payload: {
-            audio_url: mediaUrl,
-            callback_url: callbackUrl,
-            diarization_enhanced: true,
-            enable_code_switching: true,
-            subtitles: true,
-            subtitles_config: {
-                formats: ["vtt"],
+    try {
+        const res = await gladiaClient.post("/transcription", {
+            payload: {
+                audio_url: mediaUrl,
+                callback_url: callbackUrl,
+                diarization_enhanced: true,
+                enable_code_switching: true,
+                subtitles: true,
+                subtitles_config: {
+                    formats: ["vtt"],
+                }
+            }
+        }) as GladiaTranscriptRequestResponse;
+
+        log.info(`Transcription request sent to Gladia`, { objectId, res });
+        throw new CompleteAsyncError();
+
+    } catch (error: any) {
+        if (error instanceof RequestError && error.status === 422) {
+            return {
+                hasText: false,
+                objectId,
+                status: TextExtractionStatus.error,
+                error: `Gladia transcription error: ${error.message}`,
             }
+        } else {
+            log.error(`Error sending transcription request to Gladia for object ${objectId}`, { error });
+            throw error;
         }
-    }) as GladiaTranscriptRequestResponse;
-
-    log.info(`Transcription request sent to Gladia`, { objectId, res });
-
-    throw new CompleteAsyncError();
-
+    }
 }
 
-
 function generateCallbackUrlForGladia(baseUrl: string, authToken: string, taskToken: string, objectId: string) {
     return `${baseUrl}/api/v1/webhooks/gladia/${objectId}?access_token=${authToken}&task_token=${taskToken}`;
 }

package/src/activities/notifyWebhook.ts

@@ -180,7 +180,8 @@ async function createOldRequestBody(payload: WorkflowExecutionBaseParams, params
     let data = params.detail;
     if (data && data.run_id && params.event_name === "workflow_completed" && params.workflow_type === 'ExecuteInteractionWorkflow') {
         const client = getVersionedVertesiaClient(payload, null); //ensure client is using no specific version
-        const run = await client.runs.retrieve(data.run_id);
+        // Important Note: we cannot use client.runs.retrieve since it will transform the run result to the new format because of InteractionOutput
+        const run = await client.runs.get(data.run_id);
         // since we use an unversioned client the run will be in old format so we don't need to tranform the result
         const result = run.result;
         data = {

package/src/dsl/dsl-workflow.ts

@@ -225,7 +225,13 @@ async function executeChildWorkflow(step: DSLChildWorkflowStep, payload: DSLWork
 }
 
 function buildRateLimitParams(activity: DSLActivitySpec, executionPayload: DSLActivityExecutionPayload<any>): RateLimitParams {
-    const params = executionPayload.params;
+    // resolve payload params
+    const vars = new Vars({
+        ...executionPayload.params, // imported params (doesn't contain expressions)
+        ...executionPayload.activity.params, // activity params (may contain expressions)
+    });
+    const params = vars.resolve();
+
     let interactionId: string;
 
     switch (activity.name) {

package/src/errors.ts

@@ -64,9 +64,24 @@ export class ResourceExhaustedError extends ApplicationFailure {
     }
 }
 
+export class InvalidContentTypeError extends ApplicationFailure {
+    constructor(
+        public objectId: string,
+        public expectedType: string,
+        public actualType: string,
+    ) {
+        super(
+            `Document ${objectId} has invalid content type. Expected ${expectedType}, got ${actualType}`,
+            "InvalidContentTypeError",
+            true, // non-retryable
+        );
+    }
+}
+
 export const WF_NON_RETRYABLE_ERRORS = [
     "DocumentNotFoundError",
     "ActivityParamInvalidError",
     "ActivityParamNotFoundError",
     "WorkflowParamNotFoundError",
+    "InvalidContentTypeError",
 ];

package/src/index.ts

@@ -25,6 +25,7 @@ export * from "./activities/notifyWebhook.js";
 export * from "./activities/rateLimiter.js";
 export * from "./activities/setDocumentStatus.js";
 export * from "./iterative-generation/activities/index.js";
+export * from "./activities/media/prepareVideo.js";
 
 export * from "./dsl/setup/ActivityContext.js";
 export * from "./errors.js";

package/src/iterative-generation/activities/generatePart.ts

@@ -5,7 +5,6 @@ import { getVertesiaClient } from "../../utils/client.js";
 import { buildAndPublishMemoryPack, loadMemoryPack } from "../../utils/memory.js";
 import { IterativeGenerationPayload, OutputMemoryMeta, Section, TocPart, TocSection } from "../types.js";
 import { executeWithVars, expectMemoryIsConsistent } from "../utils.js";
-import { completionResultToString } from "@llumiverse/common";
 
 export async function it_gen_generatePart(payload: WorkflowExecutionPayload, path: number[]) {
     const vars = payload.vars as IterativeGenerationPayload;
@@ -57,7 +56,7 @@ export async function it_gen_generatePart(payload: WorkflowExecutionPayload, pat
         }
     });
 
-    const result = r.result.map(completionResultToString).join('\n');
+    const result = r.result.text();
     content[content.length - 1].content += result;
     meta.lastProcessedPart = path;
     await buildAndPublishMemoryPack(client, `${memory}/output`, async ({ copyText }) => {

package/src/iterative-generation/activities/generateToc.ts

@@ -3,7 +3,6 @@ import { getVertesiaClient } from "../../utils/client.js";
 import { buildAndPublishMemoryPack } from "../../utils/memory.js";
 import { IterativeGenerationPayload, OutputMemoryMeta, Toc, TocIndex } from "../types.js";
 import { executeWithVars, tocIndex } from "../utils.js";
-import { parseCompletionResultsToJson } from "@llumiverse/common";
 
 const defaultTocSchema = {
     "type": "object",
@@ -86,7 +85,7 @@ export async function it_gen_generateToc(payload: WorkflowExecutionPayload): Pro
     const run = await executeWithVars(client, vars.interaction, vars, undefined, schema);
 
     //Parse the CompletionResult[] to get a Toc object
-    const jsonResults = parseCompletionResultsToJson(run.result);
+    const jsonResults = run.result.object();
 
     const toc: Toc = {
         sections: jsonResults.sections

package/src/iterative-generation/utils.ts

@@ -1,8 +1,7 @@
-import { VertesiaClient } from "@vertesia/client";
-import { ExecutionRun } from "@vertesia/common";
+import { ModelOptions, TextFallbackOptions } from "@llumiverse/common";
 import { ApplicationFailure } from "@temporalio/workflow";
+import { VertesiaClient } from "@vertesia/client";
 import { OutputMemoryMeta, PartIndex, Toc, TocIndex, TocSection } from "./types.js";
-import { ModelOptions, TextFallbackOptions } from "@llumiverse/common";
 
 //TODO: For whole file, support for options beyond max_tokens and temperature and multiple modalities.
 export interface ExecuteOptions {
@@ -15,7 +14,7 @@ export interface ExecuteOptions {
     result_schema?: Record<string, any>;
 }
 
-export async function execute(client: VertesiaClient, options: ExecuteOptions): Promise<ExecutionRun> {
+export async function execute(client: VertesiaClient, options: ExecuteOptions) {
     return client.interactions.executeByName(options.interaction, {
         data: {
             ...options.memory_mapping,
@@ -30,7 +29,7 @@ export async function execute(client: VertesiaClient, options: ExecuteOptions):
     });
 }
 
-export function executeWithVars(client: VertesiaClient, interaction: string, vars: Record<string, any>, mapping?: Record<string, any>, result_schema?: Record<string, any>): Promise<ExecutionRun> {
+export function executeWithVars(client: VertesiaClient, interaction: string, vars: Record<string, any>, mapping?: Record<string, any>, result_schema?: Record<string, any>) {
     if (mapping) {
         mapping = { ...vars.input_mapping, ...mapping };
     } else {