@vertesia/workflow 0.80.0-dev.20251121 → 0.81.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vertesia/workflow",
-  "version": "0.80.0-dev.20251121",
+  "version": "0.81.0",
   "type": "module",
   "description": "Vertesia workflow DSL",
   "main": "./lib/esm/index.js",
@@ -44,11 +44,11 @@
     "tmp": "^0.2.4",
     "tmp-promise": "^3.0.3",
     "yaml": "^2.6.0",
-    "@vertesia/api-fetch-client": "0.80.0-dev.20251121",
-    "@vertesia/memory": "0.80.0-dev.20251121",
-    "@vertesia/common": "0.80.0-dev.20251121",
-    "@vertesia/client": "0.80.0-dev.20251121",
-    "@llumiverse/common": "0.23.0-dev.20251121"
+    "@llumiverse/common": "0.24.0",
+    "@vertesia/common": "0.81.0",
+    "@vertesia/client": "0.81.0",
+    "@vertesia/api-fetch-client": "0.81.0",
+    "@vertesia/memory": "0.81.0"
   },
   "ts_dual_module": {
     "outDir": "lib",
@@ -105,6 +105,16 @@
     "url": "https://github.com/vertesia/composableai.git",
     "directory": "packages/workflow"
   },
+  "keywords": [
+    "vertesia",
+    "workflow",
+    "dsl",
+    "temporalio",
+    "llm",
+    "ai",
+    "agents",
+    "typescript"
+  ],
   "types": "./lib/types/index.d.ts",
   "typesVersions": {
     "*": {

package/src/activities/executeInteraction.ts

@@ -1,4 +1,4 @@
-import { CompletionResult, Modalities, ModelOptions } from "@llumiverse/common";
+import { CompletionResult, ModelOptions } from "@llumiverse/common";
 import { activityInfo, log } from "@temporalio/activity";
 import { VertesiaClient } from "@vertesia/client";
 import { NodeStreamSource } from "@vertesia/client/node";
@@ -159,32 +159,37 @@ export async function executeInteraction(payload: DSLActivityExecutionPayload<Ex
         let completionResult: CompletionResult[] = res.result;
 
         // Handle image uploads if the result contains base64 images
-        if (res.output_modality === Modalities.image) {
-            const images = res.result.images();
+        const imageResults = completionResult.filter(r => r.type === "image");
+        if (imageResults.length > 0) {
             const uploadedImages = await Promise.all(
-                images.map((image: string, index: number) => {
-                    // Extract base64 data and create buffer
-                    const base64Data = image.replace(/^data:image\/[a-z]+;base64,/, "");
-                    const buffer = Buffer.from(base64Data, 'base64');
-
-                    // Generate filename
-                    const { runId } = activityInfo().workflowExecution;
-                    const { activityId } = activityInfo();
-                    const filename = `generated-image-${runId}-${activityId}-${index}.png`;
-
-                    // Create a readable stream from the buffer
-                    const stream = Readable.from(buffer);
-
-                    const source = new NodeStreamSource(
-                        stream,
-                        filename,
-                        "image/png",
-                    );
-
-                    return client.files.uploadFile(source);
+                completionResult.map(async (item, index) => {
+                    if (item.type === "image") {
+                        const image = item.value;
+                        // Extract base64 data and create buffer
+                        const base64Data = image.replace(/^data:image\/[a-z]+;base64,/, "");
+                        const buffer = Buffer.from(base64Data, 'base64');
+
+                        // Generate filename
+                        const { runId } = activityInfo().workflowExecution;
+                        const { activityId } = activityInfo();
+                        const filename = `generated-image-${runId}-${activityId}-${index}.png`;
+
+                        // Create a readable stream from the buffer
+                        const stream = Readable.from(buffer);
+
+                        const source = new NodeStreamSource(
+                            stream,
+                            filename,
+                            "image/png",
+                        );
+
+                        const file = await client.files.uploadFile(source);
+                        return { type: "image", value: file } as CompletionResult;
+                    }
+                    return item;
                 })
             );
-            completionResult = uploadedImages.map(file => ({ type: "image", value: file }));
+            completionResult = uploadedImages;
         }
 
         return projectResult(payload, params, res, {

package/src/activities/index-dsl.ts

@@ -8,14 +8,17 @@ export { executeInteraction } from "./executeInteraction.js";
 export { extractDocumentText } from "./extractDocumentText.js";
 export { generateDocumentProperties } from "./generateDocumentProperties.js";
 export { generateEmbeddings } from "./generateEmbeddings.js";
-export { generateImageRendition } from "./renditions/generateImageRendition.js";
-export { generateVideoRendition } from "./renditions/generateVideoRendition.js";
 export { generateOrAssignContentType } from "./generateOrAssignContentType.js";
 export { getObjectFromStore } from "./getObjectFromStore.js";
 export { handleDslError } from "./handleError.js";
+export { prepareVideo } from "./media/prepareVideo.js";
 export { convertPdfToStructuredText } from "./media/processPdfWithTextract.js";
+export { saveGladiaTranscription } from "./media/saveGladiaTranscription.js";
 export { transcribeMedia } from "./media/transcribeMediaWithGladia.js";
-export { prepareVideo } from "./media/prepareVideo.js";
+export type { TranscriptMediaResult } from "./media/transcribeMediaWithGladia.js";
 export { notifyWebhook } from "./notifyWebhook.js";
-export { setDocumentStatus } from "./setDocumentStatus.js";
 export { checkRateLimit } from "./rateLimiter.js";
+export { generateImageRendition } from "./renditions/generateImageRendition.js";
+export { generateVideoRendition } from "./renditions/generateVideoRendition.js";
+export { setDocumentStatus } from "./setDocumentStatus.js";
+

package/src/activities/media/saveGladiaTranscription.ts

@@ -0,0 +1,128 @@
+import { log } from "@temporalio/activity";
+import { FetchClient } from "@vertesia/api-fetch-client";
+import { AudioMetadata, DSLActivityExecutionPayload, DSLActivitySpec, GladiaConfiguration, SupportedIntegrations, TranscriptSegment, VideoMetadata } from "@vertesia/common";
+import { setupActivity } from "../../dsl/setup/ActivityContext.js";
+import { TextExtractionResult, TextExtractionStatus } from "../../result-types.js";
+
+export interface SaveGladiaTranscriptionParams {
+    gladiaTranscriptionId: string;
+}
+
+export interface SaveGladiaTranscription extends DSLActivitySpec<SaveGladiaTranscriptionParams> {
+    name: 'SaveGladiaTranscription';
+}
+
+const GLADIA_URL = "https://api.gladia.io/v2";
+
+/**
+ * Fetches transcription results from Gladia and saves them to the content object.
+ * This activity is called after transcribeMedia completes via webhook callback.
+ */
+export async function saveGladiaTranscription(payload: DSLActivityExecutionPayload<SaveGladiaTranscriptionParams>): Promise<TextExtractionResult> {
+    const { params, client, objectId } = await setupActivity<SaveGladiaTranscriptionParams>(payload);
+
+    const gladiaConfig = await client.projects.integrations.retrieve(payload.project_id, SupportedIntegrations.gladia) as GladiaConfiguration | undefined;
+    if (!gladiaConfig || !gladiaConfig.enabled) {
+        return {
+            hasText: false,
+            objectId,
+            status: TextExtractionStatus.error,
+            error: "Gladia integration not enabled",
+        };
+    }
+
+    const gladiaClient = new FetchClient(gladiaConfig.url ?? GLADIA_URL);
+    gladiaClient.withHeaders({ "x-gladia-key": gladiaConfig.api_key });
+
+    log.info(`Fetching transcription result from Gladia`, { objectId, transcriptionId: params.gladiaTranscriptionId });
+
+    const transcriptionResult = await gladiaClient.get(`/transcription/${params.gladiaTranscriptionId}`) as GladiaTranscriptionResult;
+
+    if (transcriptionResult.status === 'error') {
+        log.error(`Gladia transcription failed`, { objectId, error: transcriptionResult });
+        return {
+            hasText: false,
+            objectId,
+            status: TextExtractionStatus.error,
+            error: "Gladia transcription failed",
+        };
+    }
+
+    if (transcriptionResult.status !== 'done') {
+        log.warn(`Gladia transcription not ready`, { objectId, status: transcriptionResult.status });
+        return {
+            hasText: false,
+            objectId,
+            status: TextExtractionStatus.error,
+            error: `Gladia transcription not ready: ${transcriptionResult.status}`,
+        };
+    }
+
+    const object = await client.objects.retrieve(objectId, "+text");
+
+    const segments = processUtterances(transcriptionResult.result.transcription.utterances);
+    const fullText = transcriptionResult.result.transcription.full_transcript;
+
+    await client.objects.update(objectId, {
+        text: fullText,
+        text_etag: object.content?.etag,
+        transcript: {
+            segments,
+            etag: object.content?.etag
+        },
+        metadata: {
+            ...object.metadata,
+            duration: transcriptionResult.result.metadata.audio_duration,
+            languages: transcriptionResult.result.transcription.languages
+        } as AudioMetadata | VideoMetadata
+    });
+
+    log.info(`Saved transcription for object`, { objectId, textLength: fullText?.length, segmentCount: segments.length });
+
+    return {
+        hasText: (fullText?.length ?? 0) > 0,
+        objectId,
+        status: TextExtractionStatus.success,
+        message: `Transcription saved with ${segments.length} segments`
+    };
+}
+
+function processUtterances(utterances: GladiaUtterance[]): TranscriptSegment[] {
+    return utterances.map(u => ({
+        start: u.start,
+        end: u.end,
+        text: u.text,
+        speaker: u.speaker,
+        confidence: u.confidence,
+        language: u.language
+    }));
+}
+
+// Gladia API response types
+interface GladiaTranscriptionResult {
+    id: string;
+    status: 'queued' | 'processing' | 'done' | 'error';
+    result: {
+        metadata: {
+            audio_duration: number;
+            number_of_distinct_channels: number;
+            billing_time: number;
+            transcription_time: number;
+        };
+        transcription: {
+            full_transcript: string;
+            languages: string[];
+            utterances: GladiaUtterance[];
+        };
+    };
+}
+
+interface GladiaUtterance {
+    language: string;
+    start: number;
+    end: number;
+    confidence: number;
+    channel: number;
+    speaker: number;
+    text: string;
+}

package/src/activities/media/transcribeMediaWithGladia.ts

@@ -1,6 +1,6 @@
-import { DSLActivityExecutionPayload, DSLActivitySpec, GladiaConfiguration, SupportedIntegrations, AUDIO_RENDITION_NAME, VideoMetadata, ContentNature } from "@vertesia/common";
 import { activityInfo, CompleteAsyncError, log } from "@temporalio/activity";
 import { FetchClient, RequestError } from "@vertesia/api-fetch-client";
+import { AUDIO_RENDITION_NAME, ContentNature, DSLActivityExecutionPayload, DSLActivitySpec, GladiaConfiguration, SupportedIntegrations, VideoMetadata } from "@vertesia/common";
 import { setupActivity } from "../../dsl/setup/ActivityContext.js";
 import { DocumentNotFoundError } from "../../errors.js";
 import { TextExtractionResult, TextExtractionStatus } from "../../index.js";
@@ -17,6 +17,11 @@ export interface TranscriptMedia extends DSLActivitySpec<TranscriptMediaParams>
 
 export interface TranscriptMediaResult extends TextExtractionResult {
     message?: string;
+    /**
+     * Gladia transcription ID for fetching results in a follow-up activity.
+     * Present when async media transcription completes successfully.
+     */
+    gladiaTranscriptionId?: string;
 }
 
 const GLADIA_URL = "https://api.gladia.io/v2";
@@ -68,7 +73,7 @@ export async function transcribeMedia(payload: DSLActivityExecutionPayload<Trans
     log.info(`Using media URL for transcription`, { objectId, mediaUrl: mediaSource });
 
     const taskToken = Buffer.from(activityInfo().taskToken).toString('base64url');
-    const callbackUrl = generateCallbackUrlForGladia(client.store.baseUrl, payload.auth_token, taskToken, objectId);
+    const callbackUrl = generateCallbackUrlForGladia(client.store.baseUrl, taskToken, objectId);
 
     log.info(`Transcribing media ${mediaUrl} with Gladia`, { objectId, callbackUrl });
 
@@ -85,10 +90,7 @@ export async function transcribeMedia(payload: DSLActivityExecutionPayload<Trans
                 }
             }
         }) as GladiaTranscriptRequestResponse;
-
         log.info(`Transcription request sent to Gladia`, { objectId, res });
-        throw new CompleteAsyncError();
-
     } catch (error: any) {
         if (error instanceof RequestError && error.status === 422) {
             return {
@@ -97,15 +99,16 @@ export async function transcribeMedia(payload: DSLActivityExecutionPayload<Trans
                 status: TextExtractionStatus.error,
                 error: `Gladia transcription error: ${error.message}`,
             }
-        } else {
-            log.error(`Error sending transcription request to Gladia for object ${objectId}`, { error });
-            throw error;
         }
+        log.error(`Error sending transcription request to Gladia for object ${objectId}`, { error });
+        throw error;
     }
+
+    throw new CompleteAsyncError();
 }
 
-function generateCallbackUrlForGladia(baseUrl: string, authToken: string, taskToken: string, objectId: string) {
-    return `${baseUrl}/api/v1/webhooks/gladia/${objectId}?access_token=${authToken}&task_token=${taskToken}`;
+function generateCallbackUrlForGladia(baseUrl: string, taskToken: string, objectId: string) {
+    return `${baseUrl}/webhooks/gladia/${objectId}?task_token=${taskToken}`;
 }
 
 interface GladiaTranscriptRequestResponse {

package/src/dsl/setup/ActivityContext.ts

@@ -166,8 +166,6 @@ export async function setupActivity<ParamsT extends Record<string, any>>(
     }
 
     const params = vars.resolve() as ParamsT;
-    log.info(`Activity ${payload.activity.name} setup complete`);
-
     return new ActivityContext<ParamsT>(payload, client, params);
 }
 

package/src/utils/storage.ts

@@ -34,7 +34,6 @@ export async function saveAgentArtifact(
 
     //create the file path and append extension if needed
     const filePath = agentStoragePath(runId) + "/" + name + (ext && !name.endsWith(ext) ? "." + ext : "");
-    log.info(`Storing agent artifact ${filePath} for run ${runId}`);
 
     try {
         const source = new NodeStreamSource(fileContent, `${runId}-${basename(filePath)}`, mimeType, filePath);