@vertesia/workflow 0.24.0-dev.202601221707

package/src/activities/media/processPdfWithTextract.ts

@@ -0,0 +1,141 @@
+/**
+ * Use textract to convert a pdf into a data structure of the following format:
+ * <document>
+ *  <page number="n">
+ *   <text/>
+ *   <table/>
+ *   <text/>
+ *   <figure/>
+ *   ...
+ *  </page>
+ */
+
+import { fromWebToken } from "@aws-sdk/credential-providers";
+import { AwsConfiguration, CreateContentObjectPayload, DSLActivityExecutionPayload, DSLActivitySpec, SupportedIntegrations } from "@vertesia/common";
+import type { AwsCredentialIdentityProvider } from "@smithy/types";
+import { log } from "@temporalio/activity";
+import { TextractProcessor } from "../../conversion/TextractProcessor.js";
+import { setupActivity } from "../../dsl/setup/ActivityContext.js";
+import { DocumentNotFoundError } from "../../errors.js";
+import { TextExtractionResult, TextExtractionStatus } from "../../result-types.js";
+import { fetchBlobAsBuffer, md5 } from "../../utils/blobs.js";
+import { countTokens } from "../../utils/tokens.js";
+
+
+
+export interface ConvertPdfToStructuredTextParams {
+    force?: boolean;
+}
+
+export interface ConvertPdfToStructuredText extends DSLActivitySpec<ConvertPdfToStructuredTextParams> {
+    name: 'ConvertPdfToStructuredText';
+}
+
+export interface StructuredTextResult extends TextExtractionResult {
+    message?: string;
+}
+
+
+
+export async function convertPdfToStructuredText(payload: DSLActivityExecutionPayload<ConvertPdfToStructuredTextParams>): Promise<StructuredTextResult> {
+
+
+    const { params, client, objectId } = await setupActivity<ConvertPdfToStructuredTextParams>(payload);
+
+    const object = await client.objects.retrieve(objectId, "+text");
+
+    if (object.text && !params.force) {
+        return { hasText: true, objectId, status: TextExtractionStatus.skipped, message: "text already present and force not enabled" }
+    }
+
+    if (!object.content?.source) {
+        throw new DocumentNotFoundError(`No source found for object ${objectId}`);
+    }
+
+    const pdfUrl = await client.store.objects.getContentSource(objectId).then(res => res.source);
+
+    if (!pdfUrl) {
+        throw new DocumentNotFoundError(`Error fetching source ${object.content.source}`);
+    }
+
+    const project = await client.getProject();
+    const awsConfig = (await client.projects.integrations.retrieve(project!.id, SupportedIntegrations.aws)) as AwsConfiguration;
+    const credentials = await getS3AWSCredentials(awsConfig, payload.auth_token, project!.id);
+
+    const processor = new TextractProcessor({
+        fileKey: objectId,
+        region: "us-west-2",
+        bucket: "cp-textract-tests",
+        credentials,
+        log: log,
+        detectImages: true,
+        includeConfidenceInTables: true,
+    });
+
+
+
+    try {
+
+        if (!object.content.source.startsWith("s3://")) {
+            const buf = await fetchBlobAsBuffer(client, object.content.source);
+            await processor.upload(buf);
+        }
+
+        const jobId = await processor.startAnalysis(objectId);
+
+        let jobStatus = await processor.checkJobStatus(jobId);
+        while (jobStatus === "IN_PROGRESS") {
+            await new Promise(resolve => setTimeout(resolve, 5000));
+            jobStatus = await processor.checkJobStatus(jobId);
+        }
+
+        if (jobStatus === "SUCCEEDED") {
+            log.info(`Job ${jobId} succeeded, saving results`, { jobId });
+            const fText = await processor.processResults(jobId);
+            const tokensData = countTokens(fText);
+            const etag = object.content.etag ?? md5(fText);
+            const updateData: CreateContentObjectPayload = {
+                text: fText,
+                text_etag: etag,
+                tokens: {
+                    ...tokensData,
+                    etag: etag,
+                }
+            }
+
+            await client.objects.update(objectId, updateData);
+            console.log("Full text updated");
+
+            return { hasText: true, objectId, status: TextExtractionStatus.success, message: "Text extracted successfully" }
+
+        } else {
+            throw new Error(`Job failed with status: ${jobStatus}`);
+        }
+    } catch (error) {
+        console.error("Error processing document:", error);
+        throw error;
+    }
+
+
+}
+
+export async function getS3AWSCredentials(awsConfig: AwsConfiguration, composableAuthToken: string, projectId: string): Promise<AwsCredentialIdentityProvider> {
+
+    // fetch s3 role ARN
+    if (!awsConfig || !awsConfig.enabled) {
+        throw new DocumentNotFoundError("AWS integration is not enabled for this project");
+    }
+    if (!awsConfig.s3_role_arn) {
+        throw new DocumentNotFoundError("S3 Role ARN is not defined in AWS project integration");
+    }
+
+    log.info("Getting AWS credentials for Textract", { projectId, composableAuthToken, roleArn: awsConfig.s3_role_arn });
+
+    const credentials = fromWebToken({
+        webIdentityToken: composableAuthToken,
+        roleArn: awsConfig.s3_role_arn,
+        roleSessionName: `cp-project-textract-${projectId}`,
+    });
+
+    return credentials;
+}

package/src/activities/media/saveGladiaTranscription.ts

@@ -0,0 +1,128 @@
+import { log } from "@temporalio/activity";
+import { FetchClient } from "@vertesia/api-fetch-client";
+import { AudioMetadata, DSLActivityExecutionPayload, DSLActivitySpec, GladiaConfiguration, SupportedIntegrations, TranscriptSegment, VideoMetadata } from "@vertesia/common";
+import { setupActivity } from "../../dsl/setup/ActivityContext.js";
+import { TextExtractionResult, TextExtractionStatus } from "../../result-types.js";
+
+export interface SaveGladiaTranscriptionParams {
+    gladiaTranscriptionId: string;
+}
+
+export interface SaveGladiaTranscription extends DSLActivitySpec<SaveGladiaTranscriptionParams> {
+    name: 'SaveGladiaTranscription';
+}
+
+const GLADIA_URL = "https://api.gladia.io/v2";
+
+/**
+ * Fetches transcription results from Gladia and saves them to the content object.
+ * This activity is called after transcribeMedia completes via webhook callback.
+ */
+export async function saveGladiaTranscription(payload: DSLActivityExecutionPayload<SaveGladiaTranscriptionParams>): Promise<TextExtractionResult> {
+    const { params, client, objectId } = await setupActivity<SaveGladiaTranscriptionParams>(payload);
+
+    const gladiaConfig = await client.projects.integrations.retrieve(payload.project_id, SupportedIntegrations.gladia) as GladiaConfiguration | undefined;
+    if (!gladiaConfig || !gladiaConfig.enabled) {
+        return {
+            hasText: false,
+            objectId,
+            status: TextExtractionStatus.error,
+            error: "Gladia integration not enabled",
+        };
+    }
+
+    const gladiaClient = new FetchClient(gladiaConfig.url ?? GLADIA_URL);
+    gladiaClient.withHeaders({ "x-gladia-key": gladiaConfig.api_key });
+
+    log.info(`Fetching transcription result from Gladia`, { objectId, transcriptionId: params.gladiaTranscriptionId });
+
+    const transcriptionResult = await gladiaClient.get(`/transcription/${params.gladiaTranscriptionId}`) as GladiaTranscriptionResult;
+
+    if (transcriptionResult.status === 'error') {
+        log.error(`Gladia transcription failed`, { objectId, error: transcriptionResult });
+        return {
+            hasText: false,
+            objectId,
+            status: TextExtractionStatus.error,
+            error: "Gladia transcription failed",
+        };
+    }
+
+    if (transcriptionResult.status !== 'done') {
+        log.warn(`Gladia transcription not ready`, { objectId, status: transcriptionResult.status });
+        return {
+            hasText: false,
+            objectId,
+            status: TextExtractionStatus.error,
+            error: `Gladia transcription not ready: ${transcriptionResult.status}`,
+        };
+    }
+
+    const object = await client.objects.retrieve(objectId, "+text");
+
+    const segments = processUtterances(transcriptionResult.result.transcription.utterances);
+    const fullText = transcriptionResult.result.transcription.full_transcript;
+
+    await client.objects.update(objectId, {
+        text: fullText,
+        text_etag: object.content?.etag,
+        transcript: {
+            segments,
+            etag: object.content?.etag
+        },
+        metadata: {
+            ...object.metadata,
+            duration: transcriptionResult.result.metadata.audio_duration,
+            languages: transcriptionResult.result.transcription.languages
+        } as AudioMetadata | VideoMetadata
+    });
+
+    log.info(`Saved transcription for object`, { objectId, textLength: fullText?.length, segmentCount: segments.length });
+
+    return {
+        hasText: (fullText?.length ?? 0) > 0,
+        objectId,
+        status: TextExtractionStatus.success,
+        message: `Transcription saved with ${segments.length} segments`
+    };
+}
+
+function processUtterances(utterances: GladiaUtterance[]): TranscriptSegment[] {
+    return utterances.map(u => ({
+        start: u.start,
+        end: u.end,
+        text: u.text,
+        speaker: u.speaker,
+        confidence: u.confidence,
+        language: u.language
+    }));
+}
+
+// Gladia API response types
+interface GladiaTranscriptionResult {
+    id: string;
+    status: 'queued' | 'processing' | 'done' | 'error';
+    result: {
+        metadata: {
+            audio_duration: number;
+            number_of_distinct_channels: number;
+            billing_time: number;
+            transcription_time: number;
+        };
+        transcription: {
+            full_transcript: string;
+            languages: string[];
+            utterances: GladiaUtterance[];
+        };
+    };
+}
+
+interface GladiaUtterance {
+    language: string;
+    start: number;
+    end: number;
+    confidence: number;
+    channel: number;
+    speaker: number;
+    text: string;
+}

package/src/activities/media/transcribeMediaWithGladia.ts

@@ -0,0 +1,117 @@
+import { activityInfo, CompleteAsyncError, log } from "@temporalio/activity";
+import { FetchClient, RequestError } from "@vertesia/api-fetch-client";
+import { AUDIO_RENDITION_NAME, ContentNature, DSLActivityExecutionPayload, DSLActivitySpec, GladiaConfiguration, SupportedIntegrations, VideoMetadata } from "@vertesia/common";
+import { setupActivity } from "../../dsl/setup/ActivityContext.js";
+import { DocumentNotFoundError } from "../../errors.js";
+import { TextExtractionResult, TextExtractionStatus } from "../../index.js";
+
+
+export interface TranscriptMediaParams {
+    environmentId?: string;
+    force?: boolean;
+}
+
+export interface TranscriptMedia extends DSLActivitySpec<TranscriptMediaParams> {
+    name: 'TranscribeMedia';
+}
+
+export interface TranscriptMediaResult extends TextExtractionResult {
+    message?: string;
+    /**
+     * Gladia transcription ID for fetching results in a follow-up activity.
+     * Present when async media transcription completes successfully.
+     */
+    gladiaTranscriptionId?: string;
+}
+
+const GLADIA_URL = "https://api.gladia.io/v2";
+
+export async function transcribeMedia(payload: DSLActivityExecutionPayload<TranscriptMediaParams>): Promise<TranscriptMediaResult> {
+
+    const { params, client, objectId } = await setupActivity<TranscriptMediaParams>(payload);
+
+    const gladiaConfig = await client.projects.integrations.retrieve(payload.project_id, SupportedIntegrations.gladia) as GladiaConfiguration | undefined;
+    if (!gladiaConfig || !gladiaConfig.enabled) {
+        return {
+            hasText: false,
+            objectId,
+            status: TextExtractionStatus.error,
+            error: "Gladia integration not enabled",
+        }
+    }
+
+    const object = await client.objects.retrieve(objectId, "+text");
+    const gladiaClient = new FetchClient(gladiaConfig.url ?? GLADIA_URL);
+    gladiaClient.withHeaders({ "x-gladia-key": gladiaConfig.api_key });
+
+    if (object.text && !params.force) {
+        return { hasText: true, objectId, status: TextExtractionStatus.skipped, message: "text already present and force not enabled" }
+    }
+
+    if (!object.content?.source) {
+        throw new DocumentNotFoundError(`No source found for object ${objectId}`);
+    }
+
+    // Check for audio rendition in video metadata (preferred for videos)
+    let mediaSource: string = object.content.source;
+    if (object.metadata?.type === ContentNature.Video) {
+        const videoMetadata = object.metadata as VideoMetadata;
+        const audioRendition = videoMetadata.renditions?.find(r => r.name === AUDIO_RENDITION_NAME);
+        if (audioRendition?.content?.source) {
+            mediaSource = audioRendition.content.source;
+            log.info(`Found audio rendition for video object ${objectId}`, { mediaSource });
+        }
+    }
+
+    // Get download URL for the media source
+    const { url: mediaUrl } = await client.files.getDownloadUrl(mediaSource);
+
+    if (!mediaUrl) {
+        throw new DocumentNotFoundError(`Error fetching media URL for ${mediaSource}`);
+    }
+
+    log.info(`Using media URL for transcription`, { objectId, mediaUrl: mediaSource });
+
+    const taskToken = Buffer.from(activityInfo().taskToken).toString('base64url');
+    const callbackUrl = generateCallbackUrlForGladia(client.store.baseUrl, taskToken, objectId);
+
+    log.info(`Transcribing media ${mediaUrl} with Gladia`, { objectId, callbackUrl });
+
+    try {
+        const res = await gladiaClient.post("/transcription", {
+            payload: {
+                audio_url: mediaUrl,
+                callback_url: callbackUrl,
+                diarization_enhanced: true,
+                enable_code_switching: true,
+                subtitles: true,
+                subtitles_config: {
+                    formats: ["vtt"],
+                }
+            }
+        }) as GladiaTranscriptRequestResponse;
+        log.info(`Transcription request sent to Gladia`, { objectId, res });
+    } catch (error: any) {
+        if (error instanceof RequestError && error.status === 422) {
+            return {
+                hasText: false,
+                objectId,
+                status: TextExtractionStatus.error,
+                error: `Gladia transcription error: ${error.message}`,
+            }
+        }
+        log.error(`Error sending transcription request to Gladia for object ${objectId}`, { error });
+        throw error;
+    }
+
+    throw new CompleteAsyncError();
+}
+
+function generateCallbackUrlForGladia(baseUrl: string, taskToken: string, objectId: string) {
+    return `${baseUrl}/webhooks/gladia/${objectId}?task_token=${taskToken}`;
+}
+
+interface GladiaTranscriptRequestResponse {
+    id: string;
+    result_url: string;
+}

package/src/activities/notifyWebhook.test.ts

@@ -0,0 +1,134 @@
+import {
+  MockActivityEnvironment,
+} from "@temporalio/testing";
+import { ContentEventName, DSLActivityExecutionPayload } from "@vertesia/common";
+import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
+import { notifyWebhook, NotifyWebhookParams } from "./notifyWebhook.js";
+
+// Mock fetch globally
+vi.stubGlobal('fetch', vi.fn());
+
+let testEnv: MockActivityEnvironment;
+const mockFetch = vi.mocked(fetch);
+
+beforeAll(async () => {
+  testEnv = new MockActivityEnvironment();
+});
+
+beforeEach(() => {
+  vi.clearAllMocks();
+});
+
+const defaultParams = {
+  webhook: "https://vertesia.test",
+  method: "POST" as const,
+  detail: { message: "Hello World" },
+  workflow_id: "wf_id",
+  workflow_type: "wfFuncName",
+  workflow_run_id: "wf_run_id",
+  event_name: "completed",
+} satisfies NotifyWebhookParams;
+
+// Helper function to create test payload
+const createTestPayload = (params: Partial<NotifyWebhookParams> = {}): DSLActivityExecutionPayload<NotifyWebhookParams> => {
+  const mergedParams = { ...defaultParams, ...params };
+  return {
+    auth_token: process.env.VERTESIA_KEY || "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJodHRwOi8vbW9jay10b2tlbi1zZXJ2ZXIiLCJzdWIiOiJ0ZXN0In0.signature",
+    account_id: "unset",
+    project_id: "unset",
+    params: mergedParams,
+    config: {
+      studio_url: "http://mock-studio",
+      store_url: "http://mock-store",
+    },
+    workflow_name: "",
+    event: ContentEventName.create,
+    objectIds: [],
+    vars: {},
+    activity: { name: "notifyWebhook", params: mergedParams }
+  };
+};
+
+describe("Webhook should be notified", () => {
+  it("test POST success", async () => {
+    // Mock successful response
+    const mockResponse = {
+      ok: true,
+      status: 200,
+      statusText: 'OK',
+      url: defaultParams.webhook
+    };
+    mockFetch.mockResolvedValueOnce(mockResponse as Response);
+
+    const payload = createTestPayload();
+    const res = await testEnv.run(notifyWebhook, payload);
+
+    // Verify fetch was called with correct parameters
+    expect(mockFetch).toHaveBeenCalledWith(defaultParams.webhook, {
+      method: 'POST',
+      body: JSON.stringify({ message: 'Hello World' }),
+      headers: {
+        'Content-Type': 'application/json',
+      },
+    });
+
+    // Verify response
+    expect(res).toEqual({
+      status: 200,
+      message: 'OK',
+      url: defaultParams.webhook
+    });
+  });
+
+  it("test POST server error", async () => {
+    // Mock error response with response body
+    const mockResponse = {
+      ok: false,
+      status: 500,
+      statusText: 'Internal Server Error',
+      url: defaultParams.webhook,
+      text: vi.fn().mockResolvedValue('{"error": "Database connection failed", "code": "DB_ERROR"}')
+    } as unknown as Response;
+    mockFetch.mockResolvedValueOnce(mockResponse);
+
+    const payload = createTestPayload();
+
+    // Expect the function to throw an error with response payload
+    await expect(testEnv.run(notifyWebhook, payload)).rejects.toThrow(
+      `Webhook Notification to ${defaultParams.webhook} failed with status: 500 Internal Server Error - Response: {"error": "Database connection failed", "code": "DB_ERROR"}`
+    );
+
+    // Verify fetch was called with correct parameters
+    expect(mockFetch).toHaveBeenCalledWith(defaultParams.webhook, {
+      method: 'POST',
+      body: JSON.stringify({ message: 'Hello World' }),
+      headers: {
+        'Content-Type': 'application/json',
+      },
+    });
+
+    // Verify that text() was called to read the response
+    expect(mockResponse.text).toHaveBeenCalled();
+  });
+
+  it("test POST network error", async () => {
+    // Mock fetch to throw a network error
+    const networkError = new Error('Network request failed');
+    mockFetch.mockRejectedValueOnce(networkError);
+
+    const payload = createTestPayload();
+
+    // Expect the function to throw the network error
+    await expect(testEnv.run(notifyWebhook, payload)).rejects.toThrow('Network request failed');
+
+    // Verify fetch was called with correct parameters
+    expect(mockFetch).toHaveBeenCalledWith(defaultParams.webhook, {
+      method: 'POST',
+      body: JSON.stringify({ message: 'Hello World' }),
+      headers: {
+        'Content-Type': 'application/json',
+      },
+    });
+  });
+
+});