@vertesia/workflow 0.42.2

package/src/activities/media/processPdfWithTextract.ts

@@ -0,0 +1,141 @@
+/**
+ * Use textract to convert a pdf into a data structure of the following format:
+ * <document>
+ *  <page number="n">
+ *   <text/>
+ *   <table/>
+ *   <text/>
+ *   <figure/>
+ *   ...
+ *  </page>
+ */
+
+import { fromWebToken } from "@aws-sdk/credential-providers";
+import { AwsConfiguration, CreateContentObjectPayload, DSLActivityExecutionPayload, DSLActivitySpec, SupportedIntegrations } from "@vertesia/common";
+import type { AwsCredentialIdentityProvider } from "@smithy/types";
+import { log } from "@temporalio/activity";
+import { TextractProcessor } from "../../conversion/TextractProcessor.js";
+import { setupActivity } from "../../dsl/setup/ActivityContext.js";
+import { NoDocumentFound } from "../../errors.js";
+import { TextExtractionResult, TextExtractionStatus } from "../../result-types.js";
+import { fetchBlobAsBuffer, md5 } from "../../utils/blobs.js";
+import { countTokens } from "../../utils/tokens.js";
+
+
+
+export interface ConvertPdfToStructuredTextParams {
+    force?: boolean;
+}
+
+export interface ConvertPdfToStructuredText extends DSLActivitySpec<ConvertPdfToStructuredTextParams> {
+    name: 'ConvertPdfToStructuredText';
+}
+
+export interface StructuredTextResult extends TextExtractionResult {
+    message?: string;
+}
+
+
+
+export async function convertPdfToStructuredText(payload: DSLActivityExecutionPayload): Promise<StructuredTextResult> {
+
+
+    const { params, client, objectId } = await setupActivity<ConvertPdfToStructuredTextParams>(payload);
+
+    const object = await client.objects.retrieve(objectId, "+text");
+
+    if (object.text && !params.force) {
+        return { hasText: true, objectId, status: TextExtractionStatus.skipped, message: "text already present and force not enabled" }
+    }
+
+    if (!object.content?.source) {
+        throw new NoDocumentFound(`No source found for object ${objectId}`);
+    }
+
+    const pdfUrl = await client.store.objects.getContentSource(objectId).then(res => res.source);
+
+    if (!pdfUrl) {
+        throw new NoDocumentFound(`Error fetching source ${object.content.source}`);
+    }
+
+
+    const awsConfig = (await client.projects.integrations.retrieve(client.project!, SupportedIntegrations.aws)) as AwsConfiguration;
+    const credentials = await getS3AWSCredentials(awsConfig, payload.auth_token, client.project!);
+
+    const processor = new TextractProcessor({
+        fileKey: objectId,
+        region: "us-west-2",
+        bucket: "cp-textract-tests",
+        credentials,
+        log: log,
+        detectImages: true,
+        includeConfidenceInTables: true,
+    });
+
+
+
+    try {
+
+        if (!object.content.source.startsWith("s3://")) {
+            const buf = await fetchBlobAsBuffer(client, object.content.source);
+            await processor.upload(buf);
+        }
+
+        const jobId = await processor.startAnalysis(objectId);
+
+        let jobStatus = await processor.checkJobStatus(jobId);
+        while (jobStatus === "IN_PROGRESS") {
+            await new Promise(resolve => setTimeout(resolve, 5000));
+            jobStatus = await processor.checkJobStatus(jobId);
+        }
+
+        if (jobStatus === "SUCCEEDED") {
+            log.info(`Job ${jobId} succeeded, saving results`, { jobId });
+            const ftext = await processor.processResults(jobId);
+            const tokensData = countTokens(ftext);
+            const etag = object.content.etag ?? md5(ftext);
+            const updateData: CreateContentObjectPayload = {
+                text: ftext,
+                text_etag: etag,
+                tokens: {
+                    ...tokensData,
+                    etag: etag,
+                }
+            }
+
+            await client.objects.update(objectId, updateData);
+            console.log("Full text updated");
+
+            return { hasText: true, objectId, status: TextExtractionStatus.success, message: "Text extracted successfully" }
+
+        } else {
+            throw new Error(`Job failed with status: ${jobStatus}`);
+        }
+    } catch (error) {
+        console.error("Error processing document:", error);
+        throw error;
+    }
+
+
+}
+
+export async function getS3AWSCredentials(awsConfig: AwsConfiguration, composableAuthToken: string, projectId: string): Promise<AwsCredentialIdentityProvider> {
+
+    // fetch s3 role ARN
+    if (!awsConfig || !awsConfig.enabled) {
+        throw new NoDocumentFound("AWS integration is not enabled for this project");
+    }
+    if (!awsConfig.s3_role_arn) {
+        throw new NoDocumentFound("S3 Role ARN is not defined in AWS project integration");
+    }
+
+    log.info("Getting AWS credentials for Textract", { projectId, composableAuthToken, roleArn: awsConfig.s3_role_arn });
+
+    const credentials = fromWebToken({
+        webIdentityToken: composableAuthToken,
+        roleArn: awsConfig.s3_role_arn,
+        roleSessionName: `cp-project-textract-${projectId}`,
+    });
+
+    return credentials;
+}

package/src/activities/media/transcribeMediaWithGladia.ts

@@ -0,0 +1,83 @@
+import { DSLActivityExecutionPayload, DSLActivitySpec, GladiaConfiguration, SupportedIntegrations } from "@vertesia/common";
+import { activityInfo, CompleteAsyncError, log } from "@temporalio/activity";
+import { FetchClient } from "api-fetch-client";
+import { setupActivity } from "../../dsl/setup/ActivityContext.js";
+import { NoDocumentFound } from "../../errors.js";
+import { TextExtractionResult, TextExtractionStatus } from "../../index.js";
+
+
+export interface TranscriptMediaParams {
+    environmentId?: string;
+    force?: boolean;
+}
+
+export interface TranscriptMedia extends DSLActivitySpec<TranscriptMediaParams> {
+    name: 'TranscribeMedia';
+}
+
+export interface TranscriptMediaResult extends TextExtractionResult {
+    message?: string;
+}
+
+const GLADIA_URL = "https://api.gladia.io/v2";
+
+export async function transcribeMedia(payload: DSLActivityExecutionPayload): Promise<TranscriptMediaResult> {
+
+    const { params, client, objectId } = await setupActivity<TranscriptMediaParams>(payload);
+
+    const gladiaConfig = await client.projects.integrations.retrieve(payload.project_id, SupportedIntegrations.gladia) as GladiaConfiguration | undefined;
+    if (!gladiaConfig || !gladiaConfig.enabled) {
+        throw new NoDocumentFound("Gladia integration not enabled");
+    }
+
+    const object = await client.objects.retrieve(objectId, "+text");
+    const gladiaClient = new FetchClient(gladiaConfig.url ?? GLADIA_URL);
+    gladiaClient.withHeaders({ "x-gladia-key": gladiaConfig.api_key });
+
+    if (object.text && !params.force) {
+        return { hasText: true, objectId, status: TextExtractionStatus.skipped, message: "text already present and force not enabled" }
+    }
+
+    if (!object.content?.source) {
+        throw new NoDocumentFound(`No source found for object ${objectId}`);
+    }
+
+    const mediaUrl = await client.store.objects.getContentSource(objectId).then(res => res.source);
+
+    if (!mediaUrl) {
+        throw new NoDocumentFound(`Error fetching source ${object.content.source}`);
+    }
+
+    const taskToken = Buffer.from(activityInfo().taskToken).toString('base64url');
+    const callbackUrl = generateCallbackUrlForGladia(client.store.baseUrl, payload.auth_token, taskToken, objectId);
+
+    log.info(`Transcribing media ${mediaUrl} with Gladia`, { objectId, callbackUrl });
+
+    const res = await gladiaClient.post("/transcription", {
+        payload: {
+            audio_url: mediaUrl,
+            callback_url: callbackUrl,
+            diarization_enhanced: true,
+            enable_code_switching: true,
+            subtitles: true,
+            subtitles_config: {
+                formats: ["vtt"],
+            }
+        }
+    }) as GladiaTranscriptRequestResponse;
+
+    log.info(`Transcription request sent to Gladia`, { objectId, res });
+
+    throw new CompleteAsyncError();
+
+}
+
+
+function generateCallbackUrlForGladia(baseUrl: string, authToken: string, taskToken: string, objectId: string) {
+    return `${baseUrl}/api/v1/webhooks/gladia/${objectId}?auth_token=${authToken}&task_token=${taskToken}`;
+}
+
+interface GladiaTranscriptRequestResponse {
+    id: string;
+    result_url: string;
+}

package/src/activities/notifyWebhook.test.ts

@@ -0,0 +1,32 @@
+import {
+  MockActivityEnvironment,
+  TestWorkflowEnvironment,
+} from "@temporalio/testing";
+import { beforeAll, describe, expect, test } from "vitest";
+import { notifyWebhook, NotifyWebhook } from "./notifyWebhook.js";
+
+let testEnv: TestWorkflowEnvironment;
+let activityContext: MockActivityEnvironment;
+
+beforeAll(async () => {
+  testEnv = await TestWorkflowEnvironment.createLocal();
+  activityContext = new MockActivityEnvironment();
+});
+
+// https://github.com/becomposable/studio/issues/432 Skip tests
+// Cannot read properties of undefined (reading 'params')
+describe("Webhook should be notified", () => {
+  test.skip("test POST", async () => {
+    const activityConfig = {
+      name: "notifyWebhook",
+      params: {
+        target_url: "https://en5zdcyvn4dc3.x.pipedream.net",
+        method: "POST",
+        payload: { message: "Hello World" },
+      },
+    } satisfies NotifyWebhook;
+
+    const res = await activityContext.run(notifyWebhook, activityConfig);
+    expect(res).toBeDefined();
+  });
+});

package/src/activities/notifyWebhook.ts

@@ -0,0 +1,51 @@
+import { DSLActivityExecutionPayload, DSLActivitySpec } from "@vertesia/common";
+import { log } from "@temporalio/activity";
+import { setupActivity } from "../dsl/setup/ActivityContext.js";
+import { WorkflowParamNotFound } from "../errors.js";
+
+interface NotifyWebhookParams {
+    target_url: string; //URL to send the notification to
+    method: 'GET' | 'POST'; //HTTP method to use
+    payload: Record<string, any>; //payload to send (if POST then as JSON body, if GET then as query string)
+    headers?: Record<string, string>; //additional headers to send
+}
+
+
+export interface NotifyWebhook extends DSLActivitySpec<NotifyWebhookParams> {
+    name: 'notifyWebhook';
+}
+
+
+export async function notifyWebhook(payload: DSLActivityExecutionPayload) {
+
+    const { params } = await setupActivity<NotifyWebhookParams>(payload);
+    const { target_url, method, payload: requestPayload, headers } = params
+
+    if (!target_url) throw new WorkflowParamNotFound('target_url');
+
+    const body = method === 'POST' ? JSON.stringify({
+        ...requestPayload,
+        ...params
+    }) : undefined
+
+    log.info(`Notifying webhook at ${target_url}`);
+    const res = await fetch(target_url, {
+        method,
+        body,
+        headers: {
+            'Content-Type': 'application/json',
+            ...headers
+        },
+    }).catch(err => {
+        log.warn(`Failed to notify webhook ${target_url}: ${err}`);
+        throw new Error(`Failed to notify webhook ${target_url}: ${err}`);
+    });
+
+    if (!res.ok) {
+        log.warn(`Failed to notify webhook ${target_url} - ${res.status}: ${res.statusText}`, { res });
+        throw new Error(`Failed to notify webhook ${target_url}: ${res.statusText}`);
+    }
+
+    return {status: res.status, message: res.statusText, url: res.url }
+
+}

package/src/activities/setDocumentStatus.ts

@@ -0,0 +1,25 @@
+import { ContentObjectStatus, DSLActivityExecutionPayload, DSLActivitySpec } from "@vertesia/common";
+import { setupActivity } from "../dsl/setup/ActivityContext.js";
+
+export interface SetDocumentStatusParams {
+    status: ContentObjectStatus;
+}
+
+export interface SetDocumentStatus extends DSLActivitySpec<SetDocumentStatusParams> {
+    name: 'setDocumentStatus';
+    projection?: never;
+}
+
+/**
+ * We are using a union type for the status parameter since typescript enumbs breaks the workflow code generation
+ * @param objectId
+ * @param status
+ */
+export async function setDocumentStatus(payload: DSLActivityExecutionPayload) {
+    const { client, params, objectId } = await setupActivity<SetDocumentStatusParams>(payload);
+
+    const res = await client.objects.update(objectId, { status: params.status });
+
+    return res.status;
+
+}