@vertesia/workflow 0.24.0-dev.202601221707

package/src/activities/generateOrAssignContentType.ts

@@ -0,0 +1,218 @@
+import { log } from "@temporalio/activity";
+import {
+  ContentObjectTypeItem,
+  CreateContentObjectTypePayload,
+  DSLActivityExecutionPayload,
+  DSLActivitySpec,
+  ImageRenditionFormat,
+} from "@vertesia/common";
+import {
+  ActivityContext,
+  setupActivity,
+} from "../dsl/setup/ActivityContext.js";
+import { TruncateSpec, truncByMaxTokens } from "../utils/tokens.js";
+import {
+  InteractionExecutionParams,
+  executeInteractionFromActivity,
+} from "./executeInteraction.js";
+
+const INT_SELECT_DOCUMENT_TYPE = "sys:SelectDocumentType";
+const INT_GENERATE_METADATA_MODEL = "sys:GenerateMetadataModel";
+
+export interface GenerateOrAssignContentTypeParams
+  extends InteractionExecutionParams {
+  typesHint?: string[];
+  /**
+   * truncate the input doc text to the specified max_tokens
+   */
+  truncate?: TruncateSpec;
+
+  /**
+   * The name of the interaction to execute
+   * @default SelectDocumentType
+   */
+  interactionNames?: {
+    selectDocumentType?: string;
+    generateMetadataModel?: string;
+  };
+}
+
+export interface GenerateOrAssignContentType
+  extends DSLActivitySpec<GenerateOrAssignContentTypeParams> {
+  name: "generateOrAssignContentType";
+}
+
+export async function generateOrAssignContentType(
+  payload: DSLActivityExecutionPayload<GenerateOrAssignContentTypeParams>,
+) {
+  const context =
+    await setupActivity<GenerateOrAssignContentTypeParams>(payload);
+  const { params, client, objectId } = context;
+
+  const interactionName =
+    params.interactionNames?.selectDocumentType ?? INT_SELECT_DOCUMENT_TYPE;
+
+  log.info("SelectDocumentType for object: " + objectId, { payload });
+
+  const object = await client.objects.retrieve(objectId, "+text");
+
+  //Expects object.type to be null on first ingestion of content
+  //User initiated Content Type change via the Composable UI,
+  //sets object.type to null when they let Composable choose for them.
+  //sets object.type to chosen type (thus non-null) when user picks a type.
+  if (object.type) {
+    log.warn(`Object ${objectId} has already a type. Skipping type creation.`);
+    return {
+      status: "skipped",
+      message: "Object already has a type: " + object.type.name,
+    };
+  }
+
+  if (
+    !object ||
+    (!object.text &&
+      !object.content?.type?.startsWith("image/") &&
+      !object.content?.type?.startsWith("application/pdf"))
+  ) {
+    log.info(`Object ${objectId} not found or text is empty and not an image`, {
+      object,
+    });
+    return { status: "failed", error: "no-text" };
+  }
+
+  const types = await client.types.list(undefined, {
+    schema: true,
+  });
+
+  //make a list of all existing types, and add hints if any
+  const existing_types = types.filter(
+    (t) => !["DocumentPart", "Rendition"].includes(t.name),
+  );
+  const content = object.text
+    ? truncByMaxTokens(object.text, params.truncate || 30000)
+    : undefined;
+
+  const getImage = async () => {
+    if (
+      object.content?.type?.includes("pdf") &&
+      object.text?.length &&
+      object.text?.length < 100
+    ) {
+      return "store:" + objectId;
+    }
+    if (!object.content?.type?.startsWith("image/")) {
+      return undefined;
+    }
+    const res = await client.objects.getRendition(objectId, {
+      format: ImageRenditionFormat.jpeg,
+      generate_if_missing: true,
+    });
+    if (!res.renditions?.length && res.status === "generating") {
+      //throw to try again
+      throw new Error(`Rendition for object ${objectId} is in progress`);
+    } else if (res.renditions) {
+      return "store:" + objectId;
+    }
+  };
+
+  const fileRef = await getImage();
+
+  log.info(
+    "Execute SelectDocumentType interaction on content with \nexisting types - passing full types: " +
+    existing_types.filter((t) => !t.tags?.includes("system")),
+  );
+
+  const res = await executeInteractionFromActivity(
+    client,
+    interactionName,
+    params,
+    {
+      existing_types,
+      content,
+      image: fileRef,
+    },
+  );
+
+  const jsonResult = res.result.object();
+
+  log.info("Selected Content Type Result: " + JSON.stringify(jsonResult));
+
+
+  //if type is not identified or not present in the database, generate a new type
+  let selectedType: { id: string; name: string } | undefined = undefined;
+
+  selectedType = types.find((t) => t.name === jsonResult.document_type);
+
+  if (!selectedType) {
+    log.warn("Document type not identified: starting type generation");
+    const newType = await generateNewType(
+      context,
+      existing_types,
+      content,
+      fileRef,
+    );
+    selectedType = { id: newType.id, name: newType.name };
+  }
+
+  if (!selectedType) {
+    log.error("Type not found: ", res.result);
+    throw new Error("Type not found: " + jsonResult.document_type);
+  }
+
+  //update object with selected type
+  await client.objects.update(objectId, {
+    type: selectedType.id,
+  });
+
+  return {
+    id: selectedType.id,
+    name: selectedType.name,
+    isNew: !types.find((t) => t.name === selectedType.name),
+  };
+}
+
+async function generateNewType(
+  context: ActivityContext<GenerateOrAssignContentTypeParams>,
+  existing_types: ContentObjectTypeItem[],
+  content?: string,
+  fileRef?: string,
+) {
+  const { client, params } = context;
+
+  const project = await context.fetchProject();
+  const interactionName =
+    params.interactionNames?.generateMetadataModel ??
+    INT_GENERATE_METADATA_MODEL;
+
+  const genTypeRes = await executeInteractionFromActivity(
+    client,
+    interactionName,
+    params,
+    {
+      existing_types,
+      content: content,
+      human_context: project?.configuration?.human_context ?? undefined,
+      image: fileRef ? fileRef : undefined,
+    },
+  );
+
+  const jsonResult = genTypeRes.result.object();
+
+  if (!jsonResult.document_type) {
+    log.error("No name generated for type", genTypeRes);
+    throw new Error("No name generated for type");
+  }
+
+  log.info("Generated schema for type", jsonResult.metadata_schema);
+  const typeData: CreateContentObjectTypePayload = {
+    name: jsonResult.document_type,
+    description: jsonResult.document_type_description,
+    object_schema: jsonResult.metadata_schema,
+    is_chunkable: jsonResult.is_chunkable,
+    table_layout: jsonResult.table_layout,
+  };
+
+  const type = await client.types.create(typeData);
+
+  return type;
+}

package/src/activities/getObjectFromStore.ts

@@ -0,0 +1,31 @@
+import { ContentObject, DSLActivityExecutionPayload, DSLActivitySpec } from "@vertesia/common";
+import { projectResult } from "../dsl/projections.js";
+import { setupActivity } from "../dsl/setup/ActivityContext.js";
+
+
+export interface GetObjectParams {
+    select?: string;
+}
+
+export interface GetObject extends DSLActivitySpec<GetObjectParams> {
+    name: 'getObject';
+}
+
+/**
+ * We are using a union type for the status parameter since typescript enums breaks the workflow code generation
+ * @param objectId
+ * @param status
+ */
+export async function getObjectFromStore(payload: DSLActivityExecutionPayload<GetObjectParams>): Promise<ContentObject> {
+    const { client, params, objectId } = await setupActivity<GetObjectParams>(payload);
+
+    const obj = await client.objects.retrieve(objectId, params.select);
+
+    const projection = projectResult(payload, params, obj, obj);
+
+    return {
+        ...projection,
+        id: obj.id,
+    }
+
+}

package/src/activities/handleError.ts

@@ -0,0 +1,25 @@
+import { ContentObjectStatus, DSLActivityExecutionPayload } from "@vertesia/common";
+import { setupActivity } from "../dsl/setup/ActivityContext.js";
+import { log } from "@temporalio/activity"
+
+export interface HandleDslErrorParams {
+    errorMessage: string;
+}
+
+export async function handleDslError(payload: DSLActivityExecutionPayload<HandleDslErrorParams>): Promise<void> {
+    const { client, params, objectId } = await setupActivity<HandleDslErrorParams>(payload);
+    const isIntake = payload.workflow_name === "StandardDocumentIntake" || payload.workflow_name === "StandardImageIntake";
+    if (!isIntake) {
+        log.warn(`Workflow execution failed, but no error handler registered for this workflow: ${payload.workflow_name}`,
+            { error: params.errorMessage },
+        );
+        return;
+    }
+
+    try {
+        await client.objects.update(objectId, { status: ContentObjectStatus.failed });
+    } catch (e) {
+        log.error("Failed to handle error", { error: e });
+    }
+    return;
+}

package/src/activities/index-dsl.ts

@@ -0,0 +1,25 @@
+// Export here DSL activities
+export { createDocumentTypeFromInteractionRun } from "./advanced/createDocumentTypeFromInteractionRun.js";
+export { createOrUpdateDocumentFromInteractionRun } from "./advanced/createOrUpdateDocumentFromInteractionRun.js";
+export { updateDocumentFromInteractionRun } from "./advanced/updateDocumentFromInteractionRun.js";
+export { chunkDocument } from "./chunkDocument.js";
+export { createPdfDocumentFromSource } from "./createDocumentFromOther.js";
+export { executeInteraction } from "./executeInteraction.js";
+export { extractDocumentText } from "./extractDocumentText.js";
+export { generateDocumentProperties } from "./generateDocumentProperties.js";
+export { generateEmbeddings } from "./generateEmbeddings.js";
+export { generateOrAssignContentType } from "./generateOrAssignContentType.js";
+export { getObjectFromStore } from "./getObjectFromStore.js";
+export { handleDslError } from "./handleError.js";
+export { prepareVideo } from "./media/prepareVideo.js";
+export { prepareAudio } from "./media/prepareAudio.js";
+export { convertPdfToStructuredText } from "./media/processPdfWithTextract.js";
+export { saveGladiaTranscription } from "./media/saveGladiaTranscription.js";
+export { transcribeMedia } from "./media/transcribeMediaWithGladia.js";
+export type { TranscriptMediaResult } from "./media/transcribeMediaWithGladia.js";
+export { notifyWebhook } from "./notifyWebhook.js";
+export { checkRateLimit } from "./rateLimiter.js";
+export { copyParentArtifacts } from "./copyParentArtifacts.js";
+export { generateImageRendition } from "./renditions/generateImageRendition.js";
+export { generateVideoRendition } from "./renditions/generateVideoRendition.js";
+export { setDocumentStatus } from "./setDocumentStatus.js";

package/src/activities/index.ts

@@ -0,0 +1,4 @@
+/**
+ * Here we export all activities to be registered with the temporal worker
+ */
+export * from "./index-dsl.js";

package/src/activities/media/prepareAudio.ts

@@ -0,0 +1,334 @@
+import { log } from '@temporalio/activity';
+import { DSLActivityExecutionPayload, DSLActivitySpec, AudioMetadata, AUDIO_RENDITION_NAME, ContentNature, Rendition } from '@vertesia/common';
+import { exec } from 'child_process';
+import fs from 'fs';
+import os from 'os';
+import path from 'path';
+import { promisify } from 'util';
+import { setupActivity } from '../../dsl/setup/ActivityContext.js';
+import { DocumentNotFoundError, InvalidContentTypeError } from '../../errors.js';
+import { saveBlobToTempFile } from '../../utils/blobs.js';
+import { VertesiaClient } from '@vertesia/client';
+import { RequestError } from '@vertesia/api-fetch-client';
+
+const execAsync = promisify(exec);
+
+// Default configuration constants
+const DEFAULT_AUDIO_BITRATE = '128k'; // Default audio bitrate for AAC encoding
+const FFMPEG_MAX_BUFFER = 1024 * 1024 * 10; // 10MB buffer for ffmpeg output
+
+export interface PrepareAudioParams {
+    audioBitrate?: string; // Audio bitrate for AAC encoding, default '128k'
+}
+
+export interface PrepareAudio extends DSLActivitySpec<PrepareAudioParams> {
+    name: 'prepareAudio';
+}
+
+interface AudioMetadataExtended {
+    duration: number;
+    codec: string;
+    bitrate: number;
+    sampleRate: number;
+    channels: number;
+}
+
+interface FFProbeStream {
+    codec_type: string;
+    codec_name?: string;
+    sample_rate?: string;
+    channels?: number;
+    bit_rate?: string;
+}
+
+interface FFProbeFormat {
+    duration?: string;
+    bit_rate?: string;
+}
+
+interface FFProbeOutput {
+    streams: FFProbeStream[];
+    format: FFProbeFormat;
+}
+
+export interface PrepareAudioMetadata {
+    duration: number;
+    codec: string;
+    bitrate: number;
+    sampleRate: number;
+    channels: number;
+}
+
+export interface PrepareAudioResult {
+    objectId: string;
+    metadata: PrepareAudioMetadata;
+    renditions: Rendition[];
+    status: 'success';
+}
+
+/**
+ * Extract comprehensive audio metadata using ffprobe
+ */
+async function getAudioMetadata(audioPath: string): Promise<AudioMetadataExtended> {
+    try {
+        const command = `ffprobe -v quiet -print_format json -show_format -show_streams "${audioPath}"`;
+        const { stdout } = await execAsync(command);
+        const metadata = JSON.parse(stdout) as FFProbeOutput;
+
+        const audioStream = metadata.streams.find(
+            (stream) => stream.codec_type === 'audio',
+        );
+
+        if (!audioStream) {
+            throw new Error('No audio stream found in file');
+        }
+
+        const duration = parseFloat(metadata.format.duration ?? '0') || 0;
+        const codec = audioStream.codec_name || 'unknown';
+        const bitrate = parseInt(audioStream.bit_rate ?? metadata.format.bit_rate ?? '0', 10) || 0;
+        const sampleRate = parseInt(audioStream.sample_rate ?? '0', 10) || 0;
+        const channels = audioStream.channels || 0;
+
+        return { duration, codec, bitrate, sampleRate, channels };
+    } catch (error) {
+        log.error(
+            `Failed to get audio metadata: ${error instanceof Error ? error.message : 'Unknown error'}`,
+        );
+        throw new Error(
+            `Failed to probe audio metadata: ${error instanceof Error ? error.message : 'Unknown error'}`,
+        );
+    }
+}
+
+/**
+ * Generate a web-compatible audio rendition (AAC in M4A container)
+ * Ensures broad browser compatibility for HTML5 audio playback
+ */
+async function generateAudioRendition(
+    audioPath: string,
+    outputDir: string,
+    audioBitrate: string,
+): Promise<string | null> {
+    const outputFile = path.join(outputDir, 'audio.m4a');
+
+    const command = [
+        'ffmpeg',
+        '-y', // Overwrite output
+        '-i', `"${audioPath}"`,
+        '-c:a', 'aac', // AAC codec
+        '-b:a', audioBitrate, // Audio bitrate
+        '-movflags', '+faststart', // Enable streaming
+        `"${outputFile}"`,
+    ].join(' ');
+
+    log.info('Generating web audio rendition (AAC M4A)', { command, audioBitrate });
+
+    try {
+        const { stderr } = await execAsync(command, { maxBuffer: FFMPEG_MAX_BUFFER });
+
+        if (stderr && !stderr.includes('frame=')) {
+            log.debug(`FFmpeg stderr for audio rendition: ${stderr}`);
+        }
+
+        // Verify output file was created
+        try {
+            await fs.promises.access(outputFile, fs.constants.F_OK);
+            log.info(`Generated web audio rendition: ${outputFile}`);
+            return outputFile;
+        } catch {
+            log.warn('Audio rendition not generated');
+            return null;
+        }
+    } catch (error) {
+        log.error(
+            `Failed to generate audio rendition: ${error instanceof Error ? error.message : 'Unknown error'}`,
+        );
+        return null;
+    }
+}
+
+/**
+ * Upload a file to the storage and return its URI
+ */
+async function uploadFile(
+    client: VertesiaClient,
+    filePath: string,
+    mimeType: string,
+    fileName: string,
+    storagePath: string,
+): Promise<string> {
+    const { NodeStreamSource } = await import('@vertesia/client/node');
+    const fileStream = fs.createReadStream(filePath);
+    const source = new NodeStreamSource(fileStream, fileName, mimeType, storagePath);
+
+    const result = await client.files.uploadFile(source);
+    log.info(`Uploaded file to ${storagePath}`, { result });
+
+    return result;
+}
+
+/**
+ * Upload audio file and create a rendition entry
+ */
+async function uploadAudioAsRendition(
+    client: VertesiaClient,
+    audioFile: string,
+    renditionName: string,
+    fileName: string,
+    mimeType: string,
+    etag: string,
+): Promise<Rendition> {
+    const storagePath = `renditions/${etag}/audio/${fileName}`;
+    const uri = await uploadFile(client, audioFile, mimeType, fileName, storagePath);
+
+    return {
+        name: renditionName,
+        content: {
+            source: uri,
+            type: mimeType,
+            name: fileName,
+        },
+    };
+}
+
+/**
+ * Main activity: Prepare audio by extracting metadata and generating web rendition
+ */
+export async function prepareAudio(
+    payload: DSLActivityExecutionPayload<PrepareAudioParams>,
+): Promise<PrepareAudioResult> {
+    const {
+        client,
+        objectId,
+        params,
+    } = await setupActivity<PrepareAudioParams>(payload);
+
+    const audioBitrate = params.audioBitrate ?? DEFAULT_AUDIO_BITRATE;
+
+    log.info(`Preparing audio for ${objectId}`, { audioBitrate });
+
+    // Retrieve the content object
+    const inputObject = await client.objects.retrieve(objectId).catch((err: unknown) => {
+        log.error(`Failed to retrieve document ${objectId}`, { err });
+        if (err instanceof RequestError && err.status === 404) {
+            throw new DocumentNotFoundError(`Document ${objectId} not found`, [objectId]);
+        }
+        throw err;
+    });
+
+    if (!inputObject.content?.source) {
+        log.error(`Document ${objectId} has no source`);
+        throw new DocumentNotFoundError(`Document ${objectId} has no source`, [objectId]);
+    }
+
+    if (!inputObject.content.type || !inputObject.content.type.startsWith('audio/')) {
+        log.error(`Document ${objectId} is not an audio file: ${inputObject.content.type}`);
+        throw new InvalidContentTypeError(
+            objectId,
+            'audio/*',
+            inputObject.content.type || 'unknown',
+        );
+    }
+
+    // Download audio to temp file
+    const audioFile = await saveBlobToTempFile(client, inputObject.content.source);
+    const tempOutputDir = fs.mkdtempSync(path.join(os.tmpdir(), 'prepare-audio-'));
+
+    try {
+        // Step 1: Extract audio metadata
+        log.info('Extracting audio metadata');
+        const metadata = await getAudioMetadata(audioFile);
+
+        // Step 2: Generate web audio rendition (AAC M4A)
+        log.info('Generating web audio rendition');
+        const renditionFile = await generateAudioRendition(
+            audioFile,
+            tempOutputDir,
+            audioBitrate,
+        );
+
+        // Step 3: Upload generated rendition
+        const renditions: Rendition[] = [];
+        const etag = inputObject.content.etag ?? inputObject.id;
+
+        if (renditionFile) {
+            const audioRendition = await uploadAudioAsRendition(
+                client,
+                renditionFile,
+                AUDIO_RENDITION_NAME,
+                'audio.m4a',
+                'audio/mp4',
+                etag,
+            );
+            renditions.push(audioRendition);
+        }
+
+        // Step 4: Update content object with metadata and renditions
+        const audioMetadata: AudioMetadata = {
+            type: ContentNature.Audio,
+            duration: metadata.duration,
+            renditions,
+            generation_runs: inputObject.metadata?.generation_runs || [],
+        };
+
+        await client.objects.update(objectId, {
+            metadata: audioMetadata,
+        });
+
+        log.info(`Successfully prepared audio ${objectId}`, {
+            duration: metadata.duration,
+            codec: metadata.codec,
+            bitrate: metadata.bitrate,
+            sampleRate: metadata.sampleRate,
+            channels: metadata.channels,
+            renditionsGenerated: renditions.length,
+        });
+
+        return {
+            objectId,
+            metadata: {
+                duration: metadata.duration,
+                codec: metadata.codec,
+                bitrate: metadata.bitrate,
+                sampleRate: metadata.sampleRate,
+                channels: metadata.channels,
+            },
+            renditions: renditions,
+            status: 'success',
+        };
+    } catch (error) {
+        const errorMessage = error instanceof Error ? error.message : 'Unknown error';
+        log.error(`Error preparing audio: ${errorMessage}`, { error });
+
+        // Re-throw known errors as-is
+        if (error instanceof DocumentNotFoundError || error instanceof InvalidContentTypeError) {
+            throw error;
+        }
+
+        // Wrap unknown errors in Error
+        throw new Error(
+            `Failed to prepare audio ${objectId}: ${errorMessage}`,
+        );
+    } finally {
+        // Clean up temporary files
+        const cleanupPromises: Promise<void>[] = [];
+
+        if (audioFile) {
+            cleanupPromises.push(
+                fs.promises.unlink(audioFile).catch((err) => {
+                    log.warn(`Failed to cleanup audio file: ${audioFile}`, { err });
+                }),
+            );
+        }
+
+        if (tempOutputDir) {
+            cleanupPromises.push(
+                fs.promises.rm(tempOutputDir, { recursive: true, force: true }).catch((err) => {
+                    log.warn(`Failed to cleanup temp directory: ${tempOutputDir}`, { err });
+                }),
+            );
+        }
+
+        await Promise.allSettled(cleanupPromises);
+    }
+}