@vertesia/workflow 0.56.0 → 0.58.0

package/src/activities/generateOrAssignContentType.ts

@@ -1,172 +1,213 @@
 import { log } from "@temporalio/activity";
 import {
-    ContentObjectTypeItem,
-    CreateContentObjectTypePayload,
-    DSLActivityExecutionPayload,
-    DSLActivitySpec,
+  ContentObjectTypeItem,
+  CreateContentObjectTypePayload,
+  DSLActivityExecutionPayload,
+  DSLActivitySpec,
+  ImageRenditionFormat,
 } from "@vertesia/common";
-import { ActivityContext, setupActivity } from "../dsl/setup/ActivityContext.js";
+import {
+  ActivityContext,
+  setupActivity,
+} from "../dsl/setup/ActivityContext.js";
 import { TruncateSpec, truncByMaxTokens } from "../utils/tokens.js";
-import { InteractionExecutionParams, executeInteractionFromActivity } from "./executeInteraction.js";
+import {
+  InteractionExecutionParams,
+  executeInteractionFromActivity,
+} from "./executeInteraction.js";
 
 const INT_SELECT_DOCUMENT_TYPE = "sys:SelectDocumentType";
 const INT_GENERATE_METADATA_MODEL = "sys:GenerateMetadataModel";
 
-export interface GenerateOrAssignContentTypeParams extends InteractionExecutionParams {
-    typesHint?: string[];
-    /**
-     * truncate the input doc text to the specified max_tokens
-     */
-    truncate?: TruncateSpec;
-
-    /**
-     * The name of the interaction to execute
-     * @default SelectDocumentType
-     */
-    interactionNames?: {
-        selectDocumentType?: string;
-        generateMetadataModel?: string;
-    };
+export interface GenerateOrAssignContentTypeParams
+  extends InteractionExecutionParams {
+  typesHint?: string[];
+  /**
+   * truncate the input doc text to the specified max_tokens
+   */
+  truncate?: TruncateSpec;
+
+  /**
+   * The name of the interaction to execute
+   * @default SelectDocumentType
+   */
+  interactionNames?: {
+    selectDocumentType?: string;
+    generateMetadataModel?: string;
+  };
 }
 
-export interface GenerateOrAssignContentType extends DSLActivitySpec<GenerateOrAssignContentTypeParams> {
-    name: "generateOrAssignContentType";
+export interface GenerateOrAssignContentType
+  extends DSLActivitySpec<GenerateOrAssignContentTypeParams> {
+  name: "generateOrAssignContentType";
 }
 
 export async function generateOrAssignContentType(
-    payload: DSLActivityExecutionPayload<GenerateOrAssignContentTypeParams>,
+  payload: DSLActivityExecutionPayload<GenerateOrAssignContentTypeParams>,
 ) {
-    const context = await setupActivity<GenerateOrAssignContentTypeParams>(payload);
-    const { params, client, objectId } = context;
-
-    const interactionName = params.interactionNames?.selectDocumentType ?? INT_SELECT_DOCUMENT_TYPE;
-
-    log.info("SelectDocumentType for object: " + objectId, { payload });
+  const context =
+    await setupActivity<GenerateOrAssignContentTypeParams>(payload);
+  const { params, client, objectId } = context;
 
-    const object = await client.objects.retrieve(objectId, "+text");
+  const interactionName =
+    params.interactionNames?.selectDocumentType ?? INT_SELECT_DOCUMENT_TYPE;
 
-    //Expects object.type to be null on first ingestion of content
-    //User initiated Content Type change via the Composable UI,
-    //sets object.type to null when they let Composable choose for them.
-    //sets object.type to chosen type (thus non-null) when user picks a type.
-    if (object.type) {
-        log.warn(`Object ${objectId} has already a type. Skipping type creation.`);
-        return { status: "skipped", message: "Object already has a type: " + object.type.name };
-    }
+  log.info("SelectDocumentType for object: " + objectId, { payload });
 
-    if (
-        !object ||
-        (!object.text &&
-            !object.content?.type?.startsWith("image/") &&
-            !object.content?.type?.startsWith("application/pdf"))
-    ) {
-        log.info(`Object ${objectId} not found or text is empty and not an image`, { object });
-        return { status: "failed", error: "no-text" };
-    }
+  const object = await client.objects.retrieve(objectId, "+text");
 
-    const types = await client.types.list(undefined, {
-        schema: true,
-    });
-
-    //make a list of all existing types, and add hints if any
-    const existing_types = types.filter((t) => !["DocumentPart", "Rendition"].includes(t.name));
-    const content = object.text ? truncByMaxTokens(object.text, params.truncate || 30000) : undefined;
-
-    const getImage = async () => {
-        if (object.content?.type?.includes("pdf") && object.text?.length && object.text?.length < 100) {
-            return "store:" + objectId;
-        }
-        if (!object.content?.type?.startsWith("image/")) {
-            return undefined;
-        }
-        const res = await client.objects.getRendition(objectId, {
-            max_hw: 1024,
-            format: "image/png",
-            generate_if_missing: true,
-        });
-        if (!res.rendition && res.status === "generating") {
-            //throw to try again
-            throw new Error(`Rendition for object ${objectId} is in progress`);
-        } else if (res.rendition) {
-            return "store:" + objectId;
-        }
+  //Expects object.type to be null on first ingestion of content
+  //User initiated Content Type change via the Composable UI,
+  //sets object.type to null when they let Composable choose for them.
+  //sets object.type to chosen type (thus non-null) when user picks a type.
+  if (object.type) {
+    log.warn(`Object ${objectId} has already a type. Skipping type creation.`);
+    return {
+      status: "skipped",
+      message: "Object already has a type: " + object.type.name,
     };
-
-    const fileRef = await getImage();
-
-    log.info(
-        "Execute SelectDocumentType interaction on content with \nexisting types - passing full types: " +
-            existing_types.filter((t) => !t.tags?.includes("system")),
-    );
-
-    const res = await executeInteractionFromActivity(client, interactionName, params, {
-        existing_types,
-        content,
-        image: fileRef,
+  }
+
+  if (
+    !object ||
+    (!object.text &&
+      !object.content?.type?.startsWith("image/") &&
+      !object.content?.type?.startsWith("application/pdf"))
+  ) {
+    log.info(`Object ${objectId} not found or text is empty and not an image`, {
+      object,
     });
-
-    log.info("Selected Content Type Result: " + JSON.stringify(res.result));
-
-    //if type is not identified or not present in the database, generate a new type
-    let selectedType: { id: string; name: string } | undefined = undefined;
-
-    selectedType = types.find((t) => t.name === res.result.document_type);
-
-    if (!selectedType) {
-        log.warn("Document type not identified: starting type generation");
-        const newType = await generateNewType(context, existing_types, content, fileRef);
-        selectedType = { id: newType.id, name: newType.name };
+    return { status: "failed", error: "no-text" };
+  }
+
+  const types = await client.types.list(undefined, {
+    schema: true,
+  });
+
+  //make a list of all existing types, and add hints if any
+  const existing_types = types.filter(
+    (t) => !["DocumentPart", "Rendition"].includes(t.name),
+  );
+  const content = object.text
+    ? truncByMaxTokens(object.text, params.truncate || 30000)
+    : undefined;
+
+  const getImage = async () => {
+    if (
+      object.content?.type?.includes("pdf") &&
+      object.text?.length &&
+      object.text?.length < 100
+    ) {
+      return "store:" + objectId;
     }
-
-    if (!selectedType) {
-        log.error("Type not found: ", res.result);
-        throw new Error("Type not found: " + res.result.document_type);
+    if (!object.content?.type?.startsWith("image/")) {
+      return undefined;
     }
-
-    //update object with selected type
-    await client.objects.update(objectId, {
-        type: selectedType.id,
+    const res = await client.objects.getRendition(objectId, {
+      max_hw: 1024,
+      format: ImageRenditionFormat.jpeg,
+      generate_if_missing: true,
     });
-
-    return {
-        id: selectedType.id,
-        name: selectedType.name,
-        isNew: !types.find((t) => t.name === selectedType.name),
-    };
+    if (!res.renditions?.length && res.status === "generating") {
+      //throw to try again
+      throw new Error(`Rendition for object ${objectId} is in progress`);
+    } else if (res.renditions) {
+      return "store:" + objectId;
+    }
+  };
+
+  const fileRef = await getImage();
+
+  log.info(
+    "Execute SelectDocumentType interaction on content with \nexisting types - passing full types: " +
+      existing_types.filter((t) => !t.tags?.includes("system")),
+  );
+
+  const res = await executeInteractionFromActivity(
+    client,
+    interactionName,
+    params,
+    {
+      existing_types,
+      content,
+      image: fileRef,
+    },
+  );
+
+  log.info("Selected Content Type Result: " + JSON.stringify(res.result));
+
+  //if type is not identified or not present in the database, generate a new type
+  let selectedType: { id: string; name: string } | undefined = undefined;
+
+  selectedType = types.find((t) => t.name === res.result.document_type);
+
+  if (!selectedType) {
+    log.warn("Document type not identified: starting type generation");
+    const newType = await generateNewType(
+      context,
+      existing_types,
+      content,
+      fileRef,
+    );
+    selectedType = { id: newType.id, name: newType.name };
+  }
+
+  if (!selectedType) {
+    log.error("Type not found: ", res.result);
+    throw new Error("Type not found: " + res.result.document_type);
+  }
+
+  //update object with selected type
+  await client.objects.update(objectId, {
+    type: selectedType.id,
+  });
+
+  return {
+    id: selectedType.id,
+    name: selectedType.name,
+    isNew: !types.find((t) => t.name === selectedType.name),
+  };
 }
 
 async function generateNewType(
-    context: ActivityContext<GenerateOrAssignContentTypeParams>,
-    existing_types: ContentObjectTypeItem[],
-    content?: string,
-    fileRef?: string,
+  context: ActivityContext<GenerateOrAssignContentTypeParams>,
+  existing_types: ContentObjectTypeItem[],
+  content?: string,
+  fileRef?: string,
 ) {
-    const { client, params } = context;
-
-    const project = await context.fetchProject();
-    const interactionName = params.interactionNames?.generateMetadataModel ?? INT_GENERATE_METADATA_MODEL;
-
-    const genTypeRes = await executeInteractionFromActivity(client, interactionName, params, {
-        existing_types,
-        content: content,
-        human_context: project?.configuration?.human_context ?? undefined,
-        image: fileRef ? fileRef : undefined,
-    });
-
-    if (!genTypeRes.result.document_type) {
-        log.error("No name generated for type", genTypeRes);
-        throw new Error("No name generated for type");
-    }
-
-    log.info("Generated schema for type", genTypeRes.result.metadata_schema);
-    const typeData: CreateContentObjectTypePayload = {
-        name: genTypeRes.result.document_type,
-        object_schema: genTypeRes.result.metadata_schema,
-        is_chunkable: genTypeRes.result.is_chunkable,
-        table_layout: genTypeRes.result.table_layout,
-    };
-
-    const type = await client.types.create(typeData);
-
-    return type;
+  const { client, params } = context;
+
+  const project = await context.fetchProject();
+  const interactionName =
+    params.interactionNames?.generateMetadataModel ??
+    INT_GENERATE_METADATA_MODEL;
+
+  const genTypeRes = await executeInteractionFromActivity(
+    client,
+    interactionName,
+    params,
+    {
+      existing_types,
+      content: content,
+      human_context: project?.configuration?.human_context ?? undefined,
+      image: fileRef ? fileRef : undefined,
+    },
+  );
+
+  if (!genTypeRes.result.document_type) {
+    log.error("No name generated for type", genTypeRes);
+    throw new Error("No name generated for type");
+  }
+
+  log.info("Generated schema for type", genTypeRes.result.metadata_schema);
+  const typeData: CreateContentObjectTypePayload = {
+    name: genTypeRes.result.document_type,
+    object_schema: genTypeRes.result.metadata_schema,
+    is_chunkable: genTypeRes.result.is_chunkable,
+    table_layout: genTypeRes.result.table_layout,
+  };
+
+  const type = await client.types.create(typeData);
+
+  return type;
 }

package/src/activities/index-dsl.ts

@@ -8,7 +8,8 @@ export { executeInteraction } from "./executeInteraction.js";
 export { extractDocumentText } from "./extractDocumentText.js";
 export { generateDocumentProperties } from "./generateDocumentProperties.js";
 export { generateEmbeddings } from "./generateEmbeddings.js";
-export { generateImageRendition } from "./generateImageRendition.js";
+export { generateImageRendition } from "./renditions/generateImageRendition.js";
+export { generateVideoRendition } from "./renditions/generateVideoRendition.js";
 export { generateOrAssignContentType } from "./generateOrAssignContentType.js";
 export { getObjectFromStore } from "./getObjectFromStore.js";
 export { handleDslError } from "./handleError.js";

package/src/activities/renditions/generateImageRendition.ts

@@ -0,0 +1,99 @@
+import { log } from "@temporalio/activity";
+import { DSLActivityExecutionPayload, DSLActivitySpec } from "@vertesia/common";
+import { setupActivity } from "../../dsl/setup/ActivityContext.js";
+import { NoDocumentFound, WorkflowParamNotFound } from "../../errors.js";
+import { saveBlobToTempFile } from "../../utils/blobs.js";
+import {
+  ImageRenditionParams,
+  uploadRenditionPages,
+} from "../../utils/renditions.js";
+
+interface GenerateImageRenditionParams extends ImageRenditionParams {}
+
+export interface GenerateImageRendition
+  extends DSLActivitySpec<GenerateImageRenditionParams> {
+  name: "generateImageRendition";
+}
+
+export async function generateImageRendition(
+  payload: DSLActivityExecutionPayload<GenerateImageRenditionParams>,
+) {
+  const {
+    client,
+    objectId,
+    params: originParams,
+  } = await setupActivity<GenerateImageRenditionParams>(payload);
+
+  // Fix: Use maxHeightWidth if max_hw is not provided
+  const params = {
+    ...originParams,
+    max_hw: originParams.max_hw || (originParams as any).maxHeightWidth || 1024, // Default to 1024 if both are missing
+    format: originParams.format || (originParams as any).format_output || "png", // Default to png if format is missing
+  };
+
+  log.info(`Generating image rendition for ${objectId}`, {
+    originParams,
+    params,
+  });
+
+  const inputObject = await client.objects.retrieve(objectId).catch((err) => {
+    log.error(`Failed to retrieve document ${objectId}`, { err });
+    if (err.message.includes("not found")) {
+      throw new NoDocumentFound(`Document ${objectId} not found`, [objectId]);
+    }
+    throw err;
+  });
+
+  if (!params.format) {
+    log.error(`Format not found`);
+    throw new WorkflowParamNotFound(`format`);
+  }
+
+  if (!inputObject.content?.source) {
+    log.error(`Document ${objectId} has no source`);
+    throw new NoDocumentFound(`Document ${objectId} has no source`, [objectId]);
+  }
+
+  if (
+    !inputObject.content.type ||
+    !inputObject.content.type?.startsWith("image/")
+  ) {
+    log.error(
+      `Document ${objectId} is not an image or a video: ${inputObject.content.type}`,
+    );
+    throw new NoDocumentFound(
+      `Document ${objectId} is not an image or a video: ${inputObject.content.type}`,
+      [objectId],
+    );
+  }
+
+  //array of rendition files to upload
+  let renditionPages: string[] = [];
+
+  const imageFile = await saveBlobToTempFile(
+    client,
+    inputObject.content.source,
+  );
+  log.info(`Image ${objectId} copied to ${imageFile}`);
+  renditionPages.push(imageFile);
+
+  const uploaded = await uploadRenditionPages(
+    client,
+    objectId,
+    [imageFile],
+    params,
+  );
+
+  if (!uploaded || !uploaded.length || !uploaded[0]) {
+    log.error(`Failed to upload rendition for ${objectId}`, { uploaded });
+    throw new Error(
+      `Failed to upload rendition for ${objectId} - upload object is empty`,
+    );
+  }
+
+  return {
+    uploads: uploaded.map((u) => u),
+    format: params.format,
+    status: "success",
+  };
+}