@vertesia/workflow 0.77.0 → 0.78.0-dev-9372725

package/src/activities/notifyWebhook.test.ts

@@ -1,32 +1,130 @@
 import {
   MockActivityEnvironment,
-  TestWorkflowEnvironment,
 } from "@temporalio/testing";
-import { beforeAll, describe, expect, test } from "vitest";
-import { notifyWebhook, NotifyWebhook } from "./notifyWebhook.js";
+import { beforeAll, beforeEach, describe, expect, it, test, vi } from "vitest";
+import { notifyWebhook, NotifyWebhook, NotifyWebhookParams } from "./notifyWebhook.js";
+import { ContentEventName, DSLActivityExecutionPayload } from "@vertesia/common";
 
-let testEnv: TestWorkflowEnvironment;
-let activityContext: MockActivityEnvironment;
+// Mock fetch globally
+vi.stubGlobal('fetch', vi.fn());
+
+let testEnv: MockActivityEnvironment;
+const mockFetch = vi.mocked(fetch);
 
 beforeAll(async () => {
-  testEnv = await TestWorkflowEnvironment.createLocal();
-  activityContext = new MockActivityEnvironment();
+  testEnv = new MockActivityEnvironment();
+});
+
+beforeEach(() => {
+  vi.clearAllMocks();
 });
 
-// https://github.com/becomposable/studio/issues/432 Skip tests
-// Cannot read properties of undefined (reading 'params')
+const defaultParams = {
+  target_url: "https://vertesia.test",
+  method: "POST" as const,
+  payload: { message: "Hello World" }
+};
+
+// Helper function to create test payload
+const createTestPayload = (params: Partial<NotifyWebhookParams> = {}): DSLActivityExecutionPayload<NotifyWebhookParams> => {
+  const mergedParams = { ...defaultParams, ...params };
+  return {
+    auth_token: process.env.VERTESIA_KEY || "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJodHRwOi8vbW9jay10b2tlbi1zZXJ2ZXIiLCJzdWIiOiJ0ZXN0In0.signature",
+    account_id: "unset",
+    project_id: "unset",
+    params: mergedParams,
+    config: {
+      studio_url: "http://mock-studio",
+      store_url: "http://mock-store",
+    },
+    workflow_name: "",
+    event: ContentEventName.create,
+    objectIds: [],
+    vars: {},
+    activity: { name: "notifyWebhook", params: mergedParams }
+  };
+};
+
 describe("Webhook should be notified", () => {
-  test.skip("test POST", async () => {
-    const activityConfig = {
-      name: "notifyWebhook",
-      params: {
-        target_url: "https://en5zdcyvn4dc3.x.pipedream.net",
-        method: "POST",
-        payload: { message: "Hello World" },
+  it("test POST success", async () => {
+    // Mock successful response
+    const mockResponse = {
+      ok: true,
+      status: 200,
+      statusText: 'OK',
+      url: defaultParams.target_url
+    };
+    mockFetch.mockResolvedValueOnce(mockResponse as Response);
+
+    const payload = createTestPayload();
+    const res = await testEnv.run(notifyWebhook, payload);
+    
+    // Verify fetch was called with correct parameters
+    expect(mockFetch).toHaveBeenCalledWith(defaultParams.target_url, {
+      method: 'POST',
+      body: JSON.stringify({ message: 'Hello World' }),
+      headers: {
+        'Content-Type': 'application/json',
+      },
+    });
+
+    // Verify response
+    expect(res).toEqual({
+      status: 200,
+      message: 'OK',
+      url: defaultParams.target_url
+    });
+  });
+
+  it("test POST server error", async () => {
+    // Mock error response with response body
+    const mockResponse = {
+      ok: false,
+      status: 500,
+      statusText: 'Internal Server Error',
+      url: defaultParams.target_url,
+      text: vi.fn().mockResolvedValue('{"error": "Database connection failed", "code": "DB_ERROR"}')
+    } as unknown as Response;
+    mockFetch.mockResolvedValueOnce(mockResponse);
+
+    const payload = createTestPayload();
+
+    // Expect the function to throw an error with response payload
+    await expect(testEnv.run(notifyWebhook, payload)).rejects.toThrow(
+      `Webhook Notification to ${defaultParams.target_url} failed with status: 500 Internal Server Error - Response: {"error": "Database connection failed", "code": "DB_ERROR"}`
+    );
+
+    // Verify fetch was called with correct parameters
+    expect(mockFetch).toHaveBeenCalledWith(defaultParams.target_url, {
+      method: 'POST',
+      body: JSON.stringify({ message: 'Hello World' }),
+      headers: {
+        'Content-Type': 'application/json',
       },
-    } satisfies NotifyWebhook;
+    });
+
+    // Verify that text() was called to read the response
+    expect(mockResponse.text).toHaveBeenCalled();
+  });
 
-    const res = await activityContext.run(notifyWebhook, activityConfig);
-    expect(res).toBeDefined();
+  it("test POST network error", async () => {
+    // Mock fetch to throw a network error
+    const networkError = new Error('Network request failed');
+    mockFetch.mockRejectedValueOnce(networkError);
+
+    const payload = createTestPayload();
+
+    // Expect the function to throw the network error
+    await expect(testEnv.run(notifyWebhook, payload)).rejects.toThrow('Network request failed');
+
+    // Verify fetch was called with correct parameters
+    expect(mockFetch).toHaveBeenCalledWith(defaultParams.target_url, {
+      method: 'POST',
+      body: JSON.stringify({ message: 'Hello World' }),
+      headers: {
+        'Content-Type': 'application/json',
+      },
+    });
   });
+
 });

package/src/activities/notifyWebhook.ts

@@ -36,13 +36,26 @@ export async function notifyWebhook(payload: DSLActivityExecutionPayload<NotifyW
             ...headers
         },
     }).catch(err => {
-        log.warn(`Failed to notify webhook ${target_url}: ${err}`);
-        throw new Error(`Failed to notify webhook ${target_url}: ${err}`);
+        log.error(`An error occurred while notifying webhook at ${target_url}`, { err });
+        throw err;
     });
 
     if (!res.ok) {
-        log.warn(`Failed to notify webhook ${target_url} - ${res.status}: ${res.statusText}`, { res });
-        throw new Error(`Failed to notify webhook ${target_url}: ${res.statusText}`);
+        log.warn(`Webhook endpoint ${target_url} returned an error - ${res.status} ${res.statusText}`, { fetchResponse: res });
+        
+        // Try to get response payload for error message
+        let errorMessage = `Webhook Notification to ${target_url} failed with status: ${res.status} ${res.statusText}`;
+        try {
+            const responseText = await res.text();
+            if (responseText) {
+                errorMessage += ` - Response: ${responseText}`;
+            }
+        } catch (readError) {
+            // If we can't read the response, just use the basic error message
+            log.debug('Could not read response body for error', { readError });
+        }
+        
+        throw new Error(errorMessage);
     }
 
     return { status: res.status, message: res.statusText, url: res.url }

package/src/activities/rateLimiter.ts

@@ -0,0 +1,41 @@
+import { DSLActivityExecutionPayload, RateLimitRequestPayload } from "@vertesia/common";
+import { activityInfo, log } from "@temporalio/activity";
+import { setupActivity } from "../dsl/setup/ActivityContext.js";
+
+export interface RateLimitParams {
+  interactionIdOrEndpoint: string;
+  environmentId?: string;
+  modelId?: string;
+}
+
+export interface RateLimitResult {
+  delayMs: number;
+}
+
+export async function checkRateLimit(payload: DSLActivityExecutionPayload<RateLimitParams>): Promise<RateLimitResult> {
+  const { client, params } = await setupActivity<RateLimitParams>(payload);
+  const { environmentId, modelId } = params;
+  
+  const result: RateLimitResult = {
+    delayMs: 0,
+  }
+
+  try {
+    // Call the studio-server endpoint to get rate limit delay using the Vertesia client
+    const info = activityInfo();
+    const requestPayload: RateLimitRequestPayload = {
+      interaction: params.interactionIdOrEndpoint,
+      workflow_run_id: info.workflowExecution.runId,
+      environment_id: environmentId,
+      model_id: modelId,
+    };
+    
+    const response = await client.interactions.requestSlot(requestPayload);
+    result.delayMs = response.delay_ms;
+  } catch (error) {
+    log.warn('Failed to call rate limit API:', {error});
+    throw error;
+  }
+
+  return result;
+}

package/src/conversion/image.ts

@@ -78,6 +78,12 @@ export async function imageResizer(
         const command = `convert`
         let args = [inputPath];
 
+        // Add JPEG shrink-on-load optimization
+        args.push("-define", `jpeg:size=${max_hw * 3}x${max_hw * 3}`);
+
+        // Remove metadata
+        args.push("-strip");
+
         // https://usage.imagemagick.org/filter/nicolas/#downsample
         // Add colorspace correction if enabled
         if (colorspaceCorrection) {
@@ -110,9 +116,6 @@ export async function imageResizer(
             }
         }
 
-        // Add JPEG shrink-on-load optimization
-        args.push("-define", `jpeg:size=${max_hw * 3}x${max_hw * 3}`);
-
         // Resize operation
         args.push("-resize", `${max_hw}x${max_hw}>`);
 

package/src/dsl/dsl-workflow.ts

@@ -7,6 +7,7 @@ import {
     log,
     patched,
     proxyActivities,
+    sleep,
     startChild,
     UntypedActivities,
 } from "@temporalio/workflow";
@@ -18,6 +19,7 @@ import {
     DSLWorkflowExecutionPayload,
     DSLWorkflowSpec,
     getDocumentIds,
+    getTenantId,
     WorkflowExecutionPayload
 } from "@vertesia/common";
 import ms, { StringValue } from 'ms';
@@ -25,6 +27,8 @@ import { HandleDslErrorParams } from "../activities/handleError.js";
 import * as activities from "../activities/index.js";
 import { WF_NON_RETRYABLE_ERRORS, WorkflowParamNotFoundError } from "../errors.js";
 import { Vars } from "./vars.js";
+import { RateLimitParams } from "../activities/rateLimiter.js";
+
 
 interface BaseActivityPayload extends WorkflowExecutionPayload {
     workflow_name: string;
@@ -172,6 +176,7 @@ async function startChildWorkflow(step: DSLChildWorkflowStep, payload: DSLWorkfl
             AccountId: [payload.account_id],
             DocumentId: getDocumentIds(payload),
             ProjectId: [payload.project_id],
+            TenantId: [getTenantId(payload.account_id, payload.project_id)],
             InitiatedBy: payload.initiated_by ? [payload.initiated_by] : [],
         },
     });
@@ -204,6 +209,7 @@ async function executeChildWorkflow(step: DSLChildWorkflowStep, payload: DSLWork
             AccountId: [payload.account_id],
             DocumentId: getDocumentIds(payload),
             ProjectId: [payload.project_id],
+            TenantId: [getTenantId(payload.account_id, payload.project_id)],
             InitiatedBy: payload.initiated_by ? [payload.initiated_by] : [],
         },
     });
@@ -218,6 +224,48 @@ async function executeChildWorkflow(step: DSLChildWorkflowStep, payload: DSLWork
     }
 }
 
+function buildRateLimitParams(activity: DSLActivitySpec, executionPayload: DSLActivityExecutionPayload<any>): RateLimitParams {
+    const params = executionPayload.params;
+    let interactionId: string;
+
+    switch (activity.name) {
+        case "executeInteraction":
+            interactionId = params.interactionName;
+            break;
+
+        case "generateDocumentProperties":
+            interactionId = params.interactionName || "sys:ExtractInformation";
+            break;
+
+        case "identifyTextSections":
+            interactionId = params.interactionName || "sys:IdentifyTextSections";
+            break;
+
+        case "generateOrAssignContentType":
+            interactionId = params.interactionNames?.selectDocumentType || "sys:SelectDocumentType";
+            break;
+
+        case "chunkDocument":
+            interactionId = params.interactionName || "sys:ChunkDocument";
+            break;
+
+        default:
+            // For any other rate-limited activities, try to extract what we can
+            interactionId = params.interactionName;
+            break;
+    }
+
+    if (!interactionId) {
+        throw new Error(`No interaction ID could be determined for activity ${activity.name}`);
+    }
+
+    return {
+        interactionIdOrEndpoint: interactionId,
+        environmentId: params.environment,
+        modelId: params.model,
+    };
+}
+
 async function runActivity(activity: DSLActivitySpec, basePayload: BaseActivityPayload, vars: Vars, defaultProxy: ActivityInterfaceFor<UntypedActivities>, defaultOptions: ActivityOptions) {
     if (basePayload.debug_mode) {
         log.debug(`Workflow vars before executing activity ${activity.name}`, { vars: vars.resolve() });
@@ -245,6 +293,38 @@ async function runActivity(activity: DSLActivitySpec, basePayload: BaseActivityP
         });
     }
 
+    if (patched('system-activity-taskqueue')) {
+        // hack: do nothing, remove later
+        // https://github.com/vertesia/composableai/pull/544/files
+    }
+
+    // call rate limiter depending on the activity type
+    const rateLimitedActivities = [
+        "chunkDocument",
+        "executeInteraction",
+        "generateDocumentProperties",
+        "generateOrAssignContentType",
+        "identifyTextSections",
+    ];
+
+    if (activity.name && rateLimitedActivities.includes(activity.name)) {
+        log.info(`Applying rate limit for activity ${activity.name}`);
+        // Apply rate limiting logic here
+         // Check rate limit first - loop until no delay
+        const rateLimitParams = buildRateLimitParams(activity, executionPayload);
+
+        const rateLimitPayload = dslActivityPayload(basePayload, activity, rateLimitParams);
+        let rateLimitResult = await proxy.checkRateLimit(rateLimitPayload);
+    
+        while (rateLimitResult.delayMs > 0) {
+            log.info(`Rate limit delay applied: ${rateLimitResult.delayMs}ms`);
+            await sleep(rateLimitResult.delayMs);
+
+            // Check again after sleeping
+            rateLimitResult = await proxy.checkRateLimit(rateLimitPayload);
+        }
+    }
+
     const fn = proxy[activity.name];
     if (activity.parallel) {
         //TODO execute in parallel

package/src/dsl/setup/ActivityContext.ts

@@ -6,16 +6,32 @@ import {
     Project,
     WorkflowExecutionPayload,
 } from "@vertesia/common";
-import { DocumentNotFoundError, WorkflowParamNotFoundError } from "../../errors.js";
+import {
+    DocumentNotFoundError,
+    WorkflowParamNotFoundError,
+} from "../../errors.js";
 import { getProjectFromToken } from "../../utils/auth.js";
 import { getVertesiaClient } from "../../utils/client.js";
 import { Vars } from "../vars.js";
-import { getFetchProvider, registerFetchProviderFactory } from "./fetch/index.js";
-import { DocumentProvider, DocumentTypeProvider, InteractionRunProvider } from "./fetch/providers.js";
+import {
+    getFetchProvider,
+    registerFetchProviderFactory,
+} from "./fetch/index.js";
+import {
+    DocumentProvider,
+    DocumentTypeProvider,
+    InteractionRunProvider,
+} from "./fetch/providers.js";
 
 registerFetchProviderFactory(DocumentProvider.ID, DocumentProvider.factory);
-registerFetchProviderFactory(DocumentTypeProvider.ID, DocumentTypeProvider.factory);
-registerFetchProviderFactory(InteractionRunProvider.ID, InteractionRunProvider.factory);
+registerFetchProviderFactory(
+    DocumentTypeProvider.ID,
+    DocumentTypeProvider.factory,
+);
+registerFetchProviderFactory(
+    InteractionRunProvider.ID,
+    InteractionRunProvider.factory,
+);
 
 export class ActivityContext<ParamsT extends Record<string, any>> {
     client: VertesiaClient;
@@ -40,7 +56,10 @@ export class ActivityContext<ParamsT extends Record<string, any>> {
             log.error("No objectId found in payload");
             throw new WorkflowParamNotFoundError(
                 "objectIds[0]",
-                (this.payload as WorkflowExecutionPayload as DSLWorkflowExecutionPayload).workflow,
+                (
+                    this
+                        .payload as WorkflowExecutionPayload as DSLWorkflowExecutionPayload
+                ).workflow,
             );
         }
         return objectId;
@@ -56,7 +75,10 @@ export class ActivityContext<ParamsT extends Record<string, any>> {
             log.error("No runId found in activityInfo");
             throw new WorkflowParamNotFoundError(
                 "runId",
-                (this.payload as WorkflowExecutionPayload as DSLWorkflowExecutionPayload).workflow,
+                (
+                    this
+                        .payload as WorkflowExecutionPayload as DSLWorkflowExecutionPayload
+                ).workflow,
             );
         }
         return runId;
@@ -68,7 +90,10 @@ export class ActivityContext<ParamsT extends Record<string, any>> {
             log.error("No workflowId found in activityInfo");
             throw new WorkflowParamNotFoundError(
                 "workflowId",
-                (this.payload as WorkflowExecutionPayload as DSLWorkflowExecutionPayload).workflow,
+                (
+                    this
+                        .payload as WorkflowExecutionPayload as DSLWorkflowExecutionPayload
+                ).workflow,
             );
         }
         return workflowId;
@@ -102,7 +127,7 @@ export async function setupActivity<ParamsT extends Record<string, any>>(
         });
     }
 
-    const client = getVertesiaClient(payload);
+    const client = await getVertesiaClient(payload);
     const fetchSpecs = payload.activity.fetch;
     if (fetchSpecs) {
         const keys = Object.keys(fetchSpecs);
@@ -118,7 +143,10 @@ export async function setupActivity<ParamsT extends Record<string, any>>(
 
                 const provider = getFetchProvider(client, fetchSpec);
 
-                log.info(`Fetching data for ${key} with provider ${provider.name}`, { fetchSpec });
+                log.info(
+                    `Fetching data for ${key} with provider ${provider.name}`,
+                    { fetchSpec },
+                );
                 const result = await provider.fetch(fetchSpec);
                 if (result && result.length > 0) {
                     if (fetchSpec.limit === 1) {
@@ -127,7 +155,9 @@ export async function setupActivity<ParamsT extends Record<string, any>>(
                         vars.setValue(key, result);
                     }
                 } else if (fetchSpec.on_not_found === "throw") {
-                    throw new DocumentNotFoundError("No documents found for: " + JSON.stringify(fetchSpec));
+                    throw new DocumentNotFoundError(
+                        "No documents found for: " + JSON.stringify(fetchSpec),
+                    );
                 } else {
                     vars.setValue(key, null);
                 }
@@ -141,7 +171,10 @@ export async function setupActivity<ParamsT extends Record<string, any>>(
     return new ActivityContext<ParamsT>(payload, client, params);
 }
 
-async function _fetchProject(client: VertesiaClient, payload: WorkflowExecutionPayload) {
+async function _fetchProject(
+    client: VertesiaClient,
+    payload: WorkflowExecutionPayload,
+) {
     const project = await getProjectFromToken(payload.auth_token);
     return project ? await client.projects.retrieve(project.id) : undefined;
 }

package/src/dsl/workflow-exec-child.test.ts

@@ -129,6 +129,7 @@ describe('DSL Workflow with child workflows', () => {
                 AccountId: protos.temporal.api.enums.v1.IndexedValueType.INDEXED_VALUE_TYPE_KEYWORD,
                 DocumentId: protos.temporal.api.enums.v1.IndexedValueType.INDEXED_VALUE_TYPE_KEYWORD,
                 ProjectId: protos.temporal.api.enums.v1.IndexedValueType.INDEXED_VALUE_TYPE_KEYWORD,
+                TenantId: protos.temporal.api.enums.v1.IndexedValueType.INDEXED_VALUE_TYPE_KEYWORD,
                 InitiatedBy: protos.temporal.api.enums.v1.IndexedValueType.INDEXED_VALUE_TYPE_KEYWORD,
             },
         });
@@ -158,7 +159,7 @@ describe('DSL Workflow with child workflows', () => {
             account_id: '123',
             project_id: '123',
             wf_rule_name: 'test',
-            auth_token: 'test',
+            auth_token: process.env.VERTESIA_KEY || 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJodHRwOi8vbW9jay10b2tlbi1zZXJ2ZXIiLCJzdWIiOiJ0ZXN0In0.signature',
             config: {
                 studio_url: process.env.CP_STUDIO_URL || "http://localhost:8081",
                 store_url: process.env.CP_STORE_URL || "http://localhost:8082",
@@ -202,7 +203,7 @@ describe('DSL Workflow with child workflows', () => {
             account_id: '123',
             project_id: '123',
             wf_rule_name: 'test',
-            auth_token: 'test',
+            auth_token: process.env.VERTESIA_KEY || 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJodHRwOi8vbW9jay10b2tlbi1zZXJ2ZXIiLCJzdWIiOiJ0ZXN0In0.signature',
             config: {
                 studio_url: process.env.CP_STUDIO_URL || "http://localhost:8081",
                 store_url: process.env.CP_STORE_URL || "http://localhost:8082",
@@ -246,7 +247,7 @@ describe('DSL Workflow with child workflows', () => {
             account_id: '123',
             project_id: '123',
             wf_rule_name: 'test',
-            auth_token: 'test',
+            auth_token: process.env.VERTESIA_KEY || 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJodHRwOi8vbW9jay10b2tlbi1zZXJ2ZXIiLCJzdWIiOiJ0ZXN0In0.signature',
             config: {
                 studio_url: process.env.CP_STUDIO_URL || "http://localhost:8081",
                 store_url: process.env.CP_STORE_URL || "http://localhost:8082",

package/src/dsl/workflow-fetch.test.ts

@@ -107,7 +107,7 @@ describe('DSL Workflow', () => {
             project_id: '123',
             timestamp: Date.now(),
             wf_rule_name: 'test',
-            auth_token: 'test',
+            auth_token: process.env.VERTESIA_KEY || 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJodHRwOi8vbW9jay10b2tlbi1zZXJ2ZXIiLCJzdWIiOiJ0ZXN0In0.signature',
             config: {
                 studio_url: process.env.CP_STUDIO_URL || "http://localhost:8081",
                 store_url: process.env.CP_STORE_URL || "http://localhost:8082",

package/src/dsl/workflow-import.test.ts

@@ -61,7 +61,7 @@ describe('DSL Workflow import vars', () => {
             project_id: '123',
             timestamp: Date.now(),
             wf_rule_name: 'test',
-            auth_token: 'test',
+            auth_token: process.env.VERTESIA_KEY || 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJodHRwOi8vbW9jay10b2tlbi1zZXJ2ZXIiLCJzdWIiOiJ0ZXN0In0.signature',
             config: {
                 studio_url: process.env.CP_STUDIO_URL || "http://localhost:8081",
                 store_url: process.env.CP_STORE_URL || "http://localhost:8082",

package/src/dsl/workflow.test.ts

@@ -60,6 +60,7 @@ describe('DSL Workflow', () => {
                 AccountId: protos.temporal.api.enums.v1.IndexedValueType.INDEXED_VALUE_TYPE_KEYWORD,
                 DocumentId: protos.temporal.api.enums.v1.IndexedValueType.INDEXED_VALUE_TYPE_KEYWORD,
                 ProjectId: protos.temporal.api.enums.v1.IndexedValueType.INDEXED_VALUE_TYPE_KEYWORD,
+                TenantId: protos.temporal.api.enums.v1.IndexedValueType.INDEXED_VALUE_TYPE_KEYWORD,
                 InitiatedBy: protos.temporal.api.enums.v1.IndexedValueType.INDEXED_VALUE_TYPE_KEYWORD,
             },
         });
@@ -90,7 +91,7 @@ describe('DSL Workflow', () => {
             project_id: '123',
             timestamp: Date.now(),
             wf_rule_name: 'test',
-            auth_token: 'test',
+            auth_token: process.env.VERTESIA_KEY || 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJodHRwOi8vbW9jay10b2tlbi1zZXJ2ZXIiLCJzdWIiOiJ0ZXN0In0.signature',
             config: {
                 studio_url: process.env.CP_STUDIO_URL || "http://localhost:8081",
                 store_url: process.env.CP_STORE_URL || "http://localhost:8082",

package/src/errors.ts

@@ -51,6 +51,19 @@ export class WorkflowParamNotFoundError extends ApplicationFailure {
     }
 }
 
+export class ResourceExhaustedError extends ApplicationFailure {
+    constructor(
+        public statusCode: number = 429,
+        message?: string,
+    ) {
+        super(
+            message || "Resource exhausted - rate limit exceeded",
+            "ResourceExhaustedError",
+            true, // non-retryable
+        );
+    }
+}
+
 export const WF_NON_RETRYABLE_ERRORS = [
     "DocumentNotFoundError",
     "ActivityParamInvalidError",

package/src/index.ts

@@ -18,11 +18,11 @@ export * from "./activities/executeInteraction.js";
 export * from "./activities/extractDocumentText.js";
 export * from "./activities/generateDocumentProperties.js";
 export * from "./activities/generateEmbeddings.js";
-export * from "./activities/identifyTextSections.js";
 export * from "./activities/renditions/generateImageRendition.js";
 export * from "./activities/renditions/generateVideoRendition.js";
 export * from "./activities/generateOrAssignContentType.js";
 export * from "./activities/notifyWebhook.js";
+export * from "./activities/rateLimiter.js";
 export * from "./activities/setDocumentStatus.js";
 export * from "./iterative-generation/activities/index.js";
 

package/src/iterative-generation/activities/extractToc.ts

@@ -2,8 +2,16 @@ import { log } from "@temporalio/activity";
 import { WorkflowExecutionPayload } from "@vertesia/common";
 import { parse as parseYaml } from "yaml";
 import { getVertesiaClient } from "../../utils/client.js";
-import { buildAndPublishMemoryPack, loadMemoryPack } from "../../utils/memory.js";
-import { IterativeGenerationPayload, OutputMemoryMeta, Toc, TocIndex } from "../types.js";
+import {
+    buildAndPublishMemoryPack,
+    loadMemoryPack,
+} from "../../utils/memory.js";
+import {
+    IterativeGenerationPayload,
+    OutputMemoryMeta,
+    Toc,
+    TocIndex,
+} from "../types.js";
 import { tocIndex } from "../utils.js";
 
 /**
@@ -12,10 +20,12 @@ import { tocIndex } from "../utils.js";
  *
  * @param payload
  */
-export async function it_gen_extractToc(payload: WorkflowExecutionPayload): Promise<TocIndex | null> {
+export async function it_gen_extractToc(
+    payload: WorkflowExecutionPayload,
+): Promise<TocIndex | null> {
     const vars = payload.vars as IterativeGenerationPayload;
     const memory = vars.memory;
-    const client = getVertesiaClient(payload);
+    const client = await getVertesiaClient(payload);
 
     const inMemory = await loadMemoryPack(client, `${memory}/input`);
     let tocJson: string | null = null;
@@ -37,13 +47,17 @@ export async function it_gen_extractToc(payload: WorkflowExecutionPayload): Prom
 
     log.info(`Found a TOC in the input memory pack`);
 
-    await buildAndPublishMemoryPack(client, `${vars.memory}/output`, async () => {
-        return {
-            toc,
-            lastProcessedPart: undefined, // the part index (a number array)
-            previouslyGenerated: ""
-        } as OutputMemoryMeta
-    });
+    await buildAndPublishMemoryPack(
+        client,
+        `${vars.memory}/output`,
+        async () => {
+            return {
+                toc,
+                lastProcessedPart: undefined, // the part index (a number array)
+                previouslyGenerated: "",
+            } as OutputMemoryMeta;
+        },
+    );
 
     return tocIndex(toc);
-}
+}