@vertesia/workflow 0.79.1 → 0.80.0-dev-20251118

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@vertesia/workflow",
-    "version": "0.79.1",
+    "version": "0.80.0-dev-20251118",
     "type": "module",
     "description": "Vertesia workflow DSL",
     "main": "./lib/esm/index.js",
@@ -16,11 +16,6 @@
         "build": "pnpm exec tsmod build && node ./bin/bundle-workflows.mjs lib/esm/workflows.js lib/workflows-bundle.js",
         "clean": "rm -rf ./lib tsconfig.tsbuildinfo"
     },
-    "repository": {
-        "type": "git",
-        "url": "https://github.com/vertesia/composableai.git",
-        "directory": "zeno/workflow"
-    },
     "license": "Apache-2.0",
     "devDependencies": {
         "@smithy/types": "^3.7.2",
@@ -56,7 +51,7 @@
         "seedrandom": "^3.0.5",
         "sharp": "^0.33.4",
         "tiktoken": "^1.0.15",
-        "tmp": "^0.2.3",
+        "tmp": "^0.2.4",
         "tmp-promise": "^3.0.3",
         "yaml": "^2.6.0"
     },
@@ -110,6 +105,11 @@
             "require": "./lib/cjs/errors.js"
         }
     },
+    "repository": {
+        "type": "git",
+        "url": "https://github.com/vertesia/composableai.git",
+        "directory": "packages/workflow"
+    },
     "types": "./lib/types/index.d.ts",
     "typesVersions": {
         "*": {

package/src/activities/notifyWebhook.test.ts

@@ -33,7 +33,7 @@ const defaultParams = {
 const createTestPayload = (params: Partial<NotifyWebhookParams> = {}): DSLActivityExecutionPayload<NotifyWebhookParams> => {
   const mergedParams = { ...defaultParams, ...params };
   return {
-    auth_token: "unset",
+    auth_token: process.env.VERTESIA_KEY || "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJodHRwOi8vbW9jay10b2tlbi1zZXJ2ZXIiLCJzdWIiOiJ0ZXN0In0.signature",
     account_id: "unset",
     project_id: "unset",
     params: mergedParams,

package/src/dsl/setup/ActivityContext.ts

@@ -6,16 +6,32 @@ import {
     Project,
     WorkflowExecutionPayload,
 } from "@vertesia/common";
-import { DocumentNotFoundError, WorkflowParamNotFoundError } from "../../errors.js";
+import {
+    DocumentNotFoundError,
+    WorkflowParamNotFoundError,
+} from "../../errors.js";
 import { getProjectFromToken } from "../../utils/auth.js";
 import { getVertesiaClient } from "../../utils/client.js";
 import { Vars } from "../vars.js";
-import { getFetchProvider, registerFetchProviderFactory } from "./fetch/index.js";
-import { DocumentProvider, DocumentTypeProvider, InteractionRunProvider } from "./fetch/providers.js";
+import {
+    getFetchProvider,
+    registerFetchProviderFactory,
+} from "./fetch/index.js";
+import {
+    DocumentProvider,
+    DocumentTypeProvider,
+    InteractionRunProvider,
+} from "./fetch/providers.js";
 
 registerFetchProviderFactory(DocumentProvider.ID, DocumentProvider.factory);
-registerFetchProviderFactory(DocumentTypeProvider.ID, DocumentTypeProvider.factory);
-registerFetchProviderFactory(InteractionRunProvider.ID, InteractionRunProvider.factory);
+registerFetchProviderFactory(
+    DocumentTypeProvider.ID,
+    DocumentTypeProvider.factory,
+);
+registerFetchProviderFactory(
+    InteractionRunProvider.ID,
+    InteractionRunProvider.factory,
+);
 
 export class ActivityContext<ParamsT extends Record<string, any>> {
     client: VertesiaClient;
@@ -40,7 +56,10 @@ export class ActivityContext<ParamsT extends Record<string, any>> {
             log.error("No objectId found in payload");
             throw new WorkflowParamNotFoundError(
                 "objectIds[0]",
-                (this.payload as WorkflowExecutionPayload as DSLWorkflowExecutionPayload).workflow,
+                (
+                    this
+                        .payload as WorkflowExecutionPayload as DSLWorkflowExecutionPayload
+                ).workflow,
             );
         }
         return objectId;
@@ -56,7 +75,10 @@ export class ActivityContext<ParamsT extends Record<string, any>> {
             log.error("No runId found in activityInfo");
             throw new WorkflowParamNotFoundError(
                 "runId",
-                (this.payload as WorkflowExecutionPayload as DSLWorkflowExecutionPayload).workflow,
+                (
+                    this
+                        .payload as WorkflowExecutionPayload as DSLWorkflowExecutionPayload
+                ).workflow,
             );
         }
         return runId;
@@ -68,7 +90,10 @@ export class ActivityContext<ParamsT extends Record<string, any>> {
             log.error("No workflowId found in activityInfo");
             throw new WorkflowParamNotFoundError(
                 "workflowId",
-                (this.payload as WorkflowExecutionPayload as DSLWorkflowExecutionPayload).workflow,
+                (
+                    this
+                        .payload as WorkflowExecutionPayload as DSLWorkflowExecutionPayload
+                ).workflow,
             );
         }
         return workflowId;
@@ -102,7 +127,7 @@ export async function setupActivity<ParamsT extends Record<string, any>>(
         });
     }
 
-    const client = getVertesiaClient(payload);
+    const client = await getVertesiaClient(payload);
     const fetchSpecs = payload.activity.fetch;
     if (fetchSpecs) {
         const keys = Object.keys(fetchSpecs);
@@ -118,7 +143,10 @@ export async function setupActivity<ParamsT extends Record<string, any>>(
 
                 const provider = getFetchProvider(client, fetchSpec);
 
-                log.info(`Fetching data for ${key} with provider ${provider.name}`, { fetchSpec });
+                log.info(
+                    `Fetching data for ${key} with provider ${provider.name}`,
+                    { fetchSpec },
+                );
                 const result = await provider.fetch(fetchSpec);
                 if (result && result.length > 0) {
                     if (fetchSpec.limit === 1) {
@@ -127,7 +155,9 @@ export async function setupActivity<ParamsT extends Record<string, any>>(
                         vars.setValue(key, result);
                     }
                 } else if (fetchSpec.on_not_found === "throw") {
-                    throw new DocumentNotFoundError("No documents found for: " + JSON.stringify(fetchSpec));
+                    throw new DocumentNotFoundError(
+                        "No documents found for: " + JSON.stringify(fetchSpec),
+                    );
                 } else {
                     vars.setValue(key, null);
                 }
@@ -141,7 +171,10 @@ export async function setupActivity<ParamsT extends Record<string, any>>(
     return new ActivityContext<ParamsT>(payload, client, params);
 }
 
-async function _fetchProject(client: VertesiaClient, payload: WorkflowExecutionPayload) {
+async function _fetchProject(
+    client: VertesiaClient,
+    payload: WorkflowExecutionPayload,
+) {
     const project = await getProjectFromToken(payload.auth_token);
     return project ? await client.projects.retrieve(project.id) : undefined;
 }

package/src/dsl/workflow-exec-child.test.ts

@@ -159,7 +159,7 @@ describe('DSL Workflow with child workflows', () => {
             account_id: '123',
             project_id: '123',
             wf_rule_name: 'test',
-            auth_token: 'test',
+            auth_token: process.env.VERTESIA_KEY || 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJodHRwOi8vbW9jay10b2tlbi1zZXJ2ZXIiLCJzdWIiOiJ0ZXN0In0.signature',
             config: {
                 studio_url: process.env.CP_STUDIO_URL || "http://localhost:8081",
                 store_url: process.env.CP_STORE_URL || "http://localhost:8082",
@@ -203,7 +203,7 @@ describe('DSL Workflow with child workflows', () => {
             account_id: '123',
             project_id: '123',
             wf_rule_name: 'test',
-            auth_token: 'test',
+            auth_token: process.env.VERTESIA_KEY || 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJodHRwOi8vbW9jay10b2tlbi1zZXJ2ZXIiLCJzdWIiOiJ0ZXN0In0.signature',
             config: {
                 studio_url: process.env.CP_STUDIO_URL || "http://localhost:8081",
                 store_url: process.env.CP_STORE_URL || "http://localhost:8082",
@@ -247,7 +247,7 @@ describe('DSL Workflow with child workflows', () => {
             account_id: '123',
             project_id: '123',
             wf_rule_name: 'test',
-            auth_token: 'test',
+            auth_token: process.env.VERTESIA_KEY || 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJodHRwOi8vbW9jay10b2tlbi1zZXJ2ZXIiLCJzdWIiOiJ0ZXN0In0.signature',
             config: {
                 studio_url: process.env.CP_STUDIO_URL || "http://localhost:8081",
                 store_url: process.env.CP_STORE_URL || "http://localhost:8082",

package/src/dsl/workflow-fetch.test.ts

@@ -107,7 +107,7 @@ describe('DSL Workflow', () => {
             project_id: '123',
             timestamp: Date.now(),
             wf_rule_name: 'test',
-            auth_token: 'test',
+            auth_token: process.env.VERTESIA_KEY || 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJodHRwOi8vbW9jay10b2tlbi1zZXJ2ZXIiLCJzdWIiOiJ0ZXN0In0.signature',
             config: {
                 studio_url: process.env.CP_STUDIO_URL || "http://localhost:8081",
                 store_url: process.env.CP_STORE_URL || "http://localhost:8082",

package/src/dsl/workflow-import.test.ts

@@ -61,7 +61,7 @@ describe('DSL Workflow import vars', () => {
             project_id: '123',
             timestamp: Date.now(),
             wf_rule_name: 'test',
-            auth_token: 'test',
+            auth_token: process.env.VERTESIA_KEY || 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJodHRwOi8vbW9jay10b2tlbi1zZXJ2ZXIiLCJzdWIiOiJ0ZXN0In0.signature',
             config: {
                 studio_url: process.env.CP_STUDIO_URL || "http://localhost:8081",
                 store_url: process.env.CP_STORE_URL || "http://localhost:8082",

package/src/dsl/workflow.test.ts

@@ -91,7 +91,7 @@ describe('DSL Workflow', () => {
             project_id: '123',
             timestamp: Date.now(),
             wf_rule_name: 'test',
-            auth_token: 'test',
+            auth_token: process.env.VERTESIA_KEY || 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJodHRwOi8vbW9jay10b2tlbi1zZXJ2ZXIiLCJzdWIiOiJ0ZXN0In0.signature',
             config: {
                 studio_url: process.env.CP_STUDIO_URL || "http://localhost:8081",
                 store_url: process.env.CP_STORE_URL || "http://localhost:8082",

package/src/iterative-generation/activities/extractToc.ts

@@ -2,8 +2,16 @@ import { log } from "@temporalio/activity";
 import { WorkflowExecutionPayload } from "@vertesia/common";
 import { parse as parseYaml } from "yaml";
 import { getVertesiaClient } from "../../utils/client.js";
-import { buildAndPublishMemoryPack, loadMemoryPack } from "../../utils/memory.js";
-import { IterativeGenerationPayload, OutputMemoryMeta, Toc, TocIndex } from "../types.js";
+import {
+    buildAndPublishMemoryPack,
+    loadMemoryPack,
+} from "../../utils/memory.js";
+import {
+    IterativeGenerationPayload,
+    OutputMemoryMeta,
+    Toc,
+    TocIndex,
+} from "../types.js";
 import { tocIndex } from "../utils.js";
 
 /**
@@ -12,10 +20,12 @@ import { tocIndex } from "../utils.js";
  *
  * @param payload
  */
-export async function it_gen_extractToc(payload: WorkflowExecutionPayload): Promise<TocIndex | null> {
+export async function it_gen_extractToc(
+    payload: WorkflowExecutionPayload,
+): Promise<TocIndex | null> {
     const vars = payload.vars as IterativeGenerationPayload;
     const memory = vars.memory;
-    const client = getVertesiaClient(payload);
+    const client = await getVertesiaClient(payload);
 
     const inMemory = await loadMemoryPack(client, `${memory}/input`);
     let tocJson: string | null = null;
@@ -37,13 +47,17 @@ export async function it_gen_extractToc(payload: WorkflowExecutionPayload): Prom
 
     log.info(`Found a TOC in the input memory pack`);
 
-    await buildAndPublishMemoryPack(client, `${vars.memory}/output`, async () => {
-        return {
-            toc,
-            lastProcessedPart: undefined, // the part index (a number array)
-            previouslyGenerated: ""
-        } as OutputMemoryMeta
-    });
+    await buildAndPublishMemoryPack(
+        client,
+        `${vars.memory}/output`,
+        async () => {
+            return {
+                toc,
+                lastProcessedPart: undefined, // the part index (a number array)
+                previouslyGenerated: "",
+            } as OutputMemoryMeta;
+        },
+    );
 
     return tocIndex(toc);
-}
+}

package/src/iterative-generation/activities/finalizeOutput.ts

@@ -2,14 +2,24 @@ import { log } from "@temporalio/activity";
 import { WorkflowExecutionPayload } from "@vertesia/common";
 import { getVertesiaClient } from "../../utils/client.js";
 import { expandVars } from "../../utils/expand-vars.js";
-import { buildAndPublishMemoryPack, loadMemoryPack } from "../../utils/memory.js";
-import { IterativeGenerationPayload, Section, SECTION_ID_PLACEHOLDER, TocSection } from "../types.js";
+import {
+    buildAndPublishMemoryPack,
+    loadMemoryPack,
+} from "../../utils/memory.js";
+import {
+    IterativeGenerationPayload,
+    Section,
+    SECTION_ID_PLACEHOLDER,
+    TocSection,
+} from "../types.js";
 
-export async function it_gen_finalizeOutput(payload: WorkflowExecutionPayload): Promise<string> {
+export async function it_gen_finalizeOutput(
+    payload: WorkflowExecutionPayload,
+): Promise<string> {
     const vars = payload.vars as IterativeGenerationPayload;
 
     const memory = vars.memory;
-    const client = getVertesiaClient(payload);
+    const client = await getVertesiaClient(payload);
     const inMemory = await loadMemoryPack(client, `${memory}/input`);
     const outMemory = await loadMemoryPack(client, `${memory}/output`);
 
@@ -35,43 +45,56 @@ export async function it_gen_finalizeOutput(payload: WorkflowExecutionPayload):
     }
     const sections = JSON.parse(content) as Section[];
 
-    await buildAndPublishMemoryPack(client, `${memory}/output`, async ({ copyText }) => {
-        // copy the input toc file if any
-        if (toc) {
-            copyText(toc, tocName);
-        }
-        // copy the raw JSON content
-        copyText(content, "content.json");
-        if (vars.section_file_pattern) {
-            log.info(`Saving sections to files using pattern: ${vars.section_file_pattern}`);
-            // save sections to files
-            for (const section of sections) {
-                let content = section.content;
-                if (vars.section_file_header) {
-                    content = getSectionFileHeader(section, vars.section_file_header) + '\n\n' + content;
+    await buildAndPublishMemoryPack(
+        client,
+        `${memory}/output`,
+        async ({ copyText }) => {
+            // copy the input toc file if any
+            if (toc) {
+                copyText(toc, tocName);
+            }
+            // copy the raw JSON content
+            copyText(content, "content.json");
+            if (vars.section_file_pattern) {
+                log.info(
+                    `Saving sections to files using pattern: ${vars.section_file_pattern}`,
+                );
+                // save sections to files
+                for (const section of sections) {
+                    let content = section.content;
+                    if (vars.section_file_header) {
+                        content =
+                            getSectionFileHeader(
+                                section,
+                                vars.section_file_header,
+                            ) +
+                            "\n\n" +
+                            content;
+                    }
+                    copyText(
+                        content,
+                        getSectionFileName(section, vars.section_file_pattern),
+                    );
                 }
-                copyText(content, getSectionFileName(section, vars.section_file_pattern));
             }
-        }
-        return {
-            ...inMeta,
-            vars
-        };
-    });
+            return {
+                ...inMeta,
+                vars,
+            };
+        },
+    );
 
     return `Processing done. Extracted files to: ${vars.section_file_pattern}`;
 }
 
-
-
 function getSectionFileHeader(section: TocSection, header: string): string {
     const date = new Date().toISOString();
     return expandVars(header, {
         section,
-        date
+        date,
     });
 }
 
 function getSectionFileName(section: TocSection, pattern: string): string {
     return pattern.replace(SECTION_ID_PLACEHOLDER, section.id);
-}
+}

package/src/iterative-generation/activities/generatePart.ts

@@ -2,29 +2,49 @@ import { ApplicationFailure } from "@temporalio/workflow";
 import { WorkflowExecutionPayload } from "@vertesia/common";
 import { MemoryPack } from "@vertesia/memory";
 import { getVertesiaClient } from "../../utils/client.js";
-import { buildAndPublishMemoryPack, loadMemoryPack } from "../../utils/memory.js";
-import { IterativeGenerationPayload, OutputMemoryMeta, Section, TocPart, TocSection } from "../types.js";
+import {
+    buildAndPublishMemoryPack,
+    loadMemoryPack,
+} from "../../utils/memory.js";
+import {
+    IterativeGenerationPayload,
+    OutputMemoryMeta,
+    Section,
+    TocPart,
+    TocSection,
+} from "../types.js";
 import { executeWithVars, expectMemoryIsConsistent } from "../utils.js";
 
-export async function it_gen_generatePart(payload: WorkflowExecutionPayload, path: number[]) {
+export async function it_gen_generatePart(
+    payload: WorkflowExecutionPayload,
+    path: number[],
+) {
     const vars = payload.vars as IterativeGenerationPayload;
-    const client = getVertesiaClient(payload);
+    const client = await getVertesiaClient(payload);
     const memory = vars.memory;
 
     const [sectionIndex, partIndex] = path;
     const outMemory = await loadMemoryPack(client, `${memory}/output`);
-    const meta = await outMemory.getMetadata() as OutputMemoryMeta;
+    const meta = (await outMemory.getMetadata()) as OutputMemoryMeta;
 
     // the section we build is the section at the given index
     const section: TocSection = meta.toc.sections[sectionIndex];
     if (!section) {
-        throw ApplicationFailure.nonRetryable('Section not found in the TOC', 'SectionNotFound', { memory, path });
+        throw ApplicationFailure.nonRetryable(
+            "Section not found in the TOC",
+            "SectionNotFound",
+            { memory, path },
+        );
     }
     let part: TocPart | undefined;
     if (partIndex !== undefined) {
         part = section.parts?.[partIndex];
         if (!part) {
-            throw ApplicationFailure.nonRetryable('Part not found in the TOC section', 'PartNotFound', { memory, path });
+            throw ApplicationFailure.nonRetryable(
+                "Part not found in the TOC section",
+                "PartNotFound",
+                { memory, path },
+            );
         }
     }
 
@@ -32,17 +52,26 @@ export async function it_gen_generatePart(payload: WorkflowExecutionPayload, pat
 
     const content = await loadGeneratedContent(outMemory);
 
-    let previously_generated = getPreviouslyGeneratedContent(content, !part, vars.remembrance_strategy);
+    let previously_generated = getPreviouslyGeneratedContent(
+        content,
+        !part,
+        vars.remembrance_strategy,
+    );
 
-    if (!part) { // a new section
+    if (!part) {
+        // a new section
         content.push({
             id: section.id,
             name: section.name,
             description: section.description,
-            content: ''
-        })
+            content: "",
+        });
     } else if (!content.length) {
-        throw ApplicationFailure.nonRetryable('content.json is empty while generating a part', 'InvalidIterationState', { memory, path });
+        throw ApplicationFailure.nonRetryable(
+            "content.json is empty while generating a part",
+            "InvalidIterationState",
+            { memory, path },
+        );
     }
 
     const interaction = vars.iterative_interaction || vars.interaction;
@@ -53,30 +82,42 @@ export async function it_gen_generatePart(payload: WorkflowExecutionPayload, pat
             section: section.name,
             part: part?.name,
             path: path,
-        }
+        },
     });
 
     const result = r.result.text();
     content[content.length - 1].content += result;
     meta.lastProcessedPart = path;
-    await buildAndPublishMemoryPack(client, `${memory}/output`, async ({ copyText }) => {
-        copyText(JSON.stringify(content, null, 2), "content.json");
-        return meta;
-    });
+    await buildAndPublishMemoryPack(
+        client,
+        `${memory}/output`,
+        async ({ copyText }) => {
+            copyText(JSON.stringify(content, null, 2), "content.json");
+            return meta;
+        },
+    );
 }
 
 async function loadGeneratedContent(memory: MemoryPack): Promise<Section[]> {
-    const content = await memory.getEntryText('content.json');
+    const content = await memory.getEntryText("content.json");
     return content ? JSON.parse(content) : [];
 }
 
-function getPreviouslyGeneratedContent(sections: Section[], isNewSection: boolean, strategy?: "document" | "section" | "none"): string {
+function getPreviouslyGeneratedContent(
+    sections: Section[],
+    isNewSection: boolean,
+    strategy?: "document" | "section" | "none",
+): string {
     switch (strategy) {
         case "document":
-            return sections.map((section: Section) => section.content || '').join('\n\n');
+            return sections
+                .map((section: Section) => section.content || "")
+                .join("\n\n");
         case "none":
-            return '';
+            return "";
         default:
-            return isNewSection ? '' : sections[sections.length - 1]?.content || '';
+            return isNewSection
+                ? ""
+                : sections[sections.length - 1]?.content || "";
     }
-}
+}

package/src/iterative-generation/activities/generateToc.ts

@@ -1,88 +1,97 @@
 import { WorkflowExecutionPayload } from "@vertesia/common";
 import { getVertesiaClient } from "../../utils/client.js";
 import { buildAndPublishMemoryPack } from "../../utils/memory.js";
-import { IterativeGenerationPayload, OutputMemoryMeta, Toc, TocIndex } from "../types.js";
+import {
+    IterativeGenerationPayload,
+    OutputMemoryMeta,
+    Toc,
+    TocIndex,
+} from "../types.js";
 import { executeWithVars, tocIndex } from "../utils.js";
 
 const defaultTocSchema = {
-    "type": "object",
-    "properties": {
-        "sections": {
-            "type": "array",
-            "items": {
-                "type": "object",
-                "properties": {
-                    "id": {
-                        "type": "string",
-                        "description": "the id of the section, can be a filename if working on a file, a slug if working on a document or path, or a unique identifier if working on a model."
+    type: "object",
+    properties: {
+        sections: {
+            type: "array",
+            items: {
+                type: "object",
+                properties: {
+                    id: {
+                        type: "string",
+                        description:
+                            "the id of the section, can be a filename if working on a file, a slug if working on a document or path, or a unique identifier if working on a model.",
                     },
-                    "operation": {
-                        "type": "string",
-                        "enum": ["create", "update", "delete"],
-                        "description": "The operation to perform on the section, create, update or delete. If update, you will be requested later to provide the list of change operation to perform."
+                    operation: {
+                        type: "string",
+                        enum: ["create", "update", "delete"],
+                        description:
+                            "The operation to perform on the section, create, update or delete. If update, you will be requested later to provide the list of change operation to perform.",
                     },
-                    "name": {
-                        "type": "string",
-                        "description": "The name or title of the section, should be the path in the OpenAPI spec, of the title of the section/part."
+                    name: {
+                        type: "string",
+                        description:
+                            "The name or title of the section, should be the path in the OpenAPI spec, of the title of the section/part.",
                     },
-                    "description": {
-                        "type": "string"
+                    description: {
+                        type: "string",
                     },
-                    "instructions": {
-                        "type": "string"
+                    instructions: {
+                        type: "string",
                     },
-                    "parts":
-                    {
-                        "type": "array",
-                        "description": "when the section is too large, you can split it into parts, each part should have a title and description. Use it to split the section into subsection. When doing an API documentation, you can do one part for each path. When generating code, you can do one part for each method. When generating an OpenAPI spec, you can do one part for each operation.",
-                        "items": {
-                            "type": "object",
-                            "properties": {
-                                "id": {
-                                    "type": "string",
-                                    "description": "the id of the part, can be a filename if working on a file, a slug if working on a document or path, or a unique identifier if working on a model."
+                    parts: {
+                        type: "array",
+                        description:
+                            "when the section is too large, you can split it into parts, each part should have a title and description. Use it to split the section into subsection. When doing an API documentation, you can do one part for each path. When generating code, you can do one part for each method. When generating an OpenAPI spec, you can do one part for each operation.",
+                        items: {
+                            type: "object",
+                            properties: {
+                                id: {
+                                    type: "string",
+                                    description:
+                                        "the id of the part, can be a filename if working on a file, a slug if working on a document or path, or a unique identifier if working on a model.",
                                 },
-                                "name": {
-                                    "type": "string",
-                                    "description": "The name or title of the part, should be the path in the OpenAPI spec, of the title of the section/part."
+                                name: {
+                                    type: "string",
+                                    description:
+                                        "The name or title of the part, should be the path in the OpenAPI spec, of the title of the section/part.",
                                 },
                                 /*
                                 "description": {
                                     "type": "string"
                                 },
                                 */
-                                "instructions": {
-                                    "type": "string"
-                                }
+                                instructions: {
+                                    type: "string",
+                                },
                             },
-                            "required": [
-                                "id",
-                                "name",
-                            ]
-                        }
-                    }
+                            required: ["id", "name"],
+                        },
+                    },
                 },
-                "required": [
-                    "id",
-                    "name",
-                    "operation"
-                ]
-            }
-        }
+                required: ["id", "name", "operation"],
+            },
+        },
     },
-    "required": [
-        "sections"
-    ]
-}
+    required: ["sections"],
+};
 
-export async function it_gen_generateToc(payload: WorkflowExecutionPayload): Promise<TocIndex> {
+export async function it_gen_generateToc(
+    payload: WorkflowExecutionPayload,
+): Promise<TocIndex> {
     const vars = payload.vars as IterativeGenerationPayload;
 
     const schema = vars.toc_schema || defaultTocSchema;
 
-    const client = getVertesiaClient(payload);
+    const client = await getVertesiaClient(payload);
 
-    const run = await executeWithVars(client, vars.interaction, vars, undefined, schema);
+    const run = await executeWithVars(
+        client,
+        vars.interaction,
+        vars,
+        undefined,
+        schema,
+    );
 
     //Parse the CompletionResult[] to get a Toc object
     const jsonResults = run.result.object();
@@ -91,13 +100,17 @@ export async function it_gen_generateToc(payload: WorkflowExecutionPayload): Pro
         sections: jsonResults.sections
     };
 
-    await buildAndPublishMemoryPack(client, `${vars.memory}/output`, async () => {
-        return {
-            toc,
-            lastProcessedPart: undefined, // the part index (a number array)
-            previouslyGenerated: ""
-        } satisfies OutputMemoryMeta
-    });
+    await buildAndPublishMemoryPack(
+        client,
+        `${vars.memory}/output`,
+        async () => {
+            return {
+                toc,
+                lastProcessedPart: undefined, // the part index (a number array)
+                previouslyGenerated: "",
+            } satisfies OutputMemoryMeta;
+        },
+    );
 
     return tocIndex(toc);
 }

package/src/utils/blobs.ts

@@ -14,7 +14,9 @@ export async function fetchBlobAsStream(client: VertesiaClient, blobUri: string)
     } catch (err: any) {
         if (err.message.includes("not found")) {
             //TODO improve error handling with a fetch fail error class in the client
-            throw new DocumentNotFoundError(`Failed to download blob ${blobUri}: ${err.message}`, []);
+            throw new DocumentNotFoundError(`Not found at ${blobUri}: ${err.message}`, []);
+        } else if (err.message.includes("forbidden")) {
+            throw new DocumentNotFoundError(`Forbidden at ${blobUri}: ${err.message}`);
         } else {
             throw new Error(`Failed to download blob ${blobUri}: ${err.message}`);
         }

package/src/utils/client.ts

@@ -2,31 +2,45 @@
  * get a zeno client for a given token
  */
 
-import { VertesiaClient } from "@vertesia/client";
+import {
+    decodeJWT,
+    VertesiaClient,
+    VertesiaClientProps,
+} from "@vertesia/client";
 import { WorkflowExecutionBaseParams } from "@vertesia/common";
 import { WorkflowParamNotFoundError } from "../errors.js";
 
-
 export function getVertesiaClient(payload: WorkflowExecutionBaseParams) {
     return new VertesiaClient(getVertesiaClientOptions(payload));
 }
 
-export function getVertesiaClientOptions(payload: WorkflowExecutionBaseParams) {
+export function getVertesiaClientOptions(
+    payload: WorkflowExecutionBaseParams,
+): VertesiaClientProps {
     if (!payload.auth_token) {
-        throw new WorkflowParamNotFoundError("Authentication Token is missing from WorkflowExecutionPayload.authToken");
+        throw new WorkflowParamNotFoundError(
+            "Authentication Token is missing from WorkflowExecutionPayload.authToken",
+        );
     }
 
     if (!payload.config?.studio_url) {
-        throw new WorkflowParamNotFoundError("Content Store URL is missing from WorkflowExecutionPayload.servers.storeUrl");
+        throw new WorkflowParamNotFoundError(
+            "Content Store URL is missing from WorkflowExecutionPayload.servers.storeUrl",
+        );
     }
 
     if (!payload.config?.store_url) {
-        throw new WorkflowParamNotFoundError("Content Store URL is missing from WorkflowExecutionPayload.servers.storeUrl");
+        throw new WorkflowParamNotFoundError(
+            "Content Store URL is missing from WorkflowExecutionPayload.servers.storeUrl",
+        );
     }
 
+    const token = decodeJWT(payload.auth_token);
+
     return {
         serverUrl: payload.config.studio_url,
         storeUrl: payload.config.store_url,
-        apikey: payload.auth_token
+        tokenServerUrl: token.iss,
+        apikey: payload.auth_token,
     };
-}
+}