@vertesia/workflow 0.42.2

package/src/dsl/workflow-fetch.test.ts

@@ -0,0 +1,135 @@
+import { ComposableClient } from '@vertesia/client';
+import { ContentEventName, DSLActivityExecutionPayload, DSLActivitySpec, DSLWorkflowExecutionPayload, FindPayload } from '@vertesia/common';
+import { TestWorkflowEnvironment } from '@temporalio/testing';
+import { Worker } from '@temporalio/worker';
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+import { dslWorkflow } from './dsl-workflow.js';
+import { setupActivity } from "./setup/ActivityContext.js";
+import { DataProvider } from './setup/fetch/DataProvider.js';
+import { registerFetchProviderFactory } from './setup/fetch/index.js';
+
+
+class DocumentTestProvider extends DataProvider {
+
+    static ID = "document";
+
+    constructor() {
+        super(DocumentTestProvider.ID, true);
+    }
+
+    doFetch(payload: FindPayload): Promise<Record<string, any>[]> {
+        const query = payload.query;
+        console.log('query', query);
+        if (query.lang === 'en') {
+            return Promise.resolve([query.greeting ? { text: 'Hello' } : { text: "World" }])
+        } else {
+            return Promise.resolve([query.greeting ? { text: 'Bonjour' } : { text: "Monde" }])
+        }
+    }
+
+    static factory(_context: ComposableClient) {
+        return new DocumentTestProvider();
+    }
+}
+
+registerFetchProviderFactory(DocumentTestProvider.ID, DocumentTestProvider.factory);
+
+
+async function sayMessage(payload: DSLActivityExecutionPayload): Promise<string> {
+    const { params } = await setupActivity(payload);
+    return params.message;
+}
+
+
+const activities: DSLActivitySpec[] = [
+    {
+        name: 'sayMessage',
+        output: 'result',
+        import: ["lang"],
+        params: {
+            message: "${greeting.text}, ${name.text}!"
+        },
+        fetch: {
+            name: {
+                type: "document",
+                query: {
+                    lang: "${lang}", greeting: false
+                },
+                limit: 1,
+            },
+            greeting: {
+                type: "document",
+                query: {
+                    lang: "${lang}", greeting: true
+                },
+                limit: 1,
+            }
+        },
+    },
+]
+
+// ========== test env setup ==========
+
+
+describe('DSL Workflow', () => {
+
+    let testEnv: TestWorkflowEnvironment;
+
+    beforeAll(async () => {
+        testEnv = await TestWorkflowEnvironment.createLocal();
+    });
+
+    afterAll(async () => {
+        await testEnv?.teardown();
+    });
+
+    it('successfully completes a mock workflow', async () => {
+        const { client, nativeConnection } = testEnv;
+        const taskQueue = 'test';
+
+        const lang = 'en';
+
+        const worker = await Worker.create({
+            connection: nativeConnection,
+            taskQueue,
+            workflowsPath: new URL("./dsl-workflow.ts", import.meta.url).pathname,
+            activities: { sayMessage },
+        });
+
+        const payload: DSLWorkflowExecutionPayload = {
+            event: ContentEventName.create,
+            objectIds: ['123'],
+            vars: {},
+            account_id: '123',
+            project_id: '123',
+            timestamp: Date.now(),
+            wf_rule_name: 'test',
+            auth_token: 'test',
+            config: {
+                studio_url: process.env.CP_STUDIO_URL || "http://localhost:8081",
+                store_url: process.env.CP_STODRE_URL || "http://localhost:8082",
+            },
+            workflow: {
+                activities,
+                vars: {
+                    lang,
+                },
+                name: 'test',
+            }
+        }
+
+        let result = await worker.runUntil(client.workflow.execute(dslWorkflow, {
+            args: [payload],
+            workflowId: 'test',
+            taskQueue,
+        }));
+
+        expect(result).toBe('Hello, World!');
+
+    });
+
+
+
+
+
+});

package/src/dsl/workflow-import.test.ts

@@ -0,0 +1,89 @@
+import { ContentEventName, DSLActivityExecutionPayload, DSLActivitySpec, DSLWorkflowExecutionPayload } from '@vertesia/common';
+import { TestWorkflowEnvironment } from '@temporalio/testing';
+import { Worker } from '@temporalio/worker';
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+import { dslWorkflow } from './dsl-workflow.js';
+import { setupActivity } from "./setup/ActivityContext.js";
+
+
+async function testImportedVars(payload: DSLActivityExecutionPayload) {
+    const { params } = await setupActivity(payload);
+    if (!params.object_name) throw new Error('object_name is required');
+    console.log('!!!!!!!!!!@@@@@@@@@@@@@@!!!!!!!!!!!!!!', params.object_name);
+    return params.object_name;
+}
+
+
+const activities: DSLActivitySpec[] = [
+    {
+        name: 'testImportedVars',
+        import: ["object_name"],
+        output: 'result',
+    },
+]
+
+// ========== test env setup ==========
+
+
+describe('DSL Workflow import vars', () => {
+
+    let testEnv: TestWorkflowEnvironment;
+
+    beforeAll(async () => {
+        testEnv = await TestWorkflowEnvironment.createLocal();
+    });
+
+    afterAll(async () => {
+        await testEnv?.teardown();
+    });
+
+
+    it('import vars are part of activity params', async () => {
+
+
+        const { client, nativeConnection } = testEnv;
+        const taskQueue = 'test';
+
+        const object_name = 'object_name';
+
+        const worker = await Worker.create({
+            connection: nativeConnection,
+            taskQueue,
+            workflowsPath: new URL("./dsl-workflow.ts", import.meta.url).pathname,
+            activities: { testImportedVars },
+        });
+
+        const payload: DSLWorkflowExecutionPayload = {
+            event: ContentEventName.create,
+            objectIds: ['123'],
+            vars: {},
+            account_id: '123',
+            project_id: '123',
+            timestamp: Date.now(),
+            wf_rule_name: 'test',
+            auth_token: 'test',
+            config: {
+                studio_url: process.env.CP_STUDIO_URL || "http://localhost:8081",
+                store_url: process.env.CP_STODRE_URL || "http://localhost:8082",
+            },
+            workflow: {
+                activities,
+                vars: {
+                    object_name,
+                },
+                name: 'test',
+            }
+        }
+
+        let result = await worker.runUntil(client.workflow.execute(dslWorkflow, {
+            args: [payload],
+            workflowId: 'test',
+            taskQueue,
+        }));
+
+        expect(result).toBe(object_name);
+
+    });
+
+
+});

package/src/dsl/workflow.test.ts

@@ -0,0 +1,110 @@
+import { ContentEventName, DSLActivityExecutionPayload, DSLActivitySpec, DSLWorkflowExecutionPayload } from '@vertesia/common';
+import { TestWorkflowEnvironment } from '@temporalio/testing';
+import { Worker } from '@temporalio/worker';
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+import { dslWorkflow } from './dsl-workflow.js';
+import { setupActivity } from "./setup/ActivityContext.js";
+
+async function sayHello(payload: DSLActivityExecutionPayload): Promise<string> {
+    const { params } = await setupActivity(payload);
+    return params.lang === 'fr' ? "Bonjour" : "Hello";
+}
+
+async function sayName(payload: DSLActivityExecutionPayload): Promise<string> {
+    const { params } = await setupActivity(payload);
+    return params.lang === 'fr' ? "Monde" : "World";
+}
+
+async function sayGreeting(payload: DSLActivityExecutionPayload): Promise<string> {
+    const { params } = await setupActivity(payload);
+    return `${params.hello}, ${params.name}!`;
+}
+
+
+const activities: DSLActivitySpec[] = [
+    {
+        name: 'sayHello',
+        output: 'hello',
+        import: ["lang"],
+    },
+    {
+        name: 'sayName',
+        output: 'name',
+        import: ["lang"],
+    },
+    {
+        name: 'sayGreeting',
+        import: ["hello", "name"],
+        condition: {
+            hello: { $null: false },
+            name: { $null: false }
+        },
+        output: 'result',
+    },
+]
+
+// ========== test env setup ==========
+
+
+describe('DSL Workflow', () => {
+
+    let testEnv: TestWorkflowEnvironment;
+
+    beforeAll(async () => {
+        testEnv = await TestWorkflowEnvironment.createLocal();
+    });
+
+    afterAll(async () => {
+        await testEnv?.teardown();
+    });
+
+    it('successfully completes a mock workflow', async () => {
+        const { client, nativeConnection } = testEnv;
+        const taskQueue = 'test';
+
+        const lang = 'fr';
+
+        const worker = await Worker.create({
+            connection: nativeConnection,
+            taskQueue,
+            workflowsPath: new URL("./dsl-workflow.ts", import.meta.url).pathname,
+            activities: { sayHello, sayName, sayGreeting },
+        });
+
+        const payload: DSLWorkflowExecutionPayload = {
+            event: ContentEventName.create,
+            objectIds: ['123'],
+            vars: {},
+            account_id: '123',
+            project_id: '123',
+            timestamp: Date.now(),
+            wf_rule_name: 'test',
+            auth_token: 'test',
+            config: {
+                studio_url: process.env.CP_STUDIO_URL || "http://localhost:8081",
+                store_url: process.env.CP_STODRE_URL || "http://localhost:8082",
+            },
+            workflow: {
+                activities,
+                vars: {
+                    lang,
+                },
+                name: 'test',
+            }
+        }
+
+        let result = await worker.runUntil(client.workflow.execute(dslWorkflow, {
+            args: [payload],
+            workflowId: 'test',
+            taskQueue,
+        }));
+
+        expect(result).toBe('Bonjour, Monde!');
+
+    });
+
+
+
+
+
+});

package/src/errors.ts

@@ -0,0 +1,24 @@
+import { DSLActivitySpec, DSLWorkflowSpec } from "@vertesia/common";
+
+
+export class NoDocumentFound extends Error {
+    constructor(message: string, public ids?: string[]) {
+        super(message);
+        this.name = "NoDocumentFound";
+        this.ids = ids
+    }
+}
+
+export class ActivityParamNotFound extends Error {
+    constructor(public paramName: string, public activity: DSLActivitySpec) {
+        super(`Required parameter ${paramName} not found in activity ${activity.name}`);
+        this.name = "ActivityParamNotFound";
+    }
+}
+
+export class WorkflowParamNotFound extends Error {
+    constructor(public paramName: string, public workflow?: DSLWorkflowSpec) {
+        super(`Required parameter ${paramName} not found in workflow ${workflow?.name}`);
+        this.name = "WorkflowParamNotFound";
+    }
+}

package/src/index.ts

@@ -0,0 +1,27 @@
+/**
+ * 1. Activities are exported through the '/activities' named export in package.json.
+ * Modify the `./activities/index.ts` if you want to modify activity exports
+ * 2. Workflows are exported through the '/workflows' named export in package.json.
+ * Modify the `./workflows.ts` file if you want to modify workflow exports
+ * 3. Here we export the API to be used to validate workflows and the types reuired to create workflow TS definitions.
+ */
+
+//TODO remove this - it is only for backward compat - iot is used from old workflows
+export { dslWorkflow } from "./dsl/dsl-workflow.js";
+export * from "./iterative-generation/iterativeGenerationWorkflow.js";
+
+export * from "./activities/advanced/createDocumentTypeFromInteractionRun.js";
+export * from "./activities/advanced/createOrUpdateDocumentFromInteractionRun.js";
+export * from "./activities/advanced/updateDocumentFromInteractionRun.js";
+export * from "./activities/chunkDocument.js";
+export * from "./activities/executeInteraction.js";
+export * from "./activities/extractDocumentText.js";
+export * from "./activities/generateDocumentProperties.js";
+export * from "./activities/generateEmbeddings.js";
+export * from "./activities/generateImageRendition.js";
+export * from "./activities/generateOrAssignContentType.js";
+export * from "./activities/setDocumentStatus.js";
+export * from "./activities/notifyWebhook.js";
+export * from "./iterative-generation/activities/index.js";
+
+export * from "./result-types.js";

package/src/iterative-generation/activities/extractToc.ts

@@ -0,0 +1,49 @@
+import { WorkflowExecutionPayload } from "@vertesia/common";
+import { parse as parseYaml } from "yaml";
+import { getClient } from "../../utils/client.js";
+import { buildAndPublishMemoryPack, loadMemoryPack } from "../../utils/memory.js";
+import { IterativeGenerationPayload, OutputMemoryMeta, Toc, TocIndex } from "../types.js";
+import { tocIndex } from "../utils.js";
+import { log } from "@temporalio/activity";
+
+/**
+ * This activity is called if the toc was provided in the payload. Otherwise
+ * the generateToc is called.
+ *
+ * @param payload
+ */
+export async function it_gen_extractToc(payload: WorkflowExecutionPayload): Promise<TocIndex | null> {
+    const vars = payload.vars as IterativeGenerationPayload;
+    const memory = vars.memory;
+    const client = getClient(payload);
+
+    const inMemory = await loadMemoryPack(client, `${memory}/input`);
+    let tocJson: string | null = null;
+    let tocYaml: string | null = null;
+    let toc: any;
+    tocJson = await inMemory.getEntryText("toc.json");
+    if (!tocJson) {
+        tocYaml = await inMemory.getEntryText("toc.yaml");
+        if (tocYaml) {
+            toc = parseYaml(tocYaml) as Toc;
+        }
+    } else {
+        toc = JSON.parse(tocJson) as Toc;
+    }
+    if (!toc) {
+        log.info(`Nothing to extract: no TOC found in the input memory pack.`);
+        return null; // no toc found
+    }
+
+    log.info(`Found a TOC in the input memory pack`);
+
+    await buildAndPublishMemoryPack(client, `${vars.memory}/output`, async () => {
+        return {
+            toc,
+            lastProcessdPart: undefined, // the part index (a number array)
+            previouslyGenerated: ""
+        } as OutputMemoryMeta
+    });
+
+    return tocIndex(toc);
+}

package/src/iterative-generation/activities/finalizeOutput.ts

@@ -0,0 +1,77 @@
+import { WorkflowExecutionPayload } from "@vertesia/common";
+import { log } from "@temporalio/activity";
+import { getClient } from "../../utils/client.js";
+import { expandVars } from "../../utils/expand-vars.js";
+import { buildAndPublishMemoryPack, loadMemoryPack } from "../../utils/memory.js";
+import { IterativeGenerationPayload, Section, SECTION_ID_PLACEHOLDER, TocSection } from "../types.js";
+
+export async function it_gen_finalizeOutput(payload: WorkflowExecutionPayload): Promise<string> {
+    const vars = payload.vars as IterativeGenerationPayload;
+
+    const memory = vars.memory;
+    const client = getClient(payload);
+    const inMemory = await loadMemoryPack(client, `${memory}/input`);
+    const outMemory = await loadMemoryPack(client, `${memory}/output`);
+
+    const content = await outMemory.getEntryText("content.json");
+    if (!content) {
+        log.info(`Nothing to do. No content.json file found`);
+        return "No content.json file found";
+    }
+
+    log.info(`Creating the final output memory pack.`);
+
+    const inMeta = await inMemory.getMetadata();
+    let tocName = "toc.json";
+    let toc = await inMemory.getEntryText(tocName);
+    if (!toc) {
+        tocName = "toc.yaml";
+        toc = await inMemory.getEntryText(tocName);
+    }
+    if (!toc) {
+        const outMeta = await outMemory.getMetadata();
+        tocName = "toc.json";
+        toc = JSON.stringify(outMeta.toc);
+    }
+    const sections = JSON.parse(content) as Section[];
+
+    await buildAndPublishMemoryPack(client, `${memory}/output`, async ({ copyText }) => {
+        // copy the input toc file if any
+        if (toc) {
+            copyText(toc, tocName);
+        }
+        // copy the raw JSON content
+        copyText(content, "content.json");
+        if (vars.section_file_pattern) {
+            log.info(`Saving sections to files using pattern: ${vars.section_file_pattern}`);
+            // save sections to files
+            for (const section of sections) {
+                let content = section.content;
+                if (vars.section_file_header) {
+                    content = getSectionFileHeader(section, vars.section_file_header) + '\n\n' + content;
+                }
+                copyText(content, getSectionFileName(section, vars.section_file_pattern));
+            }
+        }
+        return {
+            ...inMeta,
+            vars
+        };
+    });
+
+    return `Processing done. Extracted files to: ${vars.section_file_pattern}`;
+}
+
+
+
+function getSectionFileHeader(section: TocSection, header: string): string {
+    const date = new Date().toISOString();
+    return expandVars(header, {
+        section,
+        date
+    });
+}
+
+function getSectionFileName(section: TocSection, pattern: string): string {
+    return pattern.replace(SECTION_ID_PLACEHOLDER, section.id);
+}

package/src/iterative-generation/activities/generatePart.ts

@@ -0,0 +1,82 @@
+import { WorkflowExecutionPayload } from "@vertesia/common";
+import { MemoryPack } from "@becomposable/memory";
+import { ApplicationFailure } from "@temporalio/workflow";
+import { getClient } from "../../utils/client.js";
+import { buildAndPublishMemoryPack, loadMemoryPack } from "../../utils/memory.js";
+import { IterativeGenerationPayload, OutputMemoryMeta, Section, TocPart, TocSection } from "../types.js";
+import { executeWithVars, expectMemoryIsConsistent } from "../utils.js";
+
+export async function it_gen_generatePart(payload: WorkflowExecutionPayload, path: number[]) {
+    const vars = payload.vars as IterativeGenerationPayload;
+    const client = getClient(payload);
+    const memory = vars.memory;
+
+    const [sectionIndex, partIndex] = path;
+    const outMemory = await loadMemoryPack(client, `${memory}/output`);
+    const meta = await outMemory.getMetadata() as OutputMemoryMeta;
+
+    // the section we build is the section at the given index
+    const section: TocSection = meta.toc.sections[sectionIndex];
+    if (!section) {
+        throw ApplicationFailure.nonRetryable('Section not found in the TOC', 'SectionNotFound', { memory, path });
+    }
+    let part: TocPart | undefined;
+    if (partIndex !== undefined) {
+        part = section.parts?.[partIndex];
+        if (!part) {
+            throw ApplicationFailure.nonRetryable('Part not found in the TOC section', 'PartNotFound', { memory, path });
+        }
+    }
+
+    expectMemoryIsConsistent(meta, path);
+
+    const content = await loadGeneratedContent(outMemory);
+
+    let previously_generated = getPreviouslyGeneratedContent(content, !part, vars.rememberance_strategy);
+
+    if (!part) { // a new section
+        content.push({
+            id: section.id,
+            name: section.name,
+            description: section.description,
+            content: ''
+        })
+    } else if (!content.length) {
+        throw ApplicationFailure.nonRetryable('content.json is empty while generating a part', 'InvalidIterationState', { memory, path });
+    }
+
+    const interaction = vars.iterative_interaction || vars.interaction;
+    const r = await executeWithVars(client, interaction, vars, {
+        iteration: {
+            toc: meta.toc,
+            previously_generated,
+            section: section.name,
+            part: part?.name,
+            path: path,
+        }
+    });
+
+    const result = r.result as string;
+    content[content.length - 1].content += result;
+    meta.lastProcessdPart = path;
+    await buildAndPublishMemoryPack(client, `${memory}/output`, async ({ copyText }) => {
+        copyText(JSON.stringify(content, null, 2), "content.json");
+        return meta;
+    });
+}
+
+async function loadGeneratedContent(memory: MemoryPack): Promise<Section[]> {
+    const content = await memory.getEntryText('content.json');
+    return content ? JSON.parse(content) : [];
+}
+
+function getPreviouslyGeneratedContent(sections: Section[], isNewSection: boolean, strategy?: "document" | "section" | "none"): string {
+    switch (strategy) {
+        case "document":
+            return sections.map((section: Section) => section.content || '').join('\n\n');
+        case "none":
+            return '';
+        default:
+            return isNewSection ? '' : sections[sections.length - 1]?.content || '';
+    }
+}