@vertesia/workflow 0.24.0-dev.202601221707

package/src/iterative-generation/iterativeGenerationWorkflow.ts

@@ -0,0 +1,68 @@
+import { WorkflowExecutionPayload } from "@vertesia/common";
+
+import { log, proxyActivities } from "@temporalio/workflow";
+import { WF_NON_RETRYABLE_ERRORS } from "../errors.js";
+import * as activities from "./activities/index.js";
+import { IterativeGenerationPayload, PartIndex, SECTION_ID_PLACEHOLDER } from "./types.js";
+
+const {
+    it_gen_extractToc,
+    it_gen_generateToc,
+    it_gen_generatePart,
+    it_gen_finalizeOutput
+} = proxyActivities<typeof activities>({
+    startToCloseTimeout: "15 minute",
+    retry: {
+        initialInterval: '30s',
+        backoffCoefficient: 2,
+        maximumAttempts: 20,
+        maximumInterval: 100 * 30 * 1000, //ms
+        nonRetryableErrorTypes: WF_NON_RETRYABLE_ERRORS,
+    },
+});
+
+export async function iterativeGenerationWorkflow(payload: WorkflowExecutionPayload) {
+    log.info(`Executing Iterative generation workflow.`);
+
+    const vars = payload.vars as IterativeGenerationPayload;
+    if (vars.section_file_pattern && !vars.section_file_pattern.includes(SECTION_ID_PLACEHOLDER)) {
+        throw new Error(`Invalid section_file_pattern: ${vars.section_file_pattern}. It must include the ${SECTION_ID_PLACEHOLDER} placeholder.`);
+    }
+
+    // extractToc tries to extract the toc from the input memory pack (toc.json or toc.yaml)
+    // the generateToc activity is returning the toc hierarchy.
+    // It doesn't include extra TOC details like description etc.
+    // To minimize the payload size only the hierarchy and the section/part names are returned
+    let toc = await it_gen_extractToc(payload);
+    if (!toc) {
+        log.info(`No TOC was specified in the input memory pack. Generating one.`);
+        toc = await it_gen_generateToc(payload);
+    } else {
+        log.info(`Using the TOC specified in the input memory pack.`);
+    }
+
+    if (toc.sections.length === 0) {
+        //TODO how to handle this case?
+        throw new Error("Nothing to generate: TOC is empty");
+    }
+
+    for (const section of toc.sections) {
+        log.info(`Generating section: ${formatPath(section)}`);
+        await it_gen_generatePart(payload, section.path);
+
+        if (section.parts) {
+            for (const part of section.parts) {
+                log.info(`Generating part: ${formatPath(part)}`);
+                await it_gen_generatePart(payload, part.path);
+            }
+        }
+    }
+
+    log.info(`Post-processing output memory pack`);
+    await it_gen_finalizeOutput(payload);
+}
+
+function formatPath(node: PartIndex) {
+    // we print 1 based indexes
+    return node.path.map(i => i + 1).join('.') + ' ' + node.name;
+}

package/src/iterative-generation/types.ts

@@ -0,0 +1,99 @@
+
+export const SECTION_ID_PLACEHOLDER = '%id';
+
+/**
+ * An iterative generation workflow uses 2 memory packs one for input and the other for output.
+ * The input memory packs must be available in the project blobs bucket at `${tenant_id/memories/${memory_name}/input.tar.gz`.
+ * The output memory pack will be generated at `${tenant_id/memories/${memory_name}/output.tar.gz`.
+ * Each iteration is overwriting the output memory pack with the new generated content.
+ * The complete name of the input and output memory packs are: "${name}/input" and "${name}/output" where name is the base memory name.
+ */
+export interface IterativeGenerationPayload {
+    // the main interaction to execute. If iterative_generation is defined
+    // the main interaction will only be used to prepare the iteration (to generate the TOC)
+    // otherwise it will be used for the iterative generation too.
+    interaction: string;
+    // if defined this will be used for the iterative interaction which will generate parts.
+    // otherwise the main interaction will be used for iterative generation.
+    iterative_interaction?: string;
+    // the environment to use
+    environment?: string;
+    // the model to use
+    model?: string;
+    // A custom max tokens
+    max_tokens?: number;
+    // A custom temperature
+    temperature?: number;
+    // the memory pack group name
+    memory: string;
+    // the input memory pack mapping
+    input_mapping?: Record<string, string>;
+    // custom toc schema if any TODO remove this
+    toc_schema?: Record<string, any>
+    /**
+     * If not set to "none" the previously generated content will be passed to the iteration.
+     * If not set at all defaults to "section".
+     * If "section" is used only the section content previously generated will be passed to the next iteration.
+     * If "document" is used the whole previously generated document content will be passed to the next iteration.
+     * Defaults to section.
+     */
+    remembrance_strategy?: "document" | "section" | "none";
+    /**
+     * If present will save sections in files using the pattern
+     * The pattern must include a placeholder for the section id: %id.
+     * Examples: `sections/%id.md`, `%id/page.mdx` etc.
+     * @see SECTION_ID_PLACEHOLDER
+     */
+    section_file_pattern?: string;
+    /**
+     * An optional header to prepend to the section files.
+     * The header can contain the following variables:
+     * - ${section} - the section object
+     * - ${date} - the date when the file was generated
+     */
+    section_file_header?: string;
+}
+
+export interface TocPart {
+    id: string;
+    name: string;
+    description?: string;
+    instructions?: string;
+}
+
+export interface TocSection {
+    id: string;
+    name: string;
+    description?: string;
+    instructions?: string;
+    parts?: TocPart[];
+}
+
+export interface Toc {
+    sections: TocSection[];
+}
+
+
+export interface SectionIndex extends PartIndex {
+    parts?: PartIndex[];
+}
+export interface PartIndex {
+    path: number[];
+    name: string;
+}
+export interface TocIndex {
+    sections: SectionIndex[];
+}
+
+export interface OutputMemoryMeta {
+    toc: Toc;
+    previouslyGenerated: string;
+    lastProcessedPart?: number[] | undefined;
+}
+
+export interface Section {
+    id: string;
+    name: string;
+    description?: string;
+    content: string;
+}

package/src/iterative-generation/utils.ts

@@ -0,0 +1,126 @@
+import { ModelOptions, TextFallbackOptions } from "@llumiverse/common";
+import { ApplicationFailure } from "@temporalio/workflow";
+import { VertesiaClient } from "@vertesia/client";
+import { OutputMemoryMeta, PartIndex, Toc, TocIndex, TocSection } from "./types.js";
+
+//TODO: For whole file, support for options beyond max_tokens and temperature and multiple modalities.
+export interface ExecuteOptions {
+    interaction: string;
+    memory: string;
+    memory_mapping?: Record<string, any>;
+    environment?: string;
+    model?: string;
+    model_options?: ModelOptions;
+    result_schema?: Record<string, any>;
+}
+
+export async function execute(client: VertesiaClient, options: ExecuteOptions) {
+    return client.interactions.executeByName(options.interaction, {
+        data: {
+            ...options.memory_mapping,
+            "@memory": options.memory
+        },
+        result_schema: options.result_schema,
+        config: {
+            environment: options.environment,
+            model: options.model,
+            model_options: options.model_options,
+        }
+    });
+}
+
+export function executeWithVars(client: VertesiaClient, interaction: string, vars: Record<string, any>, mapping?: Record<string, any>, result_schema?: Record<string, any>) {
+    if (mapping) {
+        mapping = { ...vars.input_mapping, ...mapping };
+    } else {
+        mapping = vars.input_mapping;
+    }
+    const model_options: TextFallbackOptions = {
+        _option_id: "text-fallback",
+        max_tokens: vars.max_tokens,
+        temperature: vars.temperature
+    }
+    return execute(client, {
+        interaction: interaction,
+        memory: `${vars.memory}/input`,
+        memory_mapping: mapping,
+        environment: vars.environment,
+        model: vars.model,
+        model_options: model_options,
+        result_schema: result_schema
+    });
+}
+
+export function isSamePartIndex(part1: number[], part2: number[]) {
+    return part1 && part2 && part1.length === part2.length && part1.every((v, i) => v === part2[i]);
+}
+
+export function getPreviousPathIndex(toc: Toc, pathIndex: number[]): number[] | null {
+    let [sectionIdx, partIdx] = pathIndex;
+    if (partIdx === undefined) {
+        let prevSectionIdx = sectionIdx - 1;
+        if (prevSectionIdx < 0) {
+            return null;
+        } else {
+            const prevParts = toc.sections[prevSectionIdx].parts;
+            if (prevParts && prevParts.length > 0) { // return the last part of the previous section
+                return [prevSectionIdx, prevParts.length - 1];
+            } else { // if no parts return the section itself
+                return [prevSectionIdx];
+            }
+        }
+    } else if (partIdx > 0) { // return the previous part in the same section
+        return [sectionIdx, partIdx - 1];
+    } else { // if the first part return the section itself
+        return [sectionIdx];
+    }
+}
+
+
+export function expectMemoryIsConsistent(meta: OutputMemoryMeta, pathIndex: number[]) {
+    const metaLastProcessedPart = meta.lastProcessedPart;
+    if (!metaLastProcessedPart) {
+        if (pathIndex.length > 1 && pathIndex[0] !== 0) {
+            throw ApplicationFailure.nonRetryable('Memory last processed part is not consistent with the workflow.', 'MemoryPackNotConsistent', { currentIndex: pathIndex, expectedPreviousIndex: [pathIndex[0], pathIndex[1] - 1], previousIndex: metaLastProcessedPart });
+        } else {
+            return;
+        }
+    }
+    const prevPathIndex = getPreviousPathIndex(meta.toc, pathIndex);
+    if (!prevPathIndex) {
+        throw ApplicationFailure.nonRetryable('Memory last processed part is not consistent with the workflow', 'MemoryPackNotConsistent', { currentIndex: pathIndex, expectedPreviousIndex: prevPathIndex, previousIndex: metaLastProcessedPart });
+    } else if (!isSamePartIndex(prevPathIndex, metaLastProcessedPart)) {
+        throw ApplicationFailure.nonRetryable('Memory last processed part is not consistent with the workflow', 'MemoryPackNotConsistent', { currentIndex: pathIndex, expectedPreviousIndex: prevPathIndex, previousIndex: meta.lastProcessedPart });
+    }
+}
+
+export function sectionWithoutParts(section: TocSection) {
+    const clone = { ...section };
+    delete clone.parts;
+    return clone;
+}
+
+export function tocIndex(toc: Toc): TocIndex {
+    const index = { sections: [] } as TocIndex;
+    const sections = toc.sections;
+    for (let i = 0, l = sections.length; i < l; i++) {
+        const section = sections[i];
+        const indexParts: PartIndex[] = [];
+        if (section.parts) {
+            const parts = section.parts;
+            for (let k = 0, ll = section.parts.length; k < ll; k++) {
+                const part = parts[k];
+                indexParts.push({
+                    name: part.id,
+                    path: [i, k]
+                });
+            }
+        }
+        index.sections.push({
+            name: section.id,
+            path: [i],
+            parts: indexParts
+        });
+    }
+    return index;
+}

package/src/result-types.ts

@@ -0,0 +1,25 @@
+
+
+
+/**
+ * The result of a text extraction operation.
+ */
+export interface TextExtractionResult {
+    objectId: string;
+    status: TextExtractionStatus;
+    hasText: boolean;
+    message?: string;
+    tokens?: {
+        count: number;
+        encoding: string;
+        etag: string;
+    };
+    len?: number;
+    error?: string;
+}
+
+export enum TextExtractionStatus {
+    skipped = "skipped",
+    success = "success",
+    error = "error",
+}

package/src/system/notifyWebhookWorkflow.ts

@@ -0,0 +1,70 @@
+
+import { log, workflowInfo } from "@temporalio/workflow";
+import { WebHookSpec, WorkflowExecutionPayload } from "@vertesia/common";
+import * as activities from "../activities/notifyWebhook.js";
+import { dslProxyActivities } from "../dsl/dslProxyActivities.js";
+import { WF_NON_RETRYABLE_ERRORS } from "../errors.js";
+
+const {
+    notifyWebhook
+} = dslProxyActivities<typeof activities>("notifyWebhookWorkflow", {
+    startToCloseTimeout: "5 minute",
+    retry: {
+        initialInterval: '5s',
+        backoffCoefficient: 2,
+        maximumAttempts: 5,
+        maximumInterval: 100 * 30 * 1000, //ms
+        nonRetryableErrorTypes: WF_NON_RETRYABLE_ERRORS,
+    },
+});
+
+export interface NotifyWebhookWorfklowParams {
+    workflow_type: string;
+    endpoints: (string | WebHookSpec)[],
+    data: Record<string, any>
+}
+
+
+export async function notifyWebhookWorkflow(payload: WorkflowExecutionPayload<NotifyWebhookWorfklowParams>): Promise<any> {
+
+    const info = workflowInfo();
+    const { objectIds, vars } = payload;
+    const notifications = [];
+    const endpoints = vars.endpoints ?? (vars as any).webhooks ?? [];
+    const data = vars.data ?? (vars as any).webhook_data ?? undefined;
+    const workflow_type = vars.workflow_type ?? info.workflowType;
+    const eventName = payload.event;
+
+    const workflowId = info.parent?.workflowId || info.workflowId;
+    const workflowRunId = info.parent?.runId || info.runId;
+
+    if (!endpoints.length) {
+        log.info(`No webhooks to notify`);
+        return { notifications: [], message: "No webhooks to notify" };
+    }
+
+    for (const ep of endpoints) {
+        const n = notifyWebhook(payload, {
+            webhook: ep,
+            method: 'POST',
+            workflow_type,
+            workflow_id: workflowId,
+            workflow_run_id: workflowRunId,
+            event_name: eventName,
+            detail: {
+                object_ids: objectIds,
+                data
+            }
+        }).then(res => {
+            log.info(`Webhook notified at ${ep} with response code: ${res.status}`, { res });
+            return res;
+        });
+        notifications.push(n);
+    }
+
+    const res = await Promise.all(notifications);
+    log.info(`Webhooks notified`);
+
+    return { notifications: res, message: "Webhooks notified" };
+
+}

package/src/system/recalculateEmbeddingsWorkflow.ts

@@ -0,0 +1,41 @@
+
+import { SupportedEmbeddingTypes, WorkflowExecutionPayload } from "@vertesia/common";
+import * as activities from "../activities/index-dsl.js";
+import { dslProxyActivities } from "../dsl/dslProxyActivities.js";
+import { WF_NON_RETRYABLE_ERRORS } from "../errors.js";
+
+const {
+    generateEmbeddings,
+} = dslProxyActivities<typeof activities>("recalculateEmbeddingsWorkflow", {
+    startToCloseTimeout: "5 minute",
+    retry: {
+        initialInterval: '10s',
+        backoffCoefficient: 2,
+        maximumAttempts: 10,
+        maximumInterval: 100 * 30 * 1000, //ms
+        nonRetryableErrorTypes: WF_NON_RETRYABLE_ERRORS,
+    },
+});
+
+export async function recalculateEmbeddingsWorkflow(payload: WorkflowExecutionPayload) {
+
+    const embeddings = [];
+    const payloadType = payload.vars?.type as SupportedEmbeddingTypes;
+
+    if (payloadType && !Object.values(SupportedEmbeddingTypes).includes(payloadType)) {
+        throw new Error("Embedding type must be text, image, or properties");
+    }
+    const types = payloadType ? [payloadType] : Object.values(SupportedEmbeddingTypes);
+
+    for (const type of types) {
+        embeddings.push(generateEmbeddings(payload, {
+            force: true,
+            type
+        }))
+    }
+
+    const res = await Promise.all(embeddings);
+
+    return res;
+
+}

package/src/utils/auth.ts

@@ -0,0 +1,10 @@
+import { AuthTokenPayload, ProjectRef } from '@vertesia/common';
+import jwt from 'jsonwebtoken';
+
+export function decodeAuthToken(token: string): AuthTokenPayload {
+    return jwt.decode(token, { json: true }) as AuthTokenPayload;
+}
+
+export function getProjectFromToken(token: string): ProjectRef | undefined {
+    return decodeAuthToken(token)?.project;
+}

package/src/utils/blobs.ts

@@ -0,0 +1,59 @@
+import { VertesiaClient } from "@vertesia/client";
+import crypto from "crypto";
+import { createWriteStream } from "fs";
+import { Readable } from "stream";
+import { pipeline } from "stream/promises";
+import tmp from "tmp";
+import { DocumentNotFoundError } from "../errors.js";
+
+tmp.setGracefulCleanup();
+
+export async function fetchBlobAsStream(client: VertesiaClient, blobUri: string): Promise<ReadableStream<Uint8Array>> {
+    try {
+        return await client.files.downloadFile(blobUri);
+    } catch (err: any) {
+        if (err.message.includes("not found")) {
+            //TODO improve error handling with a fetch fail error class in the client
+            throw new DocumentNotFoundError(`Not found at ${blobUri}: ${err.message}`, []);
+        } else if (err.message.includes("forbidden")) {
+            throw new DocumentNotFoundError(`Forbidden at ${blobUri}: ${err.message}`);
+        } else {
+            throw new Error(`Failed to download blob ${blobUri}: ${err.message}`);
+        }
+    }
+}
+export async function fetchBlobAsBuffer(client: VertesiaClient, blobUri: string): Promise<Buffer> {
+    let stream = await fetchBlobAsStream(client, blobUri);
+    const buffers: Uint8Array[] = [];
+    for await (const data of stream) {
+        buffers.push(data);
+    }
+    return Buffer.concat(buffers);
+}
+
+export async function fetchBlobAsBase64(client: VertesiaClient, blobUri: string): Promise<string> {
+    const buffer = await fetchBlobAsBuffer(client, blobUri);
+    return buffer.toString("base64");
+}
+
+async function saveBlobToFile(client: VertesiaClient, blobUri: string, toFile: string): Promise<void> {
+    const stream = await fetchBlobAsStream(client, blobUri);
+
+    const nodeReadable = Readable.from(stream);
+    const out = createWriteStream(toFile);
+
+    await pipeline(nodeReadable, out); // Ensures completion before continuing
+}
+
+export async function saveBlobToTempFile(client: VertesiaClient, blobUri: string, fileExt?: string): Promise<string> {
+    const tmpFile = tmp.fileSync({
+        prefix: "vertesia-activity-",
+        postfix: fileExt ? "." + fileExt : "",
+    });
+    await saveBlobToFile(client, blobUri, tmpFile.name);
+    return tmpFile.name;
+}
+
+export function md5(contents: string) {
+    return crypto.createHash("md5").update(contents).digest("hex");
+}

package/src/utils/chunks.ts

@@ -0,0 +1,17 @@
+
+export interface DocPart {
+    line_number_start: number
+    line_number_end: number
+    name: string
+    type: string
+}
+
+export const getContentPart = (content: string, part: DocPart): string => {
+    const lines = content.split('\n');
+    const text = lines.filter((_l, i) => i >= part.line_number_start && i <= part.line_number_end).join('\n');
+    return text;
+}
+
+export const getContentParts = (content: string, parts: DocPart[]): string[] => {
+    return parts.map(part => getContentPart(content, part));
+}

package/src/utils/client.ts

@@ -0,0 +1,46 @@
+/**
+ * get a zeno client for a given token
+ */
+
+import {
+    decodeJWT,
+    VertesiaClient,
+    VertesiaClientProps,
+} from "@vertesia/client";
+import { WorkflowExecutionBaseParams } from "@vertesia/common";
+import { WorkflowParamNotFoundError } from "../errors.js";
+
+export function getVertesiaClient(payload: WorkflowExecutionBaseParams) {
+    return new VertesiaClient(getVertesiaClientOptions(payload));
+}
+
+export function getVertesiaClientOptions(
+    payload: WorkflowExecutionBaseParams,
+): VertesiaClientProps {
+    if (!payload.auth_token) {
+        throw new WorkflowParamNotFoundError(
+            "Authentication Token is missing from WorkflowExecutionPayload.authToken",
+        );
+    }
+
+    if (!payload.config?.studio_url) {
+        throw new WorkflowParamNotFoundError(
+            "Content Store URL is missing from WorkflowExecutionPayload.servers.storeUrl",
+        );
+    }
+
+    if (!payload.config?.store_url) {
+        throw new WorkflowParamNotFoundError(
+            "Content Store URL is missing from WorkflowExecutionPayload.servers.storeUrl",
+        );
+    }
+
+    const token = decodeJWT(payload.auth_token);
+
+    return {
+        serverUrl: payload.config.studio_url,
+        storeUrl: payload.config.store_url,
+        tokenServerUrl: token.iss,
+        apikey: payload.auth_token,
+    };
+}

package/src/utils/expand-vars.ts

@@ -0,0 +1,31 @@
+
+const VARS_RX = /\${\s*([^}]+)\s*}/g;
+
+/**
+ * Given an expression containing ${name} variables, replace them with the properties from the vars object.
+ * Nested property paths are also supported, e.g. ${section.name}
+ * @param expr
+ * @param vars
+ */
+export function expandVars(expr: string, vars: Record<string, any>) {
+    return expr.replace(VARS_RX, (_: string, name: string) => {
+        const path = name.split('.');
+        const value = resolveProp(vars, path);
+        if (value === undefined) {
+            return `${name}`; // return back the expression
+        } else {
+            return String(value);
+        }
+    });
+}
+
+function resolveProp(object: object, path: string[]) {
+    let value: any = object;
+    for (const part of path) {
+        if (value === undefined) {
+            return undefined;
+        }
+        value = value[part];
+    }
+    return value;
+}

package/src/utils/memory.ts

@@ -0,0 +1,61 @@
+import { VertesiaClient } from "@vertesia/client";
+import { NodeStreamSource } from "@vertesia/client/node";
+import { Commands, MemoryPack, buildMemoryPack as _buildMemoryPack, loadMemoryPack as _loadMemoryPack } from "@vertesia/memory";
+import { createReadStream, createWriteStream } from "fs";
+import { rm } from "fs/promises";
+import { webStreamToReadable } from "node-web-stream-adapters";
+import { pipeline } from "stream/promises";
+
+import tmp from "tmp";
+import zlib from "zlib";
+
+tmp.setGracefulCleanup();
+
+
+export async function publishMemoryPack(client: VertesiaClient, file: string, name: string): Promise<void> {
+    const stream = createReadStream(file);
+    try {
+        const source = new NodeStreamSource(stream, name);
+        await client.files.uploadMemoryPack(source);
+    } catch (err: any) {
+        stream.destroy();
+        throw err;
+    }
+}
+
+export async function buildMemoryPack(recipeFn: (commands: Commands) => Promise<Record<string, any>>): Promise<string> {
+    const tarFile = tmp.fileSync({
+        prefix: "composable-memory-pack-",
+        postfix: ".tar.gz",
+    });
+    return await _buildMemoryPack(recipeFn, {
+        out: tarFile.name,
+        gzip: true,
+    });
+}
+
+export async function buildAndPublishMemoryPack(client: VertesiaClient, name: string, recipeFn: (commands: Commands) => Promise<Record<string, any>>): Promise<void> {
+    const tarFile = await buildMemoryPack(recipeFn);
+    try {
+        await publishMemoryPack(client, tarFile, name);
+    } finally {
+        await rm(tarFile);
+    }
+}
+
+export async function fetchMemoryPack(client: VertesiaClient, name: string): Promise<string> {
+    const webStream = await client.files.downloadMemoryPack(name);
+    const tarFile = tmp.fileSync({
+        prefix: "composable-memory-pack-",
+        postfix: ".tar",
+        discardDescriptor: true,
+    });
+    const streamIn = webStreamToReadable(webStream);
+    const streamOut = createWriteStream(tarFile.name);
+    await pipeline(streamIn, zlib.createGunzip(), streamOut);
+    return tarFile.name;
+}
+
+export function loadMemoryPack(client: VertesiaClient, name: string): Promise<MemoryPack> {
+    return fetchMemoryPack(client, name).then(file => _loadMemoryPack(file));
+}