@vertesia/workflow 0.42.2

package/src/utils/blobs.ts

@@ -0,0 +1,58 @@
+import { ComposableClient } from "@vertesia/client";
+import crypto from "crypto";
+import { createWriteStream } from "fs";
+import tmp from "tmp";
+import { NoDocumentFound } from "../errors.js";
+
+tmp.setGracefulCleanup();
+
+export async function fetchBlobAsStream(client: ComposableClient, blobUri: string): Promise<ReadableStream<Uint8Array>> {
+    try {
+        return await client.files.downloadFile(blobUri);
+    } catch (err: any) {
+        throw new NoDocumentFound(`Blob ${blobUri} not found`, []);
+    }
+}
+export async function fetchBlobAsBuffer(client: ComposableClient, blobUri: string): Promise<Buffer> {
+    let stream = await fetchBlobAsStream(client, blobUri);
+    const buffers: Uint8Array[] = [];
+    for await (const data of stream) {
+        buffers.push(data);
+    }
+    return Buffer.concat(buffers);
+}
+
+export async function fetchBlobAsBase64(client: ComposableClient, blobUri: string): Promise<string> {
+    const buffer = await fetchBlobAsBuffer(client, blobUri);
+    return buffer.toString('base64');
+}
+
+export async function saveBlobToFile(client: ComposableClient, blobUri: string, toFile: string): Promise<void> {
+    let stream = await fetchBlobAsStream(client, blobUri);
+    const out = createWriteStream(toFile);
+    await writeChunksToStream(stream, out);
+}
+
+export async function saveBlobToTempFile(client: ComposableClient, blobUri: string, fileExt?: string): Promise<string> {
+    const tmpFile = tmp.fileSync({
+        prefix: "composable-activity-",
+        postfix: fileExt,
+        discardDescriptor: true
+    });
+    await saveBlobToFile(client, blobUri, tmpFile.name);
+    return tmpFile.name;
+}
+
+async function writeChunksToStream(chunks: AsyncIterable<Uint8Array>, out: NodeJS.WritableStream) {
+    for await (const chunk of chunks) {
+        if (!out.write(chunk)) {
+            // If the internal buffer is full, wait until it's drained
+            await new Promise(resolve => out.once('drain', resolve));
+        }
+    }
+    out.end(); // Close the stream when done
+}
+
+export function md5(contents: string) {
+    return crypto.createHash('md5').update(contents).digest("hex");
+}

package/src/utils/client.ts

@@ -0,0 +1,31 @@
+/**
+ * get a zeno client for a given token
+ */
+
+import { ComposableClient } from "@vertesia/client";
+import { WorkflowExecutionPayload } from "@vertesia/common";
+
+
+export function getClient(payload: WorkflowExecutionPayload) {
+
+    if (!payload.auth_token) {
+        throw new Error("Authentication Token is missing from WorkflowExecutionPayload.authToken");
+    }
+
+    if (!payload.config?.studio_url) {
+        throw new Error("Content Store URL is missing from WorkflowExecutionPayload.servers.storeUrl");
+    }
+
+    if (!payload.config?.store_url) {
+        throw new Error("Content Store URL is missing from WorkflowExecutionPayload.servers.storeUrl");
+    }
+
+    const client = new ComposableClient({
+        serverUrl: payload.config.studio_url,
+        storeUrl: payload.config.store_url,
+        apikey: payload.auth_token
+    });
+
+    return client;
+
+}

package/src/utils/expand-vars.ts

@@ -0,0 +1,31 @@
+
+const VARS_RX = /\${\s*([^}]+)\s*}/g;
+
+/**
+ * Given an expression containing ${name} variables, replace them with the properties from the vars object.
+ * Nested property paths are also supported, e.g. ${section.name}
+ * @param expr
+ * @param vars
+ */
+export function expandVars(expr: string, vars: Record<string, any>) {
+    return expr.replace(VARS_RX, (_: string, name: string) => {
+        const path = name.split('.');
+        const value = resolveProp(vars, path);
+        if (value === undefined) {
+            return `${name}`; // return back the expression
+        } else {
+            return String(value);
+        }
+    });
+}
+
+function resolveProp(object: object, path: string[]) {
+    let value: any = object;
+    for (const part of path) {
+        if (value === undefined) {
+            return undefined;
+        }
+        value = value[part];
+    }
+    return value;
+}

package/src/utils/memory.ts

@@ -0,0 +1,66 @@
+import { ComposableClient, StreamSource } from "@vertesia/client";
+import { Commands, MemoryPack, buildMemoryPack as _buildMemoryPack, loadMemoryPack as _loadMemoryPack } from "@becomposable/memory";
+import { createReadStream, createWriteStream } from "fs";
+import { rm } from "fs/promises";
+import { readableToWebStream, webStreamToReadable } from "node-web-stream-adapters";
+import { Readable } from "stream";
+import { pipeline } from "stream/promises";
+
+import tmp from "tmp";
+import zlib from "zlib";
+
+tmp.setGracefulCleanup();
+
+export class NodeStreamSource extends StreamSource {
+    constructor(stream: Readable, name: string, type?: string, id?: string) {
+        super(readableToWebStream(stream), name, type, id);
+    }
+}
+
+export async function publishMemoryPack(client: ComposableClient, file: string, name: string): Promise<void> {
+    const stream = createReadStream(file);
+    try {
+        const source = new NodeStreamSource(stream, name);
+        await client.files.uploadMemoryPack(source);
+    } catch (err: any) {
+        stream.destroy();
+        throw err;
+    }
+}
+
+export async function buildMemoryPack(recipeFn: (commands: Commands) => Promise<Record<string, any>>): Promise<string> {
+    const tarFile = tmp.fileSync({
+        prefix: "composable-memory-pack-",
+        postfix: ".tar.gz",
+    });
+    return await _buildMemoryPack(recipeFn, {
+        out: tarFile.name,
+        gzip: true,
+    });
+}
+
+export async function buildAndPublishMemoryPack(client: ComposableClient, name: string, recipeFn: (commands: Commands) => Promise<Record<string, any>>): Promise<void> {
+    const tarFile = await buildMemoryPack(recipeFn);
+    try {
+        await publishMemoryPack(client, tarFile, name);
+    } finally {
+        await rm(tarFile);
+    }
+}
+
+export async function fetchMemoryPack(client: ComposableClient, name: string): Promise<string> {
+    const webStream = await client.files.downloadMemoryPack(name);
+    const tarFile = tmp.fileSync({
+        prefix: "composable-memory-pack-",
+        postfix: ".tar",
+        discardDescriptor: true,
+    });
+    const streamIn = webStreamToReadable(webStream);
+    const streamOut = createWriteStream(tarFile.name);
+    await pipeline(streamIn, zlib.createGunzip(), streamOut);
+    return tarFile.name;
+}
+
+export function loadMemoryPack(client: ComposableClient, name: string): Promise<MemoryPack> {
+    return fetchMemoryPack(client, name).then(file => _loadMemoryPack(file));
+}

package/src/utils/tokens.ts

@@ -0,0 +1,44 @@
+
+import { TiktokenEncoding, get_encoding } from "tiktoken";
+
+
+export type TruncateSpec = number | {
+    max_tokens: number;
+    encoding?: TiktokenEncoding;
+}
+
+export function truncByMaxTokens(content: string, by: TruncateSpec) {
+    let encoding: TiktokenEncoding;
+    let maxTokens: number;
+    if (typeof by === 'number') {
+        maxTokens = by;
+        encoding = "cl100k_base";
+    } else {
+        maxTokens = by.max_tokens;
+        encoding = by.encoding || "cl100k_base";
+    }
+    const enc = get_encoding(encoding);
+    let tokens = enc.encode(content);
+    if (tokens.length > maxTokens) {
+        tokens = tokens.slice(0, maxTokens);
+        return new TextDecoder().decode(enc.decode(tokens));
+    } else {
+        return content;
+    }
+}
+
+export function countTokens(text: string, encoding: TiktokenEncoding = 'cl100k_base') {
+
+    const encoder = get_encoding(encoding);
+    if (!encoder) {
+        throw new Error(`Unknown encoding ${encoding}`);
+    }
+
+    const tokens = encoder.encode(text);
+
+    return {
+        count: tokens.length,
+        encoding: encoding,
+    };
+
+}

package/src/vars.ts

@@ -0,0 +1,3 @@
+// exported as a named export /vars
+export * from "./dsl/validation.js";
+export * from "./dsl/vars.js";

package/src/workflows.ts

@@ -0,0 +1,9 @@
+/**
+ * Export workflows to be registered on temporal workers
+ */
+export { dslWorkflow } from "./dsl/dsl-workflow.js";
+export { iterativeGenerationWorkflow } from "./iterative-generation/iterativeGenerationWorkflow.js";
+export { generateObjectText } from "./system/generateObjectText.js";
+export { notifyWebhookWorkflow } from "./system/notifyWebhookWorkflow.js";
+export { recalculateEmbeddingsWorkflow } from "./system/recalculateEmbeddingsWorkflow.js";
+