@hotmeshio/long-tail 0.1.13 → 0.1.15

package/build/services/yaml-workflow/durable-compiler/index.js

@@ -0,0 +1,168 @@
+"use strict";
+/**
+ * Durable-to-YAML compiler — converts procedural TypeScript workflows to YAML DAGs.
+ *
+ * This is "Path 3" for YAML DAG creation: static compilation from durable source
+ * code without executing the workflow. The LLM translates the procedural orchestration
+ * into an equivalent HotMesh YAML DAG that runs without replay overhead.
+ *
+ * Pipeline:
+ *   1. Read source (file or inline string)
+ *   2. Optionally resolve and read the activities module
+ *   3. Extract metadata (activity names, primitives, control flow)
+ *   4. Build LLM messages (system prompt + source + metadata)
+ *   5. Call LLM (temperature 0, retry on parse failure)
+ *   6. Fix known @pipe anti-patterns
+ *   7. Return result compatible with existing YAML workflow CRUD
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.compileDurableToYaml = compileDurableToYaml;
+const promises_1 = require("fs/promises");
+const path_1 = require("path");
+const llm_1 = require("../../llm");
+const defaults_1 = require("../../../modules/defaults");
+const logger_1 = require("../../../lib/logger");
+const utils_1 = require("../../../modules/utils");
+const parser_1 = require("./parser");
+const prompts_1 = require("./prompts");
+const MAX_COMPILE_ATTEMPTS = 3;
+const COMPILER_MAX_TOKENS = 8192;
+/**
+ * Compile a durable TypeScript workflow into a HotMesh YAML DAG.
+ */
+async function compileDurableToYaml(options) {
+    const appId = options.appId || 'longtail';
+    const graphTopic = options.subscribes || (0, utils_1.sanitizeToolName)(options.name);
+    // 1. Read source
+    const source = options.isFilePath
+        ? await (0, promises_1.readFile)(options.source, 'utf-8')
+        : options.source;
+    // 2. Extract metadata
+    const metadata = (0, parser_1.extractDurableMetadata)(source, options.workflowName);
+    // 3. Optionally resolve and read activities module
+    let activitiesSource;
+    if (options.isFilePath && metadata.activityImports.length > 0) {
+        activitiesSource = await resolveActivitiesSource(options.source, metadata.activityImports);
+    }
+    // 4. Load activity types reference
+    const activityTypesRef = await loadActivityTypesReference();
+    // 5. Build LLM messages
+    const messages = [
+        {
+            role: 'system',
+            content: (0, prompts_1.DURABLE_COMPILER_SYSTEM_PROMPT)(activityTypesRef, metadata),
+        },
+        {
+            role: 'user',
+            content: (0, prompts_1.buildUserMessage)(source, metadata, activitiesSource),
+        },
+    ];
+    // 6. Call LLM with retry on parse failure
+    let parsed = null;
+    let lastError = '';
+    for (let attempt = 1; attempt <= MAX_COMPILE_ATTEMPTS; attempt++) {
+        const t0 = Date.now();
+        const response = await (0, llm_1.callLLM)({
+            model: defaults_1.LLM_MODEL_PRIMARY,
+            messages,
+            temperature: 0,
+            max_tokens: COMPILER_MAX_TOKENS,
+        });
+        logger_1.loggerRegistry.info(`[durable-compiler] attempt=${attempt} ${Date.now() - t0}ms | in=${response.usage?.prompt_tokens} out=${response.usage?.completion_tokens}`);
+        const content = response.content || '';
+        try {
+            parsed = parseJsonResponse(content);
+            break;
+        }
+        catch (err) {
+            lastError = err.message;
+            logger_1.loggerRegistry.warn(`[durable-compiler] JSON parse failed (attempt ${attempt}): ${lastError}`);
+            // Feed error back for retry
+            messages.push({ role: 'assistant', content });
+            messages.push({
+                role: 'user',
+                content: `Your response was not valid JSON. Error: ${lastError}\n\nPlease return ONLY the JSON object with no markdown fences or surrounding text.`,
+            });
+        }
+    }
+    if (!parsed) {
+        throw new Error(`[durable-compiler] Failed to get valid JSON after ${MAX_COMPILE_ATTEMPTS} attempts. Last error: ${lastError}`);
+    }
+    // 7. Fix known @pipe anti-patterns
+    let yaml = fixPipePatterns(parsed.yaml || '');
+    // 7b. Rewrite app.id to match the target appId
+    yaml = yaml.replace(/^(\s*id:\s*)(.+)$/m, `$1${appId}`);
+    // 8. Extract the actual subscribes topic from the YAML (LLM may choose its own)
+    const subscribesMatch = yaml.match(/subscribes:\s*(.+)/);
+    const actualTopic = subscribesMatch
+        ? subscribesMatch[1].trim().replace(/^['"]|['"]$/g, '')
+        : graphTopic;
+    // 9. Build result
+    return {
+        yaml,
+        inputSchema: parsed.input_schema || {},
+        outputSchema: parsed.output_schema || {},
+        activityManifest: parsed.activity_manifest || [],
+        graphTopic: actualTopic,
+        appId,
+        tags: parsed.tags || ['durable'],
+        inputFieldMeta: [],
+        category: 'durable',
+    };
+}
+/**
+ * Parse a JSON response from the LLM, stripping markdown fences if present.
+ */
+function parseJsonResponse(content) {
+    let cleaned = content.trim();
+    // Strip markdown code fences
+    if (cleaned.startsWith('```')) {
+        cleaned = cleaned.replace(/^```(?:json)?\s*\n?/, '').replace(/\n?\s*```$/, '');
+    }
+    return JSON.parse(cleaned);
+}
+/**
+ * Fix known @pipe anti-patterns that LLMs consistently produce.
+ * Same fixes as the workflow-builder.
+ */
+function fixPipePatterns(yaml) {
+    // Fix: ['{@date.toISOString}'] then [0, 10] then ['{@string.substring}']
+    yaml = yaml.replace(/(\['\{@date\.toISOString\}'\])\s*\n(\s*)- \[0, 10\]\s*\n(\s*)- \['\{@string\.substring\}'\]/g, "['{@date.toISOString}', 0, 10]\n$2- ['{@string.substring}']");
+    return yaml;
+}
+/**
+ * Load the full activity-types.md reference file.
+ */
+async function loadActivityTypesReference() {
+    try {
+        const refPath = (0, path_1.join)(__dirname, '..', '..', '..', 'system', 'workflows', 'mcp-workflow-builder', 'reference', 'activity-types.md');
+        return await (0, promises_1.readFile)(refPath, 'utf-8');
+    }
+    catch {
+        // Fallback: return minimal reference if file not found
+        return '(Activity types reference not available — use trigger, worker, hook, await, cycle, signal, interrupt types as documented in HotMesh.)';
+    }
+}
+/**
+ * Resolve and read activities source from import paths.
+ * Returns combined source of all activity modules found.
+ */
+async function resolveActivitiesSource(workflowFilePath, importPaths) {
+    const dir = (0, path_1.dirname)(workflowFilePath);
+    const sources = [];
+    for (const importPath of importPaths) {
+        // Try common extensions
+        const basePath = (0, path_1.resolve)(dir, importPath);
+        for (const ext of ['.ts', '/index.ts', '.js', '/index.js']) {
+            try {
+                const content = await (0, promises_1.readFile)(basePath + ext, 'utf-8');
+                sources.push(`// ${importPath}\n${content}`);
+                break;
+            }
+            catch {
+                // Try next extension
+            }
+        }
+    }
+    return sources.length > 0 ? sources.join('\n\n') : undefined;
+}

package/build/services/yaml-workflow/durable-compiler/parser.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Lightweight regex-based metadata extractor for durable workflow source.
+ *
+ * Extracts structural hints (activity names, primitives, control flow)
+ * that ground the LLM during compilation. This is NOT a full AST parser —
+ * it gives the LLM better context without requiring ts-morph or the TS compiler API.
+ */
+import type { DurableSourceMetadata } from './types';
+/**
+ * Extract structural metadata from durable workflow TypeScript source.
+ */
+export declare function extractDurableMetadata(source: string, workflowFunctionName: string): DurableSourceMetadata;

package/build/services/yaml-workflow/durable-compiler/parser.js

@@ -0,0 +1,110 @@
+"use strict";
+/**
+ * Lightweight regex-based metadata extractor for durable workflow source.
+ *
+ * Extracts structural hints (activity names, primitives, control flow)
+ * that ground the LLM during compilation. This is NOT a full AST parser —
+ * it gives the LLM better context without requiring ts-morph or the TS compiler API.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.extractDurableMetadata = extractDurableMetadata;
+/**
+ * Extract structural metadata from durable workflow TypeScript source.
+ */
+function extractDurableMetadata(source, workflowFunctionName) {
+    return {
+        workflowFunctionName,
+        activityNames: extractActivityNames(source),
+        durablePrimitives: extractDurablePrimitives(source),
+        envelopeFields: extractEnvelopeFields(source),
+        activityImports: extractActivityImports(source),
+        hasForLoop: /\bfor\s*\(/.test(source) || /\.forEach\(/.test(source),
+        hasPromiseAll: /Promise\.all/.test(source),
+        hasConditionalBranch: /\bif\s*\(/.test(source),
+        hasEscalation: /type:\s*['"]escalation['"]/.test(source),
+    };
+}
+/**
+ * Extract activity function names from proxyActivities destructuring.
+ *
+ * Matches patterns like:
+ *   const { greet, fetchData } = Durable.workflow.proxyActivities<T>({ ... });
+ */
+function extractActivityNames(source) {
+    const names = [];
+    // Match destructured names from proxyActivities assignments
+    const proxyPattern = /(?:const|let)\s*\{\s*([^}]+)\}\s*=\s*(?:Durable\.workflow\.)?proxyActivities/g;
+    let match;
+    while ((match = proxyPattern.exec(source)) !== null) {
+        const destructured = match[1];
+        for (const name of destructured.split(',')) {
+            const trimmed = name.trim();
+            if (trimmed)
+                names.push(trimmed);
+        }
+    }
+    return names;
+}
+/**
+ * Extract Durable.workflow.* primitive calls.
+ */
+function extractDurablePrimitives(source) {
+    const primitives = [];
+    const seen = new Set();
+    // Durable.workflow.sleep, condition, startChild, workflowInfo, etc.
+    const durablePattern = /Durable\.workflow\.(\w+)/g;
+    let match;
+    while ((match = durablePattern.exec(source)) !== null) {
+        const prim = match[1];
+        if (prim !== 'proxyActivities' && !seen.has(prim)) {
+            seen.add(prim);
+            primitives.push(prim);
+        }
+    }
+    // conditionLT (LT-specific wrapper)
+    if (/conditionLT/.test(source) && !seen.has('conditionLT')) {
+        primitives.push('conditionLT');
+    }
+    // executeLT (orchestrator wrapper)
+    if (/executeLT/.test(source) && !seen.has('executeLT')) {
+        primitives.push('executeLT');
+    }
+    return primitives;
+}
+/**
+ * Extract envelope field names from data destructuring.
+ *
+ * Matches patterns like:
+ *   const { message, role } = envelope.data;
+ *   const { productName, stations } = envelope.data as { ... };
+ */
+function extractEnvelopeFields(source) {
+    const fields = [];
+    const envelopePattern = /(?:const|let)\s*\{\s*([^}]+)\}\s*=\s*envelope\.data/g;
+    let match;
+    while ((match = envelopePattern.exec(source)) !== null) {
+        const destructured = match[1];
+        for (const field of destructured.split(',')) {
+            // Strip defaults and type annotations
+            const name = field.trim().split(/\s*[=:]/)[0].trim();
+            if (name)
+                fields.push(name);
+        }
+    }
+    return fields;
+}
+/**
+ * Extract import paths for activity modules.
+ *
+ * Matches: import * as activities from './activities';
+ *          import * as interceptorActivities from '../../../services/interceptor/activities';
+ */
+function extractActivityImports(source) {
+    const imports = [];
+    const importPattern = /import\s+\*\s+as\s+\w+\s+from\s+['"]([^'"]+)['"]/g;
+    let match;
+    while ((match = importPattern.exec(source)) !== null) {
+        imports.push(match[1]);
+    }
+    return imports;
+}

package/build/services/yaml-workflow/durable-compiler/prompts.d.ts

@@ -0,0 +1,18 @@
+/**
+ * System prompt for the durable-to-YAML compiler.
+ *
+ * Teaches the LLM how to convert procedural durable TypeScript workflows
+ * (Temporal-like API via HotMesh Durable) into equivalent YAML DAGs.
+ */
+import type { DurableSourceMetadata } from './types';
+/**
+ * Build the system prompt for durable→YAML compilation.
+ *
+ * @param activityTypesRef - Full activity-types.md reference content
+ * @param metadata - Parsed metadata from the source workflow
+ */
+export declare function DURABLE_COMPILER_SYSTEM_PROMPT(activityTypesRef: string, metadata: DurableSourceMetadata): string;
+/**
+ * Build the user message containing the source code and context.
+ */
+export declare function buildUserMessage(source: string, metadata: DurableSourceMetadata, activitiesSource?: string): string;