@hotmeshio/long-tail 0.1.0

package/build/services/yaml-workflow/pipeline/validate.js

@@ -0,0 +1,111 @@
+"use strict";
+/**
+ * Validate stage: optional LLM review of generated YAML against intent.
+ *
+ * Checks for missing data flow connections, broken iteration sources,
+ * and session handles that get lost mid-pipeline. Records issues as
+ * informational warnings — does NOT modify the YAML.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.validate = validate;
+const llm_1 = require("../../llm");
+const defaults_1 = require("../../../modules/defaults");
+const prompts_1 = require("./prompts");
+/**
+ * Validate pipeline stage: optional LLM review.
+ */
+async function validate(ctx) {
+    ctx.validationIssues = [];
+    // Skip if no API key or no plan (no point validating mechanical output)
+    if (!(0, llm_1.hasLLMApiKey)(defaults_1.LLM_MODEL_SECONDARY) || !ctx.compilationPlan)
+        return ctx;
+    // Skip if explicitly disabled
+    if (process.env.LT_SKIP_YAML_VALIDATION === '1')
+        return ctx;
+    try {
+        // Truncate YAML if very long
+        const yamlPreview = ctx.yaml.length > 4000
+            ? ctx.yaml.slice(0, 4000) + '\n... (truncated)'
+            : ctx.yaml;
+        // Include activity manifest for cross-referencing tool requirements
+        const manifestSummary = ctx.activityManifest
+            .filter(a => a.type === 'worker')
+            .map(a => ({
+            id: a.activity_id,
+            tool: a.mcp_tool_name || a.title,
+            server: a.mcp_server_id,
+            inputs: Object.keys(a.input_mappings).filter(k => k !== 'workflowName' && k !== '_scope'),
+            outputs: a.output_fields,
+        }));
+        const userMessage = [
+            `## Intent`,
+            ctx.compilationPlan.intent,
+            ``,
+            `## Activity Manifest`,
+            JSON.stringify(manifestSummary, null, 2),
+            ``,
+            `## Generated YAML`,
+            '```yaml',
+            yamlPreview,
+            '```',
+        ].join('\n');
+        const response = await (0, llm_1.callLLM)({
+            model: defaults_1.LLM_MODEL_SECONDARY,
+            max_tokens: 500,
+            temperature: 0,
+            messages: [
+                { role: 'system', content: prompts_1.VALIDATION_PROMPT },
+                { role: 'user', content: userMessage },
+            ],
+        });
+        const raw = response.content || '';
+        const cleaned = raw
+            .replace(/^```(?:json)?\s*/m, '')
+            .replace(/\s*```$/m, '')
+            .trim();
+        const parsed = JSON.parse(cleaned);
+        if (Array.isArray(parsed.issues) && parsed.issues.length > 0) {
+            ctx.validationIssues = parsed.issues;
+            const { loggerRegistry } = await Promise.resolve().then(() => __importStar(require('../../logger')));
+            loggerRegistry.warn(`[yaml-workflow] Validation issues found: ${parsed.issues.join('; ')}`);
+        }
+    }
+    catch (err) {
+        const { loggerRegistry } = await Promise.resolve().then(() => __importStar(require('../../logger')));
+        loggerRegistry.info(`[yaml-workflow] Validation skipped: ${err}`);
+    }
+    return ctx;
+}

package/build/services/yaml-workflow/sql.d.ts

@@ -0,0 +1,19 @@
+export declare const CREATE_YAML_WORKFLOW = "\n  INSERT INTO lt_yaml_workflows\n    (name, description, app_id, app_version, source_workflow_id,\n     source_workflow_type, yaml_content, graph_topic,\n     input_schema, output_schema, activity_manifest, input_field_meta,\n     original_prompt, category, tags, metadata, content_version)\n  VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, 1)\n  RETURNING *";
+export declare const GET_YAML_WORKFLOW = "\n  SELECT * FROM lt_yaml_workflows WHERE id = $1";
+export declare const GET_YAML_WORKFLOW_BY_NAME = "\n  SELECT * FROM lt_yaml_workflows WHERE name = $1";
+export declare const UPDATE_YAML_WORKFLOW_VERSION = "\n  UPDATE lt_yaml_workflows SET app_version = $2 WHERE id = $1";
+export declare const DELETE_YAML_WORKFLOW = "\n  DELETE FROM lt_yaml_workflows WHERE id = $1";
+export declare const GET_ACTIVE_YAML_WORKFLOWS = "\n  SELECT * FROM lt_yaml_workflows\n  WHERE status = 'active'\n  ORDER BY name";
+export declare const LIST_BY_APP_ID = "\n  SELECT * FROM lt_yaml_workflows\n  WHERE app_id = $1 AND status != 'archived'\n  ORDER BY name";
+export declare const GET_DISTINCT_APP_IDS = "\n  SELECT DISTINCT app_id FROM lt_yaml_workflows\n  WHERE status != 'archived'\n  ORDER BY app_id";
+export declare const MARK_CONTENT_DEPLOYED = "\n  UPDATE lt_yaml_workflows\n  SET deployed_content_version = content_version\n  WHERE id = $1";
+export declare const MARK_APP_ID_CONTENT_DEPLOYED = "\n  UPDATE lt_yaml_workflows\n  SET deployed_content_version = content_version\n  WHERE app_id = $1 AND status != 'archived'";
+export declare const CREATE_VERSION_SNAPSHOT = "\n  INSERT INTO lt_yaml_workflow_versions\n    (workflow_id, version, yaml_content, activity_manifest, input_schema, output_schema, input_field_meta, change_summary)\n  VALUES ($1, $2, $3, $4, $5, $6, $7, $8)\n  ON CONFLICT (workflow_id, version) DO NOTHING\n  RETURNING *";
+export declare const COUNT_VERSIONS = "\n  SELECT COUNT(*) FROM lt_yaml_workflow_versions WHERE workflow_id = $1";
+export declare const LIST_VERSIONS = "\n  SELECT * FROM lt_yaml_workflow_versions\n  WHERE workflow_id = $1\n  ORDER BY version DESC\n  LIMIT $2 OFFSET $3";
+export declare const DISCOVER_WORKFLOWS = "\n  SELECT *,\n    ts_rank_cd(search_vector, plainto_tsquery('english', $1), 32) AS fts_rank\n  FROM lt_yaml_workflows\n  WHERE status = 'active'\n    AND (\n      search_vector @@ plainto_tsquery('english', $1)\n      OR tags && $2::text[]\n      OR ($3::text IS NOT NULL AND category = $3)\n    )\n  ORDER BY fts_rank DESC, activated_at DESC NULLS LAST\n  LIMIT $4";
+export declare const GET_VERSION_SNAPSHOT = "\n  SELECT * FROM lt_yaml_workflow_versions\n  WHERE workflow_id = $1 AND version = $2";
+export declare const UPDATE_STATUS_BASE = "UPDATE lt_yaml_workflows SET status = $2";
+export declare const UPDATE_STATUS_SUFFIX = " WHERE id = $1 RETURNING *";
+export declare const FIND_BY_TAGS_ANY = "\n  SELECT * FROM lt_yaml_workflows\n  WHERE status = 'active' AND tags && $1::text[]\n  ORDER BY name";
+export declare const FIND_BY_TAGS_ALL = "\n  SELECT * FROM lt_yaml_workflows\n  WHERE status = 'active' AND tags @> $1::text[]\n  ORDER BY name";

package/build/services/yaml-workflow/sql.js

@@ -0,0 +1,83 @@
+"use strict";
+// ─── YAML workflow CRUD ─────────────────────────────────────────────────────
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.FIND_BY_TAGS_ALL = exports.FIND_BY_TAGS_ANY = exports.UPDATE_STATUS_SUFFIX = exports.UPDATE_STATUS_BASE = exports.GET_VERSION_SNAPSHOT = exports.DISCOVER_WORKFLOWS = exports.LIST_VERSIONS = exports.COUNT_VERSIONS = exports.CREATE_VERSION_SNAPSHOT = exports.MARK_APP_ID_CONTENT_DEPLOYED = exports.MARK_CONTENT_DEPLOYED = exports.GET_DISTINCT_APP_IDS = exports.LIST_BY_APP_ID = exports.GET_ACTIVE_YAML_WORKFLOWS = exports.DELETE_YAML_WORKFLOW = exports.UPDATE_YAML_WORKFLOW_VERSION = exports.GET_YAML_WORKFLOW_BY_NAME = exports.GET_YAML_WORKFLOW = exports.CREATE_YAML_WORKFLOW = void 0;
+exports.CREATE_YAML_WORKFLOW = `
+  INSERT INTO lt_yaml_workflows
+    (name, description, app_id, app_version, source_workflow_id,
+     source_workflow_type, yaml_content, graph_topic,
+     input_schema, output_schema, activity_manifest, input_field_meta,
+     original_prompt, category, tags, metadata, content_version)
+  VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, 1)
+  RETURNING *`;
+exports.GET_YAML_WORKFLOW = `
+  SELECT * FROM lt_yaml_workflows WHERE id = $1`;
+exports.GET_YAML_WORKFLOW_BY_NAME = `
+  SELECT * FROM lt_yaml_workflows WHERE name = $1`;
+exports.UPDATE_YAML_WORKFLOW_VERSION = `
+  UPDATE lt_yaml_workflows SET app_version = $2 WHERE id = $1`;
+exports.DELETE_YAML_WORKFLOW = `
+  DELETE FROM lt_yaml_workflows WHERE id = $1`;
+// ─── Status and deployment ──────────────────────────────────────────────────
+exports.GET_ACTIVE_YAML_WORKFLOWS = `
+  SELECT * FROM lt_yaml_workflows
+  WHERE status = 'active'
+  ORDER BY name`;
+exports.LIST_BY_APP_ID = `
+  SELECT * FROM lt_yaml_workflows
+  WHERE app_id = $1 AND status != 'archived'
+  ORDER BY name`;
+exports.GET_DISTINCT_APP_IDS = `
+  SELECT DISTINCT app_id FROM lt_yaml_workflows
+  WHERE status != 'archived'
+  ORDER BY app_id`;
+exports.MARK_CONTENT_DEPLOYED = `
+  UPDATE lt_yaml_workflows
+  SET deployed_content_version = content_version
+  WHERE id = $1`;
+exports.MARK_APP_ID_CONTENT_DEPLOYED = `
+  UPDATE lt_yaml_workflows
+  SET deployed_content_version = content_version
+  WHERE app_id = $1 AND status != 'archived'`;
+// ─── Version history ────────────────────────────────────────────────────────
+exports.CREATE_VERSION_SNAPSHOT = `
+  INSERT INTO lt_yaml_workflow_versions
+    (workflow_id, version, yaml_content, activity_manifest, input_schema, output_schema, input_field_meta, change_summary)
+  VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
+  ON CONFLICT (workflow_id, version) DO NOTHING
+  RETURNING *`;
+exports.COUNT_VERSIONS = `
+  SELECT COUNT(*) FROM lt_yaml_workflow_versions WHERE workflow_id = $1`;
+exports.LIST_VERSIONS = `
+  SELECT * FROM lt_yaml_workflow_versions
+  WHERE workflow_id = $1
+  ORDER BY version DESC
+  LIMIT $2 OFFSET $3`;
+// ─── Discovery ─────────────────────────────────────────────────────────────
+exports.DISCOVER_WORKFLOWS = `
+  SELECT *,
+    ts_rank_cd(search_vector, plainto_tsquery('english', $1), 32) AS fts_rank
+  FROM lt_yaml_workflows
+  WHERE status = 'active'
+    AND (
+      search_vector @@ plainto_tsquery('english', $1)
+      OR tags && $2::text[]
+      OR ($3::text IS NOT NULL AND category = $3)
+    )
+  ORDER BY fts_rank DESC, activated_at DESC NULLS LAST
+  LIMIT $4`;
+exports.GET_VERSION_SNAPSHOT = `
+  SELECT * FROM lt_yaml_workflow_versions
+  WHERE workflow_id = $1 AND version = $2`;
+// ─── Status updates ─────────────────────────────────────────────────────────
+exports.UPDATE_STATUS_BASE = `UPDATE lt_yaml_workflows SET status = $2`;
+exports.UPDATE_STATUS_SUFFIX = ` WHERE id = $1 RETURNING *`;
+// ─── Tag-based lookup ────────────────────────────────────────────────────────
+exports.FIND_BY_TAGS_ANY = `
+  SELECT * FROM lt_yaml_workflows
+  WHERE status = 'active' AND tags && $1::text[]
+  ORDER BY name`;
+exports.FIND_BY_TAGS_ALL = `
+  SELECT * FROM lt_yaml_workflows
+  WHERE status = 'active' AND tags @> $1::text[]
+  ORDER BY name`;

package/build/services/yaml-workflow/types.d.ts

@@ -0,0 +1,223 @@
+/**
+ * Type definitions for the YAML workflow service.
+ *
+ * Includes types for the database layer, the 5-stage compilation pipeline
+ * (extract → analyze → compile → build → validate), and the DAG builder.
+ */
+import type { ActivityManifestEntry, InputFieldMeta } from '../../types/yaml-workflow';
+import type { WorkflowExecution } from '../../types';
+import type { PatternAnnotation } from './pattern-detector';
+export interface CreateYamlWorkflowInput {
+    name: string;
+    description?: string;
+    app_id: string;
+    app_version?: string;
+    source_workflow_id?: string;
+    source_workflow_type?: string;
+    yaml_content: string;
+    graph_topic: string;
+    input_schema?: Record<string, unknown>;
+    output_schema?: Record<string, unknown>;
+    activity_manifest?: ActivityManifestEntry[];
+    input_field_meta?: InputFieldMeta[];
+    original_prompt?: string;
+    category?: string;
+    tags?: string[];
+    metadata?: Record<string, unknown>;
+}
+/** Mutable state accumulated while building the YAML DAG. */
+export interface DagBuilder {
+    activities: Record<string, unknown>;
+    transitions: Record<string, Array<{
+        to: string;
+        conditions?: Record<string, unknown>;
+    }>>;
+    hooks: Record<string, Array<{
+        to: string;
+        conditions?: Record<string, unknown>;
+    }>>;
+    manifest: ActivityManifestEntry[];
+    stepIndexToActivityId: Map<number, string>;
+    prevActivityId: string;
+    prevResult: unknown;
+    lastPivotId: string | null;
+    triggerId: string;
+}
+/** A step extracted from an execution's event timeline. */
+export interface ExtractedStep {
+    /** Step kind: 'tool' for DB/MCP tool calls, 'llm' for LLM interpretation, 'signal' for waitFor pause/resume */
+    kind: 'tool' | 'llm' | 'signal';
+    toolName: string;
+    arguments: Record<string, unknown>;
+    result: unknown;
+    source: 'db' | 'mcp' | 'llm' | 'signal';
+    mcpServerId?: string;
+    /** For LLM steps: the system/user messages that produced this response */
+    promptMessages?: Array<{
+        role: string;
+        content: string;
+    }>;
+    /** For signal steps: the JSON Schema describing the expected signal payload */
+    signalSchema?: Record<string, unknown>;
+}
+/** Specifies how an iteration should be constructed in the YAML DAG. */
+export interface IterationSpec {
+    /** Index of the step that is the loop body (in the coreSteps list). */
+    bodyStepIndex: number;
+    /** Tool name for the iterated call. */
+    toolName: string;
+    /** Server ID for the iterated tool. */
+    serverId?: string;
+    /** Index of the step whose output produces the array to iterate. */
+    sourceStepIndex: number;
+    /** Dot-path to the array field in the source step's result. */
+    sourceField: string;
+    /** Keys that vary per iteration item. */
+    varyingKeys: string[];
+    /** Constant args shared across all iterations. */
+    constantArgs: Record<string, unknown>;
+    /**
+     * Key mappings: when the array item key doesn't match the tool's arg key.
+     * E.g., { url: 'href' } means tool wants 'url' but array items have 'href'.
+     * A null value means the key is computed/generated, not from the array.
+     */
+    keyMappings: Record<string, string | null>;
+}
+/** A directed edge in the data flow graph. */
+export interface DataFlowEdge {
+    /** Source: 'trigger' for user input, or step index. */
+    fromStep: number | 'trigger';
+    /** Source field name (dot-path in result or trigger input key). */
+    fromField: string;
+    /** Target step index. */
+    toStep: number;
+    /** Target argument key. */
+    toField: string;
+    /** Whether this is a session/handle wire (page_id, _handle). */
+    isSessionWire: boolean;
+    /**
+     * Transform spec when source format doesn't match target format.
+     * When present, the build stage inserts a reshape activity between
+     * the source and consuming step to apply field renames and defaults.
+     */
+    transform?: {
+        /** Per-field mapping: target key → source key. null = computed/not in source. */
+        fieldMap: Record<string, string | null>;
+        /** Static defaults to inject into each reshaped item. */
+        defaults?: Record<string, unknown>;
+        /** For computed fields (null in fieldMap): derivation hint. */
+        derivations?: Record<string, {
+            sourceKey: string;
+            strategy: 'slugify' | 'prefix' | 'template' | 'passthrough';
+            prefix?: string;
+            suffix?: string;
+            template?: string;
+        }>;
+    };
+}
+/** Per-step semantic annotation from the LLM. */
+export interface StepSpec {
+    /** Original step index (in the post-collapse list). */
+    index: number;
+    /** What this step accomplishes in the workflow. */
+    purpose: string;
+    /** Whether this step is core or exploratory. */
+    disposition: 'core' | 'exploratory';
+}
+/** Enhanced compilation plan — the LLM's full understanding of the workflow. */
+export interface EnhancedCompilationPlan {
+    /** Human-readable summary of workflow intent. */
+    intent: string;
+    /** Suggested workflow description for discovery. */
+    description: string;
+    /** Per-step annotations. */
+    steps: StepSpec[];
+    /** Indices of core steps to keep. */
+    coreStepIndices: number[];
+    /** Refined input field classifications. */
+    inputs: Array<{
+        key: string;
+        type: string;
+        classification: 'dynamic' | 'fixed';
+        description: string;
+        default?: unknown;
+    }>;
+    /** Iteration specifications (may be multiple for multi-loop workflows). */
+    iterations: IterationSpec[];
+    /** Data flow edges between steps. */
+    dataFlow: DataFlowEdge[];
+    /** Session/handle fields that must be threaded through the DAG. */
+    sessionFields: string[];
+    /** Whether the workflow contains iteration/looping patterns. */
+    hasIteration: boolean;
+}
+/** Options for YAML workflow generation. */
+export interface GenerateYamlOptions {
+    workflowId: string;
+    taskQueue: string;
+    workflowName: string;
+    /** User-chosen name for the YAML workflow */
+    name: string;
+    description?: string;
+    /** HotMesh app namespace (shared across flows). Defaults to 'longtail'. */
+    appId?: string;
+    /** Graph subscribes topic. Defaults to sanitized name. */
+    subscribes?: string;
+    /** Error from a prior failed deployment — triggers recompilation with this context. */
+    priorDeployError?: string;
+    /** YAML from the prior failed attempt. */
+    priorFailedYaml?: string;
+    /** User feedback describing issues with a prior compilation (wrong inputs, leaked details, etc.). */
+    compilationFeedback?: string;
+}
+/** Result from YAML workflow generation. */
+export interface GenerateYamlResult {
+    yaml: string;
+    inputSchema: Record<string, unknown>;
+    outputSchema: Record<string, unknown>;
+    activityManifest: ActivityManifestEntry[];
+    graphTopic: string;
+    appId: string;
+    tags: string[];
+    inputFieldMeta: InputFieldMeta[];
+    originalPrompt: string;
+    category: string;
+    /** LLM compilation plan (null if LLM unavailable or skipped). */
+    compilationPlan: EnhancedCompilationPlan | null;
+    /** Validation issues found after compilation (empty = clean). */
+    validationIssues: string[];
+}
+/** Shared context accumulated through pipeline stages. */
+export interface PipelineContext {
+    options: {
+        workflowId: string;
+        taskQueue: string;
+        workflowName: string;
+        name: string;
+        description?: string;
+        appId: string;
+        subscribes: string;
+    };
+    execution: WorkflowExecution;
+    originalPrompt: string;
+    rawSteps: ExtractedStep[];
+    collapsedSteps: ExtractedStep[];
+    patternAnnotations: PatternAnnotation[];
+    naiveInputs: InputFieldMeta[];
+    compilationPlan: EnhancedCompilationPlan | null;
+    /** Steps after filtering out exploratory ones per the compilation plan. */
+    coreSteps: ExtractedStep[];
+    /** Final refined input field metadata. */
+    refinedInputs: InputFieldMeta[];
+    yaml: string;
+    inputSchema: Record<string, unknown>;
+    outputSchema: Record<string, unknown>;
+    activityManifest: ActivityManifestEntry[];
+    tags: string[];
+    category: string;
+    validationIssues: string[];
+    /** Error message from a prior failed deployment attempt. */
+    priorDeployError?: string;
+    /** YAML that failed deployment (for reference in the retry prompt). */
+    priorFailedYaml?: string;
+}

package/build/services/yaml-workflow/types.js

@@ -0,0 +1,8 @@
+"use strict";
+/**
+ * Type definitions for the YAML workflow service.
+ *
+ * Includes types for the database layer, the 5-stage compilation pipeline
+ * (extract → analyze → compile → build → validate), and the DAG builder.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });

package/build/services/yaml-workflow/workers/callbacks.d.ts

@@ -0,0 +1,17 @@
+import type { StreamData, StreamDataResponse } from '@hotmeshio/hotmesh/build/types/stream';
+import type { ActivityManifestEntry } from '../../../types/yaml-workflow';
+/**
+ * Compact input data for LLM consumption: truncate large arrays and
+ * strip fields that are unhelpful for summarization (ids, trace data).
+ */
+export declare function compactForLlm(input: Record<string, unknown>): Record<string, unknown>;
+/**
+ * Build an LLM worker callback that interpolates a prompt template with
+ * input data and calls the LLM for interpretation/synthesis.
+ */
+export declare function buildLlmCallback(activity: ActivityManifestEntry): (data: StreamData) => Promise<StreamDataResponse>;
+/**
+ * Build a transform worker callback that reshapes array data between steps.
+ * Applies field renames, defaults, and derivations.
+ */
+export declare function buildTransformCallback(activity: ActivityManifestEntry): (data: StreamData) => Promise<StreamDataResponse>;

package/build/services/yaml-workflow/workers/callbacks.js

@@ -0,0 +1,194 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.compactForLlm = compactForLlm;
+exports.buildLlmCallback = buildLlmCallback;
+exports.buildTransformCallback = buildTransformCallback;
+const defaults_1 = require("../../../modules/defaults");
+const logger_1 = require("../../logger");
+const llm_1 = require("../../llm");
+/** Call the LLM with messages and optional format options. */
+async function callWorkerLLM(messages, options) {
+    return (0, llm_1.callLLM)({
+        model: defaults_1.LLM_MODEL_SECONDARY,
+        max_tokens: options?.max_tokens ?? defaults_1.LLM_MAX_TOKENS_JSON,
+        messages,
+        ...(options?.response_format ? { response_format: options.response_format } : {}),
+    });
+}
+/**
+ * Compact input data for LLM consumption: truncate large arrays and
+ * strip fields that are unhelpful for summarization (ids, trace data).
+ */
+function compactForLlm(input) {
+    const omitKeys = new Set(['trace_id', 'span_id', 'resolved_at']);
+    const compact = (val) => {
+        if (Array.isArray(val)) {
+            const mapped = val.map(compact);
+            if (mapped.length > defaults_1.LLM_MAX_ARRAY_ITEMS) {
+                return [...mapped.slice(0, defaults_1.LLM_MAX_ARRAY_ITEMS), `... (${mapped.length - defaults_1.LLM_MAX_ARRAY_ITEMS} more)`];
+            }
+            return mapped;
+        }
+        if (val && typeof val === 'object') {
+            const out = {};
+            for (const [k, v] of Object.entries(val)) {
+                if (!omitKeys.has(k))
+                    out[k] = compact(v);
+            }
+            return out;
+        }
+        return val;
+    };
+    return compact(input);
+}
+/**
+ * Build an LLM worker callback that interpolates a prompt template with
+ * input data and calls the LLM for interpretation/synthesis.
+ */
+function buildLlmCallback(activity) {
+    return async (data) => {
+        const rawInput = (data.data || {});
+        const input = compactForLlm(rawInput);
+        const template = activity.prompt_template || '';
+        const model = activity.model || defaults_1.LLM_MODEL_SECONDARY;
+        // Serialize and enforce hard character limit
+        let inputJson = JSON.stringify(input, null, 2);
+        if (inputJson.length > defaults_1.LLM_MAX_INPUT_CHARS) {
+            inputJson = inputJson.slice(0, defaults_1.LLM_MAX_INPUT_CHARS) + '\n... (truncated)';
+        }
+        // Parse the template into messages. Format: [role]\ncontent\n\n[role]\ncontent
+        const messages = [];
+        const parts = template.split(/\n\n(?=\[(?:system|user|assistant)\])/);
+        for (const part of parts) {
+            const roleMatch = part.match(/^\[(\w+)\]\n([\s\S]*)$/);
+            if (roleMatch) {
+                let content = roleMatch[2];
+                // Interpolate {field} placeholders with input data
+                // {input_data} is a special placeholder for the full JSON input
+                content = content.replace(/\{input_data\}/g, inputJson);
+                content = content.replace(/\{(\w+)\}/g, (_, key) => {
+                    if (key in input)
+                        return String(input[key]);
+                    return `{${key}}`;
+                });
+                messages.push({ role: roleMatch[1], content });
+            }
+            else if (part.trim()) {
+                messages.push({ role: 'user', content: part.trim() });
+            }
+        }
+        if (messages.length === 0) {
+            messages.push({ role: 'user', content: `Analyze the following data:\n${inputJson}` });
+        }
+        // Call the LLM with JSON mode for structured output
+        const response = await callWorkerLLM(messages, {
+            max_tokens: defaults_1.LLM_MAX_TOKENS_JSON,
+            response_format: { type: 'json_object' },
+        });
+        const content = response.content || '';
+        // Try to parse JSON from the response
+        let result;
+        try {
+            const cleaned = content
+                .replace(/^```(?:json)?\s*/m, '')
+                .replace(/\s*```$/m, '')
+                .trim();
+            result = JSON.parse(cleaned);
+        }
+        catch {
+            result = { response: content };
+        }
+        logger_1.loggerRegistry.info(`[yaml-workflow] LLM step completed (model: ${model}, topic: ${activity.topic})`);
+        return {
+            metadata: { ...data.metadata },
+            data: result,
+        };
+    };
+}
+/**
+ * Apply a derivation strategy to produce a computed value from a source string.
+ */
+function applyDerivation(value, spec) {
+    let result = value;
+    switch (spec.strategy) {
+        case 'slugify': {
+            // Extract path from URL if it looks like a URL, otherwise use raw value
+            try {
+                const url = new URL(result);
+                result = url.pathname.replace(/^\//, '').replace(/\//g, '-') || 'home';
+            }
+            catch {
+                result = result.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '');
+            }
+            if (spec.prefix)
+                result = spec.prefix + result;
+            if (spec.suffix)
+                result = result + spec.suffix;
+            break;
+        }
+        case 'prefix':
+            result = (spec.prefix || '') + result + (spec.suffix || '');
+            break;
+        case 'template':
+            result = (spec.template || '{value}').replace(/\{value\}/g, result);
+            break;
+        case 'passthrough':
+            break;
+    }
+    return result;
+}
+/**
+ * Build a transform worker callback that reshapes array data between steps.
+ * Applies field renames, defaults, and derivations.
+ */
+function buildTransformCallback(activity) {
+    const spec = activity.transform_spec;
+    if (!spec)
+        throw new Error(`Transform activity ${activity.activity_id} missing transform_spec`);
+    return async (data) => {
+        const input = (data.data || {});
+        const sourceData = input[spec.sourceField];
+        if (!Array.isArray(sourceData)) {
+            // Pass through non-array data unchanged
+            return {
+                metadata: { ...data.metadata },
+                data: { [spec.targetField]: sourceData, ...input },
+            };
+        }
+        // Resolve dynamic directory prefix from trigger inputs (e.g., screenshot_dir)
+        const dirKeys = ['screenshot_dir', 'screenshots_dir', 'output_dir'];
+        const dynamicDir = dirKeys.reduce((found, key) => found || (input[key] ? String(input[key]) : null), null);
+        const reshaped = sourceData.map((item) => {
+            const out = {};
+            // Apply field map: target key -> source key
+            for (const [targetKey, sourceKey] of Object.entries(spec.fieldMap)) {
+                if (sourceKey !== null) {
+                    out[targetKey] = item[sourceKey];
+                }
+                else if (spec.derivations?.[targetKey]) {
+                    // Computed field: use dynamic dir when available for path derivations
+                    const derivation = { ...spec.derivations[targetKey] };
+                    if (dynamicDir && targetKey.includes('path')) {
+                        derivation.prefix = dynamicDir.replace(/\/$/, '') + '/';
+                    }
+                    const sourceValue = String(item[derivation.sourceKey] || '');
+                    out[targetKey] = applyDerivation(sourceValue, derivation);
+                }
+            }
+            // Apply defaults
+            if (spec.defaults) {
+                for (const [key, value] of Object.entries(spec.defaults)) {
+                    if (!(key in out))
+                        out[key] = value;
+                }
+            }
+            return out;
+        });
+        // Return reshaped data alongside any other input fields (session handles, etc.)
+        const result = { ...input, [spec.targetField]: reshaped };
+        return {
+            metadata: { ...data.metadata },
+            data: result,
+        };
+    };
+}

package/build/services/yaml-workflow/workers/events.d.ts

@@ -0,0 +1,7 @@
+import type { StreamData, StreamDataResponse } from '@hotmeshio/hotmesh/build/types/stream';
+import type { ActivityManifestEntry } from '../../../types/yaml-workflow';
+/**
+ * Wrap a worker callback with activity lifecycle event publishing.
+ * Publishes activity.started before and activity.completed/failed after.
+ */
+export declare function wrapWithEvents(activity: ActivityManifestEntry, appId: string, stepIndex: number, totalSteps: number, callback: (data: StreamData) => Promise<StreamDataResponse>): (data: StreamData) => Promise<StreamDataResponse>;