@hotmeshio/long-tail 0.1.0

package/build/services/yaml-workflow/pipeline/build/dag.js

@@ -0,0 +1,254 @@
+"use strict";
+/**
+ * DAG construction helpers for the build pipeline stage.
+ *
+ * Encapsulates the mutable state of DAG assembly (activities, transitions,
+ * manifest) behind semantic operations so the main build() function reads
+ * as a high-level pipeline.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.initializeDag = initializeDag;
+exports.appendIterationStep = appendIterationStep;
+exports.appendNormalStep = appendNormalStep;
+exports.appendSignalStep = appendSignalStep;
+exports.serializeToYaml = serializeToYaml;
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const yaml = require('js-yaml');
+const defaults_1 = require("../../../../modules/defaults");
+const utils_1 = require("./utils");
+const wiring_1 = require("./wiring");
+const iteration_1 = require("./iteration");
+const transform_1 = require("./transform");
+// ── DAG lifecycle ────────────────────────────────────────────────────────────
+/** Create the DAG with its trigger activity. */
+function initializeDag(prefix, graphTopic, inputSchema) {
+    const triggerId = `${prefix}_t1`;
+    const activities = {};
+    const transitions = {};
+    activities[triggerId] = {
+        title: 'Trigger',
+        type: 'trigger',
+        output: { schema: { type: 'object' } },
+    };
+    const manifest = [{
+            activity_id: triggerId,
+            title: 'Trigger',
+            type: 'trigger',
+            tool_source: 'trigger',
+            topic: graphTopic,
+            input_mappings: {},
+            output_fields: Object.keys(inputSchema.properties || {}),
+        }];
+    return {
+        activities,
+        transitions,
+        hooks: {},
+        manifest,
+        stepIndexToActivityId: new Map(),
+        prevActivityId: triggerId,
+        prevResult: null,
+        lastPivotId: null,
+        triggerId,
+    };
+}
+// ── Step appenders ───────────────────────────────────────────────────────────
+/** Append an iteration pattern (hook → worker → cycle → done) to the DAG. */
+function appendIterationStep(dag, iterSpec, prefix, graphTopic, idx, step, triggerInputKeys, steps, sessionFields, plan) {
+    const iterResult = (0, iteration_1.buildIterationActivities)(iterSpec, prefix, graphTopic, idx, step, dag.stepIndexToActivityId, dag.triggerId, triggerInputKeys, steps, sessionFields, plan);
+    Object.assign(dag.activities, iterResult.activities);
+    Object.assign(dag.transitions, iterResult.transitions);
+    dag.manifest.push(...iterResult.manifest);
+    dag.transitions[dag.prevActivityId] = [{ to: iterResult.pivotId }];
+    dag.prevActivityId = iterResult.doneId;
+    dag.prevResult = step.result;
+    dag.lastPivotId = iterResult.pivotId;
+    dag.stepIndexToActivityId.set(idx, iterResult.pivotId);
+}
+/** Append a normal (non-iteration) step activity to the DAG. */
+function appendNormalStep(dag, idx, step, prefix, graphTopic, plan, steps, triggerInputKeys, collapsedToCoreIndex) {
+    const actId = `${prefix}_a${idx + 1}`;
+    dag.stepIndexToActivityId.set(idx, actId);
+    dag.lastPivotId = null;
+    const workflowName = step.kind === 'llm' ? 'interpret' : step.toolName;
+    const topic = graphTopic;
+    const title = step.kind === 'llm'
+        ? 'LLM Interpret'
+        : step.toolName.replace(/_/g, ' ').replace(/\b\w/g, (c) => c.toUpperCase());
+    // Insert any transform (reshape) activities before this step
+    if (plan) {
+        dag.prevActivityId = (0, transform_1.insertTransformActivities)(idx, plan, steps, prefix, graphTopic, dag.triggerId, triggerInputKeys, dag.stepIndexToActivityId, collapsedToCoreIndex, dag.activities, dag.transitions, dag.manifest, dag.prevActivityId);
+    }
+    // Wire input mappings from upstream steps
+    const inputMappings = (0, wiring_1.wireStepInputs)(idx, step, plan, dag.stepIndexToActivityId, dag.triggerId, triggerInputKeys, steps, prefix, dag.prevActivityId, dag.prevResult, collapsedToCoreIndex);
+    // Build the activity definition
+    const resultSchema = step.result ? (0, utils_1.inferSchema)(step.result) : { type: 'object' };
+    const outputFields = extractOutputFields(step);
+    const jobMaps = buildJobMaps(idx, steps.length, outputFields);
+    // Thread _scope from trigger through every activity for IAM context
+    inputMappings._scope = `{${dag.triggerId}.output.data._scope}`;
+    // Set workflowName for singleton consumer dispatch routing
+    inputMappings.workflowName = workflowName;
+    dag.activities[actId] = {
+        title,
+        type: 'worker',
+        topic,
+        input: {
+            schema: { type: 'object' },
+            ...(Object.keys(inputMappings).length > 0 ? { maps: inputMappings } : {}),
+        },
+        output: { schema: resultSchema },
+        ...(jobMaps ? { job: { maps: jobMaps } } : {}),
+    };
+    // Build manifest entry
+    dag.manifest.push(buildManifestEntry(actId, title, step, topic, workflowName, inputMappings, outputFields));
+    // Transition from previous step
+    dag.transitions[dag.prevActivityId] = [{ to: actId }];
+    dag.prevActivityId = actId;
+    dag.prevResult = step.result;
+}
+/**
+ * Append a signal step pair (escalation worker + hook) to the DAG.
+ *
+ * The escalation worker creates the escalation via the human-queue MCP tool.
+ * The hook pauses until the escalation is resolved and a signal arrives.
+ * A hooks section entry routes the external signal to the waiting hook.
+ */
+function appendSignalStep(dag, idx, step, prefix, graphTopic, plan, steps, triggerInputKeys, collapsedToCoreIndex) {
+    // The step before this signal step is the escalation tool call.
+    // The signal step itself represents the pause/resume point.
+    const hookId = `${prefix}_wait${idx + 1}`;
+    dag.stepIndexToActivityId.set(idx, hookId);
+    dag.lastPivotId = null;
+    const signalSchema = step.signalSchema || { type: 'object', properties: {} };
+    const outputFields = signalSchema.properties
+        ? Object.keys(signalSchema.properties)
+        : [];
+    // Build job maps: map each signal field from hook data to job state
+    const jobMaps = {};
+    for (const field of outputFields) {
+        jobMaps[field] = `{$self.hook.data.${field}}`;
+    }
+    // Hook topic: scoped per graph, per activity for multiple hooks
+    const hookTopic = `escalation.resolved.${graphTopic}.${hookId}`;
+    // The hook activity — pauses until signal arrives
+    dag.activities[hookId] = {
+        title: 'Wait for Human Input',
+        type: 'hook',
+        hook: signalSchema,
+        output: {
+            schema: signalSchema,
+        },
+        ...(Object.keys(jobMaps).length > 0 ? { job: { maps: jobMaps } } : {}),
+    };
+    // Find the escalation worker that precedes this signal step.
+    // It's the previous activity in the DAG — its output.data.escalationId
+    // is used as the match condition for routing signals.
+    const escalationActId = dag.prevActivityId;
+    // Patch the preceding escalation worker to persist escalationId to the job hash.
+    // Without job.maps, the hook's match condition can't resolve {a1.output.data.escalationId}.
+    const prevActivity = dag.activities[escalationActId];
+    if (prevActivity && prevActivity.type === 'worker') {
+        prevActivity.job = {
+            ...(prevActivity.job || {}),
+            maps: {
+                ...(prevActivity.job?.maps || {}),
+                escalationId: '{$self.output.data.escalationId}',
+                signalId: '{$self.output.data.signalId}',
+            },
+        };
+    }
+    // Hook rules: route signals to this hook with job ID match.
+    // Use {$job.metadata.jid} (always available) as the match key.
+    // The signal must include the job_id in its payload for routing.
+    dag.hooks[hookTopic] = [{
+            to: hookId,
+            conditions: {
+                match: [{
+                        expected: '{$job.metadata.jid}',
+                        actual: '{$self.hook.data.job_id}',
+                    }],
+            },
+        }];
+    // Manifest entry
+    dag.manifest.push({
+        activity_id: hookId,
+        title: 'Wait for Human Input',
+        type: 'hook',
+        tool_source: 'signal',
+        topic: graphTopic,
+        hook_topic: hookTopic,
+        signal_schema: signalSchema,
+        input_mappings: {},
+        output_fields: outputFields,
+    });
+    // Transition from previous step (the escalation worker) to the hook
+    dag.transitions[dag.prevActivityId] = [{ to: hookId }];
+    dag.prevActivityId = hookId;
+    dag.prevResult = step.result;
+}
+// ── Serialization ────────────────────────────────────────────────────────────
+/** Assemble the DAG into a HotMesh YAML document. */
+function serializeToYaml(appId, graphTopic, inputSchema, outputSchema, dag) {
+    const graph = {
+        subscribes: graphTopic,
+        expire: defaults_1.WORKFLOW_EXPIRE_SECS,
+        input: { schema: inputSchema },
+        output: { schema: outputSchema },
+        activities: dag.activities,
+        transitions: dag.transitions,
+    };
+    // Include hooks section when signal steps are present
+    if (Object.keys(dag.hooks).length > 0) {
+        graph.hooks = dag.hooks;
+    }
+    const graphDef = {
+        app: {
+            id: appId,
+            version: '1',
+            graphs: [graph],
+        },
+    };
+    return yaml.dump(graphDef, {
+        lineWidth: defaults_1.YAML_LINE_WIDTH,
+        noRefs: true,
+        sortKeys: false,
+    });
+}
+// ── Internal helpers ─────────────────────────────────────────────────────────
+function extractOutputFields(step) {
+    if (step.result && typeof step.result === 'object' && !Array.isArray(step.result)) {
+        return Object.keys(step.result);
+    }
+    return [];
+}
+function buildJobMaps(stepIdx, totalSteps, outputFields) {
+    if (stepIdx !== totalSteps - 1 || outputFields.length === 0)
+        return undefined;
+    return outputFields.reduce((acc, field) => {
+        acc[field] = `{$self.output.data.${field}}`;
+        return acc;
+    }, {});
+}
+function buildManifestEntry(actId, title, step, topic, workflowName, inputMappings, outputFields) {
+    const promptTemplate = step.kind === 'llm' && step.promptMessages
+        ? step.promptMessages.map((m) => `[${m.role}]\n${m.content}`).join('\n\n')
+        : undefined;
+    return {
+        activity_id: actId,
+        title,
+        type: 'worker',
+        tool_source: step.source,
+        topic,
+        workflow_name: workflowName,
+        ...(step.kind === 'tool' ? {
+            mcp_server_id: step.source === 'mcp' ? step.mcpServerId : 'db',
+            mcp_tool_name: step.toolName,
+            tool_arguments: Object.keys(step.arguments).length > 0
+                ? (0, utils_1.capToolArguments)(step.arguments) : undefined,
+        } : {}),
+        input_mappings: inputMappings,
+        output_fields: outputFields,
+        ...(promptTemplate ? { prompt_template: promptTemplate } : {}),
+        ...(step.kind === 'llm' ? { model: defaults_1.LLM_MODEL_SECONDARY } : {}),
+    };
+}

package/build/services/yaml-workflow/pipeline/build/index.d.ts

@@ -0,0 +1,20 @@
+/**
+ * Build stage: construct the HotMesh YAML DAG from the compilation plan.
+ *
+ * When an EnhancedCompilationPlan is available, uses its iteration specs,
+ * data flow edges, and key mappings to build a faithful deterministic DAG.
+ * Falls back to mechanical heuristics when no plan is available.
+ */
+import type { PipelineContext } from '../../types';
+export { capToolArguments, sanitizeName } from './utils';
+/**
+ * Build pipeline stage: construct the HotMesh YAML DAG.
+ *
+ * Reads as a five-phase pipeline:
+ *   1. Prepare inputs (schema, session fields, index mapping)
+ *   2. Initialize the DAG with a trigger activity
+ *   3. Append each step as an iteration or normal activity
+ *   4. Serialize to YAML
+ *   5. Derive metadata (tags, category)
+ */
+export declare function build(ctx: PipelineContext): Promise<PipelineContext>;

package/build/services/yaml-workflow/pipeline/build/index.js

@@ -0,0 +1,115 @@
+"use strict";
+/**
+ * Build stage: construct the HotMesh YAML DAG from the compilation plan.
+ *
+ * When an EnhancedCompilationPlan is available, uses its iteration specs,
+ * data flow edges, and key mappings to build a faithful deterministic DAG.
+ * Falls back to mechanical heuristics when no plan is available.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.sanitizeName = exports.capToolArguments = void 0;
+exports.build = build;
+const input_analyzer_1 = require("../../input-analyzer");
+const utils_1 = require("./utils");
+const metadata_1 = require("./metadata");
+const iteration_1 = require("./iteration");
+const dag_1 = require("./dag");
+// Re-export public utilities used by other modules
+var utils_2 = require("./utils");
+Object.defineProperty(exports, "capToolArguments", { enumerable: true, get: function () { return utils_2.capToolArguments; } });
+Object.defineProperty(exports, "sanitizeName", { enumerable: true, get: function () { return utils_2.sanitizeName; } });
+/**
+ * Build pipeline stage: construct the HotMesh YAML DAG.
+ *
+ * Reads as a five-phase pipeline:
+ *   1. Prepare inputs (schema, session fields, index mapping)
+ *   2. Initialize the DAG with a trigger activity
+ *   3. Append each step as an iteration or normal activity
+ *   4. Serialize to YAML
+ *   5. Derive metadata (tags, category)
+ */
+async function build(ctx) {
+    const { options, coreSteps: steps, refinedInputs, compilationPlan: plan } = ctx;
+    // 1. Prepare inputs
+    const inputSchema = (0, input_analyzer_1.buildEnrichedInputSchema)(refinedInputs);
+    const triggerInputKeys = new Set(refinedInputs.map(f => f.key));
+    const sessionFields = resolveSessionFields(plan);
+    const collapsedToCoreIndex = buildCoreStepIndexMap(plan, ctx.collapsedSteps);
+    const outputSchema = inferOutputSchema(steps);
+    const prefix = buildActivityPrefix(options.name);
+    // 2. Initialize DAG with trigger
+    const dag = (0, dag_1.initializeDag)(prefix, options.subscribes, inputSchema);
+    // 3. Build each step — iteration, signal, or normal
+    steps.forEach((step, idx) => {
+        if (step.kind === 'signal') {
+            (0, dag_1.appendSignalStep)(dag, idx, step, prefix, options.subscribes, plan, steps, triggerInputKeys, collapsedToCoreIndex);
+            return;
+        }
+        const iterSpec = (0, iteration_1.findIterationSpec)(idx, plan, step, collapsedToCoreIndex);
+        if (iterSpec) {
+            (0, dag_1.appendIterationStep)(dag, iterSpec, prefix, options.subscribes, idx, step, triggerInputKeys, steps, sessionFields, plan);
+        }
+        else {
+            (0, dag_1.appendNormalStep)(dag, idx, step, prefix, options.subscribes, plan, steps, triggerInputKeys, collapsedToCoreIndex);
+        }
+    });
+    // 4. Serialize to YAML
+    ctx.yaml = (0, dag_1.serializeToYaml)(options.appId, options.subscribes, inputSchema, outputSchema, dag);
+    ctx.inputSchema = inputSchema;
+    ctx.outputSchema = outputSchema;
+    ctx.activityManifest = dag.manifest;
+    // 5. Derive metadata
+    ctx.tags = (0, metadata_1.deriveTagsFromSteps)(steps, options.name, options.description);
+    ctx.category = await (0, metadata_1.deriveCategory)(steps);
+    return ctx;
+}
+// ── Preparation helpers ──────────────────────────────────────────────────────
+/** Resolve session fields from the compilation plan or use sensible defaults. */
+function resolveSessionFields(plan) {
+    return plan?.sessionFields?.length
+        ? plan.sessionFields
+        : ['page_id', '_handle', 'session_id'];
+}
+/**
+ * Build a remapping from collapsed-step indices to core-step indices.
+ *
+ * The compilation plan uses indices from collapsedSteps, but after core-step
+ * filtering those indices shift. This map translates between the two spaces.
+ */
+function buildCoreStepIndexMap(plan, collapsedSteps) {
+    const map = new Map();
+    if (plan && plan.coreStepIndices.length > 0 && plan.coreStepIndices.length < collapsedSteps.length) {
+        plan.coreStepIndices.forEach((collapsedIdx, coreIdx) => {
+            map.set(collapsedIdx, coreIdx);
+        });
+    }
+    return map;
+}
+/** Infer an output schema from the last step's result. */
+function inferOutputSchema(steps) {
+    const lastStep = steps[steps.length - 1];
+    return lastStep?.result
+        ? (0, utils_1.inferSchema)(lastStep.result)
+        : { type: 'object' };
+}
+/**
+ * Build a sanitized activity-ID prefix from the graph topic.
+ *
+ * The prefix must be unique per graph to avoid activity ID collisions
+ * when multiple graphs are merged into one app deployment. We use the
+ * full sanitized topic (no truncation) plus a 4-char hash suffix when
+ * the topic exceeds 20 chars, ensuring uniqueness even for topics that
+ * share a long common prefix.
+ */
+function buildActivityPrefix(graphTopic) {
+    const raw = graphTopic.replace(/[^a-z0-9]/g, '_');
+    if (raw.length <= 20)
+        return raw;
+    // Short hash suffix to guarantee uniqueness across similar topics
+    let hash = 0;
+    for (let i = 0; i < graphTopic.length; i++) {
+        hash = ((hash << 5) - hash + graphTopic.charCodeAt(i)) | 0;
+    }
+    const suffix = Math.abs(hash).toString(36).slice(0, 4);
+    return `${raw.slice(0, 16)}_${suffix}`;
+}

package/build/services/yaml-workflow/pipeline/build/iteration.d.ts

@@ -0,0 +1,25 @@
+/**
+ * Iteration pattern builders for the build pipeline stage.
+ */
+import type { ActivityManifestEntry } from '../../../../types/yaml-workflow';
+import type { ExtractedStep, EnhancedCompilationPlan, IterationSpec } from '../../types';
+/**
+ * Build iteration activities (hook -> worker -> cycle -> done) from an IterationSpec.
+ *
+ * When key mappings are available (from the compilation plan), uses the mapped
+ * key names in the @pipe transforms so array item keys correctly map to tool args.
+ */
+export declare function buildIterationActivities(spec: IterationSpec, prefix: string, graphTopic: string, idx: number, step: ExtractedStep, stepIndexToActivityId: Map<number, string>, triggerId: string, triggerInputKeys: Set<string>, steps: ExtractedStep[], sessionFields: string[], plan: EnhancedCompilationPlan | null): {
+    activities: Record<string, unknown>;
+    transitions: Record<string, Array<{
+        to: string;
+        conditions?: Record<string, unknown>;
+    }>>;
+    manifest: ActivityManifestEntry[];
+    pivotId: string;
+    doneId: string;
+};
+/**
+ * Check if a step should be rendered as an iteration based on the compilation plan.
+ */
+export declare function findIterationSpec(stepIdx: number, plan: EnhancedCompilationPlan | null, step: ExtractedStep, collapsedToCoreIndex?: Map<number, number>): IterationSpec | null;

package/build/services/yaml-workflow/pipeline/build/iteration.js

@@ -0,0 +1,231 @@
+"use strict";
+/**
+ * Iteration pattern builders for the build pipeline stage.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.buildIterationActivities = buildIterationActivities;
+exports.findIterationSpec = findIterationSpec;
+const utils_1 = require("./utils");
+/**
+ * Build iteration activities (hook -> worker -> cycle -> done) from an IterationSpec.
+ *
+ * When key mappings are available (from the compilation plan), uses the mapped
+ * key names in the @pipe transforms so array item keys correctly map to tool args.
+ */
+function buildIterationActivities(spec, prefix, graphTopic, idx, step, stepIndexToActivityId, triggerId, triggerInputKeys, steps, sessionFields, plan) {
+    const pivotId = `${prefix}_pivot${idx + 1}`;
+    const workerId = `${prefix}_iter${idx + 1}`;
+    const cycleId = `${prefix}_cycle${idx + 1}`;
+    const doneId = `${prefix}_done${idx + 1}`;
+    const originalTool = spec.toolName;
+    const workerTopic = graphTopic;
+    // Determine the array source reference
+    const sourceActId = stepIndexToActivityId.get(spec.sourceStepIndex);
+    const arraySourceRef = sourceActId
+        ? `{${sourceActId}.output.data.${spec.sourceField}}`
+        : `{${prefix}_a${spec.sourceStepIndex + 1}.output.data.${spec.sourceField}}`;
+    // Hook (cycle anchor): initializes index and holds the items array
+    const activities = {};
+    activities[pivotId] = {
+        title: `Iterate ${(0, utils_1.humanize)(originalTool)}`,
+        type: 'hook',
+        cycle: true,
+        output: {
+            maps: {
+                index: 0,
+                items: arraySourceRef,
+            },
+        },
+    };
+    // Worker: maps varying keys from the current array item
+    const workerInputMaps = {};
+    for (const key of spec.varyingKeys) {
+        // Use key mapping if available: tool wants 'url' but array items have 'href'
+        const arrayItemKey = spec.keyMappings[key];
+        if (arrayItemKey === null) {
+            // Computed/generated key — not directly from the array.
+            // 1. Check if the trigger provides it
+            if (triggerInputKeys.has(key)) {
+                workerInputMaps[key] = `{${triggerId}.output.data.${key}}`;
+                continue;
+            }
+            // 2. Try to find a semantically related field in the array items
+            //    e.g., tool wants 'path' -> array items have 'name' (path contains name)
+            const sourceActId = stepIndexToActivityId.get(spec.sourceStepIndex);
+            if (sourceActId) {
+                const sourceStep = steps.find((_, si) => stepIndexToActivityId.get(si) === sourceActId);
+                if (sourceStep?.result && typeof sourceStep.result === 'object') {
+                    const resultObj = sourceStep.result;
+                    const arrayField = spec.sourceField.split('.').reduce((obj, k) => obj?.[k], resultObj);
+                    if (Array.isArray(arrayField) && arrayField.length > 0 && typeof arrayField[0] === 'object') {
+                        const itemKeys = Object.keys(arrayField[0]);
+                        const match = itemKeys.find(ik => (0, utils_1.keysRelated)(key, ik));
+                        if (match) {
+                            workerInputMaps[key] = {
+                                '@pipe': [
+                                    [`{${pivotId}.output.data.items}`, `{${pivotId}.output.data.index}`],
+                                    ['{@array.get}'],
+                                    [`{@object.get}`, match],
+                                ],
+                            };
+                            continue;
+                        }
+                    }
+                }
+            }
+            continue;
+        }
+        const lookupKey = arrayItemKey || key;
+        workerInputMaps[key] = {
+            '@pipe': [
+                [`{${pivotId}.output.data.items}`, `{${pivotId}.output.data.index}`],
+                ['{@array.get}'],
+                [`{@object.get}`, lookupKey],
+            ],
+        };
+    }
+    // Wire constant args from trigger or directly
+    for (const [key, value] of Object.entries(spec.constantArgs)) {
+        if (key === '_iteration')
+            continue;
+        if (triggerInputKeys.has(key)) {
+            workerInputMaps[key] = `{${triggerId}.output.data.${key}}`;
+        }
+        else {
+            workerInputMaps[key] = value;
+        }
+    }
+    // Wire session fields — plan-driven first, then mechanical backward scan
+    for (const wiredKey of sessionFields) {
+        if (workerInputMaps[wiredKey])
+            continue; // already mapped
+        // Plan-driven: use data flow edges for session wiring
+        if (plan && plan.dataFlow.length > 0) {
+            const edge = plan.dataFlow.find(e => e.toStep === spec.bodyStepIndex && e.toField === wiredKey && e.isSessionWire);
+            if (edge && edge.fromStep !== 'trigger') {
+                const sourceActId = stepIndexToActivityId.get(edge.fromStep);
+                if (sourceActId) {
+                    workerInputMaps[wiredKey] = `{${sourceActId}.output.data.${wiredKey}}`;
+                    continue;
+                }
+            }
+        }
+        // Mechanical fallback: walk backward to find upstream producer
+        for (let si = idx - 1; si >= 0; si--) {
+            const priorResult = steps[si].result;
+            if (priorResult && typeof priorResult === 'object' && !Array.isArray(priorResult) &&
+                wiredKey in priorResult) {
+                const priorActId = stepIndexToActivityId.get(si) || `${prefix}_a${si + 1}`;
+                workerInputMaps[wiredKey] = `{${priorActId}.output.data.${wiredKey}}`;
+                break;
+            }
+        }
+    }
+    // Thread _scope from trigger for IAM context
+    workerInputMaps._scope = `{${triggerId}.output.data._scope}`;
+    // Set workflowName for singleton consumer dispatch routing
+    workerInputMaps.workflowName = originalTool;
+    const resultSchema = step.result ? (0, utils_1.inferSchema)(step.result) : { type: 'object' };
+    activities[workerId] = {
+        title: (0, utils_1.humanize)(originalTool),
+        type: 'worker',
+        topic: workerTopic,
+        input: {
+            schema: { type: 'object' },
+            maps: workerInputMaps,
+        },
+        output: { schema: resultSchema },
+    };
+    // Cycle: increment index and loop back to pivot
+    activities[cycleId] = {
+        title: 'Next Item',
+        type: 'cycle',
+        ancestor: pivotId,
+        input: {
+            maps: {
+                index: {
+                    '@pipe': [
+                        [`{${pivotId}.output.data.index}`, 1],
+                        ['{@math.add}'],
+                    ],
+                },
+            },
+        },
+    };
+    // Done: exit hook after iteration completes
+    activities[doneId] = {
+        title: 'Iteration Complete',
+        type: 'hook',
+    };
+    // Transitions
+    const transitions = {};
+    transitions[pivotId] = [{ to: workerId }];
+    transitions[workerId] = [
+        {
+            to: cycleId,
+            conditions: {
+                match: [{
+                        expected: true,
+                        actual: {
+                            '@pipe': [
+                                // Subpipe 1: compute next_index = index + 1
+                                { '@pipe': [
+                                        [`{${pivotId}.output.data.index}`, 1],
+                                        ['{@math.add}'],
+                                    ] },
+                                // Subpipe 2: compute items.length
+                                { '@pipe': [
+                                        [`{${pivotId}.output.data.items}`],
+                                        ['{@array.length}'],
+                                    ] },
+                                // Row: compare next_index < items.length
+                                ['{@conditional.less_than}'],
+                            ],
+                        },
+                    }],
+            },
+        },
+        { to: doneId },
+    ];
+    // Manifest entries
+    const manifest = [
+        { activity_id: pivotId, title: `Iterate ${(0, utils_1.humanize)(originalTool)}`, type: 'worker', tool_source: 'trigger', topic: graphTopic, input_mappings: {}, output_fields: ['index', 'items'] },
+        { activity_id: workerId, title: (0, utils_1.humanize)(originalTool), type: 'worker', tool_source: step.source, topic: workerTopic, workflow_name: originalTool, mcp_server_id: spec.serverId, mcp_tool_name: originalTool, input_mappings: workerInputMaps, output_fields: [] },
+        { activity_id: cycleId, title: 'Next Item', type: 'worker', tool_source: 'trigger', topic: graphTopic, input_mappings: {}, output_fields: [] },
+        { activity_id: doneId, title: 'Iteration Complete', type: 'worker', tool_source: 'trigger', topic: graphTopic, input_mappings: {}, output_fields: [] },
+    ];
+    return { activities, transitions, manifest, pivotId, doneId };
+}
+/**
+ * Check if a step should be rendered as an iteration based on the compilation plan.
+ */
+function findIterationSpec(stepIdx, plan, step, collapsedToCoreIndex) {
+    // Plan-driven: check if the plan specifies an iteration for this step.
+    // The plan uses collapsed-step indices; remap to core-step indices.
+    if (plan) {
+        const spec = plan.iterations.find(it => {
+            const remappedBody = collapsedToCoreIndex?.get(it.bodyStepIndex) ?? it.bodyStepIndex;
+            return remappedBody === stepIdx;
+        });
+        if (spec) {
+            // Remap sourceStepIndex from collapsed to core space
+            const remappedSource = collapsedToCoreIndex?.get(spec.sourceStepIndex) ?? spec.sourceStepIndex;
+            return { ...spec, bodyStepIndex: stepIdx, sourceStepIndex: remappedSource };
+        }
+    }
+    // Mechanical fallback: check _iteration metadata from pattern detector
+    const iteration = step.arguments?._iteration;
+    if (iteration) {
+        return {
+            bodyStepIndex: stepIdx,
+            toolName: iteration.tool.replace(/_batch$/, ''),
+            serverId: iteration.server,
+            sourceStepIndex: iteration.arraySource?.stepIndex ?? stepIdx - 1,
+            sourceField: iteration.arraySource?.field ?? 'items',
+            varyingKeys: iteration.varyingKeys,
+            constantArgs: iteration.constantArgs,
+            keyMappings: {}, // no key mappings from mechanical detection
+        };
+    }
+    return null;
+}

package/build/services/yaml-workflow/pipeline/build/metadata.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Category and tag derivation for compiled workflows.
+ *
+ * Categories are derived dynamically from registered MCP server metadata —
+ * not hardcoded. As new servers are registered, their categories and tags
+ * automatically participate in workflow classification.
+ */
+import type { ExtractedStep } from '../../types';
+/**
+ * Derive a category for a compiled workflow based on which MCP servers
+ * and tools were used in the execution trace. Queries the DB for
+ * current server metadata so new servers are automatically recognized.
+ */
+export declare function deriveCategory(steps: ExtractedStep[]): Promise<string>;
+export declare function deriveTagsFromSteps(steps: ExtractedStep[], name: string, description?: string): string[];