@hotmeshio/long-tail 0.1.0

package/build/services/yaml-workflow/pipeline/build/metadata.js

@@ -0,0 +1,126 @@
+"use strict";
+/**
+ * Category and tag derivation for compiled workflows.
+ *
+ * Categories are derived dynamically from registered MCP server metadata —
+ * not hardcoded. As new servers are registered, their categories and tags
+ * automatically participate in workflow classification.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.deriveCategory = deriveCategory;
+exports.deriveTagsFromSteps = deriveTagsFromSteps;
+const mcpDbService = __importStar(require("../../../mcp/db"));
+const defaults_1 = require("../../../../modules/defaults");
+/**
+ * Load category signals from registered MCP servers.
+ * Each server's metadata.category becomes a category, with the server's
+ * name, tags, and tool names as matching keywords.
+ */
+async function loadCategorySignals() {
+    try {
+        const { servers } = await mcpDbService.listMcpServers({ limit: 100 });
+        return servers.map(s => ({
+            category: s.metadata?.category || 'general',
+            keywords: [
+                s.name.toLowerCase(),
+                ...(s.tags || []).map(t => t.toLowerCase()),
+                ...(s.tool_manifest || []).map((t) => t.name?.toLowerCase()).filter(Boolean),
+            ],
+        }));
+    }
+    catch {
+        return [];
+    }
+}
+/**
+ * Derive a category for a compiled workflow based on which MCP servers
+ * and tools were used in the execution trace. Queries the DB for
+ * current server metadata so new servers are automatically recognized.
+ */
+async function deriveCategory(steps) {
+    const signals = await loadCategorySignals();
+    const counts = new Map();
+    const tokens = [];
+    for (const step of steps) {
+        if (step.kind === 'tool') {
+            tokens.push(step.toolName.toLowerCase());
+            if (step.mcpServerId)
+                tokens.push(step.mcpServerId.toLowerCase());
+            for (const key of Object.keys(step.arguments)) {
+                tokens.push(key.toLowerCase());
+            }
+        }
+    }
+    const signalText = tokens.join(' ');
+    for (const { category, keywords } of signals) {
+        const hits = keywords.filter(kw => signalText.includes(kw)).length;
+        if (hits > 0)
+            counts.set(category, (counts.get(category) || 0) + hits);
+    }
+    const hasLlm = steps.some(s => s.kind === 'llm');
+    if (hasLlm && (counts.get('database') || counts.get('data-extraction') || 0) > 0) {
+        counts.set('reporting', (counts.get('reporting') || 0) + 5);
+    }
+    let best = 'general';
+    let bestCount = 0;
+    for (const [cat, count] of counts) {
+        if (count > bestCount) {
+            best = cat;
+            bestCount = count;
+        }
+    }
+    return best;
+}
+function deriveTagsFromSteps(steps, name, description) {
+    const tags = new Set();
+    for (const step of steps) {
+        if (step.kind === 'tool') {
+            tags.add(step.toolName);
+            if (step.mcpServerId)
+                tags.add(step.mcpServerId);
+            tags.add(step.source);
+        }
+    }
+    const text = `${name} ${description || ''}`.toLowerCase();
+    const keywords = text
+        .replace(/[^a-z0-9\s-]/g, ' ')
+        .split(/\s+/)
+        .filter((w) => w.length > 2 && !defaults_1.STOP_WORDS.has(w));
+    for (const kw of keywords) {
+        tags.add(kw);
+    }
+    return Array.from(tags);
+}

package/build/services/yaml-workflow/pipeline/build/transform.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Transform edge insertion logic for the build pipeline stage.
+ *
+ * Handles inserting reshape activities before steps that have transform edges
+ * in their compilation plan data flow.
+ */
+import type { ActivityManifestEntry } from '../../../../types/yaml-workflow';
+import type { ExtractedStep, EnhancedCompilationPlan } from '../../types';
+/**
+ * Insert transform (reshape) activities for data flow edges that require field mapping.
+ *
+ * Returns the new prevActivityId after any transforms are inserted.
+ */
+export declare function insertTransformActivities(idx: number, plan: EnhancedCompilationPlan, steps: ExtractedStep[], prefix: string, graphTopic: string, triggerId: string, triggerInputKeys: Set<string>, stepIndexToActivityId: Map<number, string>, collapsedToCoreIndex: Map<number, number>, activities: Record<string, unknown>, transitions: Record<string, Array<{
+    to: string;
+    conditions?: Record<string, unknown>;
+}>>, activityManifest: ActivityManifestEntry[], prevActivityId: string): string;

package/build/services/yaml-workflow/pipeline/build/transform.js

@@ -0,0 +1,94 @@
+"use strict";
+/**
+ * Transform edge insertion logic for the build pipeline stage.
+ *
+ * Handles inserting reshape activities before steps that have transform edges
+ * in their compilation plan data flow.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.insertTransformActivities = insertTransformActivities;
+const utils_1 = require("./utils");
+/**
+ * Insert transform (reshape) activities for data flow edges that require field mapping.
+ *
+ * Returns the new prevActivityId after any transforms are inserted.
+ */
+function insertTransformActivities(idx, plan, steps, prefix, graphTopic, triggerId, triggerInputKeys, stepIndexToActivityId, collapsedToCoreIndex, activities, transitions, activityManifest, prevActivityId) {
+    const collapsedIdx = collapsedToCoreIndex?.size
+        ? [...collapsedToCoreIndex.entries()].find(([_, core]) => core === idx)?.[0] ?? idx
+        : idx;
+    const transformEdges = plan.dataFlow.filter(e => e.toStep === collapsedIdx && e.transform && Object.keys(e.transform.fieldMap).length > 0);
+    for (let ei = 0; ei < transformEdges.length; ei++) {
+        const edge = transformEdges[ei];
+        // Include edge index to avoid collisions when multiple transforms target the same step
+        const transformId = transformEdges.length === 1
+            ? `${prefix}_xf${idx + 1}`
+            : `${prefix}_xf${idx + 1}_${ei}`;
+        const transformTopic = graphTopic;
+        const transformWorkflowName = `reshape_${edge.toField}`;
+        // Wire transform input: source field from prior step + all trigger inputs
+        // (trigger inputs needed for dynamic derivation prefixes like screenshot_dir)
+        // Also wire ALL output fields from the source step so the transform has
+        // full context (e.g., script_result, session handles) for reshaping.
+        const transformInputMaps = {};
+        for (const triggerKey of triggerInputKeys) {
+            transformInputMaps[triggerKey] = `{${triggerId}.output.data.${triggerKey}}`;
+        }
+        if (edge.fromStep === 'trigger') {
+            transformInputMaps[edge.fromField] = `{${triggerId}.output.data.${edge.fromField}}`;
+        }
+        else {
+            const remappedFrom = collapsedToCoreIndex?.get(edge.fromStep) ?? edge.fromStep;
+            const sourceActId = stepIndexToActivityId.get(remappedFrom);
+            if (sourceActId) {
+                transformInputMaps[edge.fromField] = `{${sourceActId}.output.data.${edge.fromField}}`;
+                // Wire all other output fields from the source step so the transform
+                // can access them (e.g., script_result needed for reshaping)
+                const sourceIdx = remappedFrom;
+                if (sourceIdx >= 0 && sourceIdx < steps.length) {
+                    const sourceResult = steps[sourceIdx].result;
+                    if (sourceResult && typeof sourceResult === 'object' && !Array.isArray(sourceResult)) {
+                        for (const field of Object.keys(sourceResult)) {
+                            if (!transformInputMaps[field]) {
+                                transformInputMaps[field] = `{${sourceActId}.output.data.${field}}`;
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        // Set workflowName for singleton consumer dispatch routing
+        transformInputMaps.workflowName = transformWorkflowName;
+        activities[transformId] = {
+            title: `Reshape ${(0, utils_1.humanize)(edge.toField)}`,
+            type: 'worker',
+            topic: transformTopic,
+            input: {
+                schema: { type: 'object' },
+                maps: transformInputMaps,
+            },
+            output: { schema: { type: 'object' } },
+        };
+        activityManifest.push({
+            activity_id: transformId,
+            title: `Reshape ${(0, utils_1.humanize)(edge.toField)}`,
+            type: 'worker',
+            tool_source: 'transform',
+            topic: transformTopic,
+            workflow_name: transformWorkflowName,
+            input_mappings: transformInputMaps,
+            output_fields: [edge.toField],
+            transform_spec: {
+                sourceField: edge.fromField,
+                targetField: edge.toField,
+                fieldMap: edge.transform.fieldMap,
+                defaults: edge.transform.defaults,
+                derivations: edge.transform.derivations,
+            },
+        });
+        // Transition from previous -> transform
+        transitions[prevActivityId] = [{ to: transformId }];
+        prevActivityId = transformId;
+    }
+    return prevActivityId;
+}

package/build/services/yaml-workflow/pipeline/build/utils.d.ts

@@ -0,0 +1,22 @@
+/**
+ * Utility functions for the build pipeline stage.
+ */
+/**
+ * Check if two keys are semantically related (for computed key fallback).
+ * Replicates the logic from pattern-detector but local to the build stage.
+ */
+export declare function keysRelated(a: string, b: string): boolean;
+/** Cap `limit` in tool arguments to avoid sending huge payloads to downstream LLM steps. */
+export declare function capToolArguments(args: Record<string, unknown>): Record<string, unknown>;
+/** Sanitize a name for use in HotMesh app IDs and topics. */
+export declare function sanitizeName(name: string): string;
+/** Convert a snake_case/camelCase field name to a readable label. */
+export declare function humanize(name: string): string;
+/**
+ * Infer a JSON Schema from a sample value, recursively.
+ */
+export declare function inferSchema(value: unknown, withDefault?: boolean): Record<string, unknown>;
+/**
+ * Build data mappings from a previous activity's output fields.
+ */
+export declare function buildInputMappings(prevActivityId: string, prevResult: unknown): Record<string, string>;

package/build/services/yaml-workflow/pipeline/build/utils.js

@@ -0,0 +1,129 @@
+"use strict";
+/**
+ * Utility functions for the build pipeline stage.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.keysRelated = keysRelated;
+exports.capToolArguments = capToolArguments;
+exports.sanitizeName = sanitizeName;
+exports.humanize = humanize;
+exports.inferSchema = inferSchema;
+exports.buildInputMappings = buildInputMappings;
+const defaults_1 = require("../../../../modules/defaults");
+/**
+ * Check if two keys are semantically related (for computed key fallback).
+ * Replicates the logic from pattern-detector but local to the build stage.
+ */
+function keysRelated(a, b) {
+    const la = a.toLowerCase();
+    const lb = b.toLowerCase();
+    if (la.includes(lb) || lb.includes(la))
+        return true;
+    const urlKeys = ['url', 'href', 'link', 'src'];
+    if (urlKeys.includes(la) && urlKeys.includes(lb))
+        return true;
+    const pathKeys = ['path', 'file', 'filepath', 'filename', 'name'];
+    if (pathKeys.some(p => la.includes(p)) && pathKeys.some(p => lb.includes(p)))
+        return true;
+    return false;
+}
+/** Cap `limit` in tool arguments to avoid sending huge payloads to downstream LLM steps. */
+function capToolArguments(args) {
+    const capped = { ...args };
+    if (typeof capped.limit === 'number' && capped.limit > defaults_1.TOOL_ARG_LIMIT_CAP) {
+        capped.limit = defaults_1.TOOL_ARG_LIMIT_CAP;
+    }
+    return capped;
+}
+/** Sanitize a name for use in HotMesh app IDs and topics. */
+function sanitizeName(name) {
+    return name
+        .toLowerCase()
+        .replace(/[^a-z0-9]+/g, '-')
+        .replace(/^-|-$/g, '');
+}
+/** Convert a snake_case/camelCase field name to a readable label. */
+function humanize(name) {
+    return name
+        .replace(/([a-z])([A-Z])/g, '$1 $2')
+        .replace(/[_-]/g, ' ')
+        .replace(/\b\w/g, (c) => c.toUpperCase())
+        .trim();
+}
+/**
+ * Infer a JSON Schema from a sample value, recursively.
+ */
+function inferSchema(value, withDefault = false) {
+    if (value === null || value === undefined) {
+        return { type: 'string' };
+    }
+    if (typeof value === 'string') {
+        const schema = { type: 'string' };
+        if (withDefault)
+            schema.default = value;
+        return schema;
+    }
+    if (typeof value === 'number') {
+        const schema = { type: 'number' };
+        if (withDefault)
+            schema.default = value;
+        return schema;
+    }
+    if (typeof value === 'boolean') {
+        const schema = { type: 'boolean' };
+        if (withDefault)
+            schema.default = value;
+        return schema;
+    }
+    if (Array.isArray(value)) {
+        const schema = { type: 'array' };
+        if (withDefault)
+            schema.default = value;
+        if (value.length > 0) {
+            if (value.every((v) => typeof v === 'string')) {
+                schema.items = { type: 'string' };
+            }
+            else if (value.every((v) => typeof v === 'number')) {
+                schema.items = { type: 'number' };
+            }
+            else if (value.every((v) => v && typeof v === 'object' && !Array.isArray(v))) {
+                const allKeys = new Map();
+                for (const el of value) {
+                    for (const [k, v] of Object.entries(el)) {
+                        if (!allKeys.has(k))
+                            allKeys.set(k, v);
+                    }
+                }
+                const props = {};
+                for (const [k, v] of allKeys) {
+                    props[k] = inferSchema(v, false);
+                    props[k].description = humanize(k);
+                }
+                schema.items = { type: 'object', properties: props };
+            }
+        }
+        return schema;
+    }
+    if (typeof value === 'object') {
+        const entries = Object.entries(value);
+        const props = {};
+        for (const [k, v] of entries) {
+            props[k] = inferSchema(v, withDefault);
+            props[k].description = humanize(k);
+        }
+        return { type: 'object', properties: props };
+    }
+    return { type: 'string' };
+}
+/**
+ * Build data mappings from a previous activity's output fields.
+ */
+function buildInputMappings(prevActivityId, prevResult) {
+    const mappings = {};
+    if (prevResult && typeof prevResult === 'object' && !Array.isArray(prevResult)) {
+        for (const key of Object.keys(prevResult)) {
+            mappings[key] = `{${prevActivityId}.output.data.${key}}`;
+        }
+    }
+    return mappings;
+}

package/build/services/yaml-workflow/pipeline/build/wiring.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Step input wiring logic for the build pipeline stage.
+ */
+import type { ExtractedStep, EnhancedCompilationPlan } from '../../types';
+/**
+ * Build input mappings for a step using the compilation plan's data flow edges.
+ * Falls back to mechanical backward-scan when no plan is available.
+ */
+export declare function wireStepInputs(stepIdx: number, step: ExtractedStep, plan: EnhancedCompilationPlan | null, stepIndexToActivityId: Map<number, string>, triggerId: string, triggerInputKeys: Set<string>, steps: ExtractedStep[], prefix: string, prevActivityId: string, prevResult: unknown, collapsedToCoreIndex?: Map<number, number>): Record<string, string>;

package/build/services/yaml-workflow/pipeline/build/wiring.js

@@ -0,0 +1,148 @@
+"use strict";
+/**
+ * Step input wiring logic for the build pipeline stage.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.wireStepInputs = wireStepInputs;
+/**
+ * Build input mappings for a step using the compilation plan's data flow edges.
+ * Falls back to mechanical backward-scan when no plan is available.
+ */
+function wireStepInputs(stepIdx, step, plan, stepIndexToActivityId, triggerId, triggerInputKeys, steps, prefix, prevActivityId, prevResult, collapsedToCoreIndex) {
+    const inputMappings = {};
+    // Determine session fields for gap-fill after plan-driven wiring
+    const sessionFieldSet = new Set(plan?.sessionFields?.length
+        ? plan.sessionFields
+        : ['page_id', '_handle', 'session_id']);
+    // Plan-driven wiring: use data flow edges from the compilation plan.
+    // Remap indices from collapsed-step space to core-step space.
+    if (plan && plan.dataFlow.length > 0) {
+        // Remap the current step index back to collapsed space for edge matching
+        const collapsedIdx = collapsedToCoreIndex?.size
+            ? [...collapsedToCoreIndex.entries()].find(([_, core]) => core === stepIdx)?.[0] ?? stepIdx
+            : stepIdx;
+        const edgesForStep = plan.dataFlow.filter(e => e.toStep === collapsedIdx);
+        for (const edge of edgesForStep) {
+            if (edge.transform && Object.keys(edge.transform.fieldMap).length > 0) {
+                // This edge has a transform — the reshape activity was inserted before this step.
+                // Wire from the transform activity's output (which uses toField as the output key).
+                const transformActId = `${prefix}_xf${stepIdx + 1}`;
+                inputMappings[edge.toField] = `{${transformActId}.output.data.${edge.toField}}`;
+            }
+            else if (edge.fromStep === 'trigger') {
+                inputMappings[edge.toField] = `{${triggerId}.output.data.${edge.fromField}}`;
+            }
+            else {
+                // Remap the source step from collapsed to core index
+                const remappedFrom = collapsedToCoreIndex?.get(edge.fromStep) ?? edge.fromStep;
+                const sourceActId = stepIndexToActivityId.get(remappedFrom);
+                if (sourceActId) {
+                    inputMappings[edge.toField] = `{${sourceActId}.output.data.${edge.fromField}}`;
+                }
+            }
+        }
+        // If the plan provided wiring for this step, gap-fill any missing session
+        // fields and trigger inputs before returning — the LLM may have omitted them.
+        if (edgesForStep.length > 0) {
+            // Gap-fill session fields: backward scan for any session field the plan missed
+            if (step.kind === 'tool') {
+                for (const sf of sessionFieldSet) {
+                    if (inputMappings[sf])
+                        continue; // already wired by plan
+                    // Only add if the step actually uses this field (present in arguments or
+                    // known to be required by the tool based on prior step outputs)
+                    const stepNeedsField = sf in step.arguments ||
+                        steps.slice(0, stepIdx).some(s => s.result && typeof s.result === 'object' && !Array.isArray(s.result) &&
+                            sf in s.result);
+                    if (!stepNeedsField)
+                        continue;
+                    for (let si = stepIdx - 1; si >= 0; si--) {
+                        const priorResult = steps[si].result;
+                        if (priorResult && typeof priorResult === 'object' && !Array.isArray(priorResult) &&
+                            sf in priorResult) {
+                            const priorActId = stepIndexToActivityId.get(si) || `${prefix}_a${si + 1}`;
+                            inputMappings[sf] = `{${priorActId}.output.data.${sf}}`;
+                            break;
+                        }
+                    }
+                }
+                // Gap-fill trigger inputs: if a step argument matches a trigger input but the plan
+                // didn't explicitly wire it, add the wiring.
+                // Skip complex arguments (arrays, nested objects with >2 keys) — they are stored
+                // defaults in tool_arguments and must not be overridden by flat trigger mappings.
+                for (const key of Object.keys(step.arguments)) {
+                    if (inputMappings[key])
+                        continue; // already wired
+                    if (key === '_iteration')
+                        continue;
+                    const argValue = step.arguments[key];
+                    if (Array.isArray(argValue) || (argValue && typeof argValue === 'object' && !Array.isArray(argValue) && Object.keys(argValue).length > 2)) {
+                        continue;
+                    }
+                    if (triggerInputKeys.has(key)) {
+                        inputMappings[key] = `{${triggerId}.output.data.${key}}`;
+                    }
+                }
+            }
+            return inputMappings;
+        }
+    }
+    // Mechanical fallback: match step argument keys against trigger inputs and prior outputs.
+    // Skip complex array/object arguments — they come from stored tool_arguments, not wiring.
+    if (step.kind === 'tool') {
+        for (const key of Object.keys(step.arguments)) {
+            if (key === '_iteration')
+                continue;
+            // Don't wire complex arguments (arrays, nested objects) — these are stored defaults
+            const argValue = step.arguments[key];
+            if (Array.isArray(argValue) || (argValue && typeof argValue === 'object' && !Array.isArray(argValue) && Object.keys(argValue).length > 2)) {
+                continue;
+            }
+            if (triggerInputKeys.has(key)) {
+                inputMappings[key] = `{${triggerId}.output.data.${key}}`;
+            }
+            else if (prevResult && typeof prevResult === 'object' && !Array.isArray(prevResult) &&
+                key in prevResult) {
+                inputMappings[key] = `{${prevActivityId}.output.data.${key}}`;
+            }
+            else {
+                // Walk backward to find any upstream step that produced this field
+                for (let si = stepIdx - 1; si >= 0; si--) {
+                    const priorResult = steps[si].result;
+                    if (priorResult && typeof priorResult === 'object' && !Array.isArray(priorResult) &&
+                        key in priorResult) {
+                        const priorActId = stepIndexToActivityId.get(si) || `${prefix}_a${si + 1}`;
+                        inputMappings[key] = `{${priorActId}.output.data.${key}}`;
+                        break;
+                    }
+                }
+            }
+        }
+    }
+    else {
+        // LLM steps: wire all previous output fields
+        if (prevResult && typeof prevResult === 'object' && !Array.isArray(prevResult)) {
+            for (const key of Object.keys(prevResult)) {
+                inputMappings[key] = `{${prevActivityId}.output.data.${key}}`;
+            }
+        }
+    }
+    // Mechanical gap-fill: ensure session fields are wired even when not in step.arguments
+    // (they may be implicit inputs required by the tool but not captured in the trace args)
+    if (step.kind === 'tool') {
+        for (const sf of sessionFieldSet) {
+            if (inputMappings[sf])
+                continue; // already wired
+            for (let si = stepIdx - 1; si >= 0; si--) {
+                const priorResult = steps[si].result;
+                if (priorResult && typeof priorResult === 'object' && !Array.isArray(priorResult) &&
+                    sf in priorResult) {
+                    const priorActId = stepIndexToActivityId.get(si) || `${prefix}_a${si + 1}`;
+                    inputMappings[sf] = `{${priorActId}.output.data.${sf}}`;
+                    break;
+                }
+            }
+        }
+    }
+    return inputMappings;
+}

package/build/services/yaml-workflow/pipeline/compile/index.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Compile stage: LLM-powered intent compilation.
+ *
+ * Calls an LLM to review the execution steps + original prompt and produce
+ * an EnhancedCompilationPlan — a rich specification including iteration specs,
+ * data flow graphs, key mappings, and session field identification.
+ */
+import type { EnhancedCompilationPlan, PipelineContext } from '../../types';
+/**
+ * Compile pipeline stage: call LLM to produce an EnhancedCompilationPlan.
+ *
+ * Falls back gracefully when the LLM is unavailable — the build stage
+ * uses mechanical heuristics as before.
+ */
+export declare function compile(ctx: PipelineContext): Promise<PipelineContext>;
+/** @deprecated Use EnhancedCompilationPlan from types.ts instead. */
+export type CompilationPlan = EnhancedCompilationPlan;

package/build/services/yaml-workflow/pipeline/compile/index.js

@@ -0,0 +1,64 @@
+"use strict";
+/**
+ * Compile stage: LLM-powered intent compilation.
+ *
+ * Calls an LLM to review the execution steps + original prompt and produce
+ * an EnhancedCompilationPlan — a rich specification including iteration specs,
+ * data flow graphs, key mappings, and session field identification.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.compile = compile;
+const llm_call_1 = require("./llm-call");
+const prompts_1 = require("../prompts");
+/**
+ * Apply a compilation plan to override the naive input field metadata.
+ */
+function applyInputOverrides(planInputs, naiveInputs) {
+    const naiveByKey = new Map(naiveInputs.map(f => [f.key, f]));
+    return planInputs.map(inp => {
+        const naive = naiveByKey.get(inp.key);
+        return {
+            key: inp.key,
+            type: inp.type,
+            classification: inp.classification,
+            description: inp.description,
+            ...(inp.classification === 'fixed' && inp.default !== undefined
+                ? { default: inp.default }
+                : {}),
+            source_step_index: naive?.source_step_index ?? 0,
+            source_tool: naive?.source_tool ?? 'unknown',
+        };
+    });
+}
+/**
+ * Compile pipeline stage: call LLM to produce an EnhancedCompilationPlan.
+ *
+ * Falls back gracefully when the LLM is unavailable — the build stage
+ * uses mechanical heuristics as before.
+ */
+async function compile(ctx) {
+    // Build retry context — either user feedback or deployment error
+    const retryHint = ctx.priorDeployError
+        ? (0, prompts_1.buildRecompilationHint)(ctx.priorDeployError, ctx.priorFailedYaml)
+        : undefined;
+    // Attempt LLM compilation
+    ctx.compilationPlan = await (0, llm_call_1.callCompilationLLM)(ctx.collapsedSteps, ctx.originalPrompt, ctx.naiveInputs, ctx.patternAnnotations, retryHint);
+    if (ctx.compilationPlan) {
+        // Filter to core steps only
+        const coreIndices = new Set(ctx.compilationPlan.coreStepIndices);
+        if (coreIndices.size > 0 && coreIndices.size < ctx.collapsedSteps.length) {
+            ctx.coreSteps = ctx.collapsedSteps.filter((_, idx) => coreIndices.has(idx));
+        }
+        else {
+            ctx.coreSteps = [...ctx.collapsedSteps];
+        }
+        // Override input classifications with LLM's refined plan
+        ctx.refinedInputs = applyInputOverrides(ctx.compilationPlan.inputs, ctx.naiveInputs);
+    }
+    else {
+        // No plan — use collapsed steps and naive inputs as-is
+        ctx.coreSteps = [...ctx.collapsedSteps];
+        ctx.refinedInputs = [...ctx.naiveInputs];
+    }
+    return ctx;
+}

package/build/services/yaml-workflow/pipeline/compile/llm-call.d.ts

@@ -0,0 +1,14 @@
+/**
+ * LLM compilation call and server tool inventory gathering.
+ *
+ * Calls the LLM with summarized execution steps and parses the
+ * response into an EnhancedCompilationPlan.
+ */
+import type { InputFieldMeta } from '../../../../types/yaml-workflow';
+import type { PatternAnnotation } from '../../pattern-detector';
+import type { ExtractedStep, EnhancedCompilationPlan } from '../../types';
+/**
+ * Call the LLM to produce an EnhancedCompilationPlan.
+ * Returns null if the LLM is unavailable or the call fails.
+ */
+export declare function callCompilationLLM(steps: ExtractedStep[], originalPrompt: string, naiveInputs: InputFieldMeta[], patternAnnotations: PatternAnnotation[], retryHint?: string): Promise<EnhancedCompilationPlan | null>;