@hotmeshio/long-tail 0.1.6 → 0.1.7

package/build/services/workflow-sets/sql.js

@@ -0,0 +1,24 @@
+"use strict";
+// ─── Workflow set CRUD ──────────────────────────────────────────────────────
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LIST_WORKFLOW_SETS_BASE = exports.DELETE_WORKFLOW_SET = exports.UPDATE_WORKFLOW_SET_STATUS = exports.UPDATE_WORKFLOW_SET_PLAN = exports.GET_WORKFLOW_SET = exports.CREATE_WORKFLOW_SET = void 0;
+exports.CREATE_WORKFLOW_SET = `
+  INSERT INTO lt_workflow_sets (name, description, specification, plan, namespaces, source_workflow_id)
+  VALUES ($1, $2, $3, $4, $5, $6)
+  RETURNING *`;
+exports.GET_WORKFLOW_SET = `
+  SELECT * FROM lt_workflow_sets WHERE id = $1`;
+exports.UPDATE_WORKFLOW_SET_PLAN = `
+  UPDATE lt_workflow_sets
+  SET plan = $2, namespaces = $3, status = 'planned', updated_at = NOW()
+  WHERE id = $1
+  RETURNING *`;
+exports.UPDATE_WORKFLOW_SET_STATUS = `
+  UPDATE lt_workflow_sets
+  SET status = $2, updated_at = NOW()
+  WHERE id = $1
+  RETURNING *`;
+exports.DELETE_WORKFLOW_SET = `
+  DELETE FROM lt_workflow_sets WHERE id = $1`;
+exports.LIST_WORKFLOW_SETS_BASE = `
+  SELECT * FROM lt_workflow_sets`;

package/build/services/yaml-workflow/db-utils.d.ts

@@ -12,6 +12,7 @@ export declare function listYamlWorkflows(filters: {
     tags?: string[];
     search?: string;
     source_workflow_id?: string;
+    set_id?: string;
     limit?: number;
     offset?: number;
 }): Promise<{

package/build/services/yaml-workflow/db-utils.js

@@ -69,6 +69,10 @@ async function listYamlWorkflows(filters) {
         conditions.push(`source_workflow_id = $${idx++}`);
         values.push(filters.source_workflow_id);
     }
+    if (filters.set_id) {
+        conditions.push(`set_id = $${idx++}`);
+        values.push(filters.set_id);
+    }
     const where = conditions.length ? `WHERE ${conditions.join(' AND ')}` : '';
     const limit = filters.limit || defaults_1.YAML_LIST_LIMIT;
     const offset = filters.offset || 0;

package/build/services/yaml-workflow/db.d.ts

@@ -1,6 +1,11 @@
 import type { LTYamlWorkflowRecord, LTYamlWorkflowVersionRecord, ActivityManifestEntry } from '../../types/yaml-workflow';
 import type { CreateYamlWorkflowInput } from './types';
 export { parseVersionFromYaml, updateYamlWorkflowStatus, listYamlWorkflows, findYamlWorkflowsByTags, } from './db-utils';
+/**
+ * Check whether a graph_topic is already in use by a non-archived workflow
+ * in the same namespace. Returns the conflicting workflow name, or null.
+ */
+export declare function checkTopicConflict(appId: string, graphTopic: string, excludeId?: string): Promise<string | null>;
 export declare function createYamlWorkflow(input: CreateYamlWorkflowInput): Promise<LTYamlWorkflowRecord>;
 export declare function getYamlWorkflow(id: string): Promise<LTYamlWorkflowRecord | null>;
 export declare function getYamlWorkflowByName(name: string): Promise<LTYamlWorkflowRecord | null>;

package/build/services/yaml-workflow/db.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.findYamlWorkflowsByTags = exports.listYamlWorkflows = exports.updateYamlWorkflowStatus = exports.parseVersionFromYaml = void 0;
+exports.checkTopicConflict = checkTopicConflict;
 exports.createYamlWorkflow = createYamlWorkflow;
 exports.getYamlWorkflow = getYamlWorkflow;
 exports.getYamlWorkflowByName = getYamlWorkflowByName;
@@ -29,6 +30,19 @@ Object.defineProperty(exports, "parseVersionFromYaml", { enumerable: true, get:
 Object.defineProperty(exports, "updateYamlWorkflowStatus", { enumerable: true, get: function () { return db_utils_2.updateYamlWorkflowStatus; } });
 Object.defineProperty(exports, "listYamlWorkflows", { enumerable: true, get: function () { return db_utils_2.listYamlWorkflows; } });
 Object.defineProperty(exports, "findYamlWorkflowsByTags", { enumerable: true, get: function () { return db_utils_2.findYamlWorkflowsByTags; } });
+/**
+ * Check whether a graph_topic is already in use by a non-archived workflow
+ * in the same namespace. Returns the conflicting workflow name, or null.
+ */
+async function checkTopicConflict(appId, graphTopic, excludeId) {
+    const pool = (0, db_1.getPool)();
+    const { rows } = await pool.query(sql_1.CHECK_TOPIC_UNIQUE, [appId, graphTopic]);
+    if (rows.length === 0)
+        return null;
+    if (excludeId && rows[0].id === excludeId)
+        return null;
+    return rows[0].name;
+}
 async function createYamlWorkflow(input) {
     const pool = (0, db_1.getPool)();
     const { rows } = await pool.query(sql_1.CREATE_YAML_WORKFLOW, [
@@ -48,6 +62,9 @@ async function createYamlWorkflow(input) {
         input.category || null,
         input.tags || [],
         input.metadata ? JSON.stringify(input.metadata) : null,
+        input.set_id || null,
+        input.set_role || null,
+        input.set_build_order ?? null,
     ]);
     const record = rows[0];
     await createVersionSnapshot(record.id, 1, record.yaml_content, input.activity_manifest || [], input.input_schema || {}, input.output_schema || {}, input.input_field_meta || [], 'Initial version');

package/build/services/yaml-workflow/pipeline/build/wiring.js

@@ -6,7 +6,10 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.wireStepInputs = wireStepInputs;
 /**
  * HotMesh @pipe sub-pipe for today's date as YYYY-MM-DD.
- * Must be used as a ROW-LEVEL entry in a parent @pipe, never inside an array row.
+ * Follows RPN convention: operands row, then operator row.
+ * Row 1: date.now() → epoch (operator, no extra operands)
+ * Row 2: [isoString, 0, 10] — three operands for substring
+ * Row 3: substring(isoString, 0, 10) → "YYYY-MM-DD"
  */
 const DATE_SUB_PIPE = {
     '@pipe': [
@@ -108,6 +111,16 @@ function wireStepInputs(stepIdx, step, plan, stepIndexToActivityId, triggerId, t
             : stepIdx;
         const edgesForStep = plan.dataFlow.filter(e => e.toStep === collapsedIdx);
         for (const edge of edgesForStep) {
+            // Skip edges that target a complex nested object argument — these are stored
+            // defaults in tool_arguments (e.g., a nested `login` object with selectors)
+            // and must not be overridden by a flat scalar from an upstream step.
+            if (step.kind === 'tool' && !edge.transform) {
+                const argValue = step.arguments[edge.toField];
+                if (argValue && typeof argValue === 'object' && !Array.isArray(argValue) &&
+                    Object.keys(argValue).length > 2) {
+                    continue;
+                }
+            }
             if (edge.transform && Object.keys(edge.transform.fieldMap).length > 0) {
                 // This edge has a transform — the reshape activity was inserted before this step.
                 // Wire from the transform activity's output (which uses toField as the output key).

package/build/services/yaml-workflow/pipeline/prompts.d.ts

@@ -4,7 +4,7 @@
  * Consolidates all system/user prompt strings used by compile, validate,
  * and extract stages so they can be reviewed and tuned in one place.
  */
-export declare const COMPILATION_PROMPT = "You are a workflow compiler. You analyze MCP tool execution traces and produce a COMPILATION PLAN \u2014 a complete specification for building a deterministic YAML DAG workflow.\n\nGiven:\n1. The user's ORIGINAL PROMPT \u2014 the single most important signal for understanding intent\n2. EXECUTION STEPS \u2014 tool calls with arguments, result structure samples, and server IDs\n3. PATTERN ANNOTATIONS \u2014 pre-detected iteration candidates from static analysis\n4. NAIVE INPUT CLASSIFICATION \u2014 initial argument classification\n\nYour job: produce a plan that makes the workflow truly reusable and deterministic.\n\n## Critical: Understand Intent\n\nThe original prompt describes what the user wanted. The execution trace shows HOW an LLM accomplished it, but may include exploratory detours. Your compilation captures INTENT, not a blind replay.\n\nFor example, if the prompt says \"login to site X and take screenshots of all pages\":\n- INTENT: login \u2192 discover pages \u2192 iterate and screenshot each one\n- Execution may have included probing steps \u2014 exclude those\n- Deterministic version: accept credentials \u2192 login \u2192 extract links \u2192 iterate taking screenshots\n\n## Critical: Preserve Discovery Steps\n\nMany workflows follow a \"discover then act\" pattern: one step DISCOVERS data (e.g., extract navigation links, query a database, list files) and a later step ACTS on that data (e.g., screenshot each page, process each record, transform each file).\n\n**NEVER collapse discovery + action into a single step with the discovered data as a user input.** If the execution trace shows:\n1. Step A: extract_content \u2192 produces `links: [{text, href}, ...]`\n2. Step B: capture_pages(pages=[...array built from step A's links...])\n\nThe compiled workflow MUST keep BOTH steps: A produces the array, B consumes it. Do NOT make the array a trigger input \u2014 it was runtime-discovered, not user-provided.\n\n**How to detect**: If a step's argument contains a large array of items that closely mirrors a prior step's output array (same URLs, same items, possibly reshaped), that array was DERIVED from the prior step. Keep both steps and wire them with a data_flow edge (with a transform if formats differ).\n\n## Rules\n\n### Step Dispositions\n- **core**: Directly serves the workflow intent. Produces data consumed by later steps. **Discovery steps that produce arrays consumed by later action steps are ALWAYS core.**\n- **exploratory**: Probing/debugging/discovery steps that don't produce data needed by the workflow. Exclude these:\n  - Checking if compiled workflows exist (list_workflows, list_yaml_workflows)\n  - Listing files to see what exists (list_files, read_file)\n  - Initial tool calls that failed and were retried with different parameters\n  - Any step whose result is not consumed by a subsequent core step\n  - **NEVER mark a discovery step as exploratory if its output was used to build arguments for a later step**\n\n### Signal Steps (Human-in-the-Loop)\nSteps with kind \"signal\" represent a durable pause where the workflow waits for human input (e.g., credentials, approval). These are ALWAYS core \u2014 they are essential to the workflow's data flow. The signal step receives data from a human and makes it available to subsequent steps. Do NOT mark signal steps as exploratory. The escalation tool call (escalate_and_wait) that precedes a signal step is also always core.\n\n**Signal data flow**: The signal step result contains the human response fields (e.g., password). These MUST be wired via data_flow edges to every downstream step that needs them. Add a data_flow edge from the signal step index to the consuming step with the matching field name.\n\n**Credentials from signals**: When the signal provides a credential (format: password in the schema), downstream tools that need it should receive it as a separate named input argument. The runtime exchanges ephemeral credential tokens automatically. For tools with complex stored arguments (like run_script steps arrays), wire the credential as a top-level argument name \u2014 the runtime merges it with stored defaults.\n\n### Iteration Specifications\nWhen the execution shows repeated tool calls with varying arguments (the pattern detector may have already collapsed these):\n- Identify the SOURCE: which prior step's result contains the array being iterated. This is the step that PRODUCED the list of items \u2014 look for a step whose result contains an array field with items matching the iteration's varying values. For example, if the iteration visits multiple URLs, find the step that returned those URLs (e.g., extract_content with links).\n- The source is NEVER a step that doesn't have the array in its output. Double-check: does the source step's resultKeys include the source_field?\n- Specify the source_field: the dot-path to the array (e.g., \"links\", \"results.pages\")\n- List varying_keys (change per item) vs constant_args (shared)\n- **KEY MAPPINGS are critical**: array items often use different key names than the tool expects.\n  E.g., extract_content returns `links: [{text, href}, ...]` but the screenshot tool wants `url`.\n  Map: `{ \"url\": \"href\" }` \u2014 tool arg name \u2192 array item key name.\n  Use null for keys that are COMPUTED at runtime, not sourced from the array.\n  For example, screenshot_path is often derived from the link text or URL \u2014 it's not a field in the source array directly:\n  `{ \"screenshot_path\": null }` \u2014 the value must be computed or provided by the trigger.\n\n### Tool Simplification for Iterations (CRITICAL)\nThe iteration pattern works by extracting individual values from array items and passing them as simple key=value arguments to the iterated tool. This means:\n\n**The iterated tool MUST accept simple, flat arguments** (url, path, page_id \u2014 not complex nested structures like a `steps` array).\n\nIf the execution used a complex multi-step scripting tool (e.g., `run_script` with a `steps: [{action, url}, {action, path}]` array) for each iteration, you MUST replace it with a simpler tool from the same server that accepts flat arguments. Check the server's tool inventory for a simpler alternative.\n\nFor example:\n- `run_script(steps=[navigate, wait, screenshot])` per page \u2192 replace with `capture_page(url, path, full_page, wait_ms)` (1 call, flat args)\n- `run_script(steps=[navigate, fill, click])` per item \u2192 replace with `submit_form(url, fields)` (1 call, flat args)\n\nWhen replacing: use the same server_id but the simpler tool_name. The varying_keys and key_mappings should map directly to the simple tool's argument names.\n\n**If no simpler tool alternative exists**, use a data_flow edge with a transform to feed the array into a batch/composite tool that accepts the full array (like `capture_authenticated_pages` which takes a `pages` array).\n\n### Data Flow Graph\nSpecify directed edges showing how data flows between steps:\n- from_step: \"trigger\" (user input) or step index number\n- from_field: the output field name (or trigger input key)\n- to_step: the consuming step index\n- to_field: the argument key name\n- is_session_wire: true for session handles (page_id, _handle, session_id)\n\nSession handles are critical \u2014 they maintain authenticated browser sessions, database connections, etc. They must be threaded from their producer through ALL subsequent steps that need them.\n\n### Chain Analysis to Downstream Steps\nWhen a step produces a meaningful result (analysis, extraction, description, computed value) and a later step consumes related data (saving, storing, forwarding, reporting), there MUST be a data_flow edge connecting them. Match by semantic intent, not just field name:\n- Step produces `description` \u2192 downstream step takes `value` \u2192 edge with from_field: \"description\", to_field: \"value\"\n- Step produces `analysis.summary` \u2192 downstream step takes `content` \u2192 edge with appropriate field mapping\n\nIf the original execution trace shows that a step's output was used (even indirectly) as input to a later step, the compiled version must preserve that data chain. A broken chain means the downstream step receives no data \u2014 the worst possible compilation error.\n\n### Data Flow Transforms (CRITICAL for array reshaping)\nWhen a source step produces an array of objects in one format but the consuming step expects a DIFFERENT format, add a `transform` to the data_flow edge. Compare the source step's result structure with the consuming step's actual arguments from the trace.\n\n**Choosing the correct source field**: When a step produces multiple output fields, check the result sample to determine which field actually contains an ARRAY OF OBJECTS suitable for iteration/reshaping. Prefer structured array fields over raw/unstructured fields. Check the Tool-Specific Compilation Hints section (if present) for guidance on which fields to use for specific tools.\n\nFor example: extract_content returns `links: [{text, href}]` but capture tool expects `pages: [{url, screenshot_path, wait_ms, full_page}]`.\nAdd a transform with:\n- `field_map`: maps target keys \u2192 source keys (e.g., `{\"url\": \"href\"}`). Use null for keys not in the source.\n- `defaults`: static values to inject (e.g., `{\"wait_ms\": 3000, \"full_page\": true}`)\n- `derivations`: for computed keys (null in field_map), how to derive them from source data\n  - strategy: \"slugify\" (lowercase, replace spaces/special with hyphens), \"prefix\", \"template\", \"concat\"\n  - source_key: which source field to derive from\n  - prefix/suffix/template: string manipulation params\n\nAvailable derivation strategies:\n- **slugify**: Lowercase, replace spaces/special chars with hyphens. Optionally add prefix/suffix.\n- **prefix**: Prepend a static string.\n- **template**: Format string with `{value}` (source field) and `{date}` (today's ISO date, YYYY-MM-DD).\n- **concat**: Join multiple parts. Each part can use `{value}` and `{date}` placeholders.\n  Example: `{ \"strategy\": \"concat\", \"parts\": [\"{value}\", \"-\", \"{date}\"] }` produces `my-slug-2026-04-17`.\n- **passthrough**: No transformation.\n\nUse these when the workflow needs runtime-computed values (date-stamped filenames, slugified URLs, templated paths).\n\nExample edge with transform:\n```\n{\n  \"from_step\": 1, \"from_field\": \"links\", \"to_step\": 2, \"to_field\": \"pages\",\n  \"is_session_wire\": false,\n  \"transform\": {\n    \"field_map\": { \"url\": \"href\", \"screenshot_path\": null },\n    \"defaults\": { \"wait_ms\": 3000, \"full_page\": true },\n    \"derivations\": {\n      \"screenshot_path\": {\n        \"source_key\": \"href\",\n        \"strategy\": \"slugify\",\n        \"prefix\": \"screenshots/\",\n        \"suffix\": \".png\"\n      }\n    }\n  }\n}\n```\n\nIMPORTANT: Check EVERY array-typed data_flow edge. Compare the source step's result item keys with the consuming step's argument item keys. If they differ, add a transform. Look at the actual tool_arguments in the execution trace to determine the correct field_map, defaults, and derivations.\n\n### Scalar Derivations on Data Flow Edges\nWhen a scalar value needs runtime transformation before reaching its consumer, add a `derivation` to the data_flow edge (NOT a `transform` \u2014 transforms are for array reshaping). Derivations generate runtime expressions for string manipulation.\n\nCommon use case: the user's prompt mentions a pattern like \"save with key slug-{date}\" or \"name it {something}-{today's date}\". The trigger provides the base value, and the derivation appends or formats it at runtime.\n\nAdd a `derivation` field to the data_flow edge:\n```\n{\n  \"from_step\": \"trigger\", \"from_field\": \"key\", \"to_step\": 3, \"to_field\": \"key\",\n  \"is_session_wire\": false,\n  \"derivation\": { \"strategy\": \"concat\", \"parts\": [\"{value}\", \"-\", \"{date}\"] }\n}\n```\n\nThis produces a runtime-computed key like `my-slug-2026-04-17`. The `{value}` placeholder is the source field value; `{date}` is today's ISO date (YYYY-MM-DD).\n\nUse derivations when:\n- The user wants date-stamped keys, filenames, or identifiers\n- A value needs a prefix/suffix added at runtime\n- Two values need to be concatenated\n\n### Input Classification\n- **dynamic**: Simple values callers MUST provide: URLs, credentials, file paths, queries, search terms. These are always scalar strings, numbers, or booleans \u2014 NEVER complex objects or arrays.\n- **fixed**: Implementation details with sensible defaults: selectors, timeouts, boolean flags, AND complex structured arguments like `steps` arrays, `login` objects, or `pages` arrays. These are baked into stored tool_arguments.\n\n**Complex tool arguments (arrays of objects, nested structures) are ALWAYS fixed.** They represent the implementation recipe, not user input. For example:\n- A `steps` array describing browser actions (navigate, fill, click, screenshot) \u2192 **fixed**\n- A `login` object with selectors and credentials \u2192 flatten the credentials (username, password) as dynamic, but the selectors as fixed\n- A `pages` array of URLs to capture \u2192 **fixed** if hardcoded from the trace, or a data_flow edge if discovered at runtime\n\nFlatten nested objects containing dynamic values. E.g., `login: {url, username, password}` \u2192 separate `login_url`, `username`, `password` fields. But NEVER expose the full nested object or array as a trigger input.\n\n**Arrays that were DISCOVERED at runtime (by a prior step) are NOT inputs.** They flow between steps via data_flow edges. Only make an array a trigger input if the user explicitly provided it in their prompt. If the array was produced by a discovery step (extract_content, query, list), keep the discovery step as core and wire its output to the consuming step.\n\n### Prompt-Mentioned Values Are Dynamic\nIf a scalar value (URL, domain name, file path, key name, slug, query string) appears verbatim or closely paraphrased in the user's original prompt, classify it as **dynamic**. The user explicitly chose that value for this execution and will want to change it next time. Only classify a prompt-mentioned value as fixed if it is unambiguously an implementation constant (a CSS selector, a timeout, a boolean flag).\n\nThis is the most common compilation error: treating the user's specific request values as universal defaults. When in doubt, make it dynamic.\n\n### Data Flow Wiring Precision\n- **Only wire inputs that semantically match.** A directory name (e.g., `screenshot_dir = \"screenshots\"`) must NOT be wired to a file path argument (e.g., `screenshot_path` which expects `\"screenshots/home.png\"`). If a tool argument needs a specific file path but the trigger only provides a directory, leave that argument unwired \u2014 the stored tool_arguments default will provide the correct value.\n- **Trigger inputs should map to the EXACT tool argument they represent.** Don't reuse a trigger input for a different-purpose argument just because the names are vaguely related.\n- **When in doubt, don't wire.** An unwired argument falls back to the stored tool_arguments default from the original execution \u2014 this is always correct. An incorrectly wired argument overrides the correct default with a wrong value.\n\n### Session Fields and Threading Rules\nList all fields that represent session tokens/handles that must flow through the DAG (e.g., page_id, _handle, session_id).\n\n**Critical**: When a login/setup step produces a page_id or _handle, ALL subsequent browser/page steps must receive that session wire \u2014 including steps inside iterations. The data_flow graph must include session wire edges from the producing step to EVERY downstream step that operates on the same session, not just the immediately next one. For iterations: wire the session from the setup step directly to the iteration body step.\n\n**COMPLETENESS REQUIREMENT**: For EACH step that uses a session field (check the step's argumentKeys \u2014 if it includes page_id, _handle, or session_id), you MUST emit a data_flow edge wiring that field from its producer. If step 0 produces _handle and steps 1, 2, and 3 all use it, you need THREE edges: 0\u21921, 0\u21922, 0\u21923. Do NOT assume downstream steps will \"inherit\" session fields \u2014 each consumer needs an explicit edge.\n\n### Data Flow Completeness Check\nBefore finalizing the plan, verify:\n1. Every step that has a session field in its argumentKeys has a corresponding is_session_wire edge\n2. Every step that consumes data from a prior step has a data_flow edge for that field\n3. Every dynamic trigger input is wired to at least one step via a data_flow edge from \"trigger\"\n4. Transform edges include the source field AND the consuming step can access all fields it needs\n\n## Output Format\n\nReturn a JSON object (no markdown fences):\n{\n  \"intent\": \"Brief generic description of what this workflow does\",\n  \"description\": \"Suggested workflow description for discovery\",\n  \"steps\": [\n    { \"index\": 0, \"purpose\": \"Navigate to the target site\", \"disposition\": \"core\" },\n    { \"index\": 1, \"purpose\": \"Extract navigation links from the page\", \"disposition\": \"core\" },\n    { \"index\": 2, \"purpose\": \"List files to check directory structure\", \"disposition\": \"exploratory\" }\n  ],\n  \"core_step_indices\": [0, 1, 3],\n  \"inputs\": [\n    { \"key\": \"base_url\", \"type\": \"string\", \"classification\": \"dynamic\", \"description\": \"The base URL of the site\" },\n    { \"key\": \"username\", \"type\": \"string\", \"classification\": \"dynamic\", \"description\": \"Login username\" },\n    { \"key\": \"timeout\", \"type\": \"number\", \"classification\": \"fixed\", \"description\": \"Page load timeout\", \"default\": 30000 }\n  ],\n  \"iterations\": [\n    {\n      \"body_step_index\": 3,\n      \"tool_name\": \"screenshot\",\n      \"server_id\": \"playwright\",\n      \"source_step_index\": 1,\n      \"source_field\": \"links\",\n      \"varying_keys\": [\"url\", \"screenshot_path\"],\n      \"constant_args\": { \"full_page\": true },\n      \"key_mappings\": { \"url\": \"href\", \"screenshot_path\": null }\n    }\n  ],\n  \"data_flow\": [\n    { \"from_step\": \"trigger\", \"from_field\": \"base_url\", \"to_step\": 0, \"to_field\": \"url\", \"is_session_wire\": false },\n    { \"from_step\": 0, \"from_field\": \"page_id\", \"to_step\": 1, \"to_field\": \"page_id\", \"is_session_wire\": true },\n    { \"from_step\": 0, \"from_field\": \"_handle\", \"to_step\": 1, \"to_field\": \"_handle\", \"is_session_wire\": true }\n  ],\n  \"session_fields\": [\"page_id\", \"_handle\"]\n}";
+export declare const COMPILATION_PROMPT = "You are a workflow compiler. You analyze MCP tool execution traces and produce a COMPILATION PLAN \u2014 a complete specification for building a deterministic YAML DAG workflow.\n\nGiven:\n1. The user's ORIGINAL PROMPT \u2014 the single most important signal for understanding intent\n2. EXECUTION STEPS \u2014 tool calls with arguments, result structure samples, and server IDs\n3. PATTERN ANNOTATIONS \u2014 pre-detected iteration candidates from static analysis\n4. NAIVE INPUT CLASSIFICATION \u2014 initial argument classification\n\nYour job: produce a plan that makes the workflow truly reusable and deterministic.\n\n## Critical: Understand Intent\n\nThe original prompt describes what the user wanted. The execution trace shows HOW an LLM accomplished it, but may include exploratory detours. Your compilation captures INTENT, not a blind replay.\n\nFor example, if the prompt says \"login to site X and take screenshots of all pages\":\n- INTENT: login \u2192 discover pages \u2192 iterate and screenshot each one\n- Execution may have included probing steps \u2014 exclude those\n- Deterministic version: accept credentials \u2192 login \u2192 extract links \u2192 iterate taking screenshots\n\n## Critical: Preserve Discovery Steps\n\nMany workflows follow a \"discover then act\" pattern: one step DISCOVERS data (e.g., extract navigation links, query a database, list files) and a later step ACTS on that data (e.g., screenshot each page, process each record, transform each file).\n\n**NEVER collapse discovery + action into a single step with the discovered data as a user input.** If the execution trace shows:\n1. Step A: extract_content \u2192 produces `links: [{text, href}, ...]`\n2. Step B: capture_pages(pages=[...array built from step A's links...])\n\nThe compiled workflow MUST keep BOTH steps: A produces the array, B consumes it. Do NOT make the array a trigger input \u2014 it was runtime-discovered, not user-provided.\n\n**How to detect**: If a step's argument contains a large array of items that closely mirrors a prior step's output array (same URLs, same items, possibly reshaped), that array was DERIVED from the prior step. Keep both steps and wire them with a data_flow edge (with a transform if formats differ).\n\n## Rules\n\n### Step Dispositions\n- **core**: Directly serves the workflow intent. Produces data consumed by later steps. **Discovery steps that produce arrays consumed by later action steps are ALWAYS core.**\n- **exploratory**: Probing/debugging/discovery steps that don't produce data needed by the workflow. Exclude these:\n  - Checking if compiled workflows exist (list_workflows, list_yaml_workflows)\n  - Listing files to see what exists (list_files, read_file)\n  - Initial tool calls that failed and were retried with different parameters\n  - Any step whose result is not consumed by a subsequent core step\n  - **NEVER mark a discovery step as exploratory if its output was used to build arguments for a later step**\n\n### Signal Steps (Human-in-the-Loop)\nSteps with kind \"signal\" represent a durable pause where the workflow waits for human input (e.g., credentials, approval). These are ALWAYS core \u2014 they are essential to the workflow's data flow. The signal step receives data from a human and makes it available to subsequent steps. Do NOT mark signal steps as exploratory. The escalation tool call (escalate_and_wait) that precedes a signal step is also always core.\n\n**Signal data flow**: The signal step result contains the human response fields (e.g., password). These MUST be wired via data_flow edges to every downstream step that needs them. Add a data_flow edge from the signal step index to the consuming step with the matching field name.\n\n**Credentials from signals**: When the signal provides a credential (format: password in the schema), downstream tools that need it should receive it as a separate named input argument. The runtime exchanges ephemeral credential tokens automatically. For tools with complex stored arguments (like run_script steps arrays), wire the credential as a top-level argument name \u2014 the runtime merges it with stored defaults.\n\n### Iteration Specifications\nWhen the execution shows repeated tool calls with varying arguments (the pattern detector may have already collapsed these):\n- Identify the SOURCE: which prior step's result contains the array being iterated. This is the step that PRODUCED the list of items \u2014 look for a step whose result contains an array field with items matching the iteration's varying values. For example, if the iteration visits multiple URLs, find the step that returned those URLs (e.g., extract_content with links).\n- The source is NEVER a step that doesn't have the array in its output. Double-check: does the source step's resultKeys include the source_field?\n- Specify the source_field: the dot-path to the array (e.g., \"links\", \"results.pages\")\n- List varying_keys (change per item) vs constant_args (shared)\n- **KEY MAPPINGS are critical**: array items often use different key names than the tool expects.\n  E.g., extract_content returns `links: [{text, href}, ...]` but the screenshot tool wants `url`.\n  Map: `{ \"url\": \"href\" }` \u2014 tool arg name \u2192 array item key name.\n  Use null for keys that are COMPUTED at runtime, not sourced from the array.\n  For example, screenshot_path is often derived from the link text or URL \u2014 it's not a field in the source array directly:\n  `{ \"screenshot_path\": null }` \u2014 the value must be computed or provided by the trigger.\n\n### Tool Simplification for Iterations (CRITICAL)\nThe iteration pattern works by extracting individual values from array items and passing them as simple key=value arguments to the iterated tool. This means:\n\n**The iterated tool MUST accept simple, flat arguments** (url, path, page_id \u2014 not complex nested structures like a `steps` array).\n\nIf the execution used a complex multi-step scripting tool (e.g., `run_script` with a `steps: [{action, url}, {action, path}]` array) for each iteration, you MUST replace it with a simpler tool from the same server that accepts flat arguments. Check the server's tool inventory for a simpler alternative.\n\nFor example:\n- `run_script(steps=[navigate, wait, screenshot])` per page \u2192 replace with `capture_page(url, path, full_page, wait_ms)` (1 call, flat args)\n- `run_script(steps=[navigate, fill, click])` per item \u2192 replace with `submit_form(url, fields)` (1 call, flat args)\n\nWhen replacing: use the same server_id but the simpler tool_name. The varying_keys and key_mappings should map directly to the simple tool's argument names.\n\n**If no simpler tool alternative exists**, use a data_flow edge with a transform to feed the array into a batch/composite tool that accepts the full array (like `capture_authenticated_pages` which takes a `pages` array).\n\n### Data Flow Graph\nSpecify directed edges showing how data flows between steps:\n- from_step: \"trigger\" (user input) or step index number\n- from_field: the output field name (or trigger input key)\n- to_step: the consuming step index\n- to_field: the argument key name\n- is_session_wire: true for session handles (page_id, _handle, session_id)\n\nSession handles are critical \u2014 they maintain authenticated browser sessions, database connections, etc. They must be threaded from their producer through ALL subsequent steps that need them.\n\n### Chain Analysis to Downstream Steps\nWhen a step produces a meaningful result (analysis, extraction, description, computed value) and a later step consumes related data (saving, storing, forwarding, reporting), there MUST be a data_flow edge connecting them. Match by semantic intent, not just field name:\n- Step produces `description` \u2192 downstream step takes `value` \u2192 edge with from_field: \"description\", to_field: \"value\"\n- Step produces `analysis.summary` \u2192 downstream step takes `content` \u2192 edge with appropriate field mapping\n\nIf the original execution trace shows that a step's output was used (even indirectly) as input to a later step, the compiled version must preserve that data chain. A broken chain means the downstream step receives no data \u2014 the worst possible compilation error.\n\n### Data Flow Transforms (CRITICAL for array reshaping)\nWhen a source step produces an array of objects in one format but the consuming step expects a DIFFERENT format, add a `transform` to the data_flow edge. Compare the source step's result structure with the consuming step's actual arguments from the trace.\n\n**Choosing the correct source field**: When a step produces multiple output fields, check the result sample to determine which field actually contains an ARRAY OF OBJECTS suitable for iteration/reshaping. Prefer structured array fields over raw/unstructured fields. Check the Tool-Specific Compilation Hints section (if present) for guidance on which fields to use for specific tools.\n\nFor example: extract_content returns `links: [{text, href}]` but capture tool expects `pages: [{url, screenshot_path, wait_ms, full_page}]`.\nAdd a transform with:\n- `field_map`: maps target keys \u2192 source keys (e.g., `{\"url\": \"href\"}`). Use null for keys not in the source.\n- `defaults`: static values to inject (e.g., `{\"wait_ms\": 3000, \"full_page\": true}`)\n- `derivations`: for computed keys (null in field_map), how to derive them from source data\n  - strategy: \"slugify\" (lowercase, replace spaces/special with hyphens), \"prefix\", \"template\", \"concat\"\n  - source_key: which source field to derive from\n  - prefix/suffix/template: string manipulation params\n\nAvailable derivation strategies:\n- **slugify**: Lowercase, replace spaces/special chars with hyphens. Optionally add prefix/suffix.\n- **prefix**: Prepend a static string.\n- **template**: Format string with `{value}` (source field) and `{date}` (today's date as YYYY-MM-DD via @date.yyyymmdd).\n- **concat**: Join multiple parts. Each part can use `{value}` and `{date}` placeholders.\n  Example: `{ \"strategy\": \"concat\", \"parts\": [\"{value}\", \"-\", \"{date}\"] }` produces `my-slug-2026-04-17`.\n- **passthrough**: No transformation.\n\nUse these when the workflow needs runtime-computed values (date-stamped filenames, slugified URLs, templated paths).\n\nExample edge with transform:\n```\n{\n  \"from_step\": 1, \"from_field\": \"links\", \"to_step\": 2, \"to_field\": \"pages\",\n  \"is_session_wire\": false,\n  \"transform\": {\n    \"field_map\": { \"url\": \"href\", \"screenshot_path\": null },\n    \"defaults\": { \"wait_ms\": 3000, \"full_page\": true },\n    \"derivations\": {\n      \"screenshot_path\": {\n        \"source_key\": \"href\",\n        \"strategy\": \"slugify\",\n        \"prefix\": \"screenshots/\",\n        \"suffix\": \".png\"\n      }\n    }\n  }\n}\n```\n\nIMPORTANT: Check EVERY array-typed data_flow edge. Compare the source step's result item keys with the consuming step's argument item keys. If they differ, add a transform. Look at the actual tool_arguments in the execution trace to determine the correct field_map, defaults, and derivations.\n\n### Scalar Derivations on Data Flow Edges\nWhen a scalar value needs runtime transformation before reaching its consumer, add a `derivation` to the data_flow edge (NOT a `transform` \u2014 transforms are for array reshaping). Derivations generate runtime expressions for string manipulation.\n\nCommon use case: the user's prompt mentions a pattern like \"save with key slug-{date}\" or \"name it {something}-{today's date}\". The trigger provides the base value, and the derivation appends or formats it at runtime.\n\nAdd a `derivation` field to the data_flow edge:\n```\n{\n  \"from_step\": \"trigger\", \"from_field\": \"key\", \"to_step\": 3, \"to_field\": \"key\",\n  \"is_session_wire\": false,\n  \"derivation\": { \"strategy\": \"concat\", \"parts\": [\"{value}\", \"-\", \"{date}\"] }\n}\n```\n\nThis produces a runtime-computed key like `my-slug-2026-04-17`. The `{value}` placeholder is the source field value; `{date}` is today's ISO date (YYYY-MM-DD).\n\nUse derivations when:\n- The user wants date-stamped keys, filenames, or identifiers\n- A value needs a prefix/suffix added at runtime\n- Two values need to be concatenated\n\n### Input Classification\n- **dynamic**: Simple values callers MUST provide: URLs, credentials, file paths, queries, search terms. These are always scalar strings, numbers, or booleans \u2014 NEVER complex objects or arrays.\n- **fixed**: Implementation details with sensible defaults: selectors, timeouts, boolean flags, AND complex structured arguments like `steps` arrays, `login` objects, or `pages` arrays. These are baked into stored tool_arguments.\n\n**Complex tool arguments (arrays of objects, nested structures) are ALWAYS fixed.** They represent the implementation recipe, not user input. For example:\n- A `steps` array describing browser actions (navigate, fill, click, screenshot) \u2192 **fixed**\n- A `login` object with selectors and credentials \u2192 flatten the credentials (username, password) as dynamic, but the selectors as fixed\n- A `pages` array of URLs to capture \u2192 **fixed** if hardcoded from the trace, or a data_flow edge if discovered at runtime\n\nFlatten nested objects containing dynamic values. E.g., `login: {url, username, password}` \u2192 separate `login_url`, `username`, `password` fields. But NEVER expose the full nested object or array as a trigger input.\n\n**Arrays that were DISCOVERED at runtime (by a prior step) are NOT inputs.** They flow between steps via data_flow edges. Only make an array a trigger input if the user explicitly provided it in their prompt. If the array was produced by a discovery step (extract_content, query, list), keep the discovery step as core and wire its output to the consuming step.\n\n### Prompt-Mentioned Values Are Dynamic\nIf a scalar value (URL, domain name, file path, key name, slug, query string) appears verbatim or closely paraphrased in the user's original prompt, classify it as **dynamic**. The user explicitly chose that value for this execution and will want to change it next time. Only classify a prompt-mentioned value as fixed if it is unambiguously an implementation constant (a CSS selector, a timeout, a boolean flag).\n\nThis is the most common compilation error: treating the user's specific request values as universal defaults. When in doubt, make it dynamic.\n\n### Data Flow Wiring Precision\n- **Only wire inputs that semantically match.** A directory name (e.g., `screenshot_dir = \"screenshots\"`) must NOT be wired to a file path argument (e.g., `screenshot_path` which expects `\"screenshots/home.png\"`). If a tool argument needs a specific file path but the trigger only provides a directory, leave that argument unwired \u2014 the stored tool_arguments default will provide the correct value.\n- **Trigger inputs should map to the EXACT tool argument they represent.** Don't reuse a trigger input for a different-purpose argument just because the names are vaguely related.\n- **When in doubt, don't wire.** An unwired argument falls back to the stored tool_arguments default from the original execution \u2014 this is always correct. An incorrectly wired argument overrides the correct default with a wrong value.\n\n### Session Fields and Threading Rules\nList all fields that represent session tokens/handles that must flow through the DAG (e.g., page_id, _handle, session_id).\n\n**Critical**: When a login/setup step produces a page_id or _handle, ALL subsequent browser/page steps must receive that session wire \u2014 including steps inside iterations. The data_flow graph must include session wire edges from the producing step to EVERY downstream step that operates on the same session, not just the immediately next one. For iterations: wire the session from the setup step directly to the iteration body step.\n\n**COMPLETENESS REQUIREMENT**: For EACH step that uses a session field (check the step's argumentKeys \u2014 if it includes page_id, _handle, or session_id), you MUST emit a data_flow edge wiring that field from its producer. If step 0 produces _handle and steps 1, 2, and 3 all use it, you need THREE edges: 0\u21921, 0\u21922, 0\u21923. Do NOT assume downstream steps will \"inherit\" session fields \u2014 each consumer needs an explicit edge.\n\n### Data Flow Completeness Check\nBefore finalizing the plan, verify:\n1. Every step that has a session field in its argumentKeys has a corresponding is_session_wire edge\n2. Every step that consumes data from a prior step has a data_flow edge for that field\n3. Every dynamic trigger input is wired to at least one step via a data_flow edge from \"trigger\"\n4. Transform edges include the source field AND the consuming step can access all fields it needs\n\n## Output Format\n\nReturn a JSON object (no markdown fences):\n{\n  \"intent\": \"Brief generic description of what this workflow does\",\n  \"description\": \"Suggested workflow description for discovery\",\n  \"steps\": [\n    { \"index\": 0, \"purpose\": \"Navigate to the target site\", \"disposition\": \"core\" },\n    { \"index\": 1, \"purpose\": \"Extract navigation links from the page\", \"disposition\": \"core\" },\n    { \"index\": 2, \"purpose\": \"List files to check directory structure\", \"disposition\": \"exploratory\" }\n  ],\n  \"core_step_indices\": [0, 1, 3],\n  \"inputs\": [\n    { \"key\": \"base_url\", \"type\": \"string\", \"classification\": \"dynamic\", \"description\": \"The base URL of the site\" },\n    { \"key\": \"username\", \"type\": \"string\", \"classification\": \"dynamic\", \"description\": \"Login username\" },\n    { \"key\": \"timeout\", \"type\": \"number\", \"classification\": \"fixed\", \"description\": \"Page load timeout\", \"default\": 30000 }\n  ],\n  \"iterations\": [\n    {\n      \"body_step_index\": 3,\n      \"tool_name\": \"screenshot\",\n      \"server_id\": \"playwright\",\n      \"source_step_index\": 1,\n      \"source_field\": \"links\",\n      \"varying_keys\": [\"url\", \"screenshot_path\"],\n      \"constant_args\": { \"full_page\": true },\n      \"key_mappings\": { \"url\": \"href\", \"screenshot_path\": null }\n    }\n  ],\n  \"data_flow\": [\n    { \"from_step\": \"trigger\", \"from_field\": \"base_url\", \"to_step\": 0, \"to_field\": \"url\", \"is_session_wire\": false },\n    { \"from_step\": 0, \"from_field\": \"page_id\", \"to_step\": 1, \"to_field\": \"page_id\", \"is_session_wire\": true },\n    { \"from_step\": 0, \"from_field\": \"_handle\", \"to_step\": 1, \"to_field\": \"_handle\", \"is_session_wire\": true }\n  ],\n  \"session_fields\": [\"page_id\", \"_handle\"]\n}";
 /**
  * Build the retry hint injected into the compile stage when
  * user feedback or a prior deployment error triggers recompilation.

package/build/services/yaml-workflow/pipeline/prompts.js

@@ -120,7 +120,7 @@ Add a transform with:
 Available derivation strategies:
 - **slugify**: Lowercase, replace spaces/special chars with hyphens. Optionally add prefix/suffix.
 - **prefix**: Prepend a static string.
-- **template**: Format string with \`{value}\` (source field) and \`{date}\` (today's ISO date, YYYY-MM-DD).
+- **template**: Format string with \`{value}\` (source field) and \`{date}\` (today's date as YYYY-MM-DD via @date.yyyymmdd).
 - **concat**: Join multiple parts. Each part can use \`{value}\` and \`{date}\` placeholders.
   Example: \`{ "strategy": "concat", "parts": ["{value}", "-", "{date}"] }\` produces \`my-slug-2026-04-17\`.
 - **passthrough**: No transformation.

package/build/services/yaml-workflow/sql.d.ts

@@ -1,4 +1,5 @@
-export declare const CREATE_YAML_WORKFLOW = "\n  INSERT INTO lt_yaml_workflows\n    (name, description, app_id, app_version, source_workflow_id,\n     source_workflow_type, yaml_content, graph_topic,\n     input_schema, output_schema, activity_manifest, input_field_meta,\n     original_prompt, category, tags, metadata, content_version)\n  VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, 1)\n  RETURNING *";
+export declare const CREATE_YAML_WORKFLOW = "\n  INSERT INTO lt_yaml_workflows\n    (name, description, app_id, app_version, source_workflow_id,\n     source_workflow_type, yaml_content, graph_topic,\n     input_schema, output_schema, activity_manifest, input_field_meta,\n     original_prompt, category, tags, metadata, content_version,\n     set_id, set_role, set_build_order)\n  VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, 1, $17, $18, $19)\n  RETURNING *";
+export declare const CHECK_TOPIC_UNIQUE = "\n  SELECT id, name FROM lt_yaml_workflows\n  WHERE app_id = $1 AND graph_topic = $2 AND status != 'archived'\n  LIMIT 1";
 export declare const GET_YAML_WORKFLOW = "\n  SELECT * FROM lt_yaml_workflows WHERE id = $1";
 export declare const GET_YAML_WORKFLOW_BY_NAME = "\n  SELECT * FROM lt_yaml_workflows WHERE name = $1";
 export declare const UPDATE_YAML_WORKFLOW_VERSION = "\n  UPDATE lt_yaml_workflows SET app_version = $2 WHERE id = $1";

package/build/services/yaml-workflow/sql.js

@@ -1,15 +1,20 @@
 "use strict";
 // ─── YAML workflow CRUD ─────────────────────────────────────────────────────
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.FIND_BY_TAGS_ALL = exports.FIND_BY_TAGS_ANY = exports.GET_CRON_SCHEDULED_WORKFLOWS = exports.CLEAR_CRON_SCHEDULE = exports.UPDATE_CRON_SCHEDULE = exports.UPDATE_STATUS_SUFFIX = exports.UPDATE_STATUS_BASE = exports.GET_VERSION_SNAPSHOT = exports.DISCOVER_WORKFLOWS = exports.LIST_VERSIONS = exports.COUNT_VERSIONS = exports.CREATE_VERSION_SNAPSHOT = exports.MARK_APP_ID_CONTENT_DEPLOYED = exports.MARK_CONTENT_DEPLOYED = exports.GET_DISTINCT_APP_IDS = exports.LIST_BY_APP_ID = exports.GET_ACTIVE_YAML_WORKFLOWS = exports.DELETE_YAML_WORKFLOW = exports.UPDATE_YAML_WORKFLOW_VERSION = exports.GET_YAML_WORKFLOW_BY_NAME = exports.GET_YAML_WORKFLOW = exports.CREATE_YAML_WORKFLOW = void 0;
+exports.FIND_BY_TAGS_ALL = exports.FIND_BY_TAGS_ANY = exports.GET_CRON_SCHEDULED_WORKFLOWS = exports.CLEAR_CRON_SCHEDULE = exports.UPDATE_CRON_SCHEDULE = exports.UPDATE_STATUS_SUFFIX = exports.UPDATE_STATUS_BASE = exports.GET_VERSION_SNAPSHOT = exports.DISCOVER_WORKFLOWS = exports.LIST_VERSIONS = exports.COUNT_VERSIONS = exports.CREATE_VERSION_SNAPSHOT = exports.MARK_APP_ID_CONTENT_DEPLOYED = exports.MARK_CONTENT_DEPLOYED = exports.GET_DISTINCT_APP_IDS = exports.LIST_BY_APP_ID = exports.GET_ACTIVE_YAML_WORKFLOWS = exports.DELETE_YAML_WORKFLOW = exports.UPDATE_YAML_WORKFLOW_VERSION = exports.GET_YAML_WORKFLOW_BY_NAME = exports.GET_YAML_WORKFLOW = exports.CHECK_TOPIC_UNIQUE = exports.CREATE_YAML_WORKFLOW = void 0;
 exports.CREATE_YAML_WORKFLOW = `
   INSERT INTO lt_yaml_workflows
     (name, description, app_id, app_version, source_workflow_id,
      source_workflow_type, yaml_content, graph_topic,
      input_schema, output_schema, activity_manifest, input_field_meta,
-     original_prompt, category, tags, metadata, content_version)
-  VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, 1)
+     original_prompt, category, tags, metadata, content_version,
+     set_id, set_role, set_build_order)
+  VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, 1, $17, $18, $19)
   RETURNING *`;
+exports.CHECK_TOPIC_UNIQUE = `
+  SELECT id, name FROM lt_yaml_workflows
+  WHERE app_id = $1 AND graph_topic = $2 AND status != 'archived'
+  LIMIT 1`;
 exports.GET_YAML_WORKFLOW = `
   SELECT * FROM lt_yaml_workflows WHERE id = $1`;
 exports.GET_YAML_WORKFLOW_BY_NAME = `

package/build/services/yaml-workflow/types.d.ts

@@ -24,6 +24,9 @@ export interface CreateYamlWorkflowInput {
     category?: string;
     tags?: string[];
     metadata?: Record<string, unknown>;
+    set_id?: string;
+    set_role?: 'leaf' | 'composition' | 'router';
+    set_build_order?: number;
 }
 /** Mutable state accumulated while building the YAML DAG. */
 export interface DagBuilder {

package/build/services/yaml-workflow/workers/callbacks.js

@@ -47,6 +47,8 @@ function compactForLlm(input) {
  */
 function buildLlmCallback(activity) {
     return async (data) => {
+        const wfName = data.data?.workflowName || activity.workflow_name || activity.activity_id;
+        logger_1.loggerRegistry.debug(`[yaml-workflow:worker] entering llm wf=${wfName} argKeys=[${Object.keys(data.data || {}).join(',')}]`);
         const rawInput = (data.data || {});
         const input = compactForLlm(rawInput);
         const template = activity.prompt_template || '';
@@ -98,7 +100,7 @@ function buildLlmCallback(activity) {
         catch {
             result = { response: content };
         }
-        logger_1.loggerRegistry.info(`[yaml-workflow] LLM step completed (model: ${model}, topic: ${activity.topic})`);
+        logger_1.loggerRegistry.debug(`[yaml-workflow:worker] leaving llm wf=${wfName} resultType=${typeof result}`);
         return {
             metadata: { ...data.metadata },
             data: result,
@@ -154,9 +156,12 @@ function buildTransformCallback(activity) {
     if (!spec)
         throw new Error(`Transform activity ${activity.activity_id} missing transform_spec`);
     return async (data) => {
+        const wfName = data.data?.workflowName || activity.workflow_name || activity.activity_id;
+        logger_1.loggerRegistry.debug(`[yaml-workflow:worker] entering transform wf=${wfName} source=${spec.sourceField} target=${spec.targetField} argKeys=[${Object.keys(data.data || {}).join(',')}]`);
         const input = (data.data || {});
         const sourceData = input[spec.sourceField];
         if (!Array.isArray(sourceData)) {
+            logger_1.loggerRegistry.debug(`[yaml-workflow:worker] leaving transform wf=${wfName} passthrough (source not array)`);
             // Pass through non-array data unchanged
             return {
                 metadata: { ...data.metadata },
@@ -194,6 +199,7 @@ function buildTransformCallback(activity) {
         });
         // Return reshaped data alongside any other input fields (session handles, etc.)
         const result = { ...input, [spec.targetField]: reshaped };
+        logger_1.loggerRegistry.debug(`[yaml-workflow:worker] leaving transform wf=${wfName} reshapedCount=${reshaped.length} resultKeys=[${Object.keys(result).join(',')}]`);
         return {
             metadata: { ...data.metadata },
             data: result,

package/build/services/yaml-workflow/workers/register.js

@@ -125,12 +125,15 @@ async function registerWorkersForWorkflow(workflow) {
                 connection: (0, db_1.getConnection)(),
                 retry: defaultRetry,
                 callback: wrap(async (data) => {
+                    const wfName = data.data?.workflowName || activity.workflow_name || toolName;
+                    logger_1.loggerRegistry.debug(`[yaml-workflow:worker] entering db/${toolName} wf=${wfName} argKeys=[${Object.keys(data.data || {}).join(',')}]`);
                     const args = (data.data || {});
                     let mergedArgs = toolArgs ? { ...toolArgs, ...args } : args;
                     delete mergedArgs._scope;
                     delete mergedArgs.workflowName;
                     mergedArgs = await (0, ephemeral_1.exchangeTokensInArgs)(mergedArgs);
                     const result = await mcpClient.callServerTool(dbServerId, toolName, mergedArgs);
+                    logger_1.loggerRegistry.debug(`[yaml-workflow:worker] leaving db/${toolName} wf=${wfName} resultKeys=[${Object.keys(result || {}).join(',')}]`);
                     return { metadata: { ...data.metadata }, data: result };
                 }),
             });
@@ -158,6 +161,8 @@ async function registerWorkersForWorkflow(workflow) {
                 connection: (0, db_1.getConnection)(),
                 retry: defaultRetry,
                 callback: wrap(async (data) => {
+                    const wfName = data.data?.workflowName || activity.workflow_name || toolName;
+                    logger_1.loggerRegistry.debug(`[yaml-workflow:worker] entering mcp/${toolName} wf=${wfName} server=${serverId} argKeys=[${Object.keys(data.data || {}).join(',')}]`);
                     const args = (data.data || {});
                     // Start from stored defaults, then strip any wired keys that
                     // didn't arrive (upstream failure) so stale defaults don't leak.
@@ -173,6 +178,7 @@ async function registerWorkersForWorkflow(workflow) {
                             mergedArgs[key] = value;
                         }
                     }
+                    logger_1.loggerRegistry.debug(`[yaml-workflow:worker] merged mcp/${toolName} wf=${wfName} mergedKeys=[${Object.keys(mergedArgs).join(',')}]`);
                     // For escalate_and_wait: inject YAML signal routing so the MCP tool
                     // stores engine:'yaml' + hookTopic + jobId in the escalation metadata
                     if (yamlHookTopic) {
@@ -195,6 +201,7 @@ async function registerWorkersForWorkflow(workflow) {
                     if (result == null) {
                         logger_1.loggerRegistry.warn(`[yaml-workflow:worker] ${toolName} returned null/undefined`);
                     }
+                    logger_1.loggerRegistry.debug(`[yaml-workflow:worker] leaving mcp/${toolName} wf=${wfName} resultKeys=[${Object.keys(result || {}).join(',')}]`);
                     return { metadata: { ...data.metadata }, data: result };
                 }),
             });

package/build/start/adapters.js

@@ -8,6 +8,7 @@ const honeycomb_1 = require("../lib/telemetry/honeycomb");
 const events_1 = require("../lib/events");
 const nats_1 = require("../lib/events/nats");
 const socketio_1 = require("../lib/events/socketio");
+const callback_1 = require("../lib/events/callback");
 const maintenance_1 = require("../services/maintenance");
 const maintenance_2 = require("../modules/maintenance");
 const mcp_1 = require("../services/mcp");
@@ -47,6 +48,9 @@ function registerAdapters(startConfig) {
         }
         events_1.eventRegistry.register(new socketio_1.SocketIOEventAdapter());
     }
+    // Always register the callback adapter for SDK event subscriptions.
+    // Zero-cost when no listeners are registered.
+    events_1.eventRegistry.register(new callback_1.CallbackEventAdapter());
     // Maintenance
     if (startConfig.maintenance === false) {
         // Disabled

package/build/system/index.js

@@ -83,6 +83,12 @@ function getSystemWorkers() {
             workers.push({ taskQueue: 'long-tail-system', workflow: mcpWorkflowBuilder });
         }
         catch { /* not available */ }
+        // ── Workflow planner (multi-workflow plan mode) ──
+        try {
+            const { mcpWorkflowPlanner } = require('./workflows/mcp-workflow-planner');
+            workers.push({ taskQueue: 'long-tail-system', workflow: mcpWorkflowPlanner });
+        }
+        catch { /* not available */ }
     }
     return workers;
 }

package/build/system/mcp-servers/knowledge.js

@@ -41,7 +41,7 @@ const knowledge = __importStar(require("../activities/knowledge"));
 const storeSchema = zod_1.z.object({
     domain: zod_1.z.string().describe('Knowledge domain (namespace)'),
     key: zod_1.z.string().describe('Unique key within domain'),
-    data: zod_1.z.record(zod_1.z.any()).describe('JSONB payload to store'),
+    data: zod_1.z.record(zod_1.z.any()).describe('JSONB object payload to store (must be an object, not a string — wrap text as { "description": "..." })'),
     tags: zod_1.z.array(zod_1.z.string()).optional().describe('Categorization tags'),
 });
 const getSchema = zod_1.z.object({

package/build/system/seed/server-definitions.js

@@ -183,7 +183,8 @@ exports.SEED_MCP_SERVERS = [
         tool_manifest: tool_manifests_knowledge_1.KNOWLEDGE_TOOLS,
         metadata: { builtin: true, category: 'knowledge' },
         tags: ['knowledge', 'memory', 'state', 'storage'],
-        compile_hints: 'store_knowledge arguments: domain (string), key (string), data (the content to store — NOT "content"), data_key (optional sub-key). ' +
+        compile_hints: 'store_knowledge arguments: domain (string), key (string), data (object — MUST be a JSON object, never a string), tags (optional string[]). ' +
+            'The data field is a JSONB object. When storing text content, wrap it: { "description": "the text" }. NEVER pass a bare string as data. ' +
             'store_knowledge upserts by domain+key — it merges data if the entry exists. ' +
             'Use a consistent domain name across related workflows to build shared context. ' +
             'search_knowledge uses JSONB containment (@>) — the query object must be a subset of the stored data. ' +

package/build/system/workflows/mcp-workflow-builder/activities/index.d.ts

@@ -1,2 +1,2 @@
-export { loadBuilderTools } from './tool-loader';
+export { loadBuilderTools, loadReferenceSection } from './tool-loader';
 export { callBuilderLLM } from './llm';

package/build/system/workflows/mcp-workflow-builder/activities/index.js

@@ -1,7 +1,8 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.callBuilderLLM = exports.loadBuilderTools = void 0;
+exports.callBuilderLLM = exports.loadReferenceSection = exports.loadBuilderTools = void 0;
 var tool_loader_1 = require("./tool-loader");
 Object.defineProperty(exports, "loadBuilderTools", { enumerable: true, get: function () { return tool_loader_1.loadBuilderTools; } });
+Object.defineProperty(exports, "loadReferenceSection", { enumerable: true, get: function () { return tool_loader_1.loadReferenceSection; } });
 var llm_1 = require("./llm");
 Object.defineProperty(exports, "callBuilderLLM", { enumerable: true, get: function () { return llm_1.callBuilderLLM; } });

package/build/system/workflows/mcp-workflow-builder/activities/tool-loader.d.ts

@@ -3,3 +3,9 @@ export declare function loadBuilderTools(tags?: string[]): Promise<{
     inventory: string;
     strategy: string;
 }>;
+/**
+ * Load a specific section from the activity-types reference file.
+ * Returns the markdown for the requested activity type (await, signal, interrupt).
+ * Returns empty string if the section or file is not found.
+ */
+export declare function loadReferenceSection(section: 'await' | 'signal' | 'interrupt'): Promise<string>;

package/build/system/workflows/mcp-workflow-builder/activities/tool-loader.js

@@ -1,8 +1,34 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.loadBuilderTools = loadBuilderTools;
+exports.loadReferenceSection = loadReferenceSection;
+const promises_1 = require("fs/promises");
+const path_1 = require("path");
 const tool_loader_1 = require("../../shared/tool-loader");
 const caches_1 = require("./caches");
 async function loadBuilderTools(tags) {
     return (0, tool_loader_1.loadToolsFromServers)(tags, { toolServerMap: caches_1.toolServerMap, toolDefCache: caches_1.toolDefCache }, { logPrefix: 'workflowBuilder' });
 }
+/**
+ * Load a specific section from the activity-types reference file.
+ * Returns the markdown for the requested activity type (await, signal, interrupt).
+ * Returns empty string if the section or file is not found.
+ */
+async function loadReferenceSection(section) {
+    try {
+        const refPath = (0, path_1.join)(__dirname, '..', 'reference', 'activity-types.md');
+        const content = await (0, promises_1.readFile)(refPath, 'utf-8');
+        // Extract the section between ## <name> and the next ## or end of file
+        const sectionHeader = `## ${section}`;
+        const startIdx = content.indexOf(sectionHeader);
+        if (startIdx === -1)
+            return '';
+        const afterHeader = content.indexOf('\n', startIdx);
+        const nextSection = content.indexOf('\n## ', afterHeader + 1);
+        const endIdx = nextSection === -1 ? content.length : nextSection;
+        return content.slice(startIdx, endIdx).trim();
+    }
+    catch {
+        return '';
+    }
+}

package/build/system/workflows/mcp-workflow-builder/index.js

@@ -37,7 +37,7 @@ exports.mcpWorkflowBuilder = mcpWorkflowBuilder;
 const hotmesh_1 = require("@hotmeshio/hotmesh");
 const activities = __importStar(require("./activities"));
 const prompts_1 = require("./prompts");
-const { loadBuilderTools, callBuilderLLM, } = hotmesh_1.Durable.workflow.proxyActivities({
+const { loadBuilderTools, loadReferenceSection, callBuilderLLM, } = hotmesh_1.Durable.workflow.proxyActivities({
     activities,
     retry: {
         maximumAttempts: 3,
@@ -94,6 +94,7 @@ async function mcpWorkflowBuilder(envelope) {
     const priorYaml = envelope.data?.prior_yaml;
     const answers = envelope.data?.answers;
     const priorQuestions = envelope.data?.prior_questions;
+    const compositionContext = envelope.data?.composition_context;
     if (!prompt) {
         return {
             type: 'return',
@@ -111,10 +112,33 @@ async function mcpWorkflowBuilder(envelope) {
         raw.strategy ? `${raw.strategy}\n` : '',
         `## Available MCP Servers & Tools\n\n${raw.inventory}`,
     ].filter(Boolean).join('\n');
+    // 3. Load composition references if this workflow is part of a plan
+    const compositionSections = [];
+    if (compositionContext) {
+        if (compositionContext.requires_await) {
+            const awaitRef = await loadReferenceSection('await');
+            if (awaitRef)
+                compositionSections.push(awaitRef);
+        }
+        if (compositionContext.requires_signal) {
+            const signalRef = await loadReferenceSection('signal');
+            if (signalRef)
+                compositionSections.push(signalRef);
+        }
+        if (compositionContext.sibling_schemas?.length) {
+            const siblings = compositionContext.sibling_schemas
+                .map(s => `- **${s.name}** (topic: \`${s.graph_topic}\`)\n  Input: \`${JSON.stringify(s.input_schema)}\`\n  Output: \`${JSON.stringify(s.output_schema)}\``)
+                .join('\n');
+            compositionSections.push(`## Sibling Workflows in This Plan\n\nThis workflow is part of a multi-workflow set. The following sibling workflows exist in the same namespace and can be invoked using \`type: await\` activities:\n\n${siblings}`);
+        }
+    }
+    const fullInventory = compositionSections.length
+        ? `${serverSection}\n\n${compositionSections.join('\n\n')}`
+        : serverSection;
     const messages = [
         {
             role: 'system',
-            content: (0, prompts_1.BUILDER_SYSTEM_PROMPT)(serverSection),
+            content: (0, prompts_1.BUILDER_SYSTEM_PROMPT)(fullInventory),
         },
     ];
     // If refining a prior attempt, inject context