@dypai-ai/mcp 1.2.4 → 1.3.0

package/src/tools/sync/pull.js

@@ -105,6 +105,339 @@ Declarative snapshot of your DYPAI project's backend.
 
 Paths inside YAML (e.g. \`query_file: sql/create_invoice.sql\`) are always relative
 to this folder's root, regardless of where the YAML lives.
+
+## Reference examples
+
+When the project has no endpoints yet, \`dypai_pull\` writes three reference
+files (all with the \`.disabled\` suffix so \`dypai_push\` ignores them):
+
+  - \`endpoints/_example.yaml.disabled\`        full workflow tour
+  - \`endpoints/_example-tool.yaml.disabled\`   endpoint marked \`tool: true\`
+  - \`endpoints/_example-agent.yaml.disabled\`  endpoint with an agent node using the tool
+
+Read them to learn the canonical YAML format (placeholders, branching, error
+handling, agent tools). Copy + adapt:
+
+  cp endpoints/_example.yaml.disabled       endpoints/my-feature.yaml
+  cp endpoints/_example-tool.yaml.disabled  endpoints/search-products.yaml
+  cp endpoints/_example-agent.yaml.disabled endpoints/chat.yaml
+`
+
+// Reference endpoint that ships with empty projects so the agent can learn
+// the YAML format by reading a real, complete example. Covers: trigger, input
+// schema, output schema, multi-node workflow, ${input.x} / ${nodes.x.y} /
+// ${current_user_id} placeholders, expressions, branching with `handle`,
+// per-node error handling, multiple return paths, *_file references.
+const EXAMPLE_ENDPOINT_YAML = `# ============================================================================
+# DYPAI endpoint reference — complete tour of the YAML format.
+# This file is IGNORED by dypai_push, dypai_validate, dypai_diff and
+# dypai_test_endpoint (the .disabled extension keeps it out of every flow).
+#
+# Copy + adapt when authoring a new endpoint:
+#   cp endpoints/_example.yaml.disabled endpoints/my-feature.yaml
+# Then edit and remove the .disabled suffix.
+# ============================================================================
+
+name: example-process-order
+description: |
+  Reference endpoint demonstrating the canonical YAML format:
+  trigger, input/output schemas, multi-node workflow, placeholders,
+  branching, per-node error handling, and multiple return paths.
+method: POST
+
+# ─── Trigger ────────────────────────────────────────────────────────────────
+# Pick exactly one. Options:
+#   trigger: { http_api: { auth_mode: jwt | api_key | public } }
+#   trigger: { webhook: {} }                 # path is auto-generated
+#   trigger: { schedule: { cron: "0 9 * * 1-5", timezone: Europe/Madrid } }
+#   trigger: { telegram: {} }                # incoming Telegram messages
+#
+# auth_mode notes:
+#   jwt      — user-authenticated. \${current_user_id} / \${current_user_role} available.
+#   api_key  — server-to-server. No user context.
+#   public   — anonymous. Use only for read-only, non-sensitive data.
+trigger:
+  http_api:
+    auth_mode: jwt
+
+# ─── Input schema (JSON Schema) ─────────────────────────────────────────────
+# Validates the request body. Fields here are referenced via \${input.<field>}.
+# The validator (dypai_validate) catches typos in placeholders against this.
+input:
+  type: object
+  required: [product_id, quantity]
+  properties:
+    product_id:
+      type: string
+      format: uuid
+    quantity:
+      type: integer
+      minimum: 1
+    note:
+      type: string
+
+# ─── Output schema (optional but recommended) ───────────────────────────────
+# Describes the response shape. Helps the validator + frontend type-checkers.
+output:
+  type: object
+  properties:
+    order_id: { type: string }
+    total:    { type: number }
+    status:   { type: string }
+
+# ─── Workflow ───────────────────────────────────────────────────────────────
+# Nodes are the steps. Placeholders wire data flow between them:
+#
+#   \${input.<field>}            — the request body / query params
+#   \${nodes.<id>.<field>}       — output of a previous node
+#   \${current_user_id}          — UUID of the JWT-authenticated user
+#   \${current_user_role}        — role name from the JWT
+#
+# Expressions inside placeholders work: arithmetic, comparisons, JS-ish.
+#   \${input.qty * 2}, \${nodes.x.stock >= input.qty}
+#
+# In SQL contexts (custom_query, dypai_database queries) the engine
+# automatically casts placeholders by value shape:
+#   UUID-shaped string  →  binds as ::uuid
+#   plain object/array  →  binds as ::jsonb
+#   Date instance       →  binds as ::timestamptz
+#   text / int / bool   →  binds without an explicit cast (Postgres infers)
+# So you write SQL naturally — DON'T add '\${current_user_id}'::uuid manually.
+# Just write: WHERE user_id = \${current_user_id}
+workflow:
+  nodes:
+
+    # 1. Read the product. \`operation: query\` for ANY read — pure SQL.
+    - id: get_product
+      type: dypai_database
+      operation: query
+      query: SELECT * FROM products WHERE id = \${input.product_id} LIMIT 1
+      # Output is accessible as \${nodes.get_product.<column>} (first row's columns)
+
+    # 2. Branch on stock availability.
+    #    \`logic\` nodes with \`operation: if\` send flow down two edges
+    #    differentiated by \`handle: "true"\` and \`handle: "false"\`.
+    - id: check_stock
+      type: logic
+      operation: if
+      condition: \${nodes.get_product.stock >= input.quantity}
+
+    # 3. Insert the order. \`operation: mutation\` for trivial single-table writes
+    #    — declarative, no SQL needed. The shape (insert/update/delete) decides.
+    - id: insert_order
+      type: dypai_database
+      operation: mutation
+      table: orders
+      insert:
+        user_id: \${current_user_id}                            # JWT identity
+        product_id: \${input.product_id}                        # request input
+        quantity: \${input.quantity}
+        unit_price: \${nodes.get_product.price}                 # cross-node ref
+        total: \${nodes.get_product.price * input.quantity}     # expression
+        note: \${input.note}
+      returning: "*"
+      # For UPSERT: add  on_conflict: { columns: [id], action: update }
+      # For UPDATE: replace \`insert:\` with \`update: {...}\` + \`where: {...}\`
+      # For DELETE: replace \`insert:\` with \`delete: true\` + \`where: {...}\`
+      #
+      # Long SQL (>500 chars) inside \`operation: query\` is auto-extracted on
+      # \`dypai_pull\` to sql/<endpoint>.sql and referenced like:
+      #   query_file: sql/insert_order.sql
+      # Same for prompts (system_prompt_file) and JS/Python code (code_file).
+
+    # 4. Notify a downstream system. \`on_error_step_id\` routes failures
+    #    to \`build_response\` so a notification outage doesn't 500 the order.
+    - id: notify
+      type: http_request
+      method: POST
+      url: https://hooks.example.com/orders
+      body:
+        order_id: \${nodes.insert_order.id}
+        total: \${nodes.insert_order.total}
+      on_error_step_id: build_response
+
+    # 5. Build the success response. Exactly one node per code path needs
+    #    \`return: true\` — it marks that node's output as the HTTP body.
+    - id: build_response
+      type: set_fields
+      return: true
+      fields:
+        order_id: \${nodes.insert_order.id}
+        total: \${nodes.insert_order.total}
+        status: created
+
+    # 6. Error path: out of stock. Different shape, also \`return: true\`.
+    #    Multiple return nodes are fine as long as only one runs per execution.
+    - id: out_of_stock
+      type: set_fields
+      return: true
+      fields:
+        error: out_of_stock
+        message: Not enough stock for the requested quantity
+        available: \${nodes.get_product.stock}
+        requested: \${input.quantity}
+
+  # ─── Edges ────────────────────────────────────────────────────────────────
+  # Connect nodes. Form: { from: <id>, to: <id> }.
+  # For \`logic\` if-nodes use \`handle: "true"\` / \`handle: "false"\` to pick the branch.
+  edges:
+    - { from: get_product,  to: check_stock }
+    - { from: check_stock,  to: insert_order, handle: "true" }
+    - { from: check_stock,  to: out_of_stock, handle: "false" }
+    - { from: insert_order, to: notify }
+    - { from: notify,       to: build_response }
+
+  # \`on_error\` at the workflow level: stop (default) | continue
+  # on_error: stop
+`
+
+// Reference TOOL endpoint — an endpoint marked \`tool: true\` so an \`agent\` node
+// elsewhere can invoke it. Pair with EXAMPLE_AGENT_ENDPOINT_YAML below.
+const EXAMPLE_TOOL_ENDPOINT_YAML = `# ============================================================================
+# DYPAI reference — endpoint exposed as a TOOL for \`agent\` nodes.
+# Ignored by dypai_push/validate/diff/test_endpoint (.disabled suffix).
+#
+# Pair with _example-agent.yaml.disabled — that one has an agent node that
+# references THIS endpoint by name.
+#
+# Copy + adapt:
+#   cp endpoints/_example-tool.yaml.disabled endpoints/search-products.yaml
+# ============================================================================
+
+name: example-search-products
+description: Search the product catalog. Exposed as a tool for agent nodes.
+method: GET
+
+# ─── Tool flag + description ────────────────────────────────────────────────
+# \`tool: true\` exposes this endpoint to agent nodes (writes is_tool=true in DB).
+# \`tool_description\` is the prompt the LLM reads when deciding to call it.
+# Be SPECIFIC about when to use it and what it returns — vague descriptions
+# cause the LLM to skip useful tools or misuse them.
+tool: true
+tool_description: |
+  Search the product catalog by free-text query.
+  Use when the user asks about availability, prices, or categories.
+  Returns up to \`limit\` products with id, name, price, stock.
+
+# ─── Input schema (tight JSON Schema with descriptions) ─────────────────────
+# The LLM uses THIS schema to fabricate tool arguments. Every field benefits
+# from a \`description\`; typed and required fields keep hallucinated args out.
+input:
+  type: object
+  required: [query]
+  properties:
+    query:
+      type: string
+      description: Free-text search term (matches product name, case-insensitive).
+    limit:
+      type: integer
+      default: 10
+      minimum: 1
+      maximum: 50
+      description: Max results to return.
+
+# ─── Output schema (optional but recommended for tools) ─────────────────────
+output:
+  type: object
+  properties:
+    results:
+      type: array
+      items:
+        type: object
+        properties:
+          id:    { type: string }
+          name:  { type: string }
+          price: { type: number }
+          stock: { type: integer }
+
+# ─── Trigger ────────────────────────────────────────────────────────────────
+# Agent calls are server-to-server → \`api_key\`. Use \`jwt\` only if the tool
+# needs \${current_user_id}. NEVER \`public\` for a tool that writes data.
+trigger:
+  http_api:
+    auth_mode: api_key
+
+workflow:
+  nodes:
+    - id: run
+      type: dypai_database
+      operation: query
+      return: true
+      query: |
+        SELECT id, name, price, stock
+        FROM products
+        WHERE name ILIKE '%' || \${input.query} || '%'
+        ORDER BY name
+        LIMIT \${input.limit}
+`
+
+// Reference endpoint with an \`agent\` node that USES a tool endpoint by name.
+// Pair with EXAMPLE_TOOL_ENDPOINT_YAML above.
+const EXAMPLE_AGENT_ENDPOINT_YAML = `# ============================================================================
+# DYPAI reference — endpoint with an \`agent\` node that invokes tool endpoints.
+# Ignored by dypai_push/validate/diff/test_endpoint (.disabled suffix).
+#
+# Pair with _example-tool.yaml.disabled — this file references that one by name.
+#
+# Copy + adapt:
+#   cp endpoints/_example-agent.yaml.disabled endpoints/chat.yaml
+# ============================================================================
+
+name: example-chat-assistant
+description: Chat assistant that uses tool endpoints to answer questions about products.
+method: POST
+
+trigger:
+  http_api:
+    auth_mode: jwt                            # user-authenticated chat
+
+input:
+  type: object
+  required: [message]
+  properties:
+    message:
+      type: string
+      description: The user's message to the assistant.
+
+output:
+  type: object
+  properties:
+    reply:     { type: string }
+    tool_calls: { type: array }
+
+workflow:
+  nodes:
+    - id: chat
+      type: agent
+      provider: openai                        # openai | anthropic | google
+      model: gpt-4o                           # model id — see list via UI or search_docs
+      credential: openai-prod                 # credential NAME — must exist (get_app_credentials)
+      return: true
+
+      system_prompt: |
+        You are a helpful shop assistant. You have access to tools for searching
+        the product catalog and sending emails. When the user asks about
+        availability, prices, or categories, call example-search-products.
+        Always be concise and friendly.
+
+      input: \${input.message}                # what the user said → the agent's turn
+
+      # ─── Tools (IMPORTANT — BY NAME, not UUID) ─────────────────────────────
+      # In YAML the parameter is \`tools: [<endpoint_name>, ...]\`. The codec
+      # resolves names ↔ UUIDs automatically on push/pull. Writing
+      # \`tool_ids: [...]\` in YAML BYPASSES the codec and fails in production.
+      #
+      # Every name here MUST reference an endpoint with \`tool: true\`.
+      # dypai_validate catches typos (rule: agent_tool_not_found) and lists
+      # the available tool endpoints in its fix_hint.
+      tools:
+        - example-search-products             # ← references the other example file
+        # - example-send-email                 # add more tools as you define them
+
+      # ─── Agent behavior tuning ─────────────────────────────────────────────
+      max_iterations: 5                       # max rounds of tool-calling (anti-runaway)
+      temperature: 0.7
+      tool_timeout: 30                        # seconds per tool call
 `
 
 async function execSql(projectId, sql) {
@@ -150,10 +483,13 @@ function renderYaml(doc) {
 export const dypaiPullTool = {
   name: "dypai_pull",
   description:
-    "Serializes the remote project state to local YAML files under ./dypai/. " +
-    "Writes endpoints/<name>.yaml + sql/ and prompts/ for extracted content. " +
+    "Downloads BACKEND state (endpoints, SQL, prompts, node catalog, realtime policies, schema) to local YAML files under ./dypai/. " +
+    "Writes endpoints/<name>.yaml + sql/ + prompts/ + code/ for extracted content. " +
     "Canvas positions are stripped (regenerated by visual editor). Safe to run repeatedly. " +
-    "Use this to start editing a project locally with your editor + AI agent. " +
+    "Use this to start editing a project locally with your editor + AI agent.\n\n" +
+    "SCOPE: backend only. This does NOT download frontend React/Vite source code — for that call " +
+    "`manage_frontend(operation: \"sync\", targetDirectory: <abs>)`. The two are independent; run both when " +
+    "starting fresh on a full-stack project.\n\n" +
     "IMPORTANT: when called by an IDE-hosted MCP, the process cwd is often the user's home dir — " +
     "pass an ABSOLUTE path in out_dir (e.g. /Users/me/projects/my-app/dypai) to avoid writing to the wrong place. " +
     "If out_dir is relative, the tool tries to auto-detect the workspace via env vars and git markers, " +
@@ -321,6 +657,30 @@ export const dypaiPullTool = {
       }
     }
 
+    // Reference examples: only ship them on empty projects so the agent has
+    // canonical YAML tours to learn from. Once the project has real endpoints,
+    // those serve as the live examples — no need for the disabled stubs.
+    // .disabled suffix keeps them out of validate / diff / push / test_endpoint.
+    //
+    // Three separate files, each copy-ready for a different pattern:
+    //   _example.yaml.disabled         — full workflow tour (trigger, nodes, edges, branching)
+    //   _example-tool.yaml.disabled    — endpoint exposed as a tool for agent nodes
+    //   _example-agent.yaml.disabled   — endpoint with an agent node that uses the tool above
+    if (endpoints.length === 0) {
+      const refs = [
+        ["endpoints/_example.yaml.disabled",       EXAMPLE_ENDPOINT_YAML],
+        ["endpoints/_example-tool.yaml.disabled",  EXAMPLE_TOOL_ENDPOINT_YAML],
+        ["endpoints/_example-agent.yaml.disabled", EXAMPLE_AGENT_ENDPOINT_YAML],
+      ]
+      for (const [relPath, content] of refs) {
+        const fullPath = join(outDir, relPath)
+        try { await access(fullPath) } catch {
+          await writeFileEnsured(fullPath, content)
+          filesWritten.push(relPath)
+        }
+      }
+    }
+
     // Fetch project metadata to persist identity in the committed config
     let projectInfo = null
     if (project_id) {

package/src/tools/sync/test-endpoint.js

@@ -15,6 +15,7 @@ import YAML from "yaml"
 import { proxyToolCall } from "../proxy.js"
 import { deserializeEndpoint } from "./codec.js"
 import { readLocalConfig, fetchRemoteState } from "./planner.js"
+import { runValidation } from "./validate.js"
 import { summarizeTestWorkflowResponse } from "../trace-summarize.js"
 
 // ─── Local endpoint file discovery ──────────────────────────────────────────
@@ -43,6 +44,9 @@ async function findEndpointByName(rootDir, name) {
       const full = join(dir, e.name)
       if (e.isDirectory()) await walk(full)
       else if (e.isFile() && (e.name.endsWith(".yaml") || e.name.endsWith(".yml"))) {
+        // Skip disabled reference files (e.g. _example.yaml.disabled — already
+        // filtered by extension above, but also skip *.disabled.yaml form).
+        if (e.name.endsWith(".disabled.yaml") || e.name.endsWith(".disabled.yml")) continue
         candidates.push(full)
       }
     }
@@ -109,11 +113,16 @@ export const dypaiTestEndpointTool = {
         type: "string",
         description: "Project UUID. Auto-resolved from dypai.config.yaml.",
       },
+      skip_validation: {
+        type: "boolean",
+        description: "Skip the pre-flight validate pass. Default false — useful only when you intentionally want to send a malformed YAML to the engine to inspect its raw error.",
+        default: false,
+      },
     },
     required: ["endpoint"],
   },
 
-  async execute({ endpoint, input = {}, as_user, trace_mode = "smart", root_dir = "./dypai", project_id } = {}) {
+  async execute({ endpoint, input = {}, as_user, trace_mode = "smart", root_dir = "./dypai", project_id, skip_validation = false } = {}) {
     const rootDir = resolvePath(process.cwd(), root_dir)
 
     if (!endpoint) {
@@ -172,6 +181,35 @@ export const dypaiTestEndpointTool = {
       }
     }
 
+    // ── Pre-flight validation (skippable) ────────────────────────────────────
+    // Run the same linter dypai_push uses, but filter to errors that affect THIS
+    // endpoint only — warnings and other-endpoint diagnostics shouldn't block
+    // a focused test. Catches things like mutation+query_file, missing where,
+    // unknown node types, placeholder typos, etc. before the engine sees them.
+    if (!skip_validation) {
+      try {
+        const valResult = await runValidation(rootDir, targetProjectId)
+        const relevantErrors = (valResult.diagnostics || []).filter(d =>
+          d.severity === "error" && d.endpoint === found.doc.name
+        )
+        if (relevantErrors.length > 0) {
+          return {
+            success: false,
+            phase: "pre_flight_validation",
+            endpoint,
+            file: found.full.replace(rootDir + "/", ""),
+            error: `${relevantErrors.length} validation error(s) in this endpoint — fix them before testing.`,
+            errors: relevantErrors,
+            hint: "Each error includes a fix_hint. Address them and re-run dypai_test_endpoint. Pass skip_validation: true if you want to bypass this and see the raw engine response.",
+          }
+        }
+      } catch (e) {
+        // Non-fatal: if validation itself crashes, continue to the test so
+        // the agent at least gets the engine's response. Logged as a warning.
+        process.stderr.write(`[dypai_test_endpoint] pre-flight validation failed: ${e.message}\n`)
+      }
+    }
+
     // Deserialize the local YAML to the engine-shaped workflow_code
     const deserCtx = { ...mapsCtx, readFile: (p) => fileMap[p] ?? "" }
     let row
@@ -197,13 +235,41 @@ export const dypaiTestEndpointTool = {
 
     try {
       const result = await proxyToolCall("test_workflow", execArgs)
+
+      // Defensive: the remote sometimes returns a plain error string (not a
+      // structured object) when something fails outside the workflow trace —
+      // e.g. SQL operator errors before the trace is built. Spreading a string
+      // with `...` would explode it character-by-character into the response,
+      // which is what produced the {"0":"o","1":"p",...} bug. Detect and wrap.
+      if (typeof result === "string") {
+        return {
+          success: false,
+          endpoint,
+          file: found.full.replace(rootDir + "/", ""),
+          as_user: as_user || null,
+          error: result.length > 2000 ? result.slice(0, 2000) + "...[truncated]" : result,
+          hint: "The remote returned a raw error string (no per-node trace available). Read the error above for the root cause.",
+        }
+      }
+      if (!result || typeof result !== "object") {
+        return {
+          success: false,
+          endpoint,
+          file: found.full.replace(rootDir + "/", ""),
+          as_user: as_user || null,
+          error: `Unexpected response type from remote test_workflow: ${typeof result}`,
+          raw_response: result,
+        }
+      }
+
       // Summarize the trace just like the direct test_workflow path.
       const summarized = summarizeTestWorkflowResponse(result, trace_mode)
+      const safeSummary = (summarized && typeof summarized === "object" && !Array.isArray(summarized)) ? summarized : { result: summarized }
       return {
         endpoint,
         file: found.full.replace(rootDir + "/", ""),
         as_user: as_user || null,
-        ...summarized,
+        ...safeSummary,
       }
     } catch (e) {
       return {