@dypai-ai/mcp 1.0.9 → 1.1.0

package/src/tools/sync/test-endpoint.js

@@ -0,0 +1,210 @@
+/**
+ * dypai_test_endpoint — test an endpoint by name using its LOCAL YAML.
+ *
+ * Pre-git-first world: the agent had to pass workflow_code or endpoint_id to
+ * the remote test_workflow. Now the YAML lives on disk — so we just take a
+ * name + input, read the file, deserialize via the codec, and execute.
+ *
+ * The big win: you're testing the EDITED version before pushing. Tight
+ * iteration loop without round-tripping through dypai_push + remote lookup.
+ */
+
+import { readFile, readdir } from "fs/promises"
+import { join, resolve as resolvePath } from "path"
+import YAML from "yaml"
+import { proxyToolCall } from "../proxy.js"
+import { deserializeEndpoint } from "./codec.js"
+import { readLocalConfig, fetchRemoteState } from "./planner.js"
+import { summarizeTestWorkflowResponse } from "../trace-summarize.js"
+
+// ─── Local endpoint file discovery ──────────────────────────────────────────
+
+function collectFileRefs(doc) {
+  const refs = new Set()
+  const walk = (node) => {
+    if (!node || typeof node !== "object") return
+    if (Array.isArray(node)) return node.forEach(walk)
+    for (const [k, v] of Object.entries(node)) {
+      if (typeof v === "string" && k.endsWith("_file")) refs.add(v)
+      else walk(v)
+    }
+  }
+  walk(doc)
+  return Array.from(refs)
+}
+
+async function findEndpointByName(rootDir, name) {
+  const endpointsDir = join(rootDir, "endpoints")
+  const candidates = []
+
+  async function walk(dir) {
+    const entries = await readdir(dir, { withFileTypes: true }).catch(() => [])
+    for (const e of entries) {
+      const full = join(dir, e.name)
+      if (e.isDirectory()) await walk(full)
+      else if (e.isFile() && (e.name.endsWith(".yaml") || e.name.endsWith(".yml"))) {
+        candidates.push(full)
+      }
+    }
+  }
+  await walk(endpointsDir)
+
+  // 1. Exact doc.name match (authoritative)
+  for (const full of candidates) {
+    try {
+      const raw = await readFile(full, "utf8")
+      const doc = YAML.parse(raw)
+      if (doc?.name === name) return { full, doc }
+    } catch { /* skip malformed */ }
+  }
+
+  // 2. Fallback: filename match (handles case where the agent guessed the file)
+  const filenameMatch = candidates.find(p => p.endsWith(`/${name}.yaml`) || p.endsWith(`/${name}.yml`))
+  if (filenameMatch) {
+    const raw = await readFile(filenameMatch, "utf8")
+    const doc = YAML.parse(raw)
+    return { full: filenameMatch, doc }
+  }
+
+  return null
+}
+
+// ─── Tool ───────────────────────────────────────────────────────────────────
+
+export const dypaiTestEndpointTool = {
+  name: "dypai_test_endpoint",
+  description:
+    "Test an endpoint using its LOCAL YAML (dypai/endpoints/<name>.yaml, any group). " +
+    "You only pass the endpoint name + input — the tool reads the YAML, inlines referenced SQL/code/prompt files, " +
+    "and runs a debug execution against the engine. Use this to iterate BEFORE dypai_push. " +
+    "For jwt endpoints, pass as_user with the UUID to impersonate. " +
+    "Returns a summarized per-node trace by default; pass trace_mode: 'full' for the unabbreviated view.",
+  inputSchema: {
+    type: "object",
+    properties: {
+      endpoint: {
+        type: "string",
+        description: "Endpoint name as declared in its YAML (e.g. 'create-order'). Looked up in dypai/endpoints/** by matching doc.name.",
+      },
+      input: {
+        type: "object",
+        description: "Input body for the execution. Maps to ${input.*} placeholders inside the workflow.",
+      },
+      as_user: {
+        type: "string",
+        description: "User UUID to impersonate. Required for jwt endpoints that read ${current_user_id}.",
+      },
+      trace_mode: {
+        type: "string",
+        enum: ["smart", "full", "minimal"],
+        description: "How to summarize the returned trace. 'smart' (default) surfaces the failing node in detail; 'full' returns everything; 'minimal' returns only status + duration.",
+        default: "smart",
+      },
+      root_dir: {
+        type: "string",
+        description: "Path to the dypai/ folder (default: ./dypai).",
+        default: "./dypai",
+      },
+      project_id: {
+        type: "string",
+        description: "Project UUID. Auto-resolved from dypai.config.yaml.",
+      },
+    },
+    required: ["endpoint"],
+  },
+
+  async execute({ endpoint, input = {}, as_user, trace_mode = "smart", root_dir = "./dypai", project_id } = {}) {
+    const rootDir = resolvePath(process.cwd(), root_dir)
+
+    if (!endpoint) {
+      return { success: false, error: "endpoint name is required." }
+    }
+
+    const found = await findEndpointByName(rootDir, endpoint)
+    if (!found) {
+      return {
+        success: false,
+        error: `Endpoint '${endpoint}' not found under dypai/endpoints/.`,
+        hint: "Run dypai_pull to refresh, or check that the YAML's top-level `name` matches.",
+      }
+    }
+
+    // Pre-read any *_file references so the codec can inline them
+    const refs = collectFileRefs(found.doc)
+    const fileMap = {}
+    for (const ref of refs) {
+      try {
+        fileMap[ref] = await readFile(join(rootDir, ref), "utf8")
+      } catch (e) {
+        return {
+          success: false,
+          error: `Referenced file not readable: ${ref} (${e.message})`,
+          hint: "The YAML points to a *_file that doesn't exist on disk. Run dypai_pull or create the missing file.",
+        }
+      }
+    }
+
+    // Resolve project + credential/tool UUID maps for the codec
+    const config = await readLocalConfig(rootDir)
+    const targetProjectId = project_id || config?.project_id
+    if (!targetProjectId) {
+      return {
+        success: false,
+        error: "project_id required (set it in dypai.config.yaml or pass it explicitly).",
+      }
+    }
+
+    let mapsCtx
+    try {
+      const remote = await fetchRemoteState(targetProjectId)
+      mapsCtx = remote.mapsCtx
+    } catch (e) {
+      return {
+        success: false,
+        error: `Could not fetch remote state to resolve credential/tool references: ${e.message}`,
+      }
+    }
+
+    // Deserialize the local YAML to the engine-shaped workflow_code
+    const deserCtx = { ...mapsCtx, readFile: (p) => fileMap[p] ?? "" }
+    let row
+    try {
+      row = deserializeEndpoint(found.doc, deserCtx)
+    } catch (e) {
+      return {
+        success: false,
+        error: `Failed to deserialize '${endpoint}' from its YAML: ${e.message}`,
+        hint: "Run dypai_validate to surface the specific problem.",
+      }
+    }
+
+    // Build the call to the remote test_workflow (pass workflow_code inline so
+    // we test the LOCAL version, not whatever is currently deployed).
+    const execArgs = {
+      project_id: targetProjectId,
+      workflow_code: row.workflow_code,
+      data: input,
+      trace_mode,  // used by the local MCP enrichment layer
+    }
+    if (as_user) execArgs.impersonated_user_id = as_user
+
+    try {
+      const result = await proxyToolCall("test_workflow", execArgs)
+      // Summarize the trace just like the direct test_workflow path.
+      const summarized = summarizeTestWorkflowResponse(result, trace_mode)
+      return {
+        endpoint,
+        file: found.full.replace(rootDir + "/", ""),
+        as_user: as_user || null,
+        ...summarized,
+      }
+    } catch (e) {
+      return {
+        success: false,
+        error: `Execution failed: ${e.message}`,
+        endpoint,
+        hint: "If the error is cryptic, try trace_mode: 'full' or use dypai_trace with the execution_id from the response.",
+      }
+    }
+  },
+}

package/src/tools/sync/test.js

@@ -0,0 +1,343 @@
+/**
+ * dypai_test — YAML-defined tests against endpoints. Zero engine changes:
+ * orchestrates `execute_sql` + `test_workflow` (impersonation) + `execute_sql`
+ * using what already exists.
+ *
+ * File layout (committable under dypai/tests/):
+ *   endpoint: create-order        # or endpoint_id: <uuid>
+ *   tests:
+ *     - name: "creates order with valid product"
+ *       setup_sql: INSERT INTO products ...
+ *       as_user: 11111111-1111-1111-1111-111111111111
+ *       input: { product_id: "p1", qty: 3 }
+ *       expect:
+ *         success: true
+ *         response:
+ *           ok: true
+ *           order_id: { matches: uuid }
+ *         db_queries:
+ *           - sql: "SELECT COUNT(*)::int FROM orders WHERE product_id='p1'"
+ *             equals: 1
+ *       teardown_sql: DELETE FROM orders WHERE product_id='p1'; DELETE FROM products WHERE id='p1';
+ */
+
+import { readFile, readdir } from "fs/promises"
+import { join, resolve as resolvePath } from "path"
+import YAML from "yaml"
+import { proxyToolCall } from "../proxy.js"
+import { readLocalConfig } from "./planner.js"
+
+// ─── Test file discovery ────────────────────────────────────────────────────
+
+async function findTestFiles(testsDir) {
+  const out = []
+  async function walk(dir) {
+    const entries = await readdir(dir, { withFileTypes: true }).catch(() => [])
+    for (const e of entries) {
+      const full = join(dir, e.name)
+      if (e.isDirectory()) await walk(full)
+      else if (e.isFile() && (e.name.endsWith(".test.yaml") || e.name.endsWith(".test.yml"))) out.push(full)
+    }
+  }
+  await walk(testsDir)
+  return out
+}
+
+// ─── SQL helpers ────────────────────────────────────────────────────────────
+
+async function execSql(projectId, sql) {
+  const res = await proxyToolCall("execute_sql", { project_id: projectId, sql })
+  if (res?.error) throw new Error(res.error)
+  return res
+}
+
+function firstCellOf(res) {
+  const row = res?.rows?.[0]
+  if (!row) return undefined
+  const k = Object.keys(row)[0]
+  return k ? row[k] : undefined
+}
+
+/** Resolve an endpoint name to its UUID via system.endpoints (the remote
+ * test_workflow tool only accepts endpoint_id, not endpoint_name). */
+const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i
+async function resolveEndpointId(projectId, ref, cache) {
+  if (UUID_RE.test(ref)) return ref
+  if (cache.has(ref)) return cache.get(ref)
+  const safeName = String(ref).replace(/'/g, "''")
+  const res = await execSql(projectId,
+    `SELECT id FROM system.endpoints WHERE name = '${safeName}' LIMIT 1`)
+  const id = firstCellOf(res)
+  if (!id) throw new Error(`Endpoint '${ref}' not found (looked up in system.endpoints by name).`)
+  cache.set(ref, id)
+  return id
+}
+
+// ─── Assertion matchers ─────────────────────────────────────────────────────
+
+/**
+ * Deep match `actual` against `expected`. Expected can contain meta-matchers
+ * via special objects with keys `matches`, `equals`, `contains`, `type`,
+ * `exists`, `gte`, `lte`. Returns { ok, errors: [] }.
+ */
+function matchValue(actual, expected, path = "") {
+  if (expected && typeof expected === "object" && !Array.isArray(expected)) {
+    const metaKeys = new Set(["matches", "equals", "contains", "type", "exists", "gte", "lte"])
+    const keys = Object.keys(expected)
+    if (keys.length && keys.every(k => metaKeys.has(k))) return matchMeta(actual, expected, path)
+
+    if (actual == null || typeof actual !== "object") {
+      return { ok: false, errors: [`${path || "(root)"}: expected object, got ${JSON.stringify(actual)}`] }
+    }
+    const errors = []
+    for (const k of keys) {
+      const r = matchValue(actual[k], expected[k], path ? `${path}.${k}` : k)
+      if (!r.ok) errors.push(...r.errors)
+    }
+    return errors.length ? { ok: false, errors } : { ok: true }
+  }
+
+  if (Array.isArray(expected)) {
+    if (!Array.isArray(actual)) return { ok: false, errors: [`${path}: expected array, got ${typeof actual}`] }
+    if (actual.length !== expected.length) return { ok: false, errors: [`${path}: length ${expected.length} expected, got ${actual.length}`] }
+    const errors = []
+    expected.forEach((item, i) => {
+      const r = matchValue(actual[i], item, `${path}[${i}]`)
+      if (!r.ok) errors.push(...r.errors)
+    })
+    return errors.length ? { ok: false, errors } : { ok: true }
+  }
+
+  if (actual !== expected) {
+    return { ok: false, errors: [`${path || "(root)"}: expected ${JSON.stringify(expected)}, got ${JSON.stringify(actual)}`] }
+  }
+  return { ok: true }
+}
+
+function matchMeta(actual, meta, path) {
+  const errs = []
+  if ("equals" in meta && actual !== meta.equals) {
+    errs.push(`${path}: expected ${JSON.stringify(meta.equals)}, got ${JSON.stringify(actual)}`)
+  }
+  if ("matches" in meta) {
+    const m = meta.matches
+    const str = String(actual ?? "")
+    if (m === "uuid") {
+      if (!/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(str)) {
+        errs.push(`${path}: expected UUID, got ${JSON.stringify(actual)}`)
+      }
+    } else {
+      try {
+        if (!new RegExp(m).test(str)) errs.push(`${path}: did not match /${m}/, got ${JSON.stringify(actual)}`)
+      } catch {
+        errs.push(`${path}: invalid regex ${JSON.stringify(m)}`)
+      }
+    }
+  }
+  if ("contains" in meta) {
+    if (typeof actual !== "string" || !actual.includes(String(meta.contains))) {
+      errs.push(`${path}: expected to contain ${JSON.stringify(meta.contains)}, got ${JSON.stringify(actual)}`)
+    }
+  }
+  if ("type" in meta && typeof actual !== meta.type) {
+    errs.push(`${path}: expected type ${meta.type}, got ${typeof actual}`)
+  }
+  if ("exists" in meta) {
+    const exists = actual !== undefined && actual !== null
+    if (meta.exists !== exists) errs.push(`${path}: expected exists=${meta.exists}, got ${exists}`)
+  }
+  if ("gte" in meta && !(Number(actual) >= Number(meta.gte))) {
+    errs.push(`${path}: expected >= ${meta.gte}, got ${actual}`)
+  }
+  if ("lte" in meta && !(Number(actual) <= Number(meta.lte))) {
+    errs.push(`${path}: expected <= ${meta.lte}, got ${actual}`)
+  }
+  return errs.length ? { ok: false, errors: errs } : { ok: true }
+}
+
+// ─── Single test runner ─────────────────────────────────────────────────────
+
+async function runSingleTest(test, fileCtx) {
+  const result = { name: test.name || "(unnamed)", status: "pass", errors: [] }
+
+  // Setup
+  if (test.setup_sql) {
+    try { await execSql(fileCtx.projectId, test.setup_sql) }
+    catch (e) { return { ...result, status: "error", phase: "setup", errors: [`setup_sql failed: ${e.message}`] } }
+  }
+
+  try {
+    // Resolve endpoint → endpoint_id (remote only accepts UUIDs)
+    const endpointRef = test.endpoint || test.endpoint_id || fileCtx.endpoint
+    if (!endpointRef) {
+      return { ...result, status: "error", errors: ["No endpoint specified. Set `endpoint` at file root or test level."] }
+    }
+    const endpointId = await resolveEndpointId(fileCtx.projectId, endpointRef, fileCtx.endpointCache)
+
+    const execArgs = {
+      project_id: fileCtx.projectId,
+      endpoint_id: endpointId,
+      data: test.input || {},
+      trace_mode: "minimal",  // keep tests fast; detail on failure via dypai_trace
+    }
+    if (test.as_user) execArgs.impersonated_user_id = test.as_user
+
+    const runResponse = await proxyToolCall("test_workflow", execArgs)
+
+    // Normalize what counts as the workflow "result body" (vary by engine version)
+    const body = runResponse?.result ?? runResponse?.data ?? runResponse?.output ?? runResponse
+    const trace = runResponse?.trace
+
+    // Assertions
+    const expect = test.expect || {}
+
+    // expect.success — did the workflow complete?
+    if ("success" in expect) {
+      const status = trace?.status ?? trace?.workflow?.status
+      const actualSuccess = status
+        ? status === "completed"
+        : !runResponse?.error  // fallback: presence of error field
+      if (actualSuccess !== expect.success) {
+        result.errors.push(`expected success=${expect.success}, got ${actualSuccess}`)
+      }
+    }
+
+    // expect.response — match body
+    if (expect.response !== undefined) {
+      const r = matchValue(body, expect.response, "response")
+      if (!r.ok) result.errors.push(...r.errors)
+    }
+
+    // expect.failed_at — name of the node that should have failed
+    if (expect.failed_at) {
+      const failedAt = trace?.failed_at ?? trace?.workflow?.failed_at
+      if (failedAt !== expect.failed_at) {
+        result.errors.push(`expected failed_at='${expect.failed_at}', got ${JSON.stringify(failedAt)}`)
+      }
+    }
+
+    // expect.db_queries — each runs a SELECT and compares first cell or full rows
+    if (Array.isArray(expect.db_queries)) {
+      for (const q of expect.db_queries) {
+        try {
+          const res = await execSql(fileCtx.projectId, q.sql)
+          const first = firstCellOf(res)
+          if ("equals" in q && first !== q.equals) {
+            result.errors.push(`db_query "${q.sql.slice(0, 50)}": expected ${q.equals}, got ${JSON.stringify(first)}`)
+          }
+          if ("gte" in q && !(Number(first) >= Number(q.gte))) {
+            result.errors.push(`db_query: expected >= ${q.gte}, got ${first}`)
+          }
+          if ("lte" in q && !(Number(first) <= Number(q.lte))) {
+            result.errors.push(`db_query: expected <= ${q.lte}, got ${first}`)
+          }
+          if (q.match) {
+            const r = matchValue(res?.rows, q.match, "db_rows")
+            if (!r.ok) result.errors.push(...r.errors)
+          }
+        } catch (e) {
+          result.errors.push(`db_query execution failed: ${e.message}`)
+        }
+      }
+    }
+  } catch (e) {
+    result.errors.push(`execution error: ${e.message}`)
+  }
+
+  // Teardown — always try, even if the test failed
+  if (test.teardown_sql) {
+    try { await execSql(fileCtx.projectId, test.teardown_sql) }
+    catch (e) { result.errors.push(`teardown_sql failed: ${e.message}`) }
+  }
+
+  if (result.errors.length) result.status = "fail"
+  else delete result.errors
+  return result
+}
+
+// ─── Tool ───────────────────────────────────────────────────────────────────
+
+export const dypaiTestTool = {
+  name: "dypai_test",
+  description:
+    "Run YAML-defined tests from dypai/tests/*.test.yaml. Each test does: setup_sql → test_workflow (with impersonation) → response assertions → db_queries assertions → teardown_sql. " +
+    "Uses existing remote tools (execute_sql, test_workflow) — no engine changes needed. " +
+    "Pass `only` to run a subset (substring match on test name).",
+  inputSchema: {
+    type: "object",
+    properties: {
+      project_id: { type: "string", description: "Project UUID. Auto-resolved from dypai.config.yaml." },
+      root_dir: { type: "string", default: "./dypai" },
+      only: { type: "string", description: "Only run tests whose name includes this substring." },
+      file: { type: "string", description: "Relative path to a single test file under dypai/tests/." },
+    },
+  },
+  async execute({ project_id, root_dir = "./dypai", only, file } = {}) {
+    const rootDir = resolvePath(process.cwd(), root_dir)
+    const config = await readLocalConfig(rootDir)
+    const projectId = project_id || config?.project_id
+    if (!projectId) {
+      return {
+        success: false,
+        error: "project_id is required. Set it in dypai.config.yaml or pass it explicitly.",
+      }
+    }
+
+    const testsDir = join(rootDir, "tests")
+    let files = []
+    if (file) {
+      files = [file.startsWith("/") ? file : join(testsDir, file)]
+    } else {
+      files = await findTestFiles(testsDir)
+    }
+    if (!files.length) {
+      return {
+        success: true,
+        total: 0, pass: 0, fail: 0, error: 0,
+        results: [],
+        hint: "No test files found. Create dypai/tests/<endpoint>.test.yaml — see README for format.",
+      }
+    }
+
+    const allResults = []
+    const parseErrors = []
+    for (const f of files) {
+      let doc
+      try {
+        const raw = await readFile(f, "utf8")
+        doc = YAML.parse(raw) || {}
+      } catch (e) {
+        parseErrors.push({ file: f, error: e.message })
+        continue
+      }
+      const tests = Array.isArray(doc) ? doc : (doc.tests || [])
+      const fileCtx = {
+        projectId,
+        endpoint: doc.endpoint || doc.endpoint_id,
+        rootDir,
+        endpointCache: new Map(),
+      }
+      for (const t of tests) {
+        if (only && !(t.name || "").toLowerCase().includes(only.toLowerCase())) continue
+        const r = await runSingleTest(t, fileCtx)
+        r.file = f.replace(rootDir + "/", "")
+        allResults.push(r)
+      }
+    }
+
+    const pass = allResults.filter(r => r.status === "pass").length
+    const fail = allResults.filter(r => r.status === "fail").length
+    const error = allResults.filter(r => r.status === "error").length
+
+    return {
+      success: fail === 0 && error === 0 && parseErrors.length === 0,
+      total: allResults.length,
+      pass, fail, error,
+      results: allResults,
+      parse_errors: parseErrors.length ? parseErrors : undefined,
+      hint: (fail + error) > 0
+        ? `${fail} test(s) failed, ${error} errored. For failing ones, use dypai_trace on the reported execution_id for deeper inspection.`
+        : undefined,
+    }
+  },
+}