@dypai-ai/mcp 1.0.10 → 1.2.0

package/src/tools/sync/test.js

@@ -0,0 +1,343 @@
+/**
+ * dypai_test — YAML-defined tests against endpoints. Zero engine changes:
+ * orchestrates `execute_sql` + `test_workflow` (impersonation) + `execute_sql`
+ * using what already exists.
+ *
+ * File layout (committable under dypai/tests/):
+ *   endpoint: create-order        # or endpoint_id: <uuid>
+ *   tests:
+ *     - name: "creates order with valid product"
+ *       setup_sql: INSERT INTO products ...
+ *       as_user: 11111111-1111-1111-1111-111111111111
+ *       input: { product_id: "p1", qty: 3 }
+ *       expect:
+ *         success: true
+ *         response:
+ *           ok: true
+ *           order_id: { matches: uuid }
+ *         db_queries:
+ *           - sql: "SELECT COUNT(*)::int FROM orders WHERE product_id='p1'"
+ *             equals: 1
+ *       teardown_sql: DELETE FROM orders WHERE product_id='p1'; DELETE FROM products WHERE id='p1';
+ */
+
+import { readFile, readdir } from "fs/promises"
+import { join, resolve as resolvePath } from "path"
+import YAML from "yaml"
+import { proxyToolCall } from "../proxy.js"
+import { readLocalConfig } from "./planner.js"
+
+// ─── Test file discovery ────────────────────────────────────────────────────
+
+async function findTestFiles(testsDir) {
+  const out = []
+  async function walk(dir) {
+    const entries = await readdir(dir, { withFileTypes: true }).catch(() => [])
+    for (const e of entries) {
+      const full = join(dir, e.name)
+      if (e.isDirectory()) await walk(full)
+      else if (e.isFile() && (e.name.endsWith(".test.yaml") || e.name.endsWith(".test.yml"))) out.push(full)
+    }
+  }
+  await walk(testsDir)
+  return out
+}
+
+// ─── SQL helpers ────────────────────────────────────────────────────────────
+
+async function execSql(projectId, sql) {
+  const res = await proxyToolCall("execute_sql", { project_id: projectId, sql })
+  if (res?.error) throw new Error(res.error)
+  return res
+}
+
+function firstCellOf(res) {
+  const row = res?.rows?.[0]
+  if (!row) return undefined
+  const k = Object.keys(row)[0]
+  return k ? row[k] : undefined
+}
+
+/** Resolve an endpoint name to its UUID via system.endpoints (the remote
+ * test_workflow tool only accepts endpoint_id, not endpoint_name). */
+const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i
+async function resolveEndpointId(projectId, ref, cache) {
+  if (UUID_RE.test(ref)) return ref
+  if (cache.has(ref)) return cache.get(ref)
+  const safeName = String(ref).replace(/'/g, "''")
+  const res = await execSql(projectId,
+    `SELECT id FROM system.endpoints WHERE name = '${safeName}' LIMIT 1`)
+  const id = firstCellOf(res)
+  if (!id) throw new Error(`Endpoint '${ref}' not found (looked up in system.endpoints by name).`)
+  cache.set(ref, id)
+  return id
+}
+
+// ─── Assertion matchers ─────────────────────────────────────────────────────
+
+/**
+ * Deep match `actual` against `expected`. Expected can contain meta-matchers
+ * via special objects with keys `matches`, `equals`, `contains`, `type`,
+ * `exists`, `gte`, `lte`. Returns { ok, errors: [] }.
+ */
+function matchValue(actual, expected, path = "") {
+  if (expected && typeof expected === "object" && !Array.isArray(expected)) {
+    const metaKeys = new Set(["matches", "equals", "contains", "type", "exists", "gte", "lte"])
+    const keys = Object.keys(expected)
+    if (keys.length && keys.every(k => metaKeys.has(k))) return matchMeta(actual, expected, path)
+
+    if (actual == null || typeof actual !== "object") {
+      return { ok: false, errors: [`${path || "(root)"}: expected object, got ${JSON.stringify(actual)}`] }
+    }
+    const errors = []
+    for (const k of keys) {
+      const r = matchValue(actual[k], expected[k], path ? `${path}.${k}` : k)
+      if (!r.ok) errors.push(...r.errors)
+    }
+    return errors.length ? { ok: false, errors } : { ok: true }
+  }
+
+  if (Array.isArray(expected)) {
+    if (!Array.isArray(actual)) return { ok: false, errors: [`${path}: expected array, got ${typeof actual}`] }
+    if (actual.length !== expected.length) return { ok: false, errors: [`${path}: length ${expected.length} expected, got ${actual.length}`] }
+    const errors = []
+    expected.forEach((item, i) => {
+      const r = matchValue(actual[i], item, `${path}[${i}]`)
+      if (!r.ok) errors.push(...r.errors)
+    })
+    return errors.length ? { ok: false, errors } : { ok: true }
+  }
+
+  if (actual !== expected) {
+    return { ok: false, errors: [`${path || "(root)"}: expected ${JSON.stringify(expected)}, got ${JSON.stringify(actual)}`] }
+  }
+  return { ok: true }
+}
+
+function matchMeta(actual, meta, path) {
+  const errs = []
+  if ("equals" in meta && actual !== meta.equals) {
+    errs.push(`${path}: expected ${JSON.stringify(meta.equals)}, got ${JSON.stringify(actual)}`)
+  }
+  if ("matches" in meta) {
+    const m = meta.matches
+    const str = String(actual ?? "")
+    if (m === "uuid") {
+      if (!/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(str)) {
+        errs.push(`${path}: expected UUID, got ${JSON.stringify(actual)}`)
+      }
+    } else {
+      try {
+        if (!new RegExp(m).test(str)) errs.push(`${path}: did not match /${m}/, got ${JSON.stringify(actual)}`)
+      } catch {
+        errs.push(`${path}: invalid regex ${JSON.stringify(m)}`)
+      }
+    }
+  }
+  if ("contains" in meta) {
+    if (typeof actual !== "string" || !actual.includes(String(meta.contains))) {
+      errs.push(`${path}: expected to contain ${JSON.stringify(meta.contains)}, got ${JSON.stringify(actual)}`)
+    }
+  }
+  if ("type" in meta && typeof actual !== meta.type) {
+    errs.push(`${path}: expected type ${meta.type}, got ${typeof actual}`)
+  }
+  if ("exists" in meta) {
+    const exists = actual !== undefined && actual !== null
+    if (meta.exists !== exists) errs.push(`${path}: expected exists=${meta.exists}, got ${exists}`)
+  }
+  if ("gte" in meta && !(Number(actual) >= Number(meta.gte))) {
+    errs.push(`${path}: expected >= ${meta.gte}, got ${actual}`)
+  }
+  if ("lte" in meta && !(Number(actual) <= Number(meta.lte))) {
+    errs.push(`${path}: expected <= ${meta.lte}, got ${actual}`)
+  }
+  return errs.length ? { ok: false, errors: errs } : { ok: true }
+}
+
+// ─── Single test runner ─────────────────────────────────────────────────────
+
+async function runSingleTest(test, fileCtx) {
+  const result = { name: test.name || "(unnamed)", status: "pass", errors: [] }
+
+  // Setup
+  if (test.setup_sql) {
+    try { await execSql(fileCtx.projectId, test.setup_sql) }
+    catch (e) { return { ...result, status: "error", phase: "setup", errors: [`setup_sql failed: ${e.message}`] } }
+  }
+
+  try {
+    // Resolve endpoint → endpoint_id (remote only accepts UUIDs)
+    const endpointRef = test.endpoint || test.endpoint_id || fileCtx.endpoint
+    if (!endpointRef) {
+      return { ...result, status: "error", errors: ["No endpoint specified. Set `endpoint` at file root or test level."] }
+    }
+    const endpointId = await resolveEndpointId(fileCtx.projectId, endpointRef, fileCtx.endpointCache)
+
+    const execArgs = {
+      project_id: fileCtx.projectId,
+      endpoint_id: endpointId,
+      data: test.input || {},
+      trace_mode: "minimal",  // keep tests fast; detail on failure via dypai_trace
+    }
+    if (test.as_user) execArgs.impersonated_user_id = test.as_user
+
+    const runResponse = await proxyToolCall("test_workflow", execArgs)
+
+    // Normalize what counts as the workflow "result body" (vary by engine version)
+    const body = runResponse?.result ?? runResponse?.data ?? runResponse?.output ?? runResponse
+    const trace = runResponse?.trace
+
+    // Assertions
+    const expect = test.expect || {}
+
+    // expect.success — did the workflow complete?
+    if ("success" in expect) {
+      const status = trace?.status ?? trace?.workflow?.status
+      const actualSuccess = status
+        ? status === "completed"
+        : !runResponse?.error  // fallback: presence of error field
+      if (actualSuccess !== expect.success) {
+        result.errors.push(`expected success=${expect.success}, got ${actualSuccess}`)
+      }
+    }
+
+    // expect.response — match body
+    if (expect.response !== undefined) {
+      const r = matchValue(body, expect.response, "response")
+      if (!r.ok) result.errors.push(...r.errors)
+    }
+
+    // expect.failed_at — name of the node that should have failed
+    if (expect.failed_at) {
+      const failedAt = trace?.failed_at ?? trace?.workflow?.failed_at
+      if (failedAt !== expect.failed_at) {
+        result.errors.push(`expected failed_at='${expect.failed_at}', got ${JSON.stringify(failedAt)}`)
+      }
+    }
+
+    // expect.db_queries — each runs a SELECT and compares first cell or full rows
+    if (Array.isArray(expect.db_queries)) {
+      for (const q of expect.db_queries) {
+        try {
+          const res = await execSql(fileCtx.projectId, q.sql)
+          const first = firstCellOf(res)
+          if ("equals" in q && first !== q.equals) {
+            result.errors.push(`db_query "${q.sql.slice(0, 50)}": expected ${q.equals}, got ${JSON.stringify(first)}`)
+          }
+          if ("gte" in q && !(Number(first) >= Number(q.gte))) {
+            result.errors.push(`db_query: expected >= ${q.gte}, got ${first}`)
+          }
+          if ("lte" in q && !(Number(first) <= Number(q.lte))) {
+            result.errors.push(`db_query: expected <= ${q.lte}, got ${first}`)
+          }
+          if (q.match) {
+            const r = matchValue(res?.rows, q.match, "db_rows")
+            if (!r.ok) result.errors.push(...r.errors)
+          }
+        } catch (e) {
+          result.errors.push(`db_query execution failed: ${e.message}`)
+        }
+      }
+    }
+  } catch (e) {
+    result.errors.push(`execution error: ${e.message}`)
+  }
+
+  // Teardown — always try, even if the test failed
+  if (test.teardown_sql) {
+    try { await execSql(fileCtx.projectId, test.teardown_sql) }
+    catch (e) { result.errors.push(`teardown_sql failed: ${e.message}`) }
+  }
+
+  if (result.errors.length) result.status = "fail"
+  else delete result.errors
+  return result
+}
+
+// ─── Tool ───────────────────────────────────────────────────────────────────
+
+export const dypaiTestTool = {
+  name: "dypai_test",
+  description:
+    "Run YAML-defined tests from dypai/tests/*.test.yaml. Each test does: setup_sql → test_workflow (with impersonation) → response assertions → db_queries assertions → teardown_sql. " +
+    "Uses existing remote tools (execute_sql, test_workflow) — no engine changes needed. " +
+    "Pass `only` to run a subset (substring match on test name).",
+  inputSchema: {
+    type: "object",
+    properties: {
+      project_id: { type: "string", description: "Project UUID. Auto-resolved from dypai.config.yaml." },
+      root_dir: { type: "string", default: "./dypai" },
+      only: { type: "string", description: "Only run tests whose name includes this substring." },
+      file: { type: "string", description: "Relative path to a single test file under dypai/tests/." },
+    },
+  },
+  async execute({ project_id, root_dir = "./dypai", only, file } = {}) {
+    const rootDir = resolvePath(process.cwd(), root_dir)
+    const config = await readLocalConfig(rootDir)
+    const projectId = project_id || config?.project_id
+    if (!projectId) {
+      return {
+        success: false,
+        error: "project_id is required. Set it in dypai.config.yaml or pass it explicitly.",
+      }
+    }
+
+    const testsDir = join(rootDir, "tests")
+    let files = []
+    if (file) {
+      files = [file.startsWith("/") ? file : join(testsDir, file)]
+    } else {
+      files = await findTestFiles(testsDir)
+    }
+    if (!files.length) {
+      return {
+        success: true,
+        total: 0, pass: 0, fail: 0, error: 0,
+        results: [],
+        hint: "No test files found. Create dypai/tests/<endpoint>.test.yaml — see README for format.",
+      }
+    }
+
+    const allResults = []
+    const parseErrors = []
+    for (const f of files) {
+      let doc
+      try {
+        const raw = await readFile(f, "utf8")
+        doc = YAML.parse(raw) || {}
+      } catch (e) {
+        parseErrors.push({ file: f, error: e.message })
+        continue
+      }
+      const tests = Array.isArray(doc) ? doc : (doc.tests || [])
+      const fileCtx = {
+        projectId,
+        endpoint: doc.endpoint || doc.endpoint_id,
+        rootDir,
+        endpointCache: new Map(),
+      }
+      for (const t of tests) {
+        if (only && !(t.name || "").toLowerCase().includes(only.toLowerCase())) continue
+        const r = await runSingleTest(t, fileCtx)
+        r.file = f.replace(rootDir + "/", "")
+        allResults.push(r)
+      }
+    }
+
+    const pass = allResults.filter(r => r.status === "pass").length
+    const fail = allResults.filter(r => r.status === "fail").length
+    const error = allResults.filter(r => r.status === "error").length
+
+    return {
+      success: fail === 0 && error === 0 && parseErrors.length === 0,
+      total: allResults.length,
+      pass, fail, error,
+      results: allResults,
+      parse_errors: parseErrors.length ? parseErrors : undefined,
+      hint: (fail + error) > 0
+        ? `${fail} test(s) failed, ${error} errored. For failing ones, use dypai_trace on the reported execution_id for deeper inspection.`
+        : undefined,
+    }
+  },
+}

package/src/tools/sync/transforms.js

@@ -0,0 +1,157 @@
+/**
+ * Bidirectional transformations between the DB's executable JSON workflow
+ * format and the declarative YAML/files format.
+ *
+ * CRITICAL: every transform declares its pull (DB → file) and push (file → DB)
+ * direction together. This guarantees round-trip symmetry by construction —
+ * there's no separate "push" file that can drift out of sync with "pull".
+ *
+ * Adding a new transform = edit one object here, both directions in one commit.
+ *
+ * Context shape for pull:
+ *   {
+ *     endpointName, nodeId,
+ *     credIdToName, groupIdToName, endpointIdToName,
+ *     emitFile(path, content),   // collects files to write to disk
+ *   }
+ *
+ * Context shape for push:
+ *   {
+ *     endpointName, nodeId,
+ *     credNameToId, groupNameToId, endpointNameToId,
+ *     readFile(path),            // returns file contents
+ *   }
+ */
+
+// Only extract truly large content. Below these thresholds, SQL and prompts
+// stay inline in the YAML so the endpoint is one self-contained file.
+const SQL_INLINE_MAX_CHARS = 500
+const PROMPT_INLINE_MAX_CHARS = 800
+
+const shouldInlineSql = (q) => !q || q.length <= SQL_INLINE_MAX_CHARS
+
+// ─── Node-level field transforms ────────────────────────────────────────────
+
+export const NODE_FIELD_TRANSFORMS = [
+  {
+    name: "credential",
+    pull(params, ctx) {
+      if (params.credential_id && ctx.credIdToName[params.credential_id]) {
+        return { credential: ctx.credIdToName[params.credential_id] }
+      }
+      return {}
+    },
+    push(params, ctx) {
+      if (params.credential && ctx.credNameToId[params.credential]) {
+        return { credential_id: ctx.credNameToId[params.credential] }
+      }
+      return {}
+    },
+    pullConsumes: ["credential_id"],
+    pushConsumes: ["credential"],
+  },
+
+  {
+    name: "agent_tools",
+    appliesWhen: (nodeType) => nodeType === "agent",
+    pull(params, ctx) {
+      if (Array.isArray(params.tool_ids)) {
+        return { tools: params.tool_ids.map(id => ctx.endpointIdToName[id] || id) }
+      }
+      return {}
+    },
+    push(params, ctx) {
+      if (Array.isArray(params.tools)) {
+        return { tool_ids: params.tools.map(n => ctx.endpointNameToId[n] || n) }
+      }
+      return {}
+    },
+    pullConsumes: ["tool_ids"],
+    pushConsumes: ["tools"],
+  },
+
+  {
+    name: "sql_extraction",
+    appliesWhen: (nodeType) => nodeType === "dypai_database",
+    pull(params, ctx) {
+      if (params.query && !shouldInlineSql(params.query)) {
+        // Flat layout: sql/<endpoint>.sql if only one SQL node, else sql/<endpoint>.<node>.sql
+        const suffix = ctx.sqlNodeCount > 1 ? `.${ctx.nodeId}` : ""
+        const path = `sql/${ctx.endpointName}${suffix}.sql`
+        ctx.emitFile(path, params.query.trim() + "\n")
+        return { query_file: path }
+      }
+      return {}
+    },
+    push(params, ctx) {
+      if (params.query_file) {
+        return { query: ctx.readFile(params.query_file).trimEnd() }
+      }
+      return {}
+    },
+    pullConsumes: ["query"],
+    pushConsumes: ["query_file"],
+  },
+
+  {
+    name: "prompt_extraction",
+    appliesWhen: (nodeType) => nodeType === "agent",
+    pull(params, ctx) {
+      if (params.system_prompt && params.system_prompt.length > PROMPT_INLINE_MAX_CHARS) {
+        const suffix = ctx.promptNodeCount > 1 ? `.${ctx.nodeId}` : ""
+        const path = `prompts/${ctx.endpointName}${suffix}.md`
+        ctx.emitFile(path, params.system_prompt.trim() + "\n")
+        return { system_prompt_file: path }
+      }
+      return {}
+    },
+    push(params, ctx) {
+      if (params.system_prompt_file) {
+        return { system_prompt: ctx.readFile(params.system_prompt_file).trimEnd() }
+      }
+      return {}
+    },
+    pullConsumes: ["system_prompt"],
+    pushConsumes: ["system_prompt_file"],
+  },
+
+  {
+    name: "code_extraction",
+    appliesWhen: (nodeType) => nodeType === "javascript_code" || nodeType === "python_code",
+    pull(params, ctx) {
+      if (params.code && params.code.length > SQL_INLINE_MAX_CHARS) {
+        const ext = ctx.nodeType === "python_code" ? "py" : "js"
+        const suffix = ctx.codeNodeCount > 1 ? `.${ctx.nodeId}` : ""
+        const path = `code/${ctx.endpointName}${suffix}.${ext}`
+        ctx.emitFile(path, params.code.trim() + "\n")
+        return { code_file: path }
+      }
+      return {}
+    },
+    push(params, ctx) {
+      if (params.code_file) {
+        return { code: ctx.readFile(params.code_file).trimEnd() }
+      }
+      return {}
+    },
+    pullConsumes: ["code"],
+    pushConsumes: ["code_file"],
+  },
+]
+
+// ─── Engine ────────────────────────────────────────────────────────────────
+
+function applyTransforms(input, direction, ctx, nodeType) {
+  let result = { ...input }
+  for (const t of NODE_FIELD_TRANSFORMS) {
+    if (t.appliesWhen && !t.appliesWhen(nodeType)) continue
+    const added = t[direction](result, ctx) || {}
+    const consumes = direction === "pull" ? t.pullConsumes : t.pushConsumes
+    for (const f of consumes || []) delete result[f]
+    Object.assign(result, added)
+  }
+  return result
+}
+
+export const pullNodeParams = (params, ctx, nodeType) => applyTransforms(params, "pull", ctx, nodeType)
+export const pushNodeParams = (params, ctx, nodeType) => applyTransforms(params, "push", ctx, nodeType)