@dypai-ai/mcp 1.0.9 → 1.1.0

package/src/tools/sync/pull.js

@@ -0,0 +1,414 @@
+/**
+ * dypai_pull — snapshot remote project state to local YAML + SQL + MD files.
+ *
+ * Thin wrapper: reads DB via execute_sql, calls codec.serializeEndpoint,
+ * writes the resulting doc + extractedFiles to disk. All transformations
+ * live in transforms.js (shared with push).
+ *
+ * Also writes:
+ *   - schema.sql          — DDL of public.* (reference for writing SQL)
+ *   - .gitignore          — excludes .dypai/ cache from git
+ *   - .dypai/state.json   — per-endpoint updated_at for conflict detection on push
+ */
+
+import { writeFile, mkdir, rm, access } from "fs/promises"
+import { existsSync } from "fs"
+import { join, resolve as resolvePath, dirname, isAbsolute, delimiter, sep } from "path"
+import { homedir } from "os"
+import YAML from "yaml"
+import { proxyToolCall } from "../proxy.js"
+import { serializeEndpoint } from "./codec.js"
+import { dumpPublicSchema } from "./schema-dump.js"
+import { regenerateTypes } from "../codegen.js"
+
+/**
+ * Resolve a user-supplied out_dir. When an IDE spawns this MCP over stdio, the
+ * process cwd is usually the user's home dir, NOT the workspace the user sees.
+ * A relative `./foo` can land in ~/foo instead of the project root. We try
+ * several signals in order and warn the caller if the result still looks off.
+ */
+function resolveOutDir(outDir) {
+  if (isAbsolute(outDir)) return { path: outDir, source: "absolute" }
+
+  // 1. Env vars set by common agent hosts (Claude Code, Cursor, Cline, etc.)
+  const envCandidates = [
+    ["CLAUDE_PROJECT_DIR", process.env.CLAUDE_PROJECT_DIR],
+    ["DYPAI_WORKSPACE_ROOT", process.env.DYPAI_WORKSPACE_ROOT],
+    ["WORKSPACE_FOLDER_PATHS", process.env.WORKSPACE_FOLDER_PATHS?.split(delimiter)[0]],
+    ["PROJECT_ROOT", process.env.PROJECT_ROOT],
+  ]
+  for (const [name, val] of envCandidates) {
+    if (val && isAbsolute(val)) return { path: resolvePath(val, outDir), source: `env:${name}` }
+  }
+
+  // 2. Walk up from cwd looking for a project marker (.git, package.json, dypai/)
+  let cursor = process.cwd()
+  for (let i = 0; i < 6; i++) {
+    if (existsSync(join(cursor, ".git")) ||
+        existsSync(join(cursor, "package.json")) ||
+        existsSync(join(cursor, "dypai"))) {
+      return { path: resolvePath(cursor, outDir), source: "project_marker" }
+    }
+    const parent = dirname(cursor)
+    if (parent === cursor) break
+    cursor = parent
+  }
+
+  // 3. Last resort — use cwd (probably ~/ when launched by IDE)
+  return { path: resolvePath(process.cwd(), outDir), source: "cwd_fallback" }
+}
+
+/** Return a warning string if the resolved path looks suspicious (e.g. sits directly in $HOME). */
+function suspiciousPathWarning(resolvedPath, source) {
+  if (source === "absolute") return null
+  const home = homedir()
+  // If output_dir is ~/X (one level under home) and we had to fall back to cwd,
+  // the agent almost certainly didn't mean to write there.
+  if (source === "cwd_fallback" && resolvedPath.startsWith(home + sep)) {
+    const rel = resolvedPath.slice(home.length + 1)
+    if (!rel.includes(sep) || rel.split(sep).length <= 2) {
+      return (
+        `Output landed at ${resolvedPath}. The MCP process cwd is your home dir, not your project. ` +
+        `Re-run with an ABSOLUTE out_dir (e.g. ${home}/path/to/your-project/dypai) ` +
+        `or set the CLAUDE_PROJECT_DIR env var on the MCP entry.`
+      )
+    }
+  }
+  return null
+}
+
+// Subfolders that are always created so the layout is predictable. An agent
+// never has to check "does this folder exist?" before writing a new SQL/prompt/JS file.
+const CANONICAL_SUBDIRS = ["endpoints", "sql", "prompts", "code"]
+
+const README_CONTENT = `# dypai/
+
+Declarative snapshot of your DYPAI project's backend.
+
+## Layout
+
+- \`endpoints/\` — one YAML per endpoint (the workflow definition).
+  Subfolders represent endpoint groups, e.g. \`endpoints/Admin/foo.yaml\` → group "Admin".
+- \`sql/\` — SQL queries extracted from \`dypai_database\` nodes when longer than 500 chars.
+- \`prompts/\` — system prompts extracted from \`agent\` nodes when longer than 800 chars.
+- \`code/\` — JavaScript / Python extracted from \`javascript_code\` / \`python_code\` nodes when longer than 500 chars.
+
+## Workflow
+
+1. \`dypai_pull\` to snapshot the remote state into this folder
+2. Edit YAML, SQL, prompts, code with your editor or AI agent
+3. \`dypai_diff\` to preview changes
+4. \`dypai_push\` to apply to the remote
+
+Paths inside YAML (e.g. \`query_file: sql/create_invoice.sql\`) are always relative
+to this folder's root, regardless of where the YAML lives.
+`
+
+async function execSql(projectId, sql) {
+  const args = projectId ? { project_id: projectId, sql } : { sql }
+  const result = await proxyToolCall("execute_sql", args)
+  if (result?.error) throw new Error(`SQL error: ${result.error}`)
+  if (!result?.rows) {
+    throw new Error(`Unexpected SQL response: ${JSON.stringify(result).slice(0, 300)}`)
+  }
+  return result.rows
+}
+
+function parseMaybeJson(v) {
+  if (v == null || typeof v !== "string") return v
+  try { return JSON.parse(v) } catch { return v }
+}
+
+function hydrateRow(row) {
+  return {
+    ...row,
+    workflow_code: parseMaybeJson(row.workflow_code) || {},
+    input: parseMaybeJson(row.input),
+    output: parseMaybeJson(row.output),
+    allowed_roles: parseMaybeJson(row.allowed_roles) || row.allowed_roles,
+  }
+}
+
+async function writeFileEnsured(filePath, content) {
+  await mkdir(dirname(filePath), { recursive: true })
+  await writeFile(filePath, content, "utf8")
+}
+
+function renderYaml(doc) {
+  return YAML.stringify(doc, {
+    blockQuote: "literal",
+    lineWidth: 0,
+    singleQuote: false,
+  })
+}
+
+// schema.sql dump lives in ./schema-dump.js (shared with execute_sql auto-refresh)
+
+export const dypaiPullTool = {
+  name: "dypai_pull",
+  description:
+    "Serializes the remote project state to local YAML files under ./dypai/. " +
+    "Writes endpoints/<name>.yaml + sql/ and prompts/ for extracted content. " +
+    "Canvas positions are stripped (regenerated by visual editor). Safe to run repeatedly. " +
+    "Use this to start editing a project locally with your editor + AI agent. " +
+    "IMPORTANT: when called by an IDE-hosted MCP, the process cwd is often the user's home dir — " +
+    "pass an ABSOLUTE path in out_dir (e.g. /Users/me/projects/my-app/dypai) to avoid writing to the wrong place. " +
+    "If out_dir is relative, the tool tries to auto-detect the workspace via env vars and git markers, " +
+    "and will surface a `warning` if the result looks suspicious.",
+  inputSchema: {
+    type: "object",
+    properties: {
+      project_id: {
+        type: "string",
+        description: "Project UUID. Optional if your token is project-scoped.",
+      },
+      out_dir: {
+        type: "string",
+        description:
+          "Output directory. PREFER ABSOLUTE PATHS (e.g. /Users/me/projects/x/dypai) because IDE-hosted " +
+          "MCP servers often have the user's home dir as cwd. If relative, the tool auto-detects the " +
+          "workspace via CLAUDE_PROJECT_DIR / WORKSPACE_FOLDER_PATHS env vars or by walking up to a .git/package.json marker.",
+        default: "./dypai",
+      },
+      clean: {
+        type: "boolean",
+        description: "If true, removes existing endpoints/, sql/, prompts/ before writing.",
+        default: false,
+      },
+    },
+  },
+
+  async execute({ project_id, out_dir = "./dypai", clean = false } = {}) {
+    const { path: outDir, source: outDirSource } = resolveOutDir(out_dir)
+    const suspiciousWarning = suspiciousPathWarning(outDir, outDirSource)
+
+    if (clean) {
+      await Promise.all(
+        CANONICAL_SUBDIRS.map(sub =>
+          rm(join(outDir, sub), { recursive: true, force: true })
+        )
+      )
+    }
+
+    // Always create the canonical subdirs (empty or not) so the layout is predictable
+    await Promise.all(
+      CANONICAL_SUBDIRS.map(async sub => {
+        const dir = join(outDir, sub)
+        await mkdir(dir, { recursive: true })
+        // .gitkeep lets empty folders survive git. Non-destructive: only create if missing.
+        const keepPath = join(dir, ".gitkeep")
+        try { await access(keepPath) } catch { await writeFile(keepPath, "", "utf8") }
+      })
+    )
+
+    // Top-level README (only write if missing — don't clobber user edits)
+    const readmePath = join(outDir, "README.md")
+    try { await access(readmePath) } catch { await writeFile(readmePath, README_CONTENT, "utf8") }
+
+    // .gitignore so the .dypai/ cache + OS junk doesn't leak to git
+    const gitignorePath = join(outDir, ".gitignore")
+    try { await access(gitignorePath) } catch {
+      await writeFile(gitignorePath, "# Auto-generated by dypai_pull\n.dypai/\n.DS_Store\n", "utf8")
+    }
+
+    const [endpoints, credentials, groups, schemaSql, nodeCatalogResult, realtimePolicies] = await Promise.all([
+      execSql(project_id, `
+        SELECT id, name, method, description, workflow_code, input, output,
+               allowed_roles, is_tool, tool_description, group_id, updated_at
+        FROM system.endpoints
+        WHERE is_active = true
+        ORDER BY name
+      `),
+      execSql(project_id, "SELECT id, name, type FROM system.credentials"),
+      execSql(project_id, "SELECT id, name FROM system.endpoints_group"),
+      dumpPublicSchema(project_id),
+      // Dump the global node catalog (central control plane) in one call.
+      // The per-project system.node_catalog is a stripped copy with empty schemas.
+      // The remote endpoints_sdk is project_aware, so it needs project_id even
+      // though the catalog itself is global.
+      proxyToolCall("list_node_catalog", project_id ? { project_id } : {}).catch(e => {
+        console.error(`[dypai_pull] list_node_catalog failed: ${e.message}`)
+        return { nodes: [] }
+      }),
+      // Realtime policies — deny-safe by default. Table may not exist yet on
+      // older engines; the catch keeps pull working.
+      execSql(project_id, `
+        SELECT target_type, target_name, subscribe_filter,
+               events, required_role, auth_required
+        FROM system.realtime_policies
+        ORDER BY target_type, target_name
+      `).catch(() => []),
+    ])
+
+    // schema.sql: always regenerated (read-only reference)
+    await writeFile(join(outDir, "schema.sql"), schemaSql, "utf8")
+
+    // node-catalog.json: full schemas of every node this engine exposes.
+    // Committed so the validator works offline right after a git clone.
+    const catalogNodes = nodeCatalogResult?.nodes || []
+    const catalogDoc = {
+      cached_at: new Date().toISOString(),
+      source: "public.node_catalog (central control plane)",
+      total: catalogNodes.length,
+      nodes: Object.fromEntries(catalogNodes.map(n => [n.node_type || n.id, {
+        label: n.name,
+        description: n.description,
+        category: n.category,
+        provider: n.provider,
+        inputs: parseMaybeJson(n.input_schema) || {},
+        outputs: parseMaybeJson(n.output_schema) || {},
+      }])),
+    }
+    await writeFile(join(outDir, "node-catalog.json"), JSON.stringify(catalogDoc, null, 2) + "\n", "utf8")
+
+    const mapsCtx = {
+      credIdToName: Object.fromEntries(credentials.map(c => [c.id, c.name])),
+      groupIdToName: Object.fromEntries(groups.map(g => [g.id, g.name])),
+      endpointIdToName: Object.fromEntries(endpoints.map(e => [e.id, e.name])),
+    }
+
+    const filesWritten = []
+    const errors = []
+
+    // realtime.yaml: declarative access policies for WebSocket subscriptions.
+    // Committed so team members / CI can see who's allowed to subscribe to what.
+    // Missing → realtime is deny-by-default on the engine (only service_role).
+    if (Array.isArray(realtimePolicies) && realtimePolicies.length > 0) {
+      const realtimeDoc = { tables: {}, channels: {} }
+      for (const p of realtimePolicies) {
+        const entry = {}
+        if (p.subscribe_filter) entry.subscribe_filter = p.subscribe_filter
+        if (Array.isArray(p.events) && p.events.length) entry.events = p.events
+        if (p.required_role) entry.required_role = p.required_role
+        if (p.auth_required === false) entry.auth_required = false
+        const bucket = p.target_type === "channel" ? "channels" : "tables"
+        realtimeDoc[bucket][p.target_name] = entry
+      }
+      if (!Object.keys(realtimeDoc.channels).length) delete realtimeDoc.channels
+      if (!Object.keys(realtimeDoc.tables).length) delete realtimeDoc.tables
+      await writeFile(join(outDir, "realtime.yaml"),
+        "# Declarative realtime access policies. Tables / channels NOT listed here are deny-by-default.\n" +
+        "# Placeholders: ${current_user_id}, ${current_user_role}, ${channel_param} (for wildcard channels).\n" +
+        renderYaml(realtimeDoc),
+        "utf8")
+      filesWritten.push("realtime.yaml")
+    }
+
+    for (const rawRow of endpoints) {
+      const row = hydrateRow(rawRow)
+      try {
+        const { doc, extractedFiles } = serializeEndpoint(row, mapsCtx)
+
+        // Group → folder convention: path encodes the group, so strip the field
+        const groupName = doc.group
+        delete doc.group
+
+        for (const f of extractedFiles) {
+          await writeFileEnsured(join(outDir, f.path), f.content)
+          filesWritten.push(f.path)
+        }
+
+        const relPath = groupName
+          ? `endpoints/${groupName}/${row.name}.yaml`
+          : `endpoints/${row.name}.yaml`
+        await writeFileEnsured(join(outDir, relPath), renderYaml(doc))
+        filesWritten.push(relPath)
+      } catch (e) {
+        errors.push({ endpoint: row.name, error: e.message })
+      }
+    }
+
+    // Fetch project metadata to persist identity in the committed config
+    let projectInfo = null
+    if (project_id) {
+      try {
+        projectInfo = await proxyToolCall("get_project", { project_id })
+      } catch { /* non-fatal; we still pin the id we know */ }
+    }
+    const resolvedProjectId = projectInfo?.id || project_id || null
+
+    // dypai.config.yaml: COMMITTED — identifies which project this folder belongs to.
+    // Serves two purposes:
+    //   1. A cloner/CI knows what project to push to.
+    //   2. dypai_push refuses to push to a different project_id (hallucination guard).
+    const configPath = join(outDir, "dypai.config.yaml")
+    const configDoc = { project_id: resolvedProjectId }
+    if (projectInfo?.name) configDoc.project_name = projectInfo.name
+    if (projectInfo?.organization_name) configDoc.organization = projectInfo.organization_name
+    const configYaml =
+      "# DYPAI project configuration — commit this file.\n" +
+      "# It identifies which remote project this dypai/ folder belongs to.\n" +
+      "# Never edit by hand unless you KNOW you're retargeting to a different project.\n" +
+      renderYaml(configDoc)
+    await writeFile(configPath, configYaml, "utf8")
+
+    // .dypai/state.json: GITIGNORED — per-endpoint updated_at for conflict detection.
+    const state = {
+      pulled_at: new Date().toISOString(),
+      project_id: resolvedProjectId,
+      endpoints: Object.fromEntries(
+        endpoints.map(e => [e.name, { id: e.id, updated_at: e.updated_at }])
+      ),
+    }
+    await writeFileEnsured(join(outDir, ".dypai", "state.json"), JSON.stringify(state, null, 2) + "\n")
+
+    // Auto-regenerate TS types for the frontend (src/dypai/). Non-fatal on failure.
+    let codegen
+    try { codegen = await regenerateTypes(outDir) }
+    catch (e) { codegen = { skipped: true, reason: e.message } }
+
+    // Compact overview — replaces what dypai_describe used to print, but
+    // derived from the data we already have in memory (zero extra queries).
+    // Lets the agent skip the "call describe then pull" dance.
+    const byGroup = (endpoints || []).reduce((acc, e) => {
+      const groupName = e.group_id ? (mapsCtx.groupIdToName[e.group_id] || "(unknown)") : "(no group)"
+      if (!acc[groupName]) acc[groupName] = []
+      const nodeCount = Array.isArray(e.workflow_code?.nodes) ? e.workflow_code.nodes.length : 0
+      acc[groupName].push({ name: e.name, method: e.method, nodes: nodeCount, is_tool: !!e.is_tool })
+      return acc
+    }, {})
+    const toolEndpoints = (endpoints || [])
+      .filter(e => e.is_tool)
+      .map(e => ({ name: e.name, description: e.tool_description || null }))
+
+    const overview = {
+      project: projectInfo ? {
+        id: projectInfo.id,
+        name: projectInfo.name,
+        plan: projectInfo.plan,
+      } : { id: resolvedProjectId || "(from token)" },
+      endpoints: {
+        total: (endpoints || []).length,
+        groups: Object.keys(byGroup).filter(g => g !== "(no group)").sort(),
+        by_group: byGroup,
+        tool_endpoints: toolEndpoints,
+      },
+      credentials: (credentials || []).map(c => ({ name: c.name, type: c.type })),
+      realtime_policies: (realtimePolicies || []).length,
+      next_steps: (endpoints || []).length === 0
+        ? ["Empty project. Create tables via execute_sql, then write dypai/endpoints/<name>.yaml and dypai_push."]
+        : ["Read dypai/schema.sql before writing queries.", "Edit YAML in dypai/endpoints/, then dypai_diff → dypai_push."],
+    }
+
+    return {
+      success: errors.length === 0,
+      endpoints: endpoints.length,
+      files_written: filesWritten.length,
+      output_dir: outDir,
+      out_dir_resolved_via: outDirSource,
+      overview,
+      codegen: codegen?.regenerated
+        ? { output_dir: codegen.output_dir, files: codegen.files_written.length }
+        : (codegen?.reason || undefined),
+      errors: errors.length ? errors : undefined,
+      warning: suspiciousWarning || undefined,
+      hint: errors.length
+        ? "Some endpoints failed to serialize. Check errors[] — usually malformed workflow_code."
+        : suspiciousWarning
+          ? "Files were written but the path looks wrong. See `warning` above and re-run with an absolute out_dir."
+          : endpoints.length === 0
+            ? "Empty project. Create tables with execute_sql, then write endpoints/<name>.yaml and dypai_push."
+            : undefined,
+    }
+  },
+}