@dypai-ai/mcp 1.0.9 → 1.1.0

package/src/tools/sync/push.js

@@ -0,0 +1,411 @@
+/**
+ * dypai_push — apply the local ./dypai/ state to the remote project.
+ *
+ * Reads local YAML + referenced files, computes a plan against remote,
+ * and applies it via create_endpoint / update_endpoint / delete_endpoint.
+ *
+ * By default refuses to delete remote endpoints that aren't in local (safe).
+ * Pass delete_orphans: true to opt in.
+ */
+
+import { resolve as resolvePath } from "path"
+import { proxyToolCall } from "../proxy.js"
+import {
+  fetchRemoteState,
+  readLocalState,
+  readLocalStateSnapshot,
+  readLocalConfig,
+  readLocalRealtime,
+  fetchRemoteRealtime,
+  computePlan,
+  localToCanonical,
+} from "./planner.js"
+import { runValidation } from "./validate.js"
+import { regenerateTypes } from "../codegen.js"
+
+// ─── Apply helpers ─────────────────────────────────────────────────────────
+
+/**
+ * Run an async worker over an array with bounded concurrency. Workers never
+ * throw (they handle their own errors via the push errors[] accumulator),
+ * so we can await them all safely without Promise.all propagating rejections.
+ */
+async function runWithConcurrency(items, concurrency, worker) {
+  if (!items.length) return
+  const queue = items.slice()
+  const pumps = Array.from({ length: Math.min(concurrency, queue.length) }, async () => {
+    while (queue.length) {
+      const item = queue.shift()
+      await worker(item)
+    }
+  })
+  await Promise.all(pumps)
+}
+
+function endpointPayload(row) {
+  // The remote schemas reject null values for optional fields — only include
+  // what's actually set. workflow_code is always required.
+  const p = {
+    name: row.name,
+    method: row.method,
+    workflow_code: row.workflow_code,
+    is_tool: !!row.is_tool,
+    allowed_roles: row.allowed_roles || [],
+  }
+  if (row.description) p.description = row.description
+  if (row.tool_description) p.tool_description = row.tool_description
+  if (row.group_id) p.group_id = row.group_id
+  if (row.input) p.input = row.input
+  if (row.output) p.output = row.output
+  return p
+}
+
+async function applyCreate(canonicalRow, projectId) {
+  return proxyToolCall("create_endpoint", {
+    ...(projectId ? { project_id: projectId } : {}),
+    ...endpointPayload(canonicalRow),
+  })
+}
+
+async function applyUpdate(canonicalRow, endpointId, projectId) {
+  return proxyToolCall("update_endpoint", {
+    ...(projectId ? { project_id: projectId } : {}),
+    endpoint_id: endpointId,
+    updates: endpointPayload(canonicalRow),
+  })
+}
+
+async function applyDelete(endpointId, projectId) {
+  return proxyToolCall("delete_endpoint", {
+    ...(projectId ? { project_id: projectId } : {}),
+    endpoint_id: endpointId,
+  })
+}
+
+// ─── Realtime policies sync ────────────────────────────────────────────────
+
+/**
+ * Reconcile local realtime.yaml with system.realtime_policies remotely.
+ * Returns a summary of what was applied (or would be applied in dry_run).
+ */
+async function syncRealtimePolicies(projectId, rootDir, dryRun = false) {
+  const local = await readLocalRealtime(rootDir)
+  if (!local) return { skipped: true, reason: "no realtime.yaml found" }
+
+  const remote = await fetchRemoteRealtime(projectId)
+  if (remote === null) {
+    return { skipped: true, reason: "engine has no system.realtime_policies table yet (backward compat)" }
+  }
+
+  const key = (p) => `${p.target_type}::${p.target_name}`
+  const remoteMap = Object.fromEntries(remote.map(p => [key(p), p]))
+  const localMap = Object.fromEntries(local.rows.map(p => [key(p), p]))
+
+  const toUpsert = []
+  const toDelete = []
+  for (const k of Object.keys(localMap)) {
+    const l = localMap[k]
+    const r = remoteMap[k]
+    if (!r || !policiesEqual(l, r)) toUpsert.push(l)
+  }
+  for (const k of Object.keys(remoteMap)) {
+    if (!localMap[k]) toDelete.push(remoteMap[k])
+  }
+
+  if (dryRun) {
+    return { upsert: toUpsert.length, delete: toDelete.length, dry_run: true }
+  }
+
+  // execute_sql on the remote is parameterless (only takes a raw `sql` string).
+  // To avoid hand-escaping values, we embed the rows as JSON and let Postgres
+  // parse them via jsonb_to_recordset — JSON.stringify handles quote escaping
+  // correctly, and we only need to double single-quotes for the SQL literal.
+  if (toUpsert.length > 0) {
+    const rowsJson = JSON.stringify(toUpsert.map(p => ({
+      target_type: p.target_type,
+      target_name: p.target_name,
+      subscribe_filter: p.subscribe_filter,
+      events: p.events,
+      required_role: p.required_role,
+      auth_required: p.auth_required,
+    }))).replace(/'/g, "''")  // escape for SQL string literal
+
+    await proxyToolCall("execute_sql", {
+      project_id: projectId,
+      sql: `
+        INSERT INTO system.realtime_policies
+          (target_type, target_name, subscribe_filter, events, required_role, auth_required, updated_at)
+        SELECT target_type, target_name, subscribe_filter, events, required_role, auth_required, now()
+        FROM jsonb_to_recordset('${rowsJson}'::jsonb) AS r(
+          target_type text,
+          target_name text,
+          subscribe_filter text,
+          events text[],
+          required_role text,
+          auth_required boolean
+        )
+        ON CONFLICT (target_type, target_name) DO UPDATE SET
+          subscribe_filter = EXCLUDED.subscribe_filter,
+          events = EXCLUDED.events,
+          required_role = EXCLUDED.required_role,
+          auth_required = EXCLUDED.auth_required,
+          updated_at = now()
+      `,
+    })
+  }
+
+  if (toDelete.length > 0) {
+    // Build a safe tuple list. target_type is always 'table' or 'channel', target_name is user-provided.
+    const pairs = toDelete.map(p =>
+      `('${p.target_type}', '${String(p.target_name).replace(/'/g, "''")}')`
+    ).join(",")
+    await proxyToolCall("execute_sql", {
+      project_id: projectId,
+      sql: `
+        DELETE FROM system.realtime_policies
+        WHERE (target_type, target_name) IN (${pairs})
+      `,
+    })
+  }
+
+  return { upsert: toUpsert.length, delete: toDelete.length }
+}
+
+function policiesEqual(a, b) {
+  return a.target_type === b.target_type
+    && a.target_name === b.target_name
+    && (a.subscribe_filter || null) === (b.subscribe_filter || null)
+    && JSON.stringify(a.events || []) === JSON.stringify(b.events || [])
+    && (a.required_role || null) === (b.required_role || null)
+    && !!a.auth_required === !!b.auth_required
+}
+
+// ─── Tool ──────────────────────────────────────────────────────────────────
+
+export const dypaiPushTool = {
+  name: "dypai_push",
+  description:
+    "Apply the local ./dypai/ state to the remote project: creates, updates, and optionally deletes endpoints. " +
+    "ALWAYS run dypai_diff first to preview the plan. This tool mutates remote state. " +
+    "By default, endpoints in remote but missing locally are kept (safe). Pass delete_orphans: true to delete them.",
+  inputSchema: {
+    type: "object",
+    properties: {
+      project_id: {
+        type: "string",
+        description: "Project UUID. Optional if your token is project-scoped.",
+      },
+      root_dir: {
+        type: "string",
+        description: "Root of the dypai/ folder (default: ./dypai).",
+        default: "./dypai",
+      },
+      delete_orphans: {
+        type: "boolean",
+        description: "If true, endpoints in remote but missing locally get deleted. Default: false.",
+        default: false,
+      },
+      dry_run: {
+        type: "boolean",
+        description: "If true, compute the plan and return it without applying. Same as dypai_diff.",
+        default: false,
+      },
+      force: {
+        type: "boolean",
+        description: "Skip conflict detection. Use only if you know the remote changes you'd overwrite. Default: false.",
+        default: false,
+      },
+      skip_validation: {
+        type: "boolean",
+        description: "Skip the dypai_validate pre-flight check. Use only when you know the validator is wrong. Default: false.",
+        default: false,
+      },
+    },
+  },
+
+  async execute({ project_id, root_dir = "./dypai", delete_orphans = false, dry_run = false, force = false, skip_validation = false } = {}) {
+    const rootDir = resolvePath(process.cwd(), root_dir)
+
+    // Resolve the target project via dypai.config.yaml if the caller didn't
+    // pass an explicit project_id. If both are provided and disagree, refuse —
+    // this is the primary guard against an agent hallucinating a project_id.
+    const config = await readLocalConfig(rootDir)
+    const configProjectId = config?.project_id || null
+    const targetProjectId = project_id || configProjectId
+
+    // Pre-flight validation (lint). Blocks the push unless skip_validation is set.
+    if (!skip_validation && !dry_run) {
+      try {
+        const validation = await runValidation(rootDir, project_id || configProjectId)
+        if (!validation.success) {
+          return {
+            success: false,
+            applied: false,
+            reason: "validation_failed",
+            summary: validation.summary,
+            diagnostics: validation.diagnostics.filter(d => d.severity === "error").slice(0, 20),
+            hint: `${validation.summary.errors} error(s) would cause runtime failures. Fix them, or retry with skip_validation: true to override.`,
+          }
+        }
+      } catch (e) {
+        // Don't block the push on a validator crash — just warn
+        console.error(`[dypai_push] validator crashed: ${e.message}`)
+      }
+    }
+
+    if (project_id && configProjectId && project_id !== configProjectId) {
+      return {
+        success: false,
+        applied: false,
+        reason: "project_id_mismatch",
+        expected_project_id: configProjectId,
+        provided_project_id: project_id,
+        hint:
+          `This dypai/ folder is pinned to project ${configProjectId} (${config?.project_name || "unknown"}). ` +
+          `You passed ${project_id}. Refusing to push to a different project. ` +
+          `If retargeting is intentional, edit dypai.config.yaml first.`,
+      }
+    }
+
+    const [local, remote, stateSnapshot] = await Promise.all([
+      readLocalState(rootDir),
+      fetchRemoteState(targetProjectId),
+      readLocalStateSnapshot(rootDir),
+    ])
+
+    if (local.errors.length) {
+      return {
+        success: false,
+        phase: "read_local",
+        errors: local.errors,
+      }
+    }
+
+    const plan = computePlan(local, remote, remote.mapsCtx, {
+      deleteOrphans: delete_orphans,
+      stateSnapshot,
+    })
+    const totalChanges = plan.create.length + plan.update.length + plan.delete.length
+
+    // Block push on conflicts unless forced
+    const conflicts = (plan.warnings || []).filter(w => w.type === "remote_changed_since_pull")
+    if (conflicts.length && !force) {
+      return {
+        success: false,
+        applied: false,
+        reason: "conflicts_detected",
+        conflicts,
+        hint: "Run dypai_pull to refresh local state, resolve any overlap, then push again. To override, pass force=true.",
+      }
+    }
+
+    if (dry_run || totalChanges === 0) {
+      return {
+        success: true,
+        applied: false,
+        reason: totalChanges === 0 ? "no_changes" : "dry_run",
+        summary: summaryFromPlan(plan),
+        plan,
+      }
+    }
+
+    // Apply in order: creates → updates → deletes.
+    // Within each phase we run with bounded concurrency (5) so pushing 50
+    // endpoints doesn't take 50 * 200ms = 10s — drops to ~2s on typical RTT.
+    const CONCURRENCY = 5
+    const applied = { created: [], updated: [], deleted: [] }
+    const errors = []
+
+    await runWithConcurrency(plan.create, CONCURRENCY, async (item) => {
+      try {
+        const canonical = localToCanonical(local.byName[item.name], remote.mapsCtx)
+        await applyCreate(canonical, targetProjectId)
+        applied.created.push(item.name)
+      } catch (e) {
+        errors.push({ op: "create", endpoint: item.name, error: e.message })
+      }
+    })
+
+    await runWithConcurrency(plan.update, CONCURRENCY, async (item) => {
+      try {
+        const canonical = localToCanonical(local.byName[item.name], remote.mapsCtx)
+        const endpointId = remote.byName[item.name]?.id
+        if (!endpointId) throw new Error("endpoint_id missing from remote state")
+        await applyUpdate(canonical, endpointId, targetProjectId)
+        applied.updated.push({ name: item.name, changed_fields: item.changedFields })
+      } catch (e) {
+        errors.push({ op: "update", endpoint: item.name, error: e.message })
+      }
+    })
+
+    await runWithConcurrency(plan.delete, CONCURRENCY, async (item) => {
+      try {
+        const endpointId = remote.byName[item.name]?.id
+        if (!endpointId) throw new Error("endpoint_id missing from remote state")
+        await applyDelete(endpointId, targetProjectId)
+        applied.deleted.push(item.name)
+      } catch (e) {
+        errors.push({ op: "delete", endpoint: item.name, error: e.message })
+      }
+    })
+
+    // Also reconcile realtime policies if realtime.yaml exists
+    let realtime = null
+    try {
+      realtime = await syncRealtimePolicies(targetProjectId, rootDir, false)
+    } catch (e) {
+      errors.push({ op: "realtime", error: e.message })
+    }
+
+    const changedNames = [
+      ...applied.created,
+      ...applied.updated.map(u => u.name),
+    ]
+
+    // Regenerate TS types if endpoints changed (input/output schemas may have shifted).
+    let codegen
+    if (changedNames.length > 0) {
+      try { codegen = await regenerateTypes(rootDir) }
+      catch (e) { codegen = { skipped: true, reason: e.message } }
+    }
+
+    return {
+      success: errors.length === 0,
+      applied: true,
+      summary: {
+        created: applied.created.length,
+        updated: applied.updated.length,
+        deleted: applied.deleted.length,
+        unchanged: plan.unchanged.length,
+        errors: errors.length,
+        realtime: realtime || { skipped: true, reason: "no realtime.yaml" },
+        codegen: codegen?.regenerated
+          ? { files: codegen.files_written.length }
+          : (codegen?.reason || "no changes to regenerate"),
+      },
+      details: applied,
+      errors: errors.length ? errors : undefined,
+      // Only one next_step — and only when it's non-obvious
+      next_step: errors.length
+        ? "Fix the offending YAMLs and push again."
+        : changedNames.length
+          ? `Test changed endpoint(s) with test_workflow: ${changedNames.slice(0, 3).join(", ")}${changedNames.length > 3 ? "…" : ""}`
+          : undefined,
+      hint: errors.some(e => /credential/i.test(e.error))
+        ? "A referenced credential doesn't exist remotely. Create it in the dashboard (same name), then retry."
+        : errors.some(e => /validation/i.test(e.error))
+          ? "Field shape mismatch. Inspect the YAML of the failed endpoint."
+          : undefined,
+    }
+  },
+}
+
+function summaryFromPlan(plan) {
+  return {
+    create: plan.create.length,
+    update: plan.update.length,
+    delete: plan.delete.length,
+    unchanged: plan.unchanged.length,
+    orphans_ignored: plan.orphansIgnored?.length || 0,
+  }
+}

package/src/tools/sync/schema-dump.js

@@ -0,0 +1,96 @@
+/**
+ * Dump of the public schema as DDL-looking SQL (CREATE TABLE statements with
+ * columns + PKs + FK comments). Shared by dypai_pull and the auto-refresh
+ * triggered by execute_sql when it detects schema-affecting DDL.
+ */
+
+import { proxyToolCall } from "../proxy.js"
+
+async function execSql(projectId, sql) {
+  const args = projectId ? { project_id: projectId, sql } : { sql }
+  const result = await proxyToolCall("execute_sql", args)
+  if (result?.error) throw new Error(`SQL error: ${result.error}`)
+  if (!result?.rows) throw new Error(`Unexpected SQL response: ${JSON.stringify(result).slice(0, 300)}`)
+  return result.rows
+}
+
+function formatColumnType(c) {
+  if (c.data_type === "character varying") {
+    return c.character_maximum_length ? `varchar(${c.character_maximum_length})` : "varchar"
+  }
+  if (c.data_type === "USER-DEFINED") return c.udt_name || "user_defined"
+  if (c.data_type === "ARRAY") return `${c.udt_name || "text"}`.replace(/^_/, "") + "[]"
+  return c.data_type
+}
+
+export async function dumpPublicSchema(projectId) {
+  const [columns, pks, fks] = await Promise.all([
+    execSql(projectId, `
+      SELECT table_name, column_name, data_type, is_nullable, column_default,
+             character_maximum_length, udt_name, ordinal_position
+      FROM information_schema.columns
+      WHERE table_schema = 'public'
+      ORDER BY table_name, ordinal_position
+    `),
+    execSql(projectId, `
+      SELECT tc.table_name, kcu.column_name
+      FROM information_schema.table_constraints tc
+      JOIN information_schema.key_column_usage kcu
+        ON tc.constraint_schema = kcu.constraint_schema
+       AND tc.constraint_name = kcu.constraint_name
+      WHERE tc.constraint_type = 'PRIMARY KEY' AND tc.table_schema = 'public'
+    `),
+    execSql(projectId, `
+      SELECT tc.table_name, kcu.column_name,
+             ccu.table_name AS ref_table, ccu.column_name AS ref_column
+      FROM information_schema.table_constraints tc
+      JOIN information_schema.key_column_usage kcu
+        ON tc.constraint_schema = kcu.constraint_schema
+       AND tc.constraint_name = kcu.constraint_name
+      JOIN information_schema.constraint_column_usage ccu
+        ON tc.constraint_schema = ccu.constraint_schema
+       AND tc.constraint_name = ccu.constraint_name
+      WHERE tc.constraint_type = 'FOREIGN KEY' AND tc.table_schema = 'public'
+    `),
+  ])
+
+  const byTable = {}
+  for (const c of columns) {
+    if (!byTable[c.table_name]) byTable[c.table_name] = []
+    byTable[c.table_name].push(c)
+  }
+  const pkSet = new Set(pks.map(p => `${p.table_name}.${p.column_name}`))
+  const fksByTable = {}
+  for (const f of fks) {
+    if (!fksByTable[f.table_name]) fksByTable[f.table_name] = []
+    fksByTable[f.table_name].push(f)
+  }
+
+  const lines = [
+    "-- Auto-generated — do NOT edit by hand.",
+    "-- Regenerated on every dypai_pull and after schema-affecting execute_sql (CREATE/ALTER/DROP/RENAME TABLE).",
+    "",
+  ]
+
+  const tableNames = Object.keys(byTable).sort()
+  for (const table of tableNames) {
+    const cols = byTable[table]
+    lines.push(`CREATE TABLE public.${table} (`)
+    const colLines = cols.map(c => {
+      const type = formatColumnType(c)
+      const nullable = c.is_nullable === "NO" ? " NOT NULL" : ""
+      const pk = pkSet.has(`${table}.${c.column_name}`) ? " PRIMARY KEY" : ""
+      const dflt = c.column_default ? ` DEFAULT ${c.column_default}` : ""
+      return `  ${c.column_name} ${type}${nullable}${pk}${dflt}`
+    })
+    lines.push(colLines.join(",\n"))
+    lines.push(");")
+
+    const tableFks = fksByTable[table] || []
+    for (const fk of tableFks) {
+      lines.push(`-- FK: ${fk.column_name} -> public.${fk.ref_table}(${fk.ref_column})`)
+    }
+    lines.push("")
+  }
+  return lines.join("\n")
+}