@dypai-ai/mcp 1.0.9 → 1.1.0

package/src/tools/sync/diff.js

@@ -0,0 +1,83 @@
+/**
+ * dypai_diff — preview what dypai_push would change. Read-only, safe.
+ */
+
+import { resolve as resolvePath } from "path"
+import { fetchRemoteState, readLocalState, readLocalStateSnapshot, readLocalConfig, computePlan } from "./planner.js"
+
+export const dypaiDiffTool = {
+  name: "dypai_diff",
+  description:
+    "Compare the local ./dypai/ folder against the remote project state without applying anything. " +
+    "Returns a plan: endpoints to create, update (with changed fields), delete, and unchanged. " +
+    "Always call this before dypai_push to preview changes safely.",
+  inputSchema: {
+    type: "object",
+    properties: {
+      project_id: {
+        type: "string",
+        description: "Project UUID. Optional if your token is project-scoped.",
+      },
+      root_dir: {
+        type: "string",
+        description: "Root of the dypai/ folder (default: ./dypai).",
+        default: "./dypai",
+      },
+      delete_orphans: {
+        type: "boolean",
+        description: "If true, endpoints present in remote but missing locally are queued for deletion. Default: false (safe).",
+        default: false,
+      },
+    },
+  },
+
+  async execute({ project_id, root_dir = "./dypai", delete_orphans = false } = {}) {
+    const rootDir = resolvePath(process.cwd(), root_dir)
+
+    const config = await readLocalConfig(rootDir)
+    const targetProjectId = project_id || config?.project_id || null
+
+    const [local, remote, stateSnapshot] = await Promise.all([
+      readLocalState(rootDir),
+      fetchRemoteState(targetProjectId),
+      readLocalStateSnapshot(rootDir),
+    ])
+
+    if (local.errors.length) {
+      return {
+        success: false,
+        phase: "read_local",
+        errors: local.errors,
+        hint: "Fix the YAML/file errors before running diff again.",
+      }
+    }
+
+    const plan = computePlan(local, remote, remote.mapsCtx, {
+      deleteOrphans: delete_orphans,
+      stateSnapshot,
+    })
+
+    const totalChanges = plan.create.length + plan.update.length + plan.delete.length
+    const hasConflicts = (plan.warnings || []).some(w => w.type === "remote_changed_since_pull")
+
+    return {
+      success: true,
+      summary: {
+        create: plan.create.length,
+        update: plan.update.length,
+        delete: plan.delete.length,
+        unchanged: plan.unchanged.length,
+        orphans_ignored: plan.orphansIgnored?.length || 0,
+        warnings: plan.warnings?.length || 0,
+      },
+      // Only include plan details when there are actual changes or warnings
+      plan: (totalChanges > 0 || plan.warnings?.length) ? plan : undefined,
+      // Conflicts and missing creds deserve surfacing; no-op diffs don't
+      hint: hasConflicts
+        ? "Remote changed since your last pull — dypai_pull first, or dypai_push will be blocked."
+        : (plan.warnings || []).some(w => w.type === "missing_credential")
+          ? "Some YAMLs reference credentials missing remotely. Create them in the dashboard before pushing."
+          : undefined,
+    }
+  },
+}

package/src/tools/sync/index.js

@@ -0,0 +1,18 @@
+/**
+ * Git-first source-of-truth tooling for DYPAI projects.
+ *
+ * Architecture:
+ *   transforms.js — bidirectional field transforms (symmetric by construction)
+ *   codec.js      — endpoint-level serialize / deserialize (structural shape)
+ *   pull.js       — DB → filesystem (uses codec.serializeEndpoint)
+ *   push.js       — filesystem → DB (uses codec.deserializeEndpoint)  [pending]
+ *   diff.js       — compares both sides without writing                [pending]
+ */
+
+export { dypaiPullTool } from "./pull.js"
+export { dypaiDiffTool } from "./diff.js"
+export { dypaiPushTool } from "./push.js"
+export { dypaiDescribeTool } from "./describe.js"
+export { dypaiValidateTool } from "./validate.js"
+export { dypaiTestTool } from "./test.js"
+export { dypaiTestEndpointTool } from "./test-endpoint.js"

package/src/tools/sync/planner.js

@@ -0,0 +1,426 @@
+/**
+ * Planner — the brain behind diff and push.
+ *
+ * Reads local YAML + referenced files from disk, fetches the remote DB state,
+ * and produces a plan: { create, update, delete, unchanged }.
+ *
+ * Both sides are normalized through the same codec so canvas, default
+ * trigger blocks, and other noise don't show up as false-positive diffs.
+ */
+
+import { readFile, readdir } from "fs/promises"
+import { join } from "path"
+import YAML from "yaml"
+import { proxyToolCall } from "../proxy.js"
+import { deserializeEndpoint, serializeEndpoint } from "./codec.js"
+
+// ─── Remote ────────────────────────────────────────────────────────────────
+
+async function execSql(projectId, sql) {
+  const args = projectId ? { project_id: projectId, sql } : { sql }
+  const result = await proxyToolCall("execute_sql", args)
+  if (result?.error) throw new Error(`SQL error: ${result.error}`)
+  if (!result?.rows) throw new Error(`Unexpected SQL response: ${JSON.stringify(result).slice(0, 300)}`)
+  return result.rows
+}
+
+function parseMaybeJson(v) {
+  if (v == null || typeof v !== "string") return v
+  try { return JSON.parse(v) } catch { return v }
+}
+
+function hydrateRow(row) {
+  return {
+    ...row,
+    workflow_code: parseMaybeJson(row.workflow_code) || {},
+    input: parseMaybeJson(row.input),
+    output: parseMaybeJson(row.output),
+    allowed_roles: parseMaybeJson(row.allowed_roles) || row.allowed_roles || [],
+  }
+}
+
+/**
+ * Load .dypai/state.json if it exists. Used for conflict detection on push.
+ */
+export async function readLocalStateSnapshot(rootDir) {
+  try {
+    const raw = await readFile(join(rootDir, ".dypai", "state.json"), "utf8")
+    return JSON.parse(raw)
+  } catch {
+    return null
+  }
+}
+
+/**
+ * Load dypai.config.yaml if it exists. This is the committed identity of the
+ * project (project_id, project_name). Used to auto-resolve project_id and to
+ * guard against pushing to the wrong project by accident.
+ */
+export async function readLocalConfig(rootDir) {
+  try {
+    const raw = await readFile(join(rootDir, "dypai.config.yaml"), "utf8")
+    return YAML.parse(raw) || null
+  } catch {
+    return null
+  }
+}
+
+/**
+ * Load dypai/realtime.yaml if it exists. Returns { tables: {...}, channels: {...} }
+ * flattened into a list of { target_type, target_name, ...policy } rows so the
+ * push/diff paths can treat them uniformly.
+ */
+export async function readLocalRealtime(rootDir) {
+  try {
+    const raw = await readFile(join(rootDir, "realtime.yaml"), "utf8")
+    const doc = YAML.parse(raw) || {}
+    const rows = []
+    for (const [name, p] of Object.entries(doc.tables || {})) {
+      rows.push(normalizePolicy("table", name, p))
+    }
+    for (const [name, p] of Object.entries(doc.channels || {})) {
+      rows.push(normalizePolicy("channel", name, p))
+    }
+    return { rows, doc }
+  } catch {
+    return null
+  }
+}
+
+function normalizePolicy(target_type, target_name, p) {
+  return {
+    target_type,
+    target_name,
+    subscribe_filter: p?.subscribe_filter ?? null,
+    events: Array.isArray(p?.events) ? p.events : null,
+    required_role: p?.required_role ?? null,
+    auth_required: p?.auth_required !== false,  // default true
+  }
+}
+
+/** Fetch remote realtime policies from the DB. Tolerates the table not existing. */
+export async function fetchRemoteRealtime(projectId) {
+  try {
+    return await execSql(projectId, `
+      SELECT target_type, target_name, subscribe_filter,
+             events, required_role, auth_required
+      FROM system.realtime_policies
+      ORDER BY target_type, target_name
+    `)
+  } catch {
+    return null  // engine doesn't have the table yet
+  }
+}
+
+export async function fetchRemoteState(projectId) {
+  const [endpoints, credentials, groups] = await Promise.all([
+    execSql(projectId, `
+      SELECT id, name, method, description, workflow_code, input, output,
+             allowed_roles, is_tool, tool_description, group_id, updated_at
+      FROM system.endpoints
+      WHERE is_active = true
+      ORDER BY name
+    `),
+    execSql(projectId, "SELECT id, name, type FROM system.credentials"),
+    execSql(projectId, "SELECT id, name FROM system.endpoints_group"),
+  ])
+
+  const mapsCtx = {
+    credIdToName: Object.fromEntries(credentials.map(c => [c.id, c.name])),
+    groupIdToName: Object.fromEntries(groups.map(g => [g.id, g.name])),
+    endpointIdToName: Object.fromEntries(endpoints.map(e => [e.id, e.name])),
+    credNameToId: Object.fromEntries(credentials.map(c => [c.name, c.id])),
+    groupNameToId: Object.fromEntries(groups.map(g => [g.name, g.id])),
+    endpointNameToId: Object.fromEntries(endpoints.map(e => [e.name, e.id])),
+  }
+
+  const byName = {}
+  for (const raw of endpoints) {
+    byName[raw.name] = hydrateRow(raw)
+  }
+  return { byName, mapsCtx }
+}
+
+// ─── Local ─────────────────────────────────────────────────────────────────
+
+/**
+ * Walk a doc finding all `*_file` string fields. Used to pre-read contents.
+ */
+function collectFileRefs(doc) {
+  const refs = new Set()
+  const walk = (node) => {
+    if (!node || typeof node !== "object") return
+    if (Array.isArray(node)) return node.forEach(walk)
+    for (const [k, v] of Object.entries(node)) {
+      if (typeof v === "string" && k.endsWith("_file")) refs.add(v)
+      else walk(v)
+    }
+  }
+  walk(doc)
+  return Array.from(refs)
+}
+
+/**
+ * Walk endpoints/ recursively. The first-level subfolder name (if any) encodes
+ * the endpoint's group. Files directly under endpoints/ have no group.
+ *
+ * endpoints/foo.yaml                → no group
+ * endpoints/Admin/foo.yaml          → group "Admin"
+ * endpoints/Admin/Users/foo.yaml    → group "Admin" (deeper nesting ignored for now)
+ */
+async function findEndpointYamls(endpointsDir) {
+  const out = []  // { relFromEndpoints, group }
+  async function walk(dir, relFromRoot, group) {
+    const entries = await readdir(dir, { withFileTypes: true }).catch(() => [])
+    for (const e of entries) {
+      if (e.isDirectory()) {
+        // Only the first-level folder is the group; deeper is flattened into it
+        const sub = relFromRoot ? `${relFromRoot}/${e.name}` : e.name
+        await walk(join(dir, e.name), sub, group || e.name)
+      } else if (e.isFile() && (e.name.endsWith(".yaml") || e.name.endsWith(".yml"))) {
+        const rel = relFromRoot ? `${relFromRoot}/${e.name}` : e.name
+        out.push({ rel, group })
+      }
+    }
+  }
+  await walk(endpointsDir, "", null)
+  return out
+}
+
+export async function readLocalState(rootDir) {
+  const endpointsDir = join(rootDir, "endpoints")
+  const yamls = await findEndpointYamls(endpointsDir)
+
+  const byName = {}
+  const errors = []
+
+  for (const { rel, group } of yamls) {
+    try {
+      const raw = await readFile(join(endpointsDir, rel), "utf8")
+      const doc = YAML.parse(raw)
+      if (!doc?.name) {
+        errors.push({ file: rel, error: "missing 'name' field" })
+        continue
+      }
+
+      // Inject group from folder if present and not already set in the YAML
+      if (group && !doc.group) doc.group = group
+
+      // Pre-read all *_file references (paths are relative to dypai/ root)
+      const refs = collectFileRefs(doc)
+      const contents = await Promise.all(
+        refs.map(p => readFile(join(rootDir, p), "utf8").catch(() => {
+          errors.push({ file: rel, error: `referenced file not found: ${p}` })
+          return ""
+        }))
+      )
+      const fileMap = Object.fromEntries(refs.map((p, i) => [p, contents[i]]))
+
+      byName[doc.name] = { doc, fileMap }
+    } catch (e) {
+      errors.push({ file: rel, error: e.message })
+    }
+  }
+
+  return { byName, errors }
+}
+
+// ─── Normalization + diff ──────────────────────────────────────────────────
+
+/**
+ * Produce a canonical (fully-hydrated, DB-shaped) row from a local {doc, fileMap}.
+ * This is what we'd send to update_endpoint.
+ */
+export function localToCanonical(entry, mapsCtx) {
+  const { doc, fileMap } = entry
+  const ctx = { ...mapsCtx, readFile: (p) => fileMap[p] ?? "" }
+  return deserializeEndpoint(doc, ctx)
+}
+
+/**
+ * Inline *_file refs back into a doc using a path→content map. Mutates doc.
+ */
+function inlineFileRefs(node, fileMap) {
+  if (!node || typeof node !== "object") return
+  if (Array.isArray(node)) {
+    node.forEach(n => inlineFileRefs(n, fileMap))
+    return
+  }
+  const inlines = []
+  for (const [k, v] of Object.entries(node)) {
+    if (typeof v === "string" && k.endsWith("_file") && fileMap[v] !== undefined) {
+      inlines.push(k)
+    } else {
+      inlineFileRefs(v, fileMap)
+    }
+  }
+  for (const k of inlines) {
+    const inlineKey = k.replace(/_file$/, "")
+    node[inlineKey] = fileMap[node[k]]
+    delete node[k]
+  }
+}
+
+/**
+ * Canonical form for diff: doc with file contents inlined back in. Both sides
+ * are brought to this shape so file contents are part of the comparison
+ * (otherwise identical file paths would mask content changes).
+ */
+function remoteToComparable(row, mapsCtx) {
+  const { doc, extractedFiles } = serializeEndpoint(row, mapsCtx)
+  const fileMap = Object.fromEntries(extractedFiles.map(f => [f.path, f.content]))
+  inlineFileRefs(doc, fileMap)
+  return doc
+}
+
+function localToComparable(entry, mapsCtx) {
+  const row = localToCanonical(entry, mapsCtx)
+  return remoteToComparable(row, mapsCtx)
+}
+
+function deepEqual(a, b) {
+  if (a === b) return true
+  if (typeof a !== typeof b) return false
+  if (a === null || b === null) return a === b
+  if (Array.isArray(a) !== Array.isArray(b)) return false
+  if (Array.isArray(a)) {
+    if (a.length !== b.length) return false
+    return a.every((x, i) => deepEqual(x, b[i]))
+  }
+  if (typeof a === "object") {
+    const ka = Object.keys(a).sort()
+    const kb = Object.keys(b).sort()
+    if (ka.join(",") !== kb.join(",")) return false
+    return ka.every(k => deepEqual(a[k], b[k]))
+  }
+  return false
+}
+
+/**
+ * Return a list of top-level field paths that differ between two docs.
+ * Used for human-readable diff summaries.
+ */
+function summarizeFieldDiffs(a, b, prefix = "") {
+  const diffs = []
+  const keys = new Set([...Object.keys(a || {}), ...Object.keys(b || {})])
+  for (const k of keys) {
+    const va = a?.[k]
+    const vb = b?.[k]
+    if (deepEqual(va, vb)) continue
+    const path = prefix ? `${prefix}.${k}` : k
+    if (va && vb && typeof va === "object" && typeof vb === "object" && !Array.isArray(va) && !Array.isArray(vb)) {
+      diffs.push(...summarizeFieldDiffs(va, vb, path))
+    } else {
+      diffs.push(path)
+    }
+  }
+  return diffs
+}
+
+/**
+ * Find credential references in a doc and return a list of any that don't
+ * resolve against the remote credNameToId map.
+ */
+function findMissingCredentials(doc, credNameToId) {
+  const missing = new Set()
+  const walk = (node) => {
+    if (!node || typeof node !== "object") return
+    if (Array.isArray(node)) return node.forEach(walk)
+    for (const [k, v] of Object.entries(node)) {
+      if (k === "credential" && typeof v === "string" && !credNameToId[v]) {
+        missing.add(v)
+      } else {
+        walk(v)
+      }
+    }
+  }
+  walk(doc)
+  return Array.from(missing)
+}
+
+/**
+ * Detect conflicts using a prior snapshot (.dypai/state.json). If an endpoint
+ * was pulled at time T and its remote updated_at is now > T, someone modified
+ * it out-of-band since the pull. Report these so the user can pull first.
+ */
+function detectConflicts(stateSnapshot, remoteByName) {
+  if (!stateSnapshot?.endpoints) return []
+  const conflicts = []
+  for (const [name, snap] of Object.entries(stateSnapshot.endpoints)) {
+    const remote = remoteByName[name]
+    if (!remote) continue
+    const snapT = snap.updated_at ? new Date(snap.updated_at).getTime() : 0
+    const remoteT = remote.updated_at ? new Date(remote.updated_at).getTime() : 0
+    if (remoteT > snapT + 1000) { // 1s tolerance for clock skew
+      conflicts.push({
+        endpoint: name,
+        snapshot_at: snap.updated_at,
+        remote_updated_at: remote.updated_at,
+      })
+    }
+  }
+  return conflicts
+}
+
+/**
+ * Compute the plan to reconcile remote state with local.
+ */
+export function computePlan(local, remote, mapsCtx, { deleteOrphans = false, stateSnapshot = null } = {}) {
+  const plan = { create: [], update: [], delete: [], unchanged: [] }
+  const warnings = []
+
+  for (const name of Object.keys(local.byName)) {
+    const entry = local.byName[name]
+    const localCanonical = localToComparable(entry, mapsCtx)
+    const remoteRow = remote.byName[name]
+
+    // Credentials check: warn if a YAML references a credential name missing remotely
+    const missing = findMissingCredentials(entry.doc, mapsCtx.credNameToId || {})
+    for (const credName of missing) {
+      warnings.push({
+        type: "missing_credential",
+        endpoint: name,
+        credential: credName,
+        hint: `Create it in the dashboard (same name) before pushing, or push will fail at runtime.`,
+      })
+    }
+
+    if (!remoteRow) {
+      plan.create.push({ name, fields: Object.keys(localCanonical) })
+      continue
+    }
+
+    const remoteCanonical = remoteToComparable(remoteRow, mapsCtx)
+    if (deepEqual(localCanonical, remoteCanonical)) {
+      plan.unchanged.push(name)
+    } else {
+      const changedFields = summarizeFieldDiffs(localCanonical, remoteCanonical)
+      plan.update.push({ name, changedFields })
+    }
+  }
+
+  if (deleteOrphans) {
+    for (const name of Object.keys(remote.byName)) {
+      if (!local.byName[name]) plan.delete.push({ name })
+    }
+  } else {
+    plan.orphansIgnored = Object.keys(remote.byName).filter(n => !local.byName[n])
+  }
+
+  // Conflict detection (remote changed since last pull)
+  const conflicts = detectConflicts(stateSnapshot, remote.byName)
+  if (conflicts.length) {
+    for (const c of conflicts) {
+      warnings.push({
+        type: "remote_changed_since_pull",
+        endpoint: c.endpoint,
+        snapshot_at: c.snapshot_at,
+        remote_updated_at: c.remote_updated_at,
+        hint: "Run dypai_pull to refresh local state before pushing (otherwise you may overwrite remote changes).",
+      })
+    }
+  }
+
+  if (warnings.length) plan.warnings = warnings
+  return plan
+}