@dypai-ai/mcp 1.3.0 → 1.4.0

package/src/tools/storage.js

@@ -0,0 +1,289 @@
+/**
+ * manage_storage — local extension adding an `upload_file` operation.
+ *
+ * The remote `manage_storage` tool already covers list / create / delete buckets
+ * and list_objects / delete_object / get_signed_download_url. What it CAN'T do
+ * is upload: the binary would have to travel through the MCP stdio transport
+ * which is (a) slow and (b) saturates the remote pod on 100 MB files.
+ *
+ * `upload_file` solves this with the same pattern `dypai.api.upload` uses in
+ * the frontend SDK:
+ *
+ *   1. Ask the remote MCP → API for a signed PUT URL (op: sign_upload).
+ *   2. PUT the file binary DIRECTLY to R2 from the local machine (no infra hop).
+ *   3. Ask the remote MCP → API to register the object (op: verify_upload).
+ *
+ * Net effect: the agent just calls `manage_storage(operation:"upload_file",
+ * local_path:"...", bucket:"public")` and gets back a URL. Zero credentials on
+ * the agent side — the signed URL IS the temporary credential (15 min, one key,
+ * PUT only).
+ *
+ * NOTE: this file exports `uploadFile` plus the pieces needed to extend the
+ * `manage_storage` tool entry in index.js. The catalog-level tool definition
+ * still lives in index.js because manage_storage is a REMOTE tool with a local
+ * operation grafted on top — the dispatcher logic lives there too.
+ */
+
+import { statSync, readFileSync, existsSync } from "fs"
+import { basename } from "path"
+import { proxyToolCall } from "./proxy.js"
+import { updateMediaManifest } from "./deploy.js"
+
+// 100 MB — enforced by the API (MAX_UPLOAD_SIZE_BYTES). We check client-side too
+// so the user sees a clean error before we waste a round-trip.
+const MAX_UPLOAD_BYTES = 100 * 1024 * 1024
+
+// Extensions we know how to MIME-type without shelling out. Everything else
+// falls back to application/octet-stream, which R2 accepts fine — the browser
+// just won't preview it inline.
+const MIME_BY_EXT = {
+  // Images
+  png: "image/png", jpg: "image/jpeg", jpeg: "image/jpeg", gif: "image/gif",
+  webp: "image/webp", avif: "image/avif", svg: "image/svg+xml", ico: "image/x-icon",
+  bmp: "image/bmp", tiff: "image/tiff",
+  // Video
+  mp4: "video/mp4", webm: "video/webm", mov: "video/quicktime", mkv: "video/x-matroska",
+  // Audio
+  mp3: "audio/mpeg", wav: "audio/wav", ogg: "audio/ogg", m4a: "audio/mp4",
+  flac: "audio/flac",
+  // Documents
+  pdf: "application/pdf",
+  doc: "application/msword",
+  docx: "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+  xls: "application/vnd.ms-excel",
+  xlsx: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+  ppt: "application/vnd.ms-powerpoint",
+  pptx: "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+  // Text / data
+  txt: "text/plain", csv: "text/csv", json: "application/json",
+  xml: "application/xml", html: "text/html", md: "text/markdown",
+  yaml: "application/yaml", yml: "application/yaml",
+  // Archives
+  zip: "application/zip", gz: "application/gzip", tar: "application/x-tar",
+  // Fonts
+  woff: "font/woff", woff2: "font/woff2", ttf: "font/ttf", otf: "font/otf",
+}
+
+function mimeFor(filename) {
+  const ext = filename.includes(".") ? filename.split(".").pop().toLowerCase() : ""
+  return MIME_BY_EXT[ext] || "application/octet-stream"
+}
+
+function formatBytes(n) {
+  if (n < 1024) return `${n} B`
+  if (n < 1024 * 1024) return `${(n / 1024).toFixed(1)} KB`
+  return `${(n / 1024 / 1024).toFixed(1)} MB`
+}
+
+/**
+ * Core upload. Returns { success, bucket, name, size, content_type,
+ * signed_url, public_url, object_id, ... } or { success: false, error }.
+ *
+ * Flow:
+ *   1. stat → size + mime
+ *   2. ensure_bucket (optional) → list or create bucket if missing
+ *   3. sign_upload → presigned PUT URL
+ *   4. fetch(PUT, body=fileBuffer)
+ *   5. verify_upload → row in storage.objects
+ *   6. sign a short-lived download URL so the agent has something clickable
+ */
+export async function uploadFile({
+  local_path,
+  bucket,
+  object_name,
+  prefix,
+  content_type,
+  ensure_bucket = false,
+  bucket_public = false,
+  project_id,
+  // Optional: if provided, the manifest at <source_directory>/dypai/.dypai/media-manifest.json
+  // is updated after a successful upload so future deploys know this file is in the bucket.
+  source_directory,
+}) {
+  // ── Validation ──────────────────────────────────────────────────────────
+  if (!local_path) return { success: false, error: "`local_path` is required." }
+  if (!bucket) return { success: false, error: "`bucket` is required (e.g. 'public')." }
+  if (!existsSync(local_path)) {
+    return { success: false, error: `File not found: ${local_path}` }
+  }
+
+  const stat = statSync(local_path)
+  if (!stat.isFile()) {
+    return { success: false, error: `Not a file: ${local_path}` }
+  }
+  if (stat.size === 0) {
+    return { success: false, error: `File is empty: ${local_path}` }
+  }
+  if (stat.size > MAX_UPLOAD_BYTES) {
+    return {
+      success: false,
+      error: `File too large: ${formatBytes(stat.size)} (max ${formatBytes(MAX_UPLOAD_BYTES)}). Split the file or host it externally.`,
+    }
+  }
+
+  const filename = object_name || basename(local_path)
+  const resolvedContentType = content_type || mimeFor(filename)
+
+  // ── Ensure bucket (optional) ────────────────────────────────────────────
+  // If the agent passed ensure_bucket:true and the bucket doesn't exist yet,
+  // create it. Saves a round-trip of "bucket not found → go create → retry".
+  if (ensure_bucket) {
+    try {
+      const buckets = await proxyToolCall("manage_storage", {
+        operation: "list",
+        project_id,
+      })
+      const list = Array.isArray(buckets) ? buckets : (buckets?.data || [])
+      const exists = list.some(b => (b.name || b) === bucket)
+      if (!exists) {
+        await proxyToolCall("manage_storage", {
+          operation: "create",
+          project_id,
+          name: bucket,
+          public: bool(bucket_public),
+        })
+      }
+    } catch (e) {
+      return { success: false, error: `ensure_bucket failed: ${e.message}` }
+    }
+  }
+
+  // ── Step 1: sign_upload ─────────────────────────────────────────────────
+  let signed
+  try {
+    signed = await proxyToolCall("manage_storage", {
+      operation: "sign_upload",
+      project_id,
+      bucket,
+      filename,
+      content_type: resolvedContentType,
+      size_bytes: stat.size,
+      prefix,
+    })
+  } catch (e) {
+    return { success: false, error: `sign_upload failed: ${e.message}`, hint: bucketHint(e.message, bucket) }
+  }
+
+  const { upload_url, method = "PUT", headers = {}, file_path } = signed || {}
+  if (!upload_url || !file_path) {
+    return { success: false, error: "sign_upload returned an invalid payload (no upload_url or file_path)." }
+  }
+
+  // ── Step 2: direct PUT to R2 ────────────────────────────────────────────
+  let buf
+  try {
+    buf = readFileSync(local_path)
+  } catch (e) {
+    return { success: false, error: `Failed to read file: ${e.message}` }
+  }
+
+  try {
+    const res = await fetch(upload_url, {
+      method,
+      headers: {
+        "Content-Type": resolvedContentType,
+        ...headers,
+      },
+      body: buf,
+    })
+    if (!res.ok) {
+      const detail = await safeReadBody(res)
+      return {
+        success: false,
+        error: `R2 PUT failed with status ${res.status}. ${detail ? `Detail: ${detail.slice(0, 300)}` : ""}`.trim(),
+      }
+    }
+  } catch (e) {
+    return { success: false, error: `Direct upload to R2 failed: ${e.message}` }
+  }
+
+  // ── Step 3: verify_upload ───────────────────────────────────────────────
+  let verified
+  try {
+    verified = await proxyToolCall("manage_storage", {
+      operation: "verify_upload",
+      project_id,
+      bucket,
+      file_path,
+      original_filename: filename,
+      content_type: resolvedContentType,
+      size_bytes: stat.size,
+      prefix,
+    })
+  } catch (e) {
+    return {
+      success: false,
+      error: `verify_upload failed (the file WAS uploaded to R2 but not registered): ${e.message}. You can retry the upload — verify_upload is idempotent on (bucket, name).`,
+    }
+  }
+
+  // ── Step 4: fetch a preview signed download URL ─────────────────────────
+  // Best-effort: if it fails we just skip it, the upload succeeded regardless.
+  let signedDownloadUrl = null
+  try {
+    const dl = await proxyToolCall("manage_storage", {
+      operation: "get_signed_download_url",
+      project_id,
+      bucket,
+      name: verified?.name || filename,
+      expires_minutes: 15,
+      download: false,
+    })
+    signedDownloadUrl = dl?.signed_url || null
+  } catch {
+    // non-fatal
+  }
+
+  // ── Update manifest (if source_directory provided) ──────────────────────
+  // Tracks this upload so future deploy calls know the file is already in the
+  // bucket and don't re-recommend uploading it.
+  if (source_directory) {
+    try {
+      updateMediaManifest(source_directory, local_path, {
+        size: stat.size,
+        bucket,
+        object_name: verified?.name || filename,
+        content_type: resolvedContentType,
+        uploaded_at: new Date().toISOString(),
+      })
+    } catch {
+      // non-fatal — manifest update failure shouldn't break the upload response
+    }
+  }
+
+  return {
+    success: true,
+    bucket,
+    name: verified?.name || filename,
+    object_id: verified?.id || null,
+    size_bytes: stat.size,
+    size_human: formatBytes(stat.size),
+    content_type: resolvedContentType,
+    signed_url: signedDownloadUrl,
+    signed_url_expires_minutes: signedDownloadUrl ? 15 : null,
+    public_url: null,
+    message: `✓ Uploaded '${filename}' (${formatBytes(stat.size)}) to bucket '${bucket}'. Manifest updated — future deploys will skip this file.`,
+  }
+}
+
+function bool(v) {
+  if (typeof v === "boolean") return v
+  if (v === "true" || v === 1 || v === "1") return true
+  return false
+}
+
+function bucketHint(errMsg, bucket) {
+  if (!errMsg) return null
+  const lower = errMsg.toLowerCase()
+  if (lower.includes("not found") || lower.includes("no encontrado")) {
+    return `Bucket '${bucket}' may not exist. Either retry with ensure_bucket:true, or create it first: manage_storage({operation:"create", name:"${bucket}", public:true}).`
+  }
+  if (lower.includes("quota") || lower.includes("exceeded")) {
+    return "Storage quota exceeded for this project. Delete unused objects or upgrade the plan."
+  }
+  return null
+}
+
+async function safeReadBody(res) {
+  try { return await res.text() } catch { return null }
+}

package/src/tools/sync/describe.js

@@ -12,7 +12,10 @@
 import { proxyToolCall } from "../proxy.js"
 
 async function execSql(projectId, sql) {
-  const args = projectId ? { project_id: projectId, sql } : { sql }
+  // Bypass remote execute_sql auto-LIMIT 20 so describe counts match reality.
+  const hasLimit = /\bLIMIT\b/i.test(sql)
+  const finalSql = hasLimit ? sql : `${sql.replace(/;?\s*$/, "")} LIMIT 100000`
+  const args = projectId ? { project_id: projectId, sql: finalSql } : { sql: finalSql }
   const result = await proxyToolCall("execute_sql", args)
   if (result?.error) return null
   if (!result?.rows) return null
@@ -54,7 +57,6 @@ export const dypaiDescribeTool = {
                jsonb_array_length(e.workflow_code->'nodes') AS node_count
         FROM system.endpoints e
         LEFT JOIN system.endpoints_group g ON g.id = e.group_id
-        WHERE e.is_active = true
         ORDER BY e.name
       `),
       execSql(project_id, "SELECT name, type FROM system.credentials ORDER BY name"),

package/src/tools/sync/planner.js

@@ -17,7 +17,13 @@ import { deserializeEndpoint, serializeEndpoint } from "./codec.js"
 // ─── Remote ────────────────────────────────────────────────────────────────
 
 async function execSql(projectId, sql) {
-  const args = projectId ? { project_id: projectId, sql } : { sql }
+  // Bypass the remote execute_sql safety that injects `LIMIT 20` when no LIMIT
+  // is present. Planner needs the FULL endpoint/credential/group lists to
+  // compute an accurate diff — truncation here would mean invisible deletes
+  // (local thinks "remote has only X" when it actually has X+N).
+  const hasLimit = /\bLIMIT\b/i.test(sql)
+  const finalSql = hasLimit ? sql : `${sql.replace(/;?\s*$/, "")} LIMIT 100000`
+  const args = projectId ? { project_id: projectId, sql: finalSql } : { sql: finalSql }
   const result = await proxyToolCall("execute_sql", args)
   if (result?.error) throw new Error(`SQL error: ${result.error}`)
   if (!result?.rows) throw new Error(`Unexpected SQL response: ${JSON.stringify(result).slice(0, 300)}`)
@@ -116,9 +122,8 @@ export async function fetchRemoteState(projectId) {
   const [endpoints, credentials, groups] = await Promise.all([
     execSql(projectId, `
       SELECT id, name, method, description, workflow_code, input, output,
-             allowed_roles, is_tool, tool_description, group_id, updated_at
+             allowed_roles, is_tool, tool_description, group_id, is_active, updated_at
       FROM system.endpoints
-      WHERE is_active = true
       ORDER BY name
     `),
     execSql(projectId, "SELECT id, name, type FROM system.credentials"),

package/src/tools/sync/pull.js

@@ -441,7 +441,15 @@ workflow:
 `
 
 async function execSql(projectId, sql) {
-  const args = projectId ? { project_id: projectId, sql } : { sql }
+  // The remote `execute_sql` silently injects `LIMIT 20` whenever a query has no
+  // LIMIT of its own — intended as a guard against ad-hoc queries returning
+  // gigantic result sets. For pull we want the COMPLETE lists of endpoints,
+  // credentials, groups, realtime policies, etc. Without an explicit LIMIT
+  // here, a project with >20 of any resource would silently lose rows.
+  // We append a generous cap (100k) only when the caller didn't specify one.
+  const hasLimit = /\bLIMIT\b/i.test(sql)
+  const finalSql = hasLimit ? sql : `${sql.replace(/;?\s*$/, "")} LIMIT 100000`
+  const args = projectId ? { project_id: projectId, sql: finalSql } : { sql: finalSql }
   const result = await proxyToolCall("execute_sql", args)
   if (result?.error) throw new Error(`SQL error: ${result.error}`)
   if (!result?.rows) {
@@ -553,9 +561,8 @@ export const dypaiPullTool = {
     const [endpoints, credentials, groups, schemaSql, nodeCatalogResult, realtimePolicies] = await Promise.all([
       execSql(project_id, `
         SELECT id, name, method, description, workflow_code, input, output,
-               allowed_roles, is_tool, tool_description, group_id, updated_at
+               allowed_roles, is_tool, tool_description, group_id, is_active, updated_at
         FROM system.endpoints
-        WHERE is_active = true
         ORDER BY name
       `),
       execSql(project_id, "SELECT id, name, type FROM system.credentials"),
@@ -650,7 +657,22 @@ export const dypaiPullTool = {
         const relPath = groupName
           ? `endpoints/${groupName}/${row.name}.yaml`
           : `endpoints/${row.name}.yaml`
-        await writeFileEnsured(join(outDir, relPath), renderYaml(doc))
+
+        // Inactive endpoints ARE pulled (user needs to see them), but flagged
+        // with a visible header so the agent/user knows the endpoint won't
+        // execute until re-enabled in the dashboard. `is_active` isn't
+        // serialized into the YAML itself — push only updates content, never
+        // the active flag, so editing + pushing an inactive endpoint keeps it
+        // inactive (safe default).
+        const yamlBody = renderYaml(doc)
+        const content = row.is_active === false
+          ? "# ⚠️ INACTIVE on the engine — this endpoint will NOT execute until re-enabled in the dashboard.\n" +
+            "# Edits applied via `dypai_push` are still saved, but the endpoint stays paused.\n" +
+            "# To reactivate: go to the project dashboard → Endpoints → toggle this one back on.\n\n" +
+            yamlBody
+          : yamlBody
+
+        await writeFileEnsured(join(outDir, relPath), content)
         filesWritten.push(relPath)
       } catch (e) {
         errors.push({ endpoint: row.name, error: e.message })
@@ -730,6 +752,9 @@ export const dypaiPullTool = {
     const toolEndpoints = (endpoints || [])
       .filter(e => e.is_tool)
       .map(e => ({ name: e.name, description: e.tool_description || null }))
+    const inactiveEndpoints = (endpoints || [])
+      .filter(e => e.is_active === false)
+      .map(e => e.name)
 
     const overview = {
       project: projectInfo ? {
@@ -739,9 +764,12 @@ export const dypaiPullTool = {
       } : { id: resolvedProjectId || "(from token)" },
       endpoints: {
         total: (endpoints || []).length,
+        active: (endpoints || []).filter(e => e.is_active !== false).length,
+        inactive: inactiveEndpoints.length,
         groups: Object.keys(byGroup).filter(g => g !== "(no group)").sort(),
         by_group: byGroup,
         tool_endpoints: toolEndpoints,
+        inactive_endpoints: inactiveEndpoints.length > 0 ? inactiveEndpoints : undefined,
       },
       credentials: (credentials || []).map(c => ({ name: c.name, type: c.type })),
       realtime_policies: (realtimePolicies || []).length,

package/src/tools/sync/schema-dump.js

@@ -7,7 +7,13 @@
 import { proxyToolCall } from "../proxy.js"
 
 async function execSql(projectId, sql) {
-  const args = projectId ? { project_id: projectId, sql } : { sql }
+  // Bypass remote execute_sql auto-LIMIT 20 so the schema dump is complete.
+  // Without this, a project with >20 columns across public.* (very normal)
+  // would produce a truncated schema.sql silently — breaks the validator and
+  // any agent that reads schema.sql before writing SQL.
+  const hasLimit = /\bLIMIT\b/i.test(sql)
+  const finalSql = hasLimit ? sql : `${sql.replace(/;?\s*$/, "")} LIMIT 100000`
+  const args = projectId ? { project_id: projectId, sql: finalSql } : { sql: finalSql }
   const result = await proxyToolCall("execute_sql", args)
   if (result?.error) throw new Error(`SQL error: ${result.error}`)
   if (!result?.rows) throw new Error(`Unexpected SQL response: ${JSON.stringify(result).slice(0, 300)}`)

package/src/tools/sync.js

@@ -14,6 +14,7 @@
 
 import { writeFileSync, mkdirSync, existsSync } from "fs"
 import { join, dirname } from "path"
+import { createHash } from "crypto"
 import { api } from "../api.js"
 
 export async function syncFromRemote({ project_id, targetDirectory, overwrite = false }) {
@@ -60,6 +61,7 @@ export async function syncFromRemote({ project_id, targetDirectory, overwrite =
 
   let written = 0
   const failures = []
+  const hashMap = {}
 
   for (const file of payload.files) {
     if (!file?.path || typeof file.content !== "string") {
@@ -81,13 +83,33 @@ export async function syncFromRemote({ project_id, targetDirectory, overwrite =
       // `content` is base64 from the API (uniform encoding, binary-safe —
       // same convention as the deploy). Decode to a Buffer so binaries
       // (images, fonts, etc.) round-trip cleanly.
-      writeFileSync(fullPath, Buffer.from(file.content, "base64"))
+      const buf = Buffer.from(file.content, "base64")
+      writeFileSync(fullPath, buf)
+      hashMap[file.path] = createHash("sha256").update(buf).digest("hex")
       written++
     } catch (e) {
       failures.push({ path: file.path, reason: e.message })
     }
   }
 
+  // Seed the deploy manifest with the synced files' hashes. The next deploy
+  // sees the full project already matches the remote and only uploads what
+  // the agent actually edits — avoids a pointless 30 MB "full deploy" right
+  // after a sync when nothing has changed yet.
+  try {
+    const dypaiBase = existsSync(join(targetDirectory, "dypai"))
+      ? join(targetDirectory, "dypai", ".dypai")
+      : join(targetDirectory, ".dypai")
+    mkdirSync(dypaiBase, { recursive: true })
+    writeFileSync(
+      join(dypaiBase, "deploy-manifest.json"),
+      JSON.stringify(hashMap, null, 2) + "\n",
+    )
+  } catch (e) {
+    // Non-fatal: the first deploy will just be a full deploy.
+    failures.push({ path: ".dypai/deploy-manifest.json", reason: `Manifest seed failed: ${e.message}` })
+  }
+
   // Structured next_steps so the agent can act without re-prompting the LLM.
   // `npm install` is always suggested (idempotent and cheap). When we overwrote
   // an existing project the user's local files removed upstream are NOT