@dypai-ai/mcp 1.3.1 → 1.4.1

package/src/tools/storage.js

@@ -0,0 +1,289 @@
+/**
+ * manage_storage — local extension adding an `upload_file` operation.
+ *
+ * The remote `manage_storage` tool already covers list / create / delete buckets
+ * and list_objects / delete_object / get_signed_download_url. What it CAN'T do
+ * is upload: the binary would have to travel through the MCP stdio transport
+ * which is (a) slow and (b) saturates the remote pod on 100 MB files.
+ *
+ * `upload_file` solves this with the same pattern `dypai.api.upload` uses in
+ * the frontend SDK:
+ *
+ *   1. Ask the remote MCP → API for a signed PUT URL (op: sign_upload).
+ *   2. PUT the file binary DIRECTLY to R2 from the local machine (no infra hop).
+ *   3. Ask the remote MCP → API to register the object (op: verify_upload).
+ *
+ * Net effect: the agent just calls `manage_storage(operation:"upload_file",
+ * local_path:"...", bucket:"public")` and gets back a URL. Zero credentials on
+ * the agent side — the signed URL IS the temporary credential (15 min, one key,
+ * PUT only).
+ *
+ * NOTE: this file exports `uploadFile` plus the pieces needed to extend the
+ * `manage_storage` tool entry in index.js. The catalog-level tool definition
+ * still lives in index.js because manage_storage is a REMOTE tool with a local
+ * operation grafted on top — the dispatcher logic lives there too.
+ */
+
+import { statSync, readFileSync, existsSync } from "fs"
+import { basename } from "path"
+import { proxyToolCall } from "./proxy.js"
+import { updateMediaManifest } from "./deploy.js"
+
+// 100 MB — enforced by the API (MAX_UPLOAD_SIZE_BYTES). We check client-side too
+// so the user sees a clean error before we waste a round-trip.
+const MAX_UPLOAD_BYTES = 100 * 1024 * 1024
+
+// Extensions we know how to MIME-type without shelling out. Everything else
+// falls back to application/octet-stream, which R2 accepts fine — the browser
+// just won't preview it inline.
+const MIME_BY_EXT = {
+  // Images
+  png: "image/png", jpg: "image/jpeg", jpeg: "image/jpeg", gif: "image/gif",
+  webp: "image/webp", avif: "image/avif", svg: "image/svg+xml", ico: "image/x-icon",
+  bmp: "image/bmp", tiff: "image/tiff",
+  // Video
+  mp4: "video/mp4", webm: "video/webm", mov: "video/quicktime", mkv: "video/x-matroska",
+  // Audio
+  mp3: "audio/mpeg", wav: "audio/wav", ogg: "audio/ogg", m4a: "audio/mp4",
+  flac: "audio/flac",
+  // Documents
+  pdf: "application/pdf",
+  doc: "application/msword",
+  docx: "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+  xls: "application/vnd.ms-excel",
+  xlsx: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+  ppt: "application/vnd.ms-powerpoint",
+  pptx: "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+  // Text / data
+  txt: "text/plain", csv: "text/csv", json: "application/json",
+  xml: "application/xml", html: "text/html", md: "text/markdown",
+  yaml: "application/yaml", yml: "application/yaml",
+  // Archives
+  zip: "application/zip", gz: "application/gzip", tar: "application/x-tar",
+  // Fonts
+  woff: "font/woff", woff2: "font/woff2", ttf: "font/ttf", otf: "font/otf",
+}
+
+function mimeFor(filename) {
+  const ext = filename.includes(".") ? filename.split(".").pop().toLowerCase() : ""
+  return MIME_BY_EXT[ext] || "application/octet-stream"
+}
+
+function formatBytes(n) {
+  if (n < 1024) return `${n} B`
+  if (n < 1024 * 1024) return `${(n / 1024).toFixed(1)} KB`
+  return `${(n / 1024 / 1024).toFixed(1)} MB`
+}
+
+/**
+ * Core upload. Returns { success, bucket, name, size, content_type,
+ * signed_url, public_url, object_id, ... } or { success: false, error }.
+ *
+ * Flow:
+ *   1. stat → size + mime
+ *   2. ensure_bucket (optional) → list or create bucket if missing
+ *   3. sign_upload → presigned PUT URL
+ *   4. fetch(PUT, body=fileBuffer)
+ *   5. verify_upload → row in storage.objects
+ *   6. sign a short-lived download URL so the agent has something clickable
+ */
+export async function uploadFile({
+  local_path,
+  bucket,
+  object_name,
+  prefix,
+  content_type,
+  ensure_bucket = false,
+  bucket_public = false,
+  project_id,
+  // Optional: if provided, the manifest at <source_directory>/dypai/.dypai/media-manifest.json
+  // is updated after a successful upload so future deploys know this file is in the bucket.
+  source_directory,
+}) {
+  // ── Validation ──────────────────────────────────────────────────────────
+  if (!local_path) return { success: false, error: "`local_path` is required." }
+  if (!bucket) return { success: false, error: "`bucket` is required (e.g. 'public')." }
+  if (!existsSync(local_path)) {
+    return { success: false, error: `File not found: ${local_path}` }
+  }
+
+  const stat = statSync(local_path)
+  if (!stat.isFile()) {
+    return { success: false, error: `Not a file: ${local_path}` }
+  }
+  if (stat.size === 0) {
+    return { success: false, error: `File is empty: ${local_path}` }
+  }
+  if (stat.size > MAX_UPLOAD_BYTES) {
+    return {
+      success: false,
+      error: `File too large: ${formatBytes(stat.size)} (max ${formatBytes(MAX_UPLOAD_BYTES)}). Split the file or host it externally.`,
+    }
+  }
+
+  const filename = object_name || basename(local_path)
+  const resolvedContentType = content_type || mimeFor(filename)
+
+  // ── Ensure bucket (optional) ────────────────────────────────────────────
+  // If the agent passed ensure_bucket:true and the bucket doesn't exist yet,
+  // create it. Saves a round-trip of "bucket not found → go create → retry".
+  if (ensure_bucket) {
+    try {
+      const buckets = await proxyToolCall("manage_storage", {
+        operation: "list",
+        project_id,
+      })
+      const list = Array.isArray(buckets) ? buckets : (buckets?.data || [])
+      const exists = list.some(b => (b.name || b) === bucket)
+      if (!exists) {
+        await proxyToolCall("manage_storage", {
+          operation: "create",
+          project_id,
+          name: bucket,
+          public: bool(bucket_public),
+        })
+      }
+    } catch (e) {
+      return { success: false, error: `ensure_bucket failed: ${e.message}` }
+    }
+  }
+
+  // ── Step 1: sign_upload ─────────────────────────────────────────────────
+  let signed
+  try {
+    signed = await proxyToolCall("manage_storage", {
+      operation: "sign_upload",
+      project_id,
+      bucket,
+      filename,
+      content_type: resolvedContentType,
+      size_bytes: stat.size,
+      prefix,
+    })
+  } catch (e) {
+    return { success: false, error: `sign_upload failed: ${e.message}`, hint: bucketHint(e.message, bucket) }
+  }
+
+  const { upload_url, method = "PUT", headers = {}, file_path } = signed || {}
+  if (!upload_url || !file_path) {
+    return { success: false, error: "sign_upload returned an invalid payload (no upload_url or file_path)." }
+  }
+
+  // ── Step 2: direct PUT to R2 ────────────────────────────────────────────
+  let buf
+  try {
+    buf = readFileSync(local_path)
+  } catch (e) {
+    return { success: false, error: `Failed to read file: ${e.message}` }
+  }
+
+  try {
+    const res = await fetch(upload_url, {
+      method,
+      headers: {
+        "Content-Type": resolvedContentType,
+        ...headers,
+      },
+      body: buf,
+    })
+    if (!res.ok) {
+      const detail = await safeReadBody(res)
+      return {
+        success: false,
+        error: `R2 PUT failed with status ${res.status}. ${detail ? `Detail: ${detail.slice(0, 300)}` : ""}`.trim(),
+      }
+    }
+  } catch (e) {
+    return { success: false, error: `Direct upload to R2 failed: ${e.message}` }
+  }
+
+  // ── Step 3: verify_upload ───────────────────────────────────────────────
+  let verified
+  try {
+    verified = await proxyToolCall("manage_storage", {
+      operation: "verify_upload",
+      project_id,
+      bucket,
+      file_path,
+      original_filename: filename,
+      content_type: resolvedContentType,
+      size_bytes: stat.size,
+      prefix,
+    })
+  } catch (e) {
+    return {
+      success: false,
+      error: `verify_upload failed (the file WAS uploaded to R2 but not registered): ${e.message}. You can retry the upload — verify_upload is idempotent on (bucket, name).`,
+    }
+  }
+
+  // ── Step 4: fetch a preview signed download URL ─────────────────────────
+  // Best-effort: if it fails we just skip it, the upload succeeded regardless.
+  let signedDownloadUrl = null
+  try {
+    const dl = await proxyToolCall("manage_storage", {
+      operation: "get_signed_download_url",
+      project_id,
+      bucket,
+      name: verified?.name || filename,
+      expires_minutes: 15,
+      download: false,
+    })
+    signedDownloadUrl = dl?.signed_url || null
+  } catch {
+    // non-fatal
+  }
+
+  // ── Update manifest (if source_directory provided) ──────────────────────
+  // Tracks this upload so future deploy calls know the file is already in the
+  // bucket and don't re-recommend uploading it.
+  if (source_directory) {
+    try {
+      updateMediaManifest(source_directory, local_path, {
+        size: stat.size,
+        bucket,
+        object_name: verified?.name || filename,
+        content_type: resolvedContentType,
+        uploaded_at: new Date().toISOString(),
+      })
+    } catch {
+      // non-fatal — manifest update failure shouldn't break the upload response
+    }
+  }
+
+  return {
+    success: true,
+    bucket,
+    name: verified?.name || filename,
+    object_id: verified?.id || null,
+    size_bytes: stat.size,
+    size_human: formatBytes(stat.size),
+    content_type: resolvedContentType,
+    signed_url: signedDownloadUrl,
+    signed_url_expires_minutes: signedDownloadUrl ? 15 : null,
+    public_url: null,
+    message: `✓ Uploaded '${filename}' (${formatBytes(stat.size)}) to bucket '${bucket}'. Manifest updated — future deploys will skip this file.`,
+  }
+}
+
+function bool(v) {
+  if (typeof v === "boolean") return v
+  if (v === "true" || v === 1 || v === "1") return true
+  return false
+}
+
+function bucketHint(errMsg, bucket) {
+  if (!errMsg) return null
+  const lower = errMsg.toLowerCase()
+  if (lower.includes("not found") || lower.includes("no encontrado")) {
+    return `Bucket '${bucket}' may not exist. Either retry with ensure_bucket:true, or create it first: manage_storage({operation:"create", name:"${bucket}", public:true}).`
+  }
+  if (lower.includes("quota") || lower.includes("exceeded")) {
+    return "Storage quota exceeded for this project. Delete unused objects or upgrade the plan."
+  }
+  return null
+}
+
+async function safeReadBody(res) {
+  try { return await res.text() } catch { return null }
+}

package/src/tools/sync/transforms.js

@@ -74,14 +74,17 @@ export const NODE_FIELD_TRANSFORMS = [
     name: "sql_extraction",
     appliesWhen: (nodeType) => nodeType === "dypai_database",
     pull(params, ctx) {
-      if (params.query && !shouldInlineSql(params.query)) {
-        // Flat layout: sql/<endpoint>.sql if only one SQL node, else sql/<endpoint>.<node>.sql
-        const suffix = ctx.sqlNodeCount > 1 ? `.${ctx.nodeId}` : ""
-        const path = `sql/${ctx.endpointName}${suffix}.sql`
-        ctx.emitFile(path, params.query.trim() + "\n")
-        return { query_file: path }
+      if (!params.query) return {}
+      if (shouldInlineSql(params.query)) {
+        // Short SQL stays inline. Must be re-emitted here because
+        // `pullConsumes: ["query"]` deletes it from the base object.
+        return { query: params.query }
       }
-      return {}
+      // Flat layout: sql/<endpoint>.sql if only one SQL node, else sql/<endpoint>.<node>.sql
+      const suffix = ctx.sqlNodeCount > 1 ? `.${ctx.nodeId}` : ""
+      const path = `sql/${ctx.endpointName}${suffix}.sql`
+      ctx.emitFile(path, params.query.trim() + "\n")
+      return { query_file: path }
     },
     push(params, ctx) {
       if (params.query_file) {
@@ -97,13 +100,16 @@ export const NODE_FIELD_TRANSFORMS = [
     name: "prompt_extraction",
     appliesWhen: (nodeType) => nodeType === "agent",
     pull(params, ctx) {
-      if (params.system_prompt && params.system_prompt.length > PROMPT_INLINE_MAX_CHARS) {
-        const suffix = ctx.promptNodeCount > 1 ? `.${ctx.nodeId}` : ""
-        const path = `prompts/${ctx.endpointName}${suffix}.md`
-        ctx.emitFile(path, params.system_prompt.trim() + "\n")
-        return { system_prompt_file: path }
+      if (!params.system_prompt) return {}
+      if (params.system_prompt.length <= PROMPT_INLINE_MAX_CHARS) {
+        // Short prompt stays inline. Must be re-emitted here because
+        // `pullConsumes: ["system_prompt"]` deletes it from the base object.
+        return { system_prompt: params.system_prompt }
       }
-      return {}
+      const suffix = ctx.promptNodeCount > 1 ? `.${ctx.nodeId}` : ""
+      const path = `prompts/${ctx.endpointName}${suffix}.md`
+      ctx.emitFile(path, params.system_prompt.trim() + "\n")
+      return { system_prompt_file: path }
     },
     push(params, ctx) {
       if (params.system_prompt_file) {
@@ -119,14 +125,20 @@ export const NODE_FIELD_TRANSFORMS = [
     name: "code_extraction",
     appliesWhen: (nodeType) => nodeType === "javascript_code" || nodeType === "python_code",
     pull(params, ctx) {
-      if (params.code && params.code.length > SQL_INLINE_MAX_CHARS) {
-        const ext = ctx.nodeType === "python_code" ? "py" : "js"
-        const suffix = ctx.codeNodeCount > 1 ? `.${ctx.nodeId}` : ""
-        const path = `code/${ctx.endpointName}${suffix}.${ext}`
-        ctx.emitFile(path, params.code.trim() + "\n")
-        return { code_file: path }
+      if (!params.code) return {}
+      if (params.code.length <= SQL_INLINE_MAX_CHARS) {
+        // Short code stays inline. Must be re-emitted here because
+        // `pullConsumes: ["code"]` deletes it from the base object.
+        // Before this fix, short code was silently dropped from the YAML
+        // (neither extracted to a file nor kept inline) — endpoints pulled
+        // from the engine ended up with just `timeout_ms` and no handler.
+        return { code: params.code }
       }
-      return {}
+      const ext = ctx.nodeType === "python_code" ? "py" : "js"
+      const suffix = ctx.codeNodeCount > 1 ? `.${ctx.nodeId}` : ""
+      const path = `code/${ctx.endpointName}${suffix}.${ext}`
+      ctx.emitFile(path, params.code.trim() + "\n")
+      return { code_file: path }
     },
     push(params, ctx) {
       if (params.code_file) {

package/src/tools/sync/validate.js

@@ -580,7 +580,12 @@ function validateEndpoint(entry, ctx) {
           // Enum / range checks for primitive values
           const prop = properties[key]
           const v = node[key]
-          if (prop.enum && typeof v === "string" && !prop.enum.includes(v)) {
+          // Skip enum validation if the value contains a ${...} placeholder —
+          // the actual value is resolved at runtime, so the engine enforces
+          // the enum then. Common case: operation: ${input.operation} in
+          // dypai_storage / dypai_database nodes.
+          const hasPlaceholder = typeof v === "string" && v.includes("${")
+          if (prop.enum && typeof v === "string" && !hasPlaceholder && !prop.enum.includes(v)) {
             diagnostics.push({
               severity: "error",
               rule: "param_enum_violation",

package/src/tools/sync.js

@@ -14,6 +14,7 @@
 
 import { writeFileSync, mkdirSync, existsSync } from "fs"
 import { join, dirname } from "path"
+import { createHash } from "crypto"
 import { api } from "../api.js"
 
 export async function syncFromRemote({ project_id, targetDirectory, overwrite = false }) {
@@ -60,6 +61,7 @@ export async function syncFromRemote({ project_id, targetDirectory, overwrite =
 
   let written = 0
   const failures = []
+  const hashMap = {}
 
   for (const file of payload.files) {
     if (!file?.path || typeof file.content !== "string") {
@@ -81,13 +83,33 @@ export async function syncFromRemote({ project_id, targetDirectory, overwrite =
       // `content` is base64 from the API (uniform encoding, binary-safe —
       // same convention as the deploy). Decode to a Buffer so binaries
       // (images, fonts, etc.) round-trip cleanly.
-      writeFileSync(fullPath, Buffer.from(file.content, "base64"))
+      const buf = Buffer.from(file.content, "base64")
+      writeFileSync(fullPath, buf)
+      hashMap[file.path] = createHash("sha256").update(buf).digest("hex")
       written++
     } catch (e) {
       failures.push({ path: file.path, reason: e.message })
     }
   }
 
+  // Seed the deploy manifest with the synced files' hashes. The next deploy
+  // sees the full project already matches the remote and only uploads what
+  // the agent actually edits — avoids a pointless 30 MB "full deploy" right
+  // after a sync when nothing has changed yet.
+  try {
+    const dypaiBase = existsSync(join(targetDirectory, "dypai"))
+      ? join(targetDirectory, "dypai", ".dypai")
+      : join(targetDirectory, ".dypai")
+    mkdirSync(dypaiBase, { recursive: true })
+    writeFileSync(
+      join(dypaiBase, "deploy-manifest.json"),
+      JSON.stringify(hashMap, null, 2) + "\n",
+    )
+  } catch (e) {
+    // Non-fatal: the first deploy will just be a full deploy.
+    failures.push({ path: ".dypai/deploy-manifest.json", reason: `Manifest seed failed: ${e.message}` })
+  }
+
   // Structured next_steps so the agent can act without re-prompting the LLM.
   // `npm install` is always suggested (idempotent and cheap). When we overwrote
   // an existing project the user's local files removed upstream are NOT