@dypai-ai/mcp 1.6.20 → 1.7.1

package/src/tools/storage.js

@@ -1,5 +1,5 @@
 /**
- * manage_storage — local extension adding an `upload_file` operation.
+ * manage_storage — local extension adding `upload_file` and `upload_files`.
  *
  * The remote `manage_storage` tool already covers list / create / delete buckets
  * and list_objects / delete_object / get_signed_download_url. What it CAN'T do
@@ -24,14 +24,20 @@
  * operation grafted on top — the dispatcher logic lives there too.
  */
 
-import { statSync, readFileSync, existsSync } from "fs"
-import { basename } from "path"
+import { statSync, readFileSync, existsSync, readdirSync } from "fs"
+import { basename, join, relative } from "path"
 import { proxyToolCall } from "./proxy.js"
-import { updateMediaManifest } from "./deploy.js"
+import { updateMediaManifest, writeMediaManifestBatch } from "./deploy.js"
 
 // 100 MB — enforced by the API (MAX_UPLOAD_SIZE_BYTES). We check client-side too
 // so the user sees a clean error before we waste a round-trip.
 const MAX_UPLOAD_BYTES = 100 * 1024 * 1024
+const DEFAULT_UPLOAD_CONCURRENCY = 5
+const MAX_UPLOAD_CONCURRENCY = 20
+const DEFAULT_MAX_RETRIES = 3
+const RETRY_BASE_MS = 300
+const RETRY_MAX_MS = 4000
+const LIST_OBJECTS_PAGE_SIZE = 100
 
 // Extensions we know how to MIME-type without shelling out. Everything else
 // falls back to application/octet-stream, which the bucket accepts fine — the browser
@@ -75,6 +81,223 @@ function formatBytes(n) {
   return `${(n / 1024 / 1024).toFixed(1)} MB`
 }
 
+function sleep(ms) {
+  return new Promise(resolve => setTimeout(resolve, ms))
+}
+
+function retryDelayMs(attempt) {
+  const exp = Math.min(RETRY_BASE_MS * (2 ** attempt), RETRY_MAX_MS)
+  const jitter = Math.floor(Math.random() * 100)
+  return exp + jitter
+}
+
+/** HTTP statuses worth retrying (transient server/rate-limit errors). */
+export function isRetryableStatus(status) {
+  return [408, 429, 500, 502, 503, 504].includes(Number(status))
+}
+
+/** Classify thrown errors for retry logic. Exported for unit tests. */
+export function isRetryableError(err) {
+  if (!err) return false
+  if (err.retryable === false) return false
+  if (err.retryable === true) return true
+  if (err.status != null) return isRetryableStatus(err.status)
+
+  const code = err.code
+  if (code === "ECONNRESET" || code === "ETIMEDOUT" || code === "ENOTFOUND" || code === "EAI_AGAIN") {
+    return true
+  }
+
+  const msg = String(err.message || "").toLowerCase()
+  if (msg.includes("fetch failed") || msg.includes("network") || msg.includes("socket")) {
+    return true
+  }
+
+  const statusMatch = msg.match(/status (\d{3})/)
+  if (statusMatch) return isRetryableStatus(Number(statusMatch[1]))
+
+  // Proxy/API throws without status — assume transient unless clearly client error
+  if (msg.includes("403") || msg.includes("404") || msg.includes("413") || msg.includes("400")) {
+    return false
+  }
+
+  return true
+}
+
+function markRetryable(err, retryable) {
+  err.retryable = retryable
+  return err
+}
+
+/** Retry fn on transient failures with exponential backoff + jitter. */
+export async function withRetry(fn, { retries = DEFAULT_MAX_RETRIES } = {}) {
+  let lastErr
+  for (let attempt = 0; attempt <= retries; attempt++) {
+    try {
+      return await fn(attempt)
+    } catch (e) {
+      lastErr = e
+      if (!isRetryableError(e) || attempt >= retries) throw e
+      await sleep(retryDelayMs(attempt))
+    }
+  }
+  throw lastErr
+}
+
+function joinStorageKey(prefix, name) {
+  const p = (prefix || "").replace(/^\/+|\/+$/g, "")
+  const n = (name || "").replace(/^\/+/, "")
+  if (p && n) return `${p}/${n}`
+  return p || n
+}
+
+function matchesInclude(filename, include) {
+  if (!include?.length) return true
+  const ext = filename.includes(".") ? `.${filename.split(".").pop().toLowerCase()}` : ""
+  return include.some((item) => {
+    const normalized = item.startsWith(".") ? item.toLowerCase() : `.${item.toLowerCase()}`
+    return ext === normalized
+  })
+}
+
+function walkDirectory(rootDir, dir, recursive, include, out) {
+  let entries
+  try {
+    entries = readdirSync(dir, { withFileTypes: true })
+  } catch {
+    return
+  }
+
+  for (const ent of entries) {
+    const full = join(dir, ent.name)
+    if (ent.isDirectory()) {
+      if (recursive) walkDirectory(rootDir, full, recursive, include, out)
+      continue
+    }
+    if (!ent.isFile()) continue
+    if (ent.name.startsWith(".")) continue
+    if (!matchesInclude(ent.name, include)) continue
+    const rel = relative(rootDir, full).split("\\").join("/")
+    out.push({ local_path: full, relative_path: rel })
+  }
+}
+
+/**
+ * Resolve upload entries from either an explicit file list or a directory walk.
+ * Exported for unit tests.
+ */
+export function collectUploadEntries({
+  files,
+  directory,
+  recursive = true,
+  prefix,
+  include,
+}) {
+  if (Array.isArray(files) && files.length > 0) {
+    return files.map((file) => {
+      const objectName = file.object_name || basename(file.local_path)
+      return {
+        local_path: file.local_path,
+        object_name: joinStorageKey(file.prefix ?? prefix, objectName),
+        content_type: file.content_type || undefined,
+      }
+    })
+  }
+
+  if (directory) {
+    if (!existsSync(directory)) {
+      return { error: `Directory not found: ${directory}` }
+    }
+    const stat = statSync(directory)
+    if (!stat.isDirectory()) {
+      return { error: `Not a directory: ${directory}` }
+    }
+
+    const discovered = []
+    walkDirectory(directory, directory, bool(recursive), include, discovered)
+    return discovered.map((entry) => ({
+      local_path: entry.local_path,
+      object_name: joinStorageKey(prefix, entry.relative_path),
+      content_type: undefined,
+    }))
+  }
+
+  return { error: "Either `files` (non-empty array) or `directory` is required for upload_files." }
+}
+
+async function ensureBucketExists({ bucket, bucket_public, project_id }) {
+  try {
+    const buckets = await proxyToolCall("manage_storage", {
+      operation: "list",
+      project_id,
+    })
+    const list = Array.isArray(buckets) ? buckets : (buckets?.data || [])
+    const exists = list.some(b => (b.name || b) === bucket)
+    if (!exists) {
+      await proxyToolCall("manage_storage", {
+        operation: "create",
+        project_id,
+        name: bucket,
+        public: bool(bucket_public),
+      })
+    }
+    return null
+  } catch (e) {
+    return `ensure_bucket failed: ${e.message}`
+  }
+}
+
+/** Paginated list_objects sweep; returns null on failure (best-effort). */
+async function listExistingObjectNames({ bucket, project_id, prefix }) {
+  const names = new Set()
+  let offset = 0
+
+  try {
+    while (true) {
+      const resp = await proxyToolCall("manage_storage", {
+        operation: "list_objects",
+        project_id,
+        bucket,
+        prefix: prefix || undefined,
+        limit: LIST_OBJECTS_PAGE_SIZE,
+        offset,
+      })
+      const list = Array.isArray(resp) ? resp : (resp?.data || [])
+      for (const obj of list) {
+        const name = typeof obj === "string" ? obj : obj?.name
+        if (name) names.add(name)
+      }
+
+      const hasMore = resp?.has_more === true
+        || (resp?.has_more == null && list.length >= LIST_OBJECTS_PAGE_SIZE)
+      if (!hasMore || list.length === 0) break
+      offset += LIST_OBJECTS_PAGE_SIZE
+    }
+    return names
+  } catch {
+    return null
+  }
+}
+
+async function runPool(items, concurrency, fn) {
+  const results = new Array(items.length)
+  let next = 0
+
+  async function worker() {
+    while (next < items.length) {
+      const idx = next++
+      results[idx] = await fn(items[idx], idx)
+    }
+  }
+
+  const workers = Array.from(
+    { length: Math.min(concurrency, items.length) },
+    () => worker(),
+  )
+  await Promise.all(workers)
+  return results
+}
+
 /**
  * Core upload. Returns { success, bucket, name, size, content_type,
  * signed_url, public_url, object_id, ... } or { success: false, error }.
@@ -99,6 +322,8 @@ export async function uploadFile({
   // Optional: if provided, the manifest at <source_directory>/dypai/.dypai/media-manifest.json
   // is updated after a successful upload so future deploys know this file is in the bucket.
   source_directory,
+  skip_preview_url = false,
+  max_retries = DEFAULT_MAX_RETRIES,
 }) {
   // ── Validation ──────────────────────────────────────────────────────────
   if (!local_path) return { success: false, error: "`local_path` is required." }
@@ -123,51 +348,47 @@ export async function uploadFile({
 
   const filename = object_name || basename(local_path)
   const resolvedContentType = content_type || mimeFor(filename)
+  const retries = Math.max(0, Number(max_retries) || DEFAULT_MAX_RETRIES)
 
   // ── Ensure bucket (optional) ────────────────────────────────────────────
   // If the agent passed ensure_bucket:true and the bucket doesn't exist yet,
   // create it. Saves a round-trip of "bucket not found → go create → retry".
   if (ensure_bucket) {
-    try {
-      const buckets = await proxyToolCall("manage_storage", {
-        operation: "list",
-        project_id,
-      })
-      const list = Array.isArray(buckets) ? buckets : (buckets?.data || [])
-      const exists = list.some(b => (b.name || b) === bucket)
-      if (!exists) {
-        await proxyToolCall("manage_storage", {
-          operation: "create",
-          project_id,
-          name: bucket,
-          public: bool(bucket_public),
-        })
-      }
-    } catch (e) {
-      return { success: false, error: `ensure_bucket failed: ${e.message}` }
-    }
+    const bucketError = await ensureBucketExists({ bucket, bucket_public, project_id })
+    if (bucketError) return { success: false, error: bucketError }
   }
 
   // ── Step 1: sign_upload ─────────────────────────────────────────────────
   let signed
   try {
-    signed = await proxyToolCall("manage_storage", {
-      operation: "sign_upload",
-      project_id,
-      bucket,
-      filename,
-      content_type: resolvedContentType,
-      size_bytes: stat.size,
-      prefix,
-    })
+    signed = await withRetry(async () => {
+      try {
+        const result = await proxyToolCall("manage_storage", {
+          operation: "sign_upload",
+          project_id,
+          bucket,
+          filename,
+          content_type: resolvedContentType,
+          size_bytes: stat.size,
+          prefix,
+        })
+        if (!result?.upload_url || !result?.file_path) {
+          throw markRetryable(
+            new Error("sign_upload returned an invalid payload (no upload_url or file_path)."),
+            false,
+          )
+        }
+        return result
+      } catch (e) {
+        if (e.retryable === false) throw e
+        throw markRetryable(new Error(e.message || "sign_upload failed"), isRetryableError(e))
+      }
+    }, { retries })
   } catch (e) {
     return { success: false, error: `sign_upload failed: ${e.message}`, hint: bucketHint(e.message, bucket) }
   }
 
   const { upload_url, method = "PUT", headers = {}, file_path, public_url } = signed || {}
-  if (!upload_url || !file_path) {
-    return { success: false, error: "sign_upload returned an invalid payload (no upload_url or file_path)." }
-  }
 
   // ── Step 2: direct PUT to the bucket ────────────────────────────────────
   let buf
@@ -178,38 +399,48 @@ export async function uploadFile({
   }
 
   try {
-    const res = await fetch(upload_url, {
-      method,
-      headers: {
-        "Content-Type": resolvedContentType,
-        ...headers,
-      },
-      body: buf,
-    })
-    if (!res.ok) {
-      const detail = await safeReadBody(res)
-      return {
-        success: false,
-        error: `Storage upload failed with status ${res.status}. ${detail ? `Detail: ${detail.slice(0, 300)}` : ""}`.trim(),
+    await withRetry(async () => {
+      const res = await fetch(upload_url, {
+        method,
+        headers: {
+          "Content-Type": resolvedContentType,
+          ...headers,
+        },
+        body: buf,
+      })
+      if (!res.ok) {
+        const detail = await safeReadBody(res)
+        const err = new Error(
+          `Storage upload failed with status ${res.status}. ${detail ? `Detail: ${detail.slice(0, 300)}` : ""}`.trim(),
+        )
+        err.status = res.status
+        throw markRetryable(err, isRetryableStatus(res.status))
       }
-    }
+    }, { retries })
   } catch (e) {
-    return { success: false, error: `Direct upload to storage failed: ${e.message}` }
+    return { success: false, error: e.message || "Direct upload to storage failed." }
   }
 
   // ── Step 3: verify_upload ───────────────────────────────────────────────
   let verified
   try {
-    verified = await proxyToolCall("manage_storage", {
-      operation: "verify_upload",
-      project_id,
-      bucket,
-      file_path,
-      original_filename: filename,
-      content_type: resolvedContentType,
-      size_bytes: stat.size,
-      prefix,
-    })
+    verified = await withRetry(async () => {
+      try {
+        return await proxyToolCall("manage_storage", {
+          operation: "verify_upload",
+          project_id,
+          bucket,
+          file_path,
+          original_filename: filename,
+          content_type: resolvedContentType,
+          size_bytes: stat.size,
+          prefix,
+        })
+      } catch (e) {
+        if (e.retryable === false) throw e
+        throw markRetryable(new Error(e.message || "verify_upload failed"), isRetryableError(e))
+      }
+    }, { retries })
   } catch (e) {
     return {
       success: false,
@@ -220,18 +451,20 @@ export async function uploadFile({
   // ── Step 4: fetch a preview signed download URL ─────────────────────────
   // Best-effort: if it fails we just skip it, the upload succeeded regardless.
   let signedDownloadUrl = null
-  try {
-    const dl = await proxyToolCall("manage_storage", {
-      operation: "get_signed_download_url",
-      project_id,
-      bucket,
-      name: verified?.name || filename,
-      expires_minutes: 15,
-      download: false,
-    })
-    signedDownloadUrl = dl?.signed_url || null
-  } catch {
-    // non-fatal
+  if (!skip_preview_url) {
+    try {
+      const dl = await proxyToolCall("manage_storage", {
+        operation: "get_signed_download_url",
+        project_id,
+        bucket,
+        name: verified?.name || filename,
+        expires_minutes: 15,
+        download: false,
+      })
+      signedDownloadUrl = dl?.signed_url || null
+    } catch {
+      // non-fatal
+    }
   }
 
   // ── Update manifest (if source_directory provided) ──────────────────────
@@ -264,7 +497,139 @@ export async function uploadFile({
     signed_url: signedDownloadUrl,
     signed_url_expires_minutes: signedDownloadUrl ? 15 : null,
     public_url: public_url || null,
-    message: `✓ Uploaded '${filename}' (${formatBytes(stat.size)}) to bucket '${bucket}'. Manifest updated — future deploys will skip this file.`,
+    message: `✓ Uploaded '${filename}' (${formatBytes(stat.size)}) to bucket '${bucket}'.${source_directory ? " Manifest updated — future deploys will skip this file." : ""}`,
+  }
+}
+
+/**
+ * Batch upload many local files in one call. Reuses uploadFile() per entry with
+ * bounded concurrency. Returns an aggregated summary instead of per-file noise.
+ */
+export async function uploadFiles({
+  bucket,
+  files,
+  directory,
+  recursive = true,
+  prefix,
+  include,
+  ensure_bucket = false,
+  bucket_public = false,
+  project_id,
+  source_directory,
+  concurrency = DEFAULT_UPLOAD_CONCURRENCY,
+  max_retries = DEFAULT_MAX_RETRIES,
+  skip_existing = true,
+  overwrite = false,
+}) {
+  if (!bucket) return { success: false, error: "`bucket` is required (e.g. 'public')." }
+
+  const collected = collectUploadEntries({ files, directory, recursive, prefix, include })
+  if (collected.error) return { success: false, error: collected.error }
+
+  const entries = collected
+  if (entries.length === 0) {
+    return { success: false, error: "No files to upload. Check `files`, `directory`, or `include` filters." }
+  }
+
+  if (ensure_bucket) {
+    const bucketError = await ensureBucketExists({ bucket, bucket_public, project_id })
+    if (bucketError) return { success: false, error: bucketError }
+  }
+
+  const shouldSkip = bool(skip_existing) && !bool(overwrite)
+  const skippedResults = []
+  let toUpload = entries
+
+  if (shouldSkip) {
+    const existingNames = await listExistingObjectNames({ bucket, project_id, prefix })
+    if (existingNames) {
+      toUpload = []
+      for (const entry of entries) {
+        if (existingNames.has(entry.object_name)) {
+          skippedResults.push({
+            local_path: entry.local_path,
+            name: entry.object_name,
+            status: "skipped",
+            error: null,
+          })
+        } else {
+          toUpload.push(entry)
+        }
+      }
+    }
+  }
+
+  const poolSize = Math.max(
+    1,
+    Math.min(Number(concurrency) || DEFAULT_UPLOAD_CONCURRENCY, MAX_UPLOAD_CONCURRENCY),
+  )
+
+  const manifestEntries = {}
+
+  const poolResults = toUpload.length > 0
+    ? await runPool(toUpload, poolSize, async (entry) => {
+      const result = await uploadFile({
+        local_path: entry.local_path,
+        bucket,
+        object_name: entry.object_name,
+        content_type: entry.content_type,
+        ensure_bucket: false,
+        bucket_public,
+        project_id,
+        skip_preview_url: true,
+        max_retries,
+      })
+
+      if (result.success && source_directory) {
+        manifestEntries[entry.local_path] = {
+          size: result.size_bytes,
+          bucket,
+          object_name: result.storage_path || result.name || entry.object_name,
+          content_type: result.content_type,
+          uploaded_at: new Date().toISOString(),
+        }
+      }
+
+      return {
+        local_path: entry.local_path,
+        name: result.storage_path || result.name || entry.object_name,
+        status: result.success ? "uploaded" : "failed",
+        error: result.success ? null : (result.error || "Upload failed"),
+      }
+    })
+    : []
+
+  if (source_directory && Object.keys(manifestEntries).length > 0) {
+    try {
+      writeMediaManifestBatch(source_directory, manifestEntries)
+    } catch {
+      // non-fatal
+    }
+  }
+
+  const results = [...skippedResults, ...poolResults]
+  const uploaded = results.filter(r => r.status === "uploaded").length
+  const failed = results.filter(r => r.status === "failed").length
+  const skipped = results.filter(r => r.status === "skipped").length
+  const storage_paths = results.filter(r => r.status === "uploaded").map(r => r.name)
+
+  const parts = []
+  if (uploaded > 0) parts.push(`${uploaded} uploaded`)
+  if (skipped > 0) parts.push(`${skipped} skipped`)
+  if (failed > 0) parts.push(`${failed} failed`)
+
+  return {
+    success: failed === 0 && (uploaded + skipped) > 0,
+    bucket,
+    total: results.length,
+    uploaded,
+    skipped,
+    failed,
+    results,
+    storage_paths,
+    message: failed === 0
+      ? `✓ Batch complete for bucket '${bucket}': ${parts.join(", ")}.`
+      : `Batch for bucket '${bucket}': ${parts.join(", ")} — see results[].error.`,
   }
 }
 

package/src/tools/sync/generate-types.js

@@ -48,6 +48,10 @@ export async function runGenerateEndpointTypes(rootDir, projectId = null) {
       typesPath,
       contractPath,
       endpoints: payload.endpoints || [],
+      diagnosticCount: payload.diagnosticCount ?? 0,
+      errorCount: payload.errorCount ?? 0,
+      warningCount: payload.warningCount ?? 0,
+      diagnostics: Array.isArray(payload.diagnostics) ? payload.diagnostics.slice(0, 10) : [],
     }
   }
 
@@ -57,6 +61,10 @@ export async function runGenerateEndpointTypes(rootDir, projectId = null) {
     typesPath: payload.typesPath,
     contractPath: payload.contractPath,
     endpoints: payload.endpoints || [],
+    diagnosticCount: payload.diagnosticCount ?? 0,
+    errorCount: payload.errorCount ?? 0,
+    warningCount: payload.warningCount ?? 0,
+    diagnostics: Array.isArray(payload.diagnostics) ? payload.diagnostics.slice(0, 10) : [],
   }
 }
 
@@ -112,8 +120,16 @@ export const dypaiGenerateTypesTool = {
       types_path: "dypai/types/endpoints.gen.ts",
       contract_path: "dypai/types/endpoints.contract.json",
       endpoints: outcome.endpoints.slice(0, 50),
+      diagnostics_summary: outcome.diagnosticCount
+        ? {
+            total: outcome.diagnosticCount,
+            errors: outcome.errorCount,
+            warnings: outcome.warningCount,
+            sample: outcome.diagnostics,
+          }
+        : undefined,
       hint: outcome.endpointCount
-        ? `Types refreshed for ${outcome.endpointCount} endpoint(s). Frontend can import from dypai/types/endpoints.gen.ts.`
+        ? `Types refreshed for ${outcome.endpointCount} endpoint(s). Frontend can import from dypai/types/endpoints.gen.ts.${outcome.errorCount ? " Backend errors were found; run dypai_validate before trusting frontend call sites." : ""}`
         : "No effective endpoints found — types file is empty.",
     }
   },