@budibase/cli 3.38.5 → 3.39.16

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/cli",
-  "version": "3.38.5",
+  "version": "3.39.16",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
   "main": "dist/index.js",
   "bin": {
@@ -41,5 +41,5 @@
     "@types/pouchdb": "^6.4.0",
     "ts-node": "10.8.1"
   },
-  "gitHead": "5795ec6c95812463cba1d8cede8295f6c2b915cb"
+  "gitHead": "b1c87febc51bc49767bb6438e75a06038fea8fbf"
 }

package/src/backups/objectStore.ts

@@ -1,9 +1,10 @@
 import { objectStore } from "@budibase/backend-core"
 import fs from "fs"
-import { join } from "path"
+import { dirname, join } from "path"
 import { TEMP_DIR, MINIO_DIR } from "./utils"
 import { progressBar } from "../utils"
 import * as stream from "node:stream"
+import { pipeline } from "stream/promises"
 
 const {
   ObjectStoreBuckets,
@@ -15,10 +16,36 @@ const {
 
 const bucketList = Object.values(ObjectStoreBuckets)
 
+type BackupObject = {
+  bucket: string
+  Key: string
+}
+
+async function listBucketObjects(
+  client: ReturnType<typeof ObjectStore>,
+  bucket: string
+) {
+  const objects: BackupObject[] = []
+  let continuationToken: string | undefined
+  do {
+    const list = await client.listObjectsV2({
+      Bucket: bucket,
+      ContinuationToken: continuationToken,
+    })
+    objects.push(
+      ...(list.Contents?.filter(
+        (object): object is typeof object & { Key: string } => !!object.Key
+      ).map(object => ({ ...object, bucket, Key: object.Key })) || [])
+    )
+    continuationToken = list.NextContinuationToken
+  } while (continuationToken)
+  return objects
+}
+
 export async function exportObjects() {
   const path = join(TEMP_DIR, MINIO_DIR)
-  fs.mkdirSync(path)
-  let fullList: any[] = []
+  fs.mkdirSync(path, { recursive: true })
+  let fullList: BackupObject[] = []
   let errorCount = 0
   for (let bucket of bucketList) {
     const client = ObjectStore()
@@ -30,12 +57,7 @@ export async function exportObjects() {
       errorCount++
       continue
     }
-    const list = await client.listObjectsV2({
-      Bucket: bucket,
-    })
-    fullList = fullList.concat(
-      list.Contents?.map(el => ({ ...el, bucket })) || []
-    )
+    fullList = fullList.concat(await listBucketObjects(client, bucket))
   }
   if (errorCount === bucketList.length) {
     throw new Error("Unable to access MinIO/S3 - check environment config.")
@@ -46,16 +68,12 @@ export async function exportObjects() {
     const filename = object.Key
     const data = await retrieve(object.bucket, filename)
     const possiblePath = filename.split("/")
-    if (possiblePath.length > 1) {
-      const dirs = possiblePath.slice(0, possiblePath.length - 1)
-      fs.mkdirSync(join(path, object.bucket, ...dirs), { recursive: true })
-    }
+    const destination = join(path, object.bucket, ...possiblePath)
+    fs.mkdirSync(dirname(destination), { recursive: true })
     if (data instanceof stream.Readable) {
-      data.pipe(
-        fs.createWriteStream(join(path, object.bucket, ...possiblePath))
-      )
+      await pipeline(data, fs.createWriteStream(destination))
     } else {
-      fs.writeFileSync(join(path, object.bucket, ...possiblePath), data)
+      fs.writeFileSync(destination, data)
     }
     bar.update(++count)
   }