@uniformdev/cli 20.61.2-alpha.3 → 20.62.1-alpha.4

package/dist/{chunk-G6VN2UTK.mjs → chunk-RKOT64CV.mjs}

@@ -22,7 +22,7 @@ import { fetch as undiciFetch, ProxyAgent } from "undici";
 // package.json
 var package_default = {
   name: "@uniformdev/cli",
-  version: "20.61.1",
+  version: "20.62.0",
   description: "Uniform command line interface tool",
   license: "SEE LICENSE IN LICENSE.txt",
   main: "./cli.js",

package/dist/defaultConfig.mjs

@@ -1,4 +1,4 @@
-import "./chunk-G6VN2UTK.mjs";
+import "./chunk-RKOT64CV.mjs";
 
 // src/sync/allSerializableEntitiesConfig.ts
 var allSerializableEntitiesConfig = {

package/dist/index.mjs

@@ -18,7 +18,7 @@ import {
   withFormatOptions,
   withProjectOptions,
   withTeamOptions
-} from "./chunk-G6VN2UTK.mjs";
+} from "./chunk-RKOT64CV.mjs";
 
 // src/index.ts
 import * as dotenv from "dotenv";
@@ -721,6 +721,7 @@ function writeUniformPackage(filename, packageContents) {
 // src/sync/syncEngine.ts
 import { diffJson, diffLines } from "diff";
 import mitt from "mitt";
+import PQueue from "p-queue";
 
 // src/sync/serializedDequal.ts
 var has = Object.prototype.hasOwnProperty;
@@ -774,6 +775,7 @@ function serializedDequal(foo, bar) {
 }
 
 // src/sync/syncEngine.ts
+var DEFAULT_SYNC_ACTION_CONCURRENCY = 10;
 var syncEngineEvents = mitt();
 async function syncEngine({
   source,
@@ -787,7 +789,8 @@ async function syncEngine({
   onBeforeProcessObject,
   onBeforeCompareObjects,
   onBeforeWriteObject,
-  onError
+  onError,
+  actionConcurrency = DEFAULT_SYNC_ACTION_CONCURRENCY
   //verbose = false,
 }) {
   const status = new ReactiveStatusUpdate((status2) => syncEngineEvents.emit("statusUpdate", status2));
@@ -906,7 +909,7 @@ async function syncEngine({
           deletes.push(() => processDelete(targetObject));
         }
         actions.push(
-          () => Promise.all(deletes.map((deleteFn) => deleteFn())).then(() => processUpsert(sourceObject, ids[0]))
+          () => runSyncActions(deletes, actionConcurrency).then(() => processUpsert(sourceObject, ids[0]))
         );
       } else {
         actions.push(() => processUpsert(sourceObject, ids[0]));
@@ -925,11 +928,18 @@ async function syncEngine({
     targetItems.forEach((object4) => {
       deletes.push(() => processDelete(object4));
     });
-    await Promise.all(deletes.map((d) => d()));
+    await runSyncActions(deletes, actionConcurrency);
   }
-  await Promise.all(actions.map((a) => a()));
+  await runSyncActions(actions, actionConcurrency);
   await Promise.all([source.onSyncComplete?.(false), target.onSyncComplete?.(true)]);
 }
+var runSyncActions = async (actions, actionConcurrency) => {
+  if (actions.length === 0) {
+    return;
+  }
+  const queue = new PQueue({ concurrency: actionConcurrency });
+  await queue.addAll(actions);
+};
 var SyncEngineError = class _SyncEngineError extends Error {
   constructor(innerError, sourceObject) {
     super(
@@ -2197,20 +2207,59 @@ import {
 } from "@uniformdev/canvas";
 import { isRichTextNodeType, isRichTextValue, walkRichTextTree } from "@uniformdev/richtext";
 import fsj4 from "fs-jetpack";
-import PQueue2 from "p-queue";
+import PQueue3 from "p-queue";
 import { join as join10 } from "path";
 
 // src/files/downloadFile.ts
+import { createWriteStream } from "fs";
 import fsj2 from "fs-jetpack";
-import { join as join8 } from "path";
+import { dirname as dirname2, join as join8 } from "path";
+import { Readable } from "stream";
+import { pipeline } from "stream/promises";
+var downloadedFilePathCacheByDirectory = /* @__PURE__ */ new Map();
+var getDownloadedFilePathCache = (filesDirectory) => {
+  const cached = downloadedFilePathCacheByDirectory.get(filesDirectory);
+  if (cached) {
+    return cached;
+  }
+  const cache = fsj2.cwd(filesDirectory).findAsync({ files: true, directories: false }).then((paths) => new Set(paths)).catch(() => /* @__PURE__ */ new Set());
+  downloadedFilePathCacheByDirectory.set(filesDirectory, cache);
+  return cache;
+};
+var filePathMatchesSourceId = (filePath, sourceId) => filePath === sourceId || filePath.startsWith(`${sourceId}.`);
+var hasFilePathMatchingSourceId = (filePaths, sourceId) => {
+  for (const filePath of filePaths) {
+    if (filePathMatchesSourceId(filePath, sourceId)) {
+      return true;
+    }
+  }
+  return false;
+};
+var writeResponseBodyToFile = async (response, filePath) => {
+  if (!response.body) {
+    throw new Error("Response does not contain a body");
+  }
+  const tempFilePath = `${filePath}.${process.pid}.${Date.now()}.download`;
+  await fsj2.dirAsync(dirname2(filePath));
+  try {
+    const responseBody = response.body;
+    await pipeline(Readable.fromWeb(responseBody), createWriteStream(tempFilePath));
+    await fsj2.moveAsync(tempFilePath, filePath, { overwrite: true });
+  } catch (error) {
+    await fsj2.removeAsync(tempFilePath).catch(() => void 0);
+    throw error;
+  }
+};
 var downloadFile = async ({
   fileClient,
   fileUrl,
   directory
 }) => {
   const writeDirectory = getFilesDirectory(directory);
+  const filesDirectory = join8(writeDirectory, FILES_DIRECTORY_NAME);
   const fileName = urlToFileName(fileUrl.toString());
-  const fileAlreadyExists = await fsj2.existsAsync(join8(writeDirectory, FILES_DIRECTORY_NAME, fileName));
+  const filePath = join8(filesDirectory, fileName);
+  const fileAlreadyExists = await fsj2.existsAsync(filePath);
   if (fileAlreadyExists) {
     return { url: fileUrl };
   }
@@ -2220,11 +2269,10 @@ var downloadFile = async ({
     return null;
   }
   if (file.sourceId) {
+    const sourceId = file.sourceId;
     try {
-      const hashAlreadyExists = await fsj2.findAsync(join8(writeDirectory, FILES_DIRECTORY_NAME), {
-        matching: [file.sourceId, `${file.sourceId}.*`]
-      });
-      if (hashAlreadyExists.length > 0) {
+      const downloadedFilePaths = await getDownloadedFilePathCache(filesDirectory);
+      if (hasFilePathMatchingSourceId(downloadedFilePaths, sourceId)) {
         return { id: file.id, url: fileUrl };
       }
     } catch {
@@ -2235,8 +2283,12 @@ var downloadFile = async ({
   if (!response.ok) {
     return null;
   }
-  const fileBuffer = await response.arrayBuffer();
-  await fsj2.writeAsync(join8(writeDirectory, FILES_DIRECTORY_NAME, fileName), Buffer.from(fileBuffer));
+  await writeResponseBodyToFile(response, filePath);
+  const downloadedFilePathCache = downloadedFilePathCacheByDirectory.get(filesDirectory);
+  if (downloadedFilePathCache) {
+    const downloadedFilePaths = await downloadedFilePathCache;
+    downloadedFilePaths.add(fileName);
+  }
   return { id: file.id, url: fileUrl };
 };
 
@@ -2247,10 +2299,10 @@ import { fileTypeFromBuffer } from "file-type";
 import fsj3 from "fs-jetpack";
 import sizeOf from "image-size";
 import normalizeNewline from "normalize-newline";
-import PQueue from "p-queue";
+import PQueue2 from "p-queue";
 import { join as join9 } from "path";
 var uploadQueueByKey = /* @__PURE__ */ new Map();
-var fileUploadQueue = new PQueue({ concurrency: 10 });
+var fileUploadQueue = new PQueue2({ concurrency: 10 });
 var uploadFile = async ({
   fileClient,
   fileUrl,
@@ -2535,7 +2587,7 @@ var downloadFilesForCompositionOrEntry = async ({
   directory,
   fileClient
 }) => {
-  const fileDownloadQueue = new PQueue2({ concurrency: 10 });
+  const fileDownloadQueue = new PQueue3({ concurrency: 10 });
   await walkFileUrlsForCompositionOrEntry({
     entity,
     callback: ({ fileUrl }) => {
@@ -2551,7 +2603,7 @@ var uploadFilesForCompositionOrEntry = async ({
   directory,
   fileClient
 }) => {
-  const fileUploadQueue2 = new PQueue2({ concurrency: 10 });
+  const fileUploadQueue2 = new PQueue3({ concurrency: 10 });
   const urlReplacementMap = /* @__PURE__ */ new Map();
   walkFileUrlsForCompositionOrEntry({
     entity: entity.object,
@@ -2584,7 +2636,7 @@ var replaceRemoteUrlsWithLocalReferences = async ({
   let sourceEntityAsString = JSON.stringify(sourceEntity);
   const targetEntityAsString = JSON.stringify(targetEntity);
   const writeDirectory = getFilesDirectory(directory);
-  const fileUrlReplacementQueue = new PQueue2({ concurrency: 10 });
+  const fileUrlReplacementQueue = new PQueue3({ concurrency: 10 });
   walkFileUrlsForCompositionOrEntry({
     entity: sourceEntity.object,
     callback: ({ fileUrl }) => {
@@ -2624,7 +2676,7 @@ var replaceLocalUrlsWithRemoteReferences = async ({
   fileClient
 }) => {
   let entityAsString = JSON.stringify(entity);
-  const fileUrlReplacementQueue = new PQueue2({ concurrency: 10 });
+  const fileUrlReplacementQueue = new PQueue3({ concurrency: 10 });
   walkFileUrlsForCompositionOrEntry({
     entity: entity.object,
     callback: ({ fileUrl }) => {
@@ -12576,7 +12628,7 @@ import yargs40 from "yargs";
 
 // src/webhooksClient.ts
 import { ApiClient as ApiClient4 } from "@uniformdev/context/api";
-import PQueue3 from "p-queue";
+import PQueue4 from "p-queue";
 import { Svix } from "svix";
 import * as z3 from "zod/v3";
 var WEBHOOKS_DASHBOARD_BASE_PATH = "/api/v1/svix-dashboard";
@@ -12624,7 +12676,7 @@ var WebhooksClient = class extends ApiClient4 {
     };
   }
   async get() {
-    const webhooksAPIQueue = new PQueue3({ concurrency: 10 });
+    const webhooksAPIQueue = new PQueue4({ concurrency: 10 });
     const { appId, token } = await this.getToken();
     const svix = new Svix(token);
     const getEndpoints = async ({

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@uniformdev/cli",
-  "version": "20.61.2-alpha.3+fc997c8a8e",
+  "version": "20.62.1-alpha.4+295a1c3116",
   "description": "Uniform command line interface tool",
   "license": "SEE LICENSE IN LICENSE.txt",
   "main": "./cli.js",
@@ -27,13 +27,13 @@
   "dependencies": {
     "@inquirer/prompts": "^7.10.1",
     "@thi.ng/mime": "^2.2.23",
-    "@uniformdev/assets": "20.61.2-alpha.3+fc997c8a8e",
-    "@uniformdev/canvas": "20.61.2-alpha.3+fc997c8a8e",
-    "@uniformdev/context": "20.61.2-alpha.3+fc997c8a8e",
-    "@uniformdev/files": "20.61.2-alpha.3+fc997c8a8e",
-    "@uniformdev/project-map": "20.61.2-alpha.3+fc997c8a8e",
-    "@uniformdev/redirect": "20.61.2-alpha.3+fc997c8a8e",
-    "@uniformdev/richtext": "20.61.2-alpha.3+fc997c8a8e",
+    "@uniformdev/assets": "20.62.1-alpha.4+295a1c3116",
+    "@uniformdev/canvas": "20.62.1-alpha.4+295a1c3116",
+    "@uniformdev/context": "20.62.1-alpha.4+295a1c3116",
+    "@uniformdev/files": "20.62.1-alpha.4+295a1c3116",
+    "@uniformdev/project-map": "20.62.1-alpha.4+295a1c3116",
+    "@uniformdev/redirect": "20.62.1-alpha.4+295a1c3116",
+    "@uniformdev/richtext": "20.62.1-alpha.4+295a1c3116",
     "call-bind": "^1.0.2",
     "colorette": "2.0.20",
     "cosmiconfig": "9.0.0",
@@ -80,5 +80,5 @@
   "publishConfig": {
     "access": "public"
   },
-  "gitHead": "fc997c8a8e1939d5050cde71204a9212ccf1705f"
+  "gitHead": "295a1c31168e58eee9ca865d6c53d2cd7f6fe913"
 }