@uniformdev/cli 19.86.0 → 19.86.1-alpha.11

package/dist/index.mjs

@@ -669,30 +669,62 @@ var AssetListModule = {
 
 // src/commands/canvas/commands/asset/pull.ts
 import { UncachedAssetClient as UncachedAssetClient3 } from "@uniformdev/assets";
+import { UncachedFileClient } from "@uniformdev/files";
 
 // src/files/index.ts
 import { preferredType } from "@thi.ng/mime";
 import { FILE_READY_STATE, getFileNameFromUrl } from "@uniformdev/files";
-import { createHash } from "crypto";
 import fsj from "fs-jetpack";
 import sizeOf from "image-size";
 import PQueue from "p-queue";
 import { dirname as dirname2, join as join2 } from "path";
 var FILES_DIRECTORY_NAME = "files";
+var escapeRegExp = (string) => {
+  return string.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+};
 var urlToHash = (url) => {
-  const hash = createHash("sha256");
-  hash.update(url);
-  return hash.digest("hex");
+  return Buffer.from(
+    // We take only the first 64 characters of the pathname as
+    // that's enough to guarantee uniqueness
+    new URL(url).pathname.substring(0, 64)
+  ).toString("base64");
+};
+var hashToPartialPathname = (hash) => {
+  try {
+    return Buffer.from(hash, "base64").toString("utf8");
+  } catch {
+    return null;
+  }
+};
+var findUrlMatchingPartialPathname = (source, pathname) => {
+  const escapedPathname = escapeRegExp(pathname);
+  const regex = new RegExp(`"(https://([^"]*?)?img.uniform.(rocks|global)${escapedPathname}([^"]*?))"`);
+  const match = source.match(regex);
+  if (match && match[1]) {
+    return match[1];
+  }
+  return null;
 };
-var urlToFileName = (url) => {
-  const fileName = urlToHash(url);
+var urlToFileName = (url, hash) => {
+  const fileName = hash ?? urlToHash(url);
   const fileNameChunks = url.split(".");
   const fileExtension = fileNameChunks.length > 1 ? fileNameChunks.at(-1) : "";
   return `${fileName}${fileExtension ? `.${fileExtension}` : ""}`;
 };
+var getFilesDirectory = (directory) => {
+  const isPackage = isPathAPackageFile(directory);
+  return isPackage ? dirname2(directory) : (
+    // If we are syncing to a directory, we want to write all files into a
+    // top-lvl folder. That way any entities that contain files will sync to the
+    // same directory, so there is no duplication
+    join2(directory, "..")
+  );
+};
+var getUniformFileUrlMatches = (string) => {
+  return string.matchAll(/"(https:\/\/([^"]*?)?img\.uniform\.(rocks|global)\/([^"]*?))"/g);
+};
 var deleteDownloadedFileByUrl = async (url, options) => {
-  const isPackage = isPathAPackageFile(options.directory);
-  const writeDirectory = isPackage ? dirname2(options.directory) : options.directory;
+  const writeDirectory = getFilesDirectory(options.directory);
   const fileName = urlToFileName(url);
   const fileToDelete = join2(writeDirectory, FILES_DIRECTORY_NAME, fileName);
   try {
@@ -703,18 +735,14 @@ var deleteDownloadedFileByUrl = async (url, options) => {
 };
 var extractAndDownloadUniformFilesForObject = async (object, options) => {
   const objectAsString = JSON.stringify(object);
-  const uniformFileUrlMatches = objectAsString.matchAll(
-    /"(https:\/\/(.*)?img\.uniform\.(rocks|global)\/(.*?))"/g
-  );
-  const isPackage = isPathAPackageFile(options.directory);
-  const writeDirectory = isPackage ? dirname2(options.directory) : options.directory;
+  const uniformFileUrlMatches = getUniformFileUrlMatches(objectAsString);
+  const writeDirectory = getFilesDirectory(options.directory);
   if (uniformFileUrlMatches) {
     const fileDownloadQueue = new PQueue({ concurrency: 10 });
     for (const match of uniformFileUrlMatches) {
       const url = new URL(match[1]);
       fileDownloadQueue.add(async () => {
         try {
-          const fetchUrl = `${url.origin}${url.pathname}?format=original`;
           const fileName = urlToFileName(url.toString());
           const fileAlreadyExists = await fsj.existsAsync(
             join2(writeDirectory, FILES_DIRECTORY_NAME, fileName)
@@ -722,6 +750,23 @@ var extractAndDownloadUniformFilesForObject = async (object, options) => {
           if (fileAlreadyExists) {
             return;
           }
+          const file = await options.fileClient.get({ url: url.toString() }).catch(() => null);
+          if (!file) {
+            console.warn(`Skipping file ${url} as it does not exist in the project anymore`);
+            return;
+          }
+          if (file.sourceId) {
+            try {
+              const hashAlreadyExists = await fsj.findAsync(join2(writeDirectory, FILES_DIRECTORY_NAME), {
+                matching: [file.sourceId, `${file.sourceId}.*`]
+              });
+              if (hashAlreadyExists.length > 0) {
+                return;
+              }
+            } catch {
+            }
+          }
+          const fetchUrl = `${url.origin}${url.pathname}?format=original`;
           const response = await fetch(fetchUrl);
           if (!response.ok) {
             return;
@@ -739,11 +784,8 @@ var extractAndDownloadUniformFilesForObject = async (object, options) => {
 };
 var extractAndUploadUniformFilesForObject = async (object, options) => {
   let objectAsString = JSON.stringify(object);
-  const uniformFileUrlMatches = objectAsString.matchAll(
-    /"(https:\/\/(.*)?img\.uniform\.(rocks|global)\/(.*?))"/g
-  );
-  const isPackage = isPathAPackageFile(options.directory);
-  const writeDirectory = isPackage ? dirname2(options.directory) : options.directory;
+  const uniformFileUrlMatches = getUniformFileUrlMatches(objectAsString);
+  const writeDirectory = getFilesDirectory(options.directory);
   if (uniformFileUrlMatches) {
     const fileUploadQueue = new PQueue({ concurrency: 3 });
     for (const match of uniformFileUrlMatches) {
@@ -815,8 +857,8 @@ var extractAndUploadUniformFilesForObject = async (object, options) => {
             return file.url;
           };
           const abortTimeout = setTimeout(() => {
-            throw new Error(`Failed to upload file ${url}`);
-          }, 1e4);
+            throw new Error(`Failed to upload file ${url} (upload timed out)`);
+          }, 3e4);
           const uploadedFileUrl = await checkForFile();
           clearTimeout(abortTimeout);
           objectAsString = objectAsString.replaceAll(`"${url}"`, `"${uploadedFileUrl}"`);
@@ -831,9 +873,7 @@ var extractAndUploadUniformFilesForObject = async (object, options) => {
 };
 var swapOutUniformFileUrlsForTargetProject = async (object, options) => {
   let objectAsString = JSON.stringify(object);
-  const uniformFileUrlMatches = objectAsString.matchAll(
-    /"(https:\/\/(.*)?img\.uniform\.(rocks|global)\/(.*?))"/g
-  );
+  const uniformFileUrlMatches = getUniformFileUrlMatches(objectAsString);
   if (uniformFileUrlMatches) {
     const fileUrlReplacementQueue = new PQueue({ concurrency: 3 });
     for (const match of uniformFileUrlMatches) {
@@ -858,6 +898,45 @@ var swapOutUniformFileUrlsForTargetProject = async (object, options) => {
   }
   return JSON.parse(objectAsString);
 };
+var replaceRemoteUrlsWithLocalReferences = async (sourceObject, targetObject, options) => {
+  let sourceObjectAsString = JSON.stringify(sourceObject);
+  const targetObjectAsString = JSON.stringify(targetObject);
+  const uniformFileUrlMatches = getUniformFileUrlMatches(sourceObjectAsString);
+  const writeDirectory = getFilesDirectory(options.directory);
+  if (uniformFileUrlMatches) {
+    const fileUrlReplacementQueue = new PQueue({ concurrency: 3 });
+    for (const match of uniformFileUrlMatches) {
+      const url = match[1];
+      fileUrlReplacementQueue.add(async () => {
+        try {
+          const localFileName = urlToFileName(url);
+          const fileExistsLocally = await fsj.existsAsync(
+            join2(writeDirectory, FILES_DIRECTORY_NAME, localFileName)
+          );
+          if (fileExistsLocally) {
+            return;
+          }
+          const file = await options.fileClient.get({ url }).catch(() => null);
+          if (!file || !file.sourceId) {
+            return;
+          }
+          const originalPartialPath = hashToPartialPathname(file.sourceId);
+          if (!originalPartialPath) {
+            return;
+          }
+          const originalUrl = findUrlMatchingPartialPathname(targetObjectAsString, originalPartialPath);
+          if (!originalUrl) {
+            return;
+          }
+          sourceObjectAsString = sourceObjectAsString.replaceAll(`"${url}"`, `"${originalUrl}"`);
+        } catch {
+        }
+      });
+    }
+    await fileUrlReplacementQueue.onIdle();
+  }
+  return JSON.parse(sourceObjectAsString);
+};
 var updateAssetFileIdBasedOnUrl = async (asset, options) => {
   var _a, _b, _c;
   const fileUrl = (_b = (_a = asset.asset.fields) == null ? void 0 : _a.url) == null ? void 0 : _b.value;
@@ -1009,6 +1088,7 @@ var AssetPullModule = {
       fetch: fetch3,
       projectId
     });
+    const fileClient = new UncachedFileClient({ apiKey, apiHost, fetch: fetch3, projectId });
     const source = createAssetEngineDataSource({ client });
     let target;
     const isPackage = isPathAPackageFile(directory);
@@ -1050,14 +1130,27 @@ var AssetPullModule = {
       whatIf,
       allowEmptySource: true,
       log: createSyncEngineConsoleLogger({ diffMode }),
-      onBeforeCompareObjects: async (sourceObject) => {
+      onBeforeCompareObjects: async (sourceObject, targetObject) => {
+        var _a, _b;
         delete sourceObject.object.asset._author;
-        return sourceObject;
+        const sourceObjectWithPotentiallySwappedUrl = await replaceRemoteUrlsWithLocalReferences(
+          sourceObject,
+          targetObject,
+          {
+            directory,
+            fileClient
+          }
+        );
+        if (((_a = sourceObjectWithPotentiallySwappedUrl.object.asset.fields) == null ? void 0 : _a.url) && ((_b = targetObject.object.asset.fields) == null ? void 0 : _b.url) && sourceObjectWithPotentiallySwappedUrl.object.asset.fields.url.value === targetObject.object.asset.fields.url.value) {
+          targetObject.object.asset.fields.file = sourceObjectWithPotentiallySwappedUrl.object.asset.fields.file;
+        }
+        return sourceObjectWithPotentiallySwappedUrl;
       },
       onBeforeWriteObject: async (sourceObject) => {
         delete sourceObject.object.asset._author;
         return extractAndDownloadUniformFilesForObject(sourceObject, {
-          directory
+          directory,
+          fileClient
         });
       }
     });
@@ -1066,7 +1159,7 @@ var AssetPullModule = {
 
 // src/commands/canvas/commands/asset/push.ts
 import { UncachedAssetClient as UncachedAssetClient4 } from "@uniformdev/assets";
-import { UncachedFileClient } from "@uniformdev/files";
+import { UncachedFileClient as UncachedFileClient2 } from "@uniformdev/files";
 var AssetPushModule = {
   command: "push <directory>",
   describe: "Pushes all assets from files in a directory to Uniform",
@@ -1127,7 +1220,7 @@ var AssetPushModule = {
       });
     }
     const target = createAssetEngineDataSource({ client });
-    const fileClient = new UncachedFileClient({ apiKey, apiHost, fetch: fetch3, projectId });
+    const fileClient = new UncachedFileClient2({ apiKey, apiHost, fetch: fetch3, projectId });
     await syncEngine({
       source,
       target,
@@ -1150,10 +1243,15 @@ var AssetPushModule = {
         return sourceObjectWithNewFileUrls;
       },
       onBeforeWriteObject: async (sourceObject) => {
-        const sourceObjectWithNewFileUrls = await extractAndUploadUniformFilesForObject(sourceObject, {
-          directory,
-          fileClient
-        });
+        const sourceObjectWithNewFileUrls = await swapOutUniformFileUrlsForTargetProject(
+          await extractAndUploadUniformFilesForObject(sourceObject, {
+            directory,
+            fileClient
+          }),
+          {
+            fileClient
+          }
+        );
         sourceObjectWithNewFileUrls.object = await updateAssetFileIdBasedOnUrl(
           sourceObjectWithNewFileUrls.object,
           {
@@ -2089,6 +2187,7 @@ var CompositionPublishModule = {
 
 // src/commands/canvas/commands/composition/pull.ts
 import { UncachedCanvasClient as UncachedCanvasClient10 } from "@uniformdev/canvas";
+import { UncachedFileClient as UncachedFileClient3 } from "@uniformdev/files";
 var CompositionPullModule = {
   command: "pull <directory>",
   describe: "Pulls all compositions to local files in a directory",
@@ -2148,6 +2247,7 @@ var CompositionPullModule = {
   }) => {
     const fetch3 = nodeFetchProxy(proxy);
     const client = new UncachedCanvasClient10({ apiKey, apiHost, fetch: fetch3, projectId });
+    const fileClient = new UncachedFileClient3({ apiKey, apiHost, fetch: fetch3, projectId });
     const source = createComponentInstanceEngineDataSource({ client, state, onlyCompositions, onlyPatterns });
     const isPackage = isPathAPackageFile(directory);
     let target;
@@ -2177,9 +2277,16 @@ var CompositionPullModule = {
       whatIf,
       allowEmptySource: true,
       log: createSyncEngineConsoleLogger({ diffMode }),
+      onBeforeCompareObjects: async (sourceObject, targetObject) => {
+        return replaceRemoteUrlsWithLocalReferences(sourceObject, targetObject, {
+          directory,
+          fileClient
+        });
+      },
       onBeforeWriteObject: async (sourceObject) => {
         return extractAndDownloadUniformFilesForObject(sourceObject, {
-          directory
+          directory,
+          fileClient
         });
       }
     });
@@ -2188,7 +2295,7 @@ var CompositionPullModule = {
 
 // src/commands/canvas/commands/composition/push.ts
 import { UncachedCanvasClient as UncachedCanvasClient11 } from "@uniformdev/canvas";
-import { UncachedFileClient as UncachedFileClient2 } from "@uniformdev/files";
+import { UncachedFileClient as UncachedFileClient4 } from "@uniformdev/files";
 var CompositionPushModule = {
   command: "push <directory>",
   describe: "Pushes all compositions from files in a directory to Uniform Canvas",
@@ -2258,7 +2365,7 @@ var CompositionPushModule = {
       });
     }
     const target = createComponentInstanceEngineDataSource({ client, state, onlyCompositions, onlyPatterns });
-    const fileClient = new UncachedFileClient2({ apiKey, apiHost, fetch: fetch3, projectId });
+    const fileClient = new UncachedFileClient4({ apiKey, apiHost, fetch: fetch3, projectId });
     await syncEngine({
       source,
       target,
@@ -3061,6 +3168,7 @@ var EntryListModule = {
 
 // src/commands/canvas/commands/entry/pull.ts
 import { ContentClient as ContentClient10 } from "@uniformdev/canvas";
+import { UncachedFileClient as UncachedFileClient5 } from "@uniformdev/files";
 
 // src/commands/canvas/entryEngineDataSource.ts
 import { convertEntryToPutEntry } from "@uniformdev/canvas";
@@ -3160,6 +3268,7 @@ var EntryPullModule = {
       projectId,
       bypassCache: true
     });
+    const fileClient = new UncachedFileClient5({ apiKey, apiHost, fetch: fetch3, projectId });
     const source = createEntryEngineDataSource({ client, state });
     let target;
     const isPackage = isPathAPackageFile(directory);
@@ -3188,13 +3297,26 @@ var EntryPullModule = {
       mode,
       whatIf,
       allowEmptySource: true,
-      log: createSyncEngineConsoleLogger({ diffMode })
+      log: createSyncEngineConsoleLogger({ diffMode }),
+      onBeforeCompareObjects: async (sourceObject, targetObject) => {
+        return replaceRemoteUrlsWithLocalReferences(sourceObject, targetObject, {
+          directory,
+          fileClient
+        });
+      },
+      onBeforeWriteObject: async (sourceObject) => {
+        return extractAndDownloadUniformFilesForObject(sourceObject, {
+          directory,
+          fileClient
+        });
+      }
     });
   }
 };
 
 // src/commands/canvas/commands/entry/push.ts
 import { ContentClient as ContentClient11 } from "@uniformdev/canvas";
+import { UncachedFileClient as UncachedFileClient6 } from "@uniformdev/files";
 var EntryPushModule = {
   command: "push <directory>",
   describe: "Pushes all entries from files in a directory to Uniform",
@@ -3259,12 +3381,24 @@ var EntryPushModule = {
       });
     }
     const target = createEntryEngineDataSource({ client, state });
+    const fileClient = new UncachedFileClient6({ apiKey, apiHost, fetch: fetch3, projectId });
     await syncEngine({
       source,
       target,
       mode,
       whatIf,
-      log: createSyncEngineConsoleLogger({ diffMode })
+      log: createSyncEngineConsoleLogger({ diffMode }),
+      onBeforeCompareObjects: async (sourceObject) => {
+        return swapOutUniformFileUrlsForTargetProject(sourceObject, {
+          fileClient
+        });
+      },
+      onBeforeWriteObject: async (sourceObject) => {
+        return extractAndUploadUniformFilesForObject(sourceObject, {
+          directory,
+          fileClient
+        });
+      }
     });
   }
 };
@@ -7581,6 +7715,7 @@ var SyncPullModule = {
   handler: async ({ serialization, ...otherParams }) => {
     const config2 = serialization;
     const enabledEntities = Object.entries({
+      asset: AssetPullModule,
       category: CategoryPullModule,
       dataType: DataTypePullModule,
       prompt: PromptPullModule,
@@ -7596,8 +7731,7 @@ var SyncPullModule = {
       projectMapNode: ProjectMapNodePullModule,
       redirect: RedirectDefinitionPullModule,
       entry: EntryPullModule,
-      contentType: ContentTypePullModule,
-      asset: AssetPullModule
+      contentType: ContentTypePullModule
     }).filter(([entityType]) => {
       var _a, _b, _c, _d, _e, _f;
       return Boolean((_a = config2.entitiesConfig) == null ? void 0 : _a[entityType]) && ((_c = (_b = config2.entitiesConfig) == null ? void 0 : _b[entityType]) == null ? void 0 : _c.disabled) !== true && ((_f = (_e = (_d = config2.entitiesConfig) == null ? void 0 : _d[entityType]) == null ? void 0 : _e.pull) == null ? void 0 : _f.disabled) !== true;
@@ -7673,6 +7807,7 @@ var SyncPushModule = {
     var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l;
     const config2 = serialization;
     const enabledEntities = Object.entries({
+      asset: AssetPushModule,
       category: CategoryPushModule,
       dataType: DataTypePushModule,
       prompt: PromptPushModule,
@@ -7688,8 +7823,7 @@ var SyncPushModule = {
       projectMapNode: ProjectMapNodePushModule,
       redirect: RedirectDefinitionPushModule,
       contentType: ContentTypePushModule,
-      entry: EntryPushModule,
-      asset: AssetPushModule
+      entry: EntryPushModule
     }).filter(([entityType]) => {
       var _a2, _b2, _c2, _d2, _e2, _f2;
       return Boolean((_a2 = config2.entitiesConfig) == null ? void 0 : _a2[entityType]) && ((_c2 = (_b2 = config2.entitiesConfig) == null ? void 0 : _b2[entityType]) == null ? void 0 : _c2.disabled) !== true && ((_f2 = (_e2 = (_d2 = config2.entitiesConfig) == null ? void 0 : _d2[entityType]) == null ? void 0 : _e2.push) == null ? void 0 : _f2.disabled) !== true;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@uniformdev/cli",
-  "version": "19.86.0",
+  "version": "19.86.1-alpha.11+cb2b44bdd",
   "description": "Uniform command line interface tool",
   "license": "SEE LICENSE IN LICENSE.txt",
   "main": "./cli.js",
@@ -17,12 +17,12 @@
   },
   "dependencies": {
     "@thi.ng/mime": "^2.2.23",
-    "@uniformdev/assets": "19.86.0",
-    "@uniformdev/canvas": "19.86.0",
-    "@uniformdev/context": "19.86.0",
-    "@uniformdev/files": "19.86.0",
-    "@uniformdev/project-map": "19.86.0",
-    "@uniformdev/redirect": "19.86.0",
+    "@uniformdev/assets": "19.86.1-alpha.11+cb2b44bdd",
+    "@uniformdev/canvas": "19.86.1-alpha.11+cb2b44bdd",
+    "@uniformdev/context": "19.86.1-alpha.11+cb2b44bdd",
+    "@uniformdev/files": "19.86.1-alpha.11+cb2b44bdd",
+    "@uniformdev/project-map": "19.86.1-alpha.11+cb2b44bdd",
+    "@uniformdev/redirect": "19.86.1-alpha.11+cb2b44bdd",
     "call-bind": "^1.0.2",
     "colorette": "2.0.20",
     "cosmiconfig": "8.3.6",
@@ -69,5 +69,5 @@
   "publishConfig": {
     "access": "public"
   },
-  "gitHead": "33b5e38f48fe693bf6a4b195009222fd34b9e9f3"
+  "gitHead": "cb2b44bdd700124c26e17f1840dbb6049030af81"
 }