@uniformdev/cli 19.81.1 → 19.82.1-alpha.5

package/dist/index.mjs

@@ -669,30 +669,62 @@ var AssetListModule = {
 
 // src/commands/canvas/commands/asset/pull.ts
 import { UncachedAssetClient as UncachedAssetClient3 } from "@uniformdev/assets";
+import { UncachedFileClient } from "@uniformdev/files";
 
 // src/files/index.ts
 import { preferredType } from "@thi.ng/mime";
 import { FILE_READY_STATE, getFileNameFromUrl } from "@uniformdev/files";
-import { createHash } from "crypto";
 import fsj from "fs-jetpack";
 import sizeOf from "image-size";
 import PQueue from "p-queue";
 import { dirname as dirname2, join as join2 } from "path";
 var FILES_DIRECTORY_NAME = "files";
+var escapeRegExp = (string) => {
+  return string.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+};
 var urlToHash = (url) => {
-  const hash = createHash("sha256");
-  hash.update(url);
-  return hash.digest("hex");
+  return Buffer.from(
+    // We take only the first 64 characters of the pathname as
+    // that's enough to guarantee uniqueness
+    new URL(url).pathname.substring(0, 64)
+  ).toString("base64");
+};
+var hashToPartialPathname = (hash) => {
+  try {
+    return Buffer.from(hash, "base64").toString("utf8");
+  } catch {
+    return null;
+  }
+};
+var findUrlMatchingPartialPathname = (source, pathname) => {
+  const escapedPathname = escapeRegExp(pathname);
+  const regex = new RegExp(`"(https://([^"]*?)?img.uniform.(rocks|global)${escapedPathname}([^"]*?))"`);
+  const match = source.match(regex);
+  if (match && match[1]) {
+    return match[1];
+  }
+  return null;
 };
-var urlToFileName = (url) => {
-  const fileName = urlToHash(url);
+var urlToFileName = (url, hash) => {
+  const fileName = hash ?? urlToHash(url);
   const fileNameChunks = url.split(".");
   const fileExtension = fileNameChunks.length > 1 ? fileNameChunks.at(-1) : "";
   return `${fileName}${fileExtension ? `.${fileExtension}` : ""}`;
 };
+var getFilesDirectory = (directory) => {
+  const isPackage = isPathAPackageFile(directory);
+  return isPackage ? dirname2(directory) : (
+    // If we are syncing to a directory, we want to write all files into a
+    // top-lvl folder. That way any entities that contain files will sync to the
+    // same directory, so there is no duplication
+    join2(directory, "..")
+  );
+};
+var getUniformFileUrlMatches = (string) => {
+  return string.matchAll(/"(https:\/\/(.*?)?img\.uniform\.(rocks|global)\/(.*?))"/g);
+};
 var deleteDownloadedFileByUrl = async (url, options) => {
-  const isPackage = isPathAPackageFile(options.directory);
-  const writeDirectory = isPackage ? dirname2(options.directory) : options.directory;
+  const writeDirectory = getFilesDirectory(options.directory);
   const fileName = urlToFileName(url);
   const fileToDelete = join2(writeDirectory, FILES_DIRECTORY_NAME, fileName);
   try {
@@ -703,18 +735,14 @@ var deleteDownloadedFileByUrl = async (url, options) => {
 };
 var extractAndDownloadUniformFilesForObject = async (object, options) => {
   const objectAsString = JSON.stringify(object);
-  const uniformFileUrlMatches = objectAsString.matchAll(
-    /"(https:\/\/(.*)?img\.uniform\.(rocks|global)\/(.*?))"/g
-  );
-  const isPackage = isPathAPackageFile(options.directory);
-  const writeDirectory = isPackage ? dirname2(options.directory) : options.directory;
+  const uniformFileUrlMatches = getUniformFileUrlMatches(objectAsString);
+  const writeDirectory = getFilesDirectory(options.directory);
   if (uniformFileUrlMatches) {
     const fileDownloadQueue = new PQueue({ concurrency: 10 });
     for (const match of uniformFileUrlMatches) {
       const url = new URL(match[1]);
       fileDownloadQueue.add(async () => {
         try {
-          const fetchUrl = `${url.origin}${url.pathname}?format=original`;
           const fileName = urlToFileName(url.toString());
           const fileAlreadyExists = await fsj.existsAsync(
             join2(writeDirectory, FILES_DIRECTORY_NAME, fileName)
@@ -722,6 +750,20 @@ var extractAndDownloadUniformFilesForObject = async (object, options) => {
           if (fileAlreadyExists) {
             return;
           }
+          const file = await options.fileClient.get({ url: url.toString() }).catch(() => null);
+          if (!file) {
+            console.warn(`Skipping file ${url} as it does not exist in the project anymore`);
+            return;
+          }
+          if (file.sourceId) {
+            const hashAlreadyExists = await fsj.findAsync(join2(writeDirectory, FILES_DIRECTORY_NAME), {
+              matching: [file.sourceId, `${file.sourceId}.*`]
+            });
+            if (hashAlreadyExists.length > 0) {
+              return;
+            }
+          }
+          const fetchUrl = `${url.origin}${url.pathname}?format=original`;
           const response = await fetch(fetchUrl);
           if (!response.ok) {
             return;
@@ -739,11 +781,8 @@ var extractAndDownloadUniformFilesForObject = async (object, options) => {
 };
 var extractAndUploadUniformFilesForObject = async (object, options) => {
   let objectAsString = JSON.stringify(object);
-  const uniformFileUrlMatches = objectAsString.matchAll(
-    /"(https:\/\/(.*)?img\.uniform\.(rocks|global)\/(.*?))"/g
-  );
-  const isPackage = isPathAPackageFile(options.directory);
-  const writeDirectory = isPackage ? dirname2(options.directory) : options.directory;
+  const uniformFileUrlMatches = getUniformFileUrlMatches(objectAsString);
+  const writeDirectory = getFilesDirectory(options.directory);
   if (uniformFileUrlMatches) {
     const fileUploadQueue = new PQueue({ concurrency: 3 });
     for (const match of uniformFileUrlMatches) {
@@ -831,9 +870,7 @@ var extractAndUploadUniformFilesForObject = async (object, options) => {
 };
 var swapOutUniformFileUrlsForTargetProject = async (object, options) => {
   let objectAsString = JSON.stringify(object);
-  const uniformFileUrlMatches = objectAsString.matchAll(
-    /"(https:\/\/(.*)?img\.uniform\.(rocks|global)\/(.*?))"/g
-  );
+  const uniformFileUrlMatches = getUniformFileUrlMatches(objectAsString);
   if (uniformFileUrlMatches) {
     const fileUrlReplacementQueue = new PQueue({ concurrency: 3 });
     for (const match of uniformFileUrlMatches) {
@@ -858,6 +895,45 @@ var swapOutUniformFileUrlsForTargetProject = async (object, options) => {
   }
   return JSON.parse(objectAsString);
 };
+var replaceRemoteUrlsWithLocalReferences = async (sourceObject, targetObject, options) => {
+  let sourceObjectAsString = JSON.stringify(sourceObject);
+  const targetObjectAsString = JSON.stringify(targetObject);
+  const uniformFileUrlMatches = getUniformFileUrlMatches(sourceObjectAsString);
+  const writeDirectory = getFilesDirectory(options.directory);
+  if (uniformFileUrlMatches) {
+    const fileUrlReplacementQueue = new PQueue({ concurrency: 3 });
+    for (const match of uniformFileUrlMatches) {
+      const url = match[1];
+      fileUrlReplacementQueue.add(async () => {
+        try {
+          const localFileName = urlToFileName(url);
+          const fileExistsLocally = await fsj.existsAsync(
+            join2(writeDirectory, FILES_DIRECTORY_NAME, localFileName)
+          );
+          if (fileExistsLocally) {
+            return;
+          }
+          const file = await options.fileClient.get({ url }).catch(() => null);
+          if (!file || !file.sourceId) {
+            return;
+          }
+          const originalPartialPath = hashToPartialPathname(file.sourceId);
+          if (!originalPartialPath) {
+            return;
+          }
+          const originalUrl = findUrlMatchingPartialPathname(targetObjectAsString, originalPartialPath);
+          if (!originalUrl) {
+            return;
+          }
+          sourceObjectAsString = sourceObjectAsString.replaceAll(`"${url}"`, `"${originalUrl}"`);
+        } catch {
+        }
+      });
+    }
+    await fileUrlReplacementQueue.onIdle();
+  }
+  return JSON.parse(sourceObjectAsString);
+};
 var updateAssetFileIdBasedOnUrl = async (asset, options) => {
   var _a, _b, _c;
   const fileUrl = (_b = (_a = asset.asset.fields) == null ? void 0 : _a.url) == null ? void 0 : _b.value;
@@ -1009,6 +1085,7 @@ var AssetPullModule = {
       fetch: fetch3,
       projectId
     });
+    const fileClient = new UncachedFileClient({ apiKey, apiHost, fetch: fetch3, projectId });
     const source = createAssetEngineDataSource({ client });
     let target;
     const isPackage = isPathAPackageFile(directory);
@@ -1050,14 +1127,27 @@ var AssetPullModule = {
       whatIf,
       allowEmptySource: true,
       log: createSyncEngineConsoleLogger({ diffMode }),
-      onBeforeCompareObjects: async (sourceObject) => {
+      onBeforeCompareObjects: async (sourceObject, targetObject) => {
+        var _a, _b;
         delete sourceObject.object.asset._author;
-        return sourceObject;
+        const sourceObjectWithPotentiallySwappedUrl = await replaceRemoteUrlsWithLocalReferences(
+          sourceObject,
+          targetObject,
+          {
+            directory,
+            fileClient
+          }
+        );
+        if (((_a = sourceObjectWithPotentiallySwappedUrl.object.asset.fields) == null ? void 0 : _a.url) && ((_b = targetObject.object.asset.fields) == null ? void 0 : _b.url) && sourceObjectWithPotentiallySwappedUrl.object.asset.fields.url.value === targetObject.object.asset.fields.url.value) {
+          targetObject.object.asset.fields.file = sourceObjectWithPotentiallySwappedUrl.object.asset.fields.file;
+        }
+        return sourceObjectWithPotentiallySwappedUrl;
       },
       onBeforeWriteObject: async (sourceObject) => {
         delete sourceObject.object.asset._author;
         return extractAndDownloadUniformFilesForObject(sourceObject, {
-          directory
+          directory,
+          fileClient
         });
       }
     });
@@ -1066,7 +1156,7 @@ var AssetPullModule = {
 
 // src/commands/canvas/commands/asset/push.ts
 import { UncachedAssetClient as UncachedAssetClient4 } from "@uniformdev/assets";
-import { UncachedFileClient } from "@uniformdev/files";
+import { UncachedFileClient as UncachedFileClient2 } from "@uniformdev/files";
 var AssetPushModule = {
   command: "push <directory>",
   describe: "Pushes all assets from files in a directory to Uniform",
@@ -1127,7 +1217,7 @@ var AssetPushModule = {
       });
     }
     const target = createAssetEngineDataSource({ client });
-    const fileClient = new UncachedFileClient({ apiKey, apiHost, fetch: fetch3, projectId });
+    const fileClient = new UncachedFileClient2({ apiKey, apiHost, fetch: fetch3, projectId });
     await syncEngine({
       source,
       target,
@@ -1150,10 +1240,15 @@ var AssetPushModule = {
         return sourceObjectWithNewFileUrls;
       },
       onBeforeWriteObject: async (sourceObject) => {
-        const sourceObjectWithNewFileUrls = await extractAndUploadUniformFilesForObject(sourceObject, {
-          directory,
-          fileClient
-        });
+        const sourceObjectWithNewFileUrls = await swapOutUniformFileUrlsForTargetProject(
+          await extractAndUploadUniformFilesForObject(sourceObject, {
+            directory,
+            fileClient
+          }),
+          {
+            fileClient
+          }
+        );
         sourceObjectWithNewFileUrls.object = await updateAssetFileIdBasedOnUrl(
           sourceObjectWithNewFileUrls.object,
           {
@@ -2089,6 +2184,7 @@ var CompositionPublishModule = {
 
 // src/commands/canvas/commands/composition/pull.ts
 import { UncachedCanvasClient as UncachedCanvasClient10 } from "@uniformdev/canvas";
+import { UncachedFileClient as UncachedFileClient3 } from "@uniformdev/files";
 var CompositionPullModule = {
   command: "pull <directory>",
   describe: "Pulls all compositions to local files in a directory",
@@ -2148,6 +2244,7 @@ var CompositionPullModule = {
   }) => {
     const fetch3 = nodeFetchProxy(proxy);
     const client = new UncachedCanvasClient10({ apiKey, apiHost, fetch: fetch3, projectId });
+    const fileClient = new UncachedFileClient3({ apiKey, apiHost, fetch: fetch3, projectId });
     const source = createComponentInstanceEngineDataSource({ client, state, onlyCompositions, onlyPatterns });
     const isPackage = isPathAPackageFile(directory);
     let target;
@@ -2177,9 +2274,16 @@ var CompositionPullModule = {
       whatIf,
       allowEmptySource: true,
       log: createSyncEngineConsoleLogger({ diffMode }),
+      onBeforeCompareObjects: async (sourceObject, targetObject) => {
+        return replaceRemoteUrlsWithLocalReferences(sourceObject, targetObject, {
+          directory,
+          fileClient
+        });
+      },
       onBeforeWriteObject: async (sourceObject) => {
         return extractAndDownloadUniformFilesForObject(sourceObject, {
-          directory
+          directory,
+          fileClient
         });
       }
     });
@@ -2188,7 +2292,7 @@ var CompositionPullModule = {
 
 // src/commands/canvas/commands/composition/push.ts
 import { UncachedCanvasClient as UncachedCanvasClient11 } from "@uniformdev/canvas";
-import { UncachedFileClient as UncachedFileClient2 } from "@uniformdev/files";
+import { UncachedFileClient as UncachedFileClient4 } from "@uniformdev/files";
 var CompositionPushModule = {
   command: "push <directory>",
   describe: "Pushes all compositions from files in a directory to Uniform Canvas",
@@ -2258,7 +2362,7 @@ var CompositionPushModule = {
       });
     }
     const target = createComponentInstanceEngineDataSource({ client, state, onlyCompositions, onlyPatterns });
-    const fileClient = new UncachedFileClient2({ apiKey, apiHost, fetch: fetch3, projectId });
+    const fileClient = new UncachedFileClient4({ apiKey, apiHost, fetch: fetch3, projectId });
     await syncEngine({
       source,
       target,
@@ -3061,6 +3165,7 @@ var EntryListModule = {
 
 // src/commands/canvas/commands/entry/pull.ts
 import { ContentClient as ContentClient10 } from "@uniformdev/canvas";
+import { UncachedFileClient as UncachedFileClient5 } from "@uniformdev/files";
 
 // src/commands/canvas/entryEngineDataSource.ts
 import { convertEntryToPutEntry } from "@uniformdev/canvas";
@@ -3160,6 +3265,7 @@ var EntryPullModule = {
       projectId,
       bypassCache: true
     });
+    const fileClient = new UncachedFileClient5({ apiKey, apiHost, fetch: fetch3, projectId });
     const source = createEntryEngineDataSource({ client, state });
     let target;
     const isPackage = isPathAPackageFile(directory);
@@ -3188,13 +3294,26 @@ var EntryPullModule = {
       mode,
       whatIf,
       allowEmptySource: true,
-      log: createSyncEngineConsoleLogger({ diffMode })
+      log: createSyncEngineConsoleLogger({ diffMode }),
+      onBeforeCompareObjects: async (sourceObject, targetObject) => {
+        return replaceRemoteUrlsWithLocalReferences(sourceObject, targetObject, {
+          directory,
+          fileClient
+        });
+      },
+      onBeforeWriteObject: async (sourceObject) => {
+        return extractAndDownloadUniformFilesForObject(sourceObject, {
+          directory,
+          fileClient
+        });
+      }
     });
   }
 };
 
 // src/commands/canvas/commands/entry/push.ts
 import { ContentClient as ContentClient11 } from "@uniformdev/canvas";
+import { UncachedFileClient as UncachedFileClient6 } from "@uniformdev/files";
 var EntryPushModule = {
   command: "push <directory>",
   describe: "Pushes all entries from files in a directory to Uniform",
@@ -3259,12 +3378,24 @@ var EntryPushModule = {
       });
     }
     const target = createEntryEngineDataSource({ client, state });
+    const fileClient = new UncachedFileClient6({ apiKey, apiHost, fetch: fetch3, projectId });
     await syncEngine({
       source,
       target,
       mode,
       whatIf,
-      log: createSyncEngineConsoleLogger({ diffMode })
+      log: createSyncEngineConsoleLogger({ diffMode }),
+      onBeforeCompareObjects: async (sourceObject) => {
+        return swapOutUniformFileUrlsForTargetProject(sourceObject, {
+          fileClient
+        });
+      },
+      onBeforeWriteObject: async (sourceObject) => {
+        return extractAndUploadUniformFilesForObject(sourceObject, {
+          directory,
+          fileClient
+        });
+      }
     });
   }
 };
@@ -5455,7 +5586,7 @@ import { PostHog } from "posthog-node";
 // package.json
 var package_default = {
   name: "@uniformdev/cli",
-  version: "19.81.1",
+  version: "19.82.0",
   description: "Uniform command line interface tool",
   license: "SEE LICENSE IN LICENSE.txt",
   main: "./cli.js",
@@ -5511,7 +5642,7 @@ var package_default = {
     "@types/inquirer": "9.0.3",
     "@types/js-yaml": "4.0.9",
     "@types/jsonwebtoken": "9.0.5",
-    "@types/lodash.isequalwith": "4.4.7",
+    "@types/lodash.isequalwith": "4.4.9",
     "@types/node": "18.17.18",
     "@types/yargs": "17.0.24"
   },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@uniformdev/cli",
-  "version": "19.81.1",
+  "version": "19.82.1-alpha.5+7af3d5cdf",
   "description": "Uniform command line interface tool",
   "license": "SEE LICENSE IN LICENSE.txt",
   "main": "./cli.js",
@@ -17,12 +17,12 @@
   },
   "dependencies": {
     "@thi.ng/mime": "^2.2.23",
-    "@uniformdev/assets": "19.81.1",
-    "@uniformdev/canvas": "19.81.1",
-    "@uniformdev/context": "19.81.1",
-    "@uniformdev/files": "19.81.1",
-    "@uniformdev/project-map": "19.81.1",
-    "@uniformdev/redirect": "19.81.1",
+    "@uniformdev/assets": "19.82.1-alpha.5+7af3d5cdf",
+    "@uniformdev/canvas": "19.82.1-alpha.5+7af3d5cdf",
+    "@uniformdev/context": "19.82.1-alpha.5+7af3d5cdf",
+    "@uniformdev/files": "19.82.1-alpha.5+7af3d5cdf",
+    "@uniformdev/project-map": "19.82.1-alpha.5+7af3d5cdf",
+    "@uniformdev/redirect": "19.82.1-alpha.5+7af3d5cdf",
     "call-bind": "^1.0.2",
     "colorette": "2.0.20",
     "cosmiconfig": "8.3.6",
@@ -56,7 +56,7 @@
     "@types/inquirer": "9.0.3",
     "@types/js-yaml": "4.0.9",
     "@types/jsonwebtoken": "9.0.5",
-    "@types/lodash.isequalwith": "4.4.7",
+    "@types/lodash.isequalwith": "4.4.9",
     "@types/node": "18.17.18",
     "@types/yargs": "17.0.24"
   },
@@ -69,5 +69,5 @@
   "publishConfig": {
     "access": "public"
   },
-  "gitHead": "e72e4f188122bfe8b9944515be927d1f3febf257"
+  "gitHead": "7af3d5cdfabfe24f8a0d55b430cb2c196d5934d8"
 }