@uniformdev/cli 20.1.0 → 20.2.0

package/dist/index.mjs

@@ -615,17 +615,22 @@ var AssetListModule = {
   }
 };
 
-// src/files/index.ts
-import { preferredType } from "@thi.ng/mime";
-import { FILE_READY_STATE, getFileNameFromUrl } from "@uniformdev/files";
-import { fileTypeFromBuffer } from "file-type";
+// src/files/deleteDownloadedFileByUrl.ts
 import fsj from "fs-jetpack";
-import sizeOf from "image-size";
-import PQueue from "p-queue";
-import { dirname, join as join2 } from "path";
+import { join as join3 } from "path";
+
+// src/files/urlToFileName.ts
+import { join as join2 } from "path";
+import { dirname } from "path";
 var FILES_DIRECTORY_NAME = "files";
-var escapeRegExp = (string) => {
-  return string.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+var getFilesDirectory = (directory) => {
+  const isPackage = isPathAPackageFile(directory);
+  return isPackage ? dirname(directory) : (
+    // If we are syncing to a directory, we want to write all files into a
+    // top-lvl folder. That way any entities that contain files will sync to the
+    // same directory, so there is no duplication
+    join2(directory, "..")
+  );
 };
 var urlToHash = (url) => {
   return Buffer.from(
@@ -634,24 +639,6 @@ var urlToHash = (url) => {
     new URL(url).pathname.substring(0, 64)
   ).toString("base64");
 };
-var hashToPartialPathname = (hash) => {
-  try {
-    return Buffer.from(hash, "base64").toString("utf8");
-  } catch {
-    return null;
-  }
-};
-var findUrlMatchingPartialPathname = (source, pathname) => {
-  const escapedPathname = escapeRegExp(pathname);
-  const regex = new RegExp(
-    `"(https://([^"]*?)?(img|files).uniform.(rocks|global)${escapedPathname}([^"]*?))"`
-  );
-  const match = source.match(regex);
-  if (match && match[1]) {
-    return match[1];
-  }
-  return null;
-};
 var urlToFileExtension = (url) => {
   try {
     const urlObject = new URL(url);
@@ -666,228 +653,397 @@ var urlToFileName = (url, hash) => {
   const fileExtension = urlToFileExtension(url);
   return `${fileName}${fileExtension ? `.${fileExtension}` : ""}`;
 };
-var getFilesDirectory = (directory) => {
-  const isPackage = isPathAPackageFile(directory);
-  return isPackage ? dirname(directory) : (
-    // If we are syncing to a directory, we want to write all files into a
-    // top-lvl folder. That way any entities that contain files will sync to the
-    // same directory, so there is no duplication
-    join2(directory, "..")
-  );
-};
-var getUniformFileUrlMatches = (string) => {
-  return string.matchAll(/"(https:\/\/([^"]*?)?(img|files)\.uniform\.(rocks|global)\/([^"]*?))"/g);
+var hashToPartialPathname = (hash) => {
+  try {
+    return Buffer.from(hash, "base64").toString("utf8");
+  } catch {
+    return null;
+  }
 };
+
+// src/files/deleteDownloadedFileByUrl.ts
 var deleteDownloadedFileByUrl = async (url, options) => {
   const writeDirectory = getFilesDirectory(options.directory);
   const fileName = urlToFileName(url);
-  const fileToDelete = join2(writeDirectory, FILES_DIRECTORY_NAME, fileName);
+  const fileToDelete = join3(writeDirectory, FILES_DIRECTORY_NAME, fileName);
   try {
     await fsj.removeAsync(fileToDelete);
   } catch {
     console.warn(`Failed to delete a local file ${fileToDelete}`);
   }
 };
-var extractAndDownloadUniformFilesForObject = async (object, options) => {
-  const objectAsString = JSON.stringify(object);
-  const uniformFileUrlMatches = getUniformFileUrlMatches(objectAsString);
-  const writeDirectory = getFilesDirectory(options.directory);
-  if (uniformFileUrlMatches) {
-    const fileDownloadQueue = new PQueue({ concurrency: 10 });
-    for (const match of uniformFileUrlMatches) {
-      const url = new URL(match[1]);
-      fileDownloadQueue.add(async () => {
+
+// src/files/files.ts
+import {
+  ASSETS_SOURCE_UNIFORM,
+  getPropertiesValue as getPropertiesValue2,
+  isAssetParamValue,
+  isAssetParamValueItem,
+  walkNodeTree as walkNodeTree2,
+  walkPropertyValues as walkPropertyValues2
+} from "@uniformdev/canvas";
+import { isRichTextNodeType, isRichTextValue, walkRichTextTree } from "@uniformdev/richtext";
+import fsj4 from "fs-jetpack";
+import PQueue2 from "p-queue";
+import { join as join6 } from "path";
+
+// src/files/downloadFile.ts
+import fsj2 from "fs-jetpack";
+import { join as join4 } from "path";
+var downloadFile = async ({
+  fileClient,
+  fileUrl,
+  directory
+}) => {
+  const writeDirectory = getFilesDirectory(directory);
+  const fileName = urlToFileName(fileUrl.toString());
+  const fileAlreadyExists = await fsj2.existsAsync(join4(writeDirectory, FILES_DIRECTORY_NAME, fileName));
+  if (fileAlreadyExists) {
+    return { url: fileUrl };
+  }
+  const file = await fileClient.get({ url: fileUrl }).catch(() => null);
+  if (!file) {
+    console.warn(`Skipping file ${fileUrl} as it does not exist in the project anymore`);
+    return null;
+  }
+  if (file.sourceId) {
+    try {
+      const hashAlreadyExists = await fsj2.findAsync(join4(writeDirectory, FILES_DIRECTORY_NAME), {
+        matching: [file.sourceId, `${file.sourceId}.*`]
+      });
+      if (hashAlreadyExists.length > 0) {
+        return { id: file.id, url: fileUrl };
+      }
+    } catch {
+    }
+  }
+  const fetchUrl = `${fileUrl}?format=original`;
+  const response = await fetch(fetchUrl);
+  if (!response.ok) {
+    return null;
+  }
+  const fileBuffer = await response.arrayBuffer();
+  await fsj2.writeAsync(join4(writeDirectory, FILES_DIRECTORY_NAME, fileName), Buffer.from(fileBuffer));
+  return { id: file.id, url: fileUrl };
+};
+
+// src/files/uploadFile.ts
+import { preferredType } from "@thi.ng/mime";
+import { FILE_READY_STATE, getFileNameFromUrl } from "@uniformdev/files";
+import { fileTypeFromBuffer } from "file-type";
+import fsj3 from "fs-jetpack";
+import sizeOf from "image-size";
+import PQueue from "p-queue";
+import { join as join5 } from "path";
+var uploadQueueByKey = /* @__PURE__ */ new Map();
+var fileUploadQueue = new PQueue({ concurrency: 10 });
+var uploadFile = async ({
+  fileClient,
+  fileUrl,
+  directory,
+  fileId
+}) => {
+  const key = `${fileId}-${fileUrl}`;
+  if (uploadQueueByKey.has(key)) {
+    console.log("Already have this queued!!!");
+    const result2 = await uploadQueueByKey.get(key);
+    return result2 ?? null;
+  }
+  const promise = fileUploadQueue.add(async () => {
+    try {
+      const writeDirectory = getFilesDirectory(directory);
+      const hash = urlToHash(fileUrl);
+      const fileAlreadyExistsChecks = await Promise.all([
+        fileClient.get({ url: fileUrl }).catch(() => null),
+        fileClient.get({ sourceId: hash }).catch(() => null)
+      ]);
+      const file = fileAlreadyExistsChecks.find((check) => check !== null);
+      if (file?.url) {
+        return { id: file.id, url: file.url };
+      }
+      const localFileName = urlToFileName(fileUrl);
+      const expectedFilePath = join5(writeDirectory, FILES_DIRECTORY_NAME, localFileName);
+      const fileExistsLocally = await fsj3.existsAsync(expectedFilePath);
+      if (!fileExistsLocally) {
+        console.warn(
+          `Skipping file ${fileUrl} as we couldn't find a local copy (looked at ${expectedFilePath})`
+        );
+        return null;
+      }
+      const fileBuffer = await fsj3.readAsync(expectedFilePath, "buffer");
+      if (!fileBuffer) {
+        console.warn(`Skipping file ${fileUrl} (${expectedFilePath}) as we couldn't read it`);
+        return null;
+      }
+      const fileName = getFileNameFromUrl(fileUrl);
+      let mimeType = expectedFilePath.endsWith(".svg") ? "image/svg+xml" : (await fileTypeFromBuffer(fileBuffer))?.mime;
+      if (!mimeType) {
+        mimeType = preferredType(fileUrl.split(".").at(-1) ?? "");
+      }
+      if (mimeType === "audio/x-flac") {
+        mimeType = "audio/flac";
+      }
+      const { width, height } = (() => {
+        if (!mimeType.startsWith("image/")) {
+          return {
+            width: void 0,
+            height: void 0
+          };
+        }
         try {
-          const fileName = urlToFileName(url.toString());
-          const fileAlreadyExists = await fsj.existsAsync(
-            join2(writeDirectory, FILES_DIRECTORY_NAME, fileName)
-          );
-          if (fileAlreadyExists) {
-            return;
-          }
-          const file = await options.fileClient.get({ url: url.toString() }).catch(() => null);
-          if (!file) {
-            console.warn(`Skipping file ${url} as it does not exist in the project anymore`);
-            return;
-          }
-          if (file.sourceId) {
-            try {
-              const hashAlreadyExists = await fsj.findAsync(join2(writeDirectory, FILES_DIRECTORY_NAME), {
-                matching: [file.sourceId, `${file.sourceId}.*`]
-              });
-              if (hashAlreadyExists.length > 0) {
-                return;
-              }
-            } catch {
-            }
-          }
-          const fetchUrl = `${url.origin}${url.pathname}?format=original`;
-          const response = await fetch(fetchUrl);
-          if (!response.ok) {
-            return;
-          }
-          const fileBuffer = await response.arrayBuffer();
-          await fsj.writeAsync(join2(writeDirectory, FILES_DIRECTORY_NAME, fileName), Buffer.from(fileBuffer));
+          return sizeOf(fileBuffer);
         } catch {
-          console.warn(`Failed to download file ${url}`);
+          return {
+            width: void 0,
+            height: void 0
+          };
         }
+      })();
+      const { id, method, uploadUrl } = await fileClient.insert({
+        id: fileId,
+        name: fileName,
+        mediaType: mimeType,
+        size: fileBuffer.length,
+        width,
+        height,
+        sourceId: hash
       });
+      const uploadResponse = await fetch(uploadUrl, {
+        method,
+        body: fileBuffer,
+        headers: {
+          "Content-Type": mimeType,
+          "Content-Length": fileBuffer.length.toString()
+        }
+      });
+      if (!uploadResponse.ok) {
+        console.warn(`Failed to upload file ${fileUrl} (${expectedFilePath})`);
+        return null;
+      }
+      let error;
+      const checkForFile = async () => {
+        if (error) {
+          throw error;
+        }
+        const file2 = await fileClient.get({ id });
+        if (!file2 || file2.state !== FILE_READY_STATE || !file2.url) {
+          await new Promise((resolve) => setTimeout(resolve, 1e3));
+          return checkForFile();
+        }
+        return file2.url;
+      };
+      const abortTimeout = setTimeout(() => {
+        error = new Error(`Failed to upload file ${fileUrl} (${expectedFilePath}) - upload timed out`);
+      }, 6e4);
+      const uploadedFileUrl = await checkForFile();
+      clearTimeout(abortTimeout);
+      return { id, url: uploadedFileUrl };
+    } catch (e) {
+      console.warn(`Failed to upload file ${fileUrl}`, e);
+      return null;
+    }
+  });
+  uploadQueueByKey.set(key, promise);
+  const result = await promise;
+  return result ?? null;
+};
+
+// src/files/walkFileUrlsForCompositionOrEntry.ts
+import {
+  getPropertiesValue,
+  walkNodeTree,
+  walkPropertyValues
+} from "@uniformdev/canvas";
+var UNIFORM_FILE_MATCH = /"(https:\/\/([^"]*?)?(img|files)\.uniform\.(rocks|global)\/([^"]*?))"/g;
+var walkFileUrlsForCompositionOrEntry = ({
+  entity,
+  callback
+}) => {
+  const thumbnail = "entry" in entity ? entity.entry._thumbnail : void 0;
+  if (typeof thumbnail === "string") {
+    const isUniformFile = `"${thumbnail}"`.match(UNIFORM_FILE_MATCH) !== null;
+    if (isUniformFile) {
+      callback({ fileUrl: thumbnail });
     }
-    await fileDownloadQueue.onIdle();
   }
-  return object;
+  walkNodeTree("entry" in entity ? entity.entry : entity.composition, ({ node }) => {
+    const properties = getPropertiesValue(node);
+    if (!properties) {
+      return;
+    }
+    Object.entries(properties).forEach(([_, property]) => {
+      if (property.type !== "image") {
+        return;
+      }
+      walkPropertyValues(property, ({ value }) => {
+        if (typeof value !== "string") {
+          return;
+        }
+        const isUniformFile = `"${value}"`.match(UNIFORM_FILE_MATCH) !== null;
+        if (!isUniformFile) {
+          return;
+        }
+        callback({ fileUrl: value });
+      });
+    });
+  });
 };
-var extractAndUploadUniformFilesForObject = async (object, options) => {
-  let objectAsString = JSON.stringify(object);
-  const uniformFileUrlMatches = getUniformFileUrlMatches(objectAsString);
-  const writeDirectory = getFilesDirectory(options.directory);
-  if (uniformFileUrlMatches) {
-    const fileUploadQueue = new PQueue({ concurrency: 3 });
-    for (const match of uniformFileUrlMatches) {
-      const url = match[1];
-      const hash = urlToHash(url);
-      fileUploadQueue.add(async () => {
-        try {
-          const fileAlreadyExistsChecks = await Promise.all([
-            options.fileClient.get({ url }).catch(() => null),
-            options.fileClient.get({ sourceId: hash }).catch(() => null)
-          ]);
-          const file = fileAlreadyExistsChecks.find((check) => check !== null);
-          if (file) {
-            objectAsString = objectAsString.replaceAll(`"${url}"`, `"${file.url}"`);
-            return;
-          }
-          const localFileName = urlToFileName(url);
-          const expectedFilePath = join2(writeDirectory, FILES_DIRECTORY_NAME, localFileName);
-          const fileExistsLocally = await fsj.existsAsync(expectedFilePath);
-          if (!fileExistsLocally) {
-            console.warn(
-              `Skipping file ${url} as we couldn't find a local copy (looked at ${expectedFilePath})`
-            );
-            return;
-          }
-          const fileBuffer = await fsj.readAsync(expectedFilePath, "buffer");
-          if (!fileBuffer) {
-            console.warn(`Skipping file ${url} (${expectedFilePath}) as we couldn't read it`);
+
+// src/files/files.ts
+var downloadFileForAsset = async ({
+  asset,
+  directory,
+  fileClient
+}) => {
+  if (asset.asset.fields?.file?.value === void 0 || asset.asset.fields.url?.value === void 0) {
+    return null;
+  }
+  const fileId = asset.asset.fields?.file?.value;
+  const fileUrl = asset.asset.fields.url?.value;
+  if (fileId === "" || fileUrl === "") {
+    return null;
+  }
+  return downloadFile({ fileUrl, directory, fileClient });
+};
+var uploadFileForAsset = async ({
+  asset,
+  directory,
+  fileClient
+}) => {
+  if (asset.asset.fields?.file?.value === void 0 || asset.asset.fields.url?.value === void 0) {
+    return null;
+  }
+  const fileUrl = asset.asset.fields.url.value;
+  const fileId = asset.asset.fields.file.value;
+  return uploadFile({ fileUrl, directory, fileClient, fileId });
+};
+var removeUrlsFromAssetParameters = (entity) => {
+  walkNodeTree2("entry" in entity ? entity.entry : entity.composition, ({ node }) => {
+    const properties = getPropertiesValue2(node);
+    if (!properties) {
+      return;
+    }
+    Object.entries(properties).forEach(([_, property]) => {
+      if (property.type === "asset") {
+        walkPropertyValues2(property, ({ value }) => {
+          if (!isAssetParamValue(value)) {
             return;
           }
-          const fileName = getFileNameFromUrl(url);
-          let mimeType = expectedFilePath.endsWith(".svg") ? "image/svg+xml" : (await fileTypeFromBuffer(fileBuffer))?.mime;
-          if (!mimeType) {
-            mimeType = preferredType(url.split(".").at(-1) ?? "");
-          }
-          if (mimeType === "audio/x-flac") {
-            mimeType = "audio/flac";
-          }
-          const { width, height } = (() => {
-            if (!mimeType.startsWith("image/")) {
-              return {
-                width: void 0,
-                height: void 0
-              };
+          value.forEach((asset) => {
+            if (!isAssetParamValueItem(asset)) {
+              return;
             }
-            try {
-              return sizeOf(fileBuffer);
-            } catch {
-              return {
-                width: void 0,
-                height: void 0
-              };
+            if (asset._source !== ASSETS_SOURCE_UNIFORM || !asset.fields?.url.value) {
+              return;
             }
-          })();
-          const { id, method, uploadUrl } = await options.fileClient.insert({
-            name: fileName,
-            mediaType: mimeType,
-            size: fileBuffer.length,
-            width,
-            height,
-            sourceId: hash
+            asset.fields.url.value = "";
           });
-          const uploadResponse = await fetch(uploadUrl, {
-            method,
-            body: fileBuffer,
-            headers: {
-              "Content-Type": mimeType,
-              "Content-Length": fileBuffer.length.toString()
-            }
-          });
-          if (!uploadResponse.ok) {
-            console.warn(`Failed to upload file ${url} (${expectedFilePath})`);
+        });
+      } else if (property.type === "richText") {
+        walkPropertyValues2(property, ({ value }) => {
+          if (!isRichTextValue(value)) {
             return;
           }
-          let error;
-          const checkForFile = async () => {
-            if (error) {
-              throw error;
-            }
-            const file2 = await options.fileClient.get({ id });
-            if (!file2 || file2.state !== FILE_READY_STATE || !file2.url) {
-              await new Promise((resolve) => setTimeout(resolve, 1e3));
-              return checkForFile();
+          walkRichTextTree(value.root, (node2) => {
+            if (isRichTextNodeType(node2, "asset")) {
+              if (node2.__asset?._source !== ASSETS_SOURCE_UNIFORM || !node2.__asset.fields.url.value) {
+                return;
+              }
+              node2.__asset.fields.url.value = "";
             }
-            return file2.url;
-          };
-          const abortTimeout = setTimeout(() => {
-            error = new Error(`Failed to upload file ${url} (${expectedFilePath}) - upload timed out`);
-          }, 6e4);
-          const uploadedFileUrl = await checkForFile();
-          clearTimeout(abortTimeout);
-          objectAsString = objectAsString.replaceAll(`"${url}"`, `"${uploadedFileUrl}"`);
-        } catch (e) {
-          console.warn(`Failed to upload file ${url}`, e);
-        }
+          });
+        });
+      }
+    });
+  });
+  return entity;
+};
+var compareCompositionsOrEntriesWithoutAssetUrls = (source, target) => {
+  return serializedDequal(
+    removeUrlsFromAssetParameters(structuredClone(source.object)),
+    removeUrlsFromAssetParameters(structuredClone(target.object))
+  );
+};
+var removeUrlFromAsset = (asset) => {
+  if (asset.asset.fields?.url?.value) {
+    asset.asset.fields.url.value = "";
+  }
+  return asset;
+};
+var compareAssetsWithoutUrls = (source, target) => {
+  return serializedDequal(
+    removeUrlFromAsset(structuredClone(source.object)),
+    removeUrlFromAsset(structuredClone(target.object))
+  );
+};
+var downloadFilesForCompositionOrEntry = async ({
+  entity,
+  directory,
+  fileClient
+}) => {
+  const fileDownloadQueue = new PQueue2({ concurrency: 3 });
+  await walkFileUrlsForCompositionOrEntry({
+    entity,
+    callback: ({ fileUrl }) => {
+      fileDownloadQueue.add(async () => {
+        await downloadFile({ fileUrl, directory, fileClient });
       });
     }
-    await fileUploadQueue.onIdle();
-  }
-  return JSON.parse(objectAsString);
+  });
+  await fileDownloadQueue.onIdle();
 };
-var swapOutUniformFileUrlsForTargetProject = async (object, options) => {
-  let objectAsString = JSON.stringify(object);
-  const uniformFileUrlMatches = getUniformFileUrlMatches(objectAsString);
-  if (uniformFileUrlMatches) {
-    const fileUrlReplacementQueue = new PQueue({ concurrency: 3 });
-    for (const match of uniformFileUrlMatches) {
-      const url = match[1];
-      const hash = urlToHash(url);
-      fileUrlReplacementQueue.add(async () => {
-        try {
-          const fileAlreadyExistsChecks = await Promise.all([
-            options.fileClient.get({ url }).catch(() => null),
-            options.fileClient.get({ sourceId: hash }).catch(() => null)
-          ]);
-          const file = fileAlreadyExistsChecks.find((check) => check !== null);
-          if (!file) {
-            return;
-          }
-          objectAsString = objectAsString.replaceAll(`"${url}"`, `"${file.url}"`);
-        } catch {
+var uploadFilesForCompositionOrEntry = async ({
+  entity,
+  directory,
+  fileClient
+}) => {
+  const fileUploadQueue2 = new PQueue2({ concurrency: 3 });
+  const urlReplacementMap = /* @__PURE__ */ new Map();
+  walkFileUrlsForCompositionOrEntry({
+    entity: entity.object,
+    callback: async ({ fileUrl }) => {
+      fileUploadQueue2.add(async () => {
+        const upload = await uploadFile({
+          directory,
+          fileUrl,
+          fileClient
+        });
+        if (upload !== null) {
+          urlReplacementMap.set(fileUrl, upload.url);
         }
       });
     }
-    await fileUrlReplacementQueue.onIdle();
-  }
-  return JSON.parse(objectAsString);
-};
-var replaceRemoteUrlsWithLocalReferences = async (sourceObject, targetObject, options) => {
-  let sourceObjectAsString = JSON.stringify(sourceObject);
-  const targetObjectAsString = JSON.stringify(targetObject);
-  const uniformFileUrlMatches = getUniformFileUrlMatches(sourceObjectAsString);
-  const writeDirectory = getFilesDirectory(options.directory);
-  if (uniformFileUrlMatches) {
-    const fileUrlReplacementQueue = new PQueue({ concurrency: 3 });
-    for (const match of uniformFileUrlMatches) {
-      const url = match[1];
+  });
+  await fileUploadQueue2.onIdle();
+  let entityAsString = JSON.stringify(entity);
+  for (const [key, value] of urlReplacementMap.entries()) {
+    entityAsString = entityAsString.replaceAll(`"${key}"`, `"${value}"`);
+  }
+  return JSON.parse(entityAsString);
+};
+var replaceRemoteUrlsWithLocalReferences = async ({
+  sourceEntity,
+  targetEntity,
+  fileClient,
+  directory
+}) => {
+  let sourceEntityAsString = JSON.stringify(sourceEntity);
+  const targetEntityAsString = JSON.stringify(targetEntity);
+  const writeDirectory = getFilesDirectory(directory);
+  const fileUrlReplacementQueue = new PQueue2({ concurrency: 3 });
+  walkFileUrlsForCompositionOrEntry({
+    entity: sourceEntity.object,
+    callback: ({ fileUrl }) => {
       fileUrlReplacementQueue.add(async () => {
         try {
-          const localFileName = urlToFileName(url);
-          const fileExistsLocally = await fsj.existsAsync(
-            join2(writeDirectory, FILES_DIRECTORY_NAME, localFileName)
+          const localFileName = urlToFileName(fileUrl);
+          const fileExistsLocally = await fsj4.existsAsync(
+            join6(writeDirectory, FILES_DIRECTORY_NAME, localFileName)
           );
           if (fileExistsLocally) {
             return;
           }
-          const file = await options.fileClient.get({ url }).catch(() => null);
+          const file = await fileClient.get({ url: fileUrl }).catch(() => null);
           if (!file || !file.sourceId) {
             return;
           }
@@ -895,36 +1051,64 @@ var replaceRemoteUrlsWithLocalReferences = async (sourceObject, targetObject, op
           if (!originalPartialPath) {
             return;
           }
-          const originalUrl = findUrlMatchingPartialPathname(targetObjectAsString, originalPartialPath);
+          const originalUrl = findUrlMatchingPartialPathname(targetEntityAsString, originalPartialPath);
           if (!originalUrl) {
             return;
           }
-          sourceObjectAsString = sourceObjectAsString.replaceAll(`"${url}"`, `"${originalUrl}"`);
+          sourceEntityAsString = sourceEntityAsString.replaceAll(`"${fileUrl}"`, `"${originalUrl}"`);
         } catch {
         }
       });
+      return null;
     }
-    await fileUrlReplacementQueue.onIdle();
-  }
-  return JSON.parse(sourceObjectAsString);
+  });
+  await fileUrlReplacementQueue.onIdle();
+  return JSON.parse(sourceEntityAsString);
 };
-var updateAssetFileIdBasedOnUrl = async (asset, options) => {
-  if (!asset.asset.fields) {
-    return asset;
-  }
-  const fileUrl = asset.asset.fields.url?.value;
-  if (!fileUrl) {
-    return asset;
-  }
-  const file = await options.fileClient.get({ url: fileUrl }).catch(() => null);
-  if (!file) {
-    return asset;
+var replaceLocalUrlsWithRemoteReferences = async ({
+  entity,
+  fileClient
+}) => {
+  let entityAsString = JSON.stringify(entity);
+  const fileUrlReplacementQueue = new PQueue2({ concurrency: 3 });
+  walkFileUrlsForCompositionOrEntry({
+    entity: entity.object,
+    callback: ({ fileUrl }) => {
+      fileUrlReplacementQueue.add(async () => {
+        try {
+          const hash = urlToHash(fileUrl);
+          fileUrlReplacementQueue.add(async () => {
+            try {
+              const file = await fileClient.get({ sourceId: hash }).catch(() => null);
+              if (!file) {
+                return;
+              }
+              entityAsString = entityAsString.replaceAll(`"${fileUrl}"`, `"${file.url}"`);
+            } catch {
+            }
+          });
+        } catch {
+        }
+      });
+      return null;
+    }
+  });
+  await fileUrlReplacementQueue.onIdle();
+  return JSON.parse(entityAsString);
+};
+var escapeRegExp = (string) => {
+  return string.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+};
+var findUrlMatchingPartialPathname = (source, pathname) => {
+  const escapedPathname = escapeRegExp(pathname);
+  const regex = new RegExp(
+    `"(https://([^"]*?)?(img|files).uniform.(rocks|global)${escapedPathname}([^"]*?))"`
+  );
+  const match = source.match(regex);
+  if (match && match[1]) {
+    return match[1];
   }
-  asset.asset.fields.file = {
-    type: "file",
-    value: file.id
-  };
-  return asset;
+  return null;
 };
 
 // src/commands/canvas/assetEngineDataSource.ts
@@ -1115,27 +1299,25 @@ var AssetPullModule = {
       whatIf,
       allowEmptySource: allowEmptySource ?? true,
       log: createSyncEngineConsoleLogger({ diffMode }),
-      onBeforeCompareObjects: async (sourceObject, targetObject) => {
+      onBeforeCompareObjects: async (sourceObject) => {
         delete sourceObject.object.asset._author;
-        const sourceObjectWithPotentiallySwappedUrl = await replaceRemoteUrlsWithLocalReferences(
-          sourceObject,
-          targetObject,
-          {
-            directory,
-            fileClient
-          }
-        );
-        if (sourceObjectWithPotentiallySwappedUrl.object.asset.fields?.url && targetObject.object.asset.fields?.url && sourceObjectWithPotentiallySwappedUrl.object.asset.fields.url.value === targetObject.object.asset.fields.url.value) {
-          targetObject.object.asset.fields.file = sourceObjectWithPotentiallySwappedUrl.object.asset.fields.file;
-        }
-        return sourceObjectWithPotentiallySwappedUrl;
+        return sourceObject;
       },
+      compareContents: compareAssetsWithoutUrls,
       onBeforeWriteObject: async (sourceObject) => {
         delete sourceObject.object.asset._author;
-        return extractAndDownloadUniformFilesForObject(sourceObject, {
+        if (!sourceObject.object.asset.fields?.file) {
+          return sourceObject;
+        }
+        const downloadedFile = await downloadFileForAsset({
+          asset: sourceObject.object,
           directory,
           fileClient
         });
+        if (downloadedFile?.id) {
+          sourceObject.object.asset.fields.file.value = downloadedFile.id;
+        }
+        return sourceObject;
       }
     });
   }
@@ -1215,29 +1397,29 @@ var AssetPushModule = {
         if (targetObject) {
           delete targetObject.object.asset._author;
         }
-        const sourceObjectWithNewFileUrls = await swapOutUniformFileUrlsForTargetProject(sourceObject, {
-          fileClient
-        });
-        sourceObjectWithNewFileUrls.object = await updateAssetFileIdBasedOnUrl(
-          sourceObjectWithNewFileUrls.object,
-          {
-            fileClient
-          }
-        );
-        return sourceObjectWithNewFileUrls;
+        return sourceObject;
       },
+      compareContents: compareAssetsWithoutUrls,
       onBeforeWriteObject: async (sourceObject) => {
-        const sourceObjectWithNewFileUrls = await extractAndUploadUniformFilesForObject(sourceObject, {
+        const uploadedFile = await uploadFileForAsset({
+          asset: sourceObject.object,
           directory,
           fileClient
         });
-        sourceObjectWithNewFileUrls.object = await updateAssetFileIdBasedOnUrl(
-          sourceObjectWithNewFileUrls.object,
-          {
-            fileClient
+        if (uploadedFile !== null) {
+          if (sourceObject.object.asset.fields === void 0) {
+            sourceObject.object.asset.fields = {};
           }
-        );
-        return sourceObjectWithNewFileUrls;
+          sourceObject.object.asset.fields.file = {
+            type: "file",
+            value: uploadedFile.id
+          };
+          sourceObject.object.asset.fields.url = {
+            type: "text",
+            value: uploadedFile.url
+          };
+        }
+        return sourceObject;
       }
     });
   }
@@ -2323,7 +2505,8 @@ var CompositionPublishModule = {
     onlyCompositions,
     onlyPatterns,
     patternType,
-    verbose
+    verbose,
+    directory
   }) => {
     if (!all && !ids || all && ids) {
       console.error(`Specify --all or composition ID(s) to publish.`);
@@ -2350,6 +2533,7 @@ var CompositionPublishModule = {
       patternType,
       verbose
     });
+    const fileClient = getFileClient({ apiKey, apiHost, fetch: fetch2, projectId });
     await syncEngine({
       source,
       target,
@@ -2357,7 +2541,21 @@ var CompositionPublishModule = {
       mode: "createOrUpdate",
       whatIf,
       verbose,
-      log: createPublishStatusSyncEngineConsoleLogger({ status: "publish" })
+      log: createPublishStatusSyncEngineConsoleLogger({ status: "publish" }),
+      onBeforeCompareObjects: async (sourceObject) => {
+        return replaceLocalUrlsWithRemoteReferences({
+          entity: sourceObject,
+          fileClient
+        });
+      },
+      compareContents: compareCompositionsOrEntriesWithoutAssetUrls,
+      onBeforeWriteObject: async (sourceObject) => {
+        return uploadFilesForCompositionOrEntry({
+          entity: sourceObject,
+          directory,
+          fileClient
+        });
+      }
     });
   }
 };
@@ -2505,16 +2703,21 @@ var CompositionPullModule = {
       allowEmptySource: allowEmptySource ?? true,
       log: createSyncEngineConsoleLogger({ diffMode }),
       onBeforeCompareObjects: async (sourceObject, targetObject) => {
-        return replaceRemoteUrlsWithLocalReferences(sourceObject, targetObject, {
+        return replaceRemoteUrlsWithLocalReferences({
+          sourceEntity: sourceObject,
+          targetEntity: targetObject,
           directory,
           fileClient
         });
       },
+      compareContents: compareCompositionsOrEntriesWithoutAssetUrls,
       onBeforeWriteObject: async (sourceObject) => {
-        return extractAndDownloadUniformFilesForObject(sourceObject, {
+        await downloadFilesForCompositionOrEntry({
+          entity: sourceObject.object,
           directory,
           fileClient
         });
+        return sourceObject;
       }
     });
   }
@@ -2658,12 +2861,15 @@ var CompositionPushModule = {
       allowEmptySource,
       log: createSyncEngineConsoleLogger({ diffMode }),
       onBeforeCompareObjects: async (sourceObject) => {
-        return swapOutUniformFileUrlsForTargetProject(sourceObject, {
+        return replaceLocalUrlsWithRemoteReferences({
+          entity: sourceObject,
           fileClient
         });
       },
+      compareContents: compareCompositionsOrEntriesWithoutAssetUrls,
       onBeforeWriteObject: async (sourceObject) => {
-        return extractAndUploadUniformFilesForObject(sourceObject, {
+        return uploadFilesForCompositionOrEntry({
+          entity: sourceObject,
           directory,
           fileClient
         });
@@ -4140,7 +4346,18 @@ var EntryPublishModule = {
       )
     )
   ),
-  handler: async ({ apiHost, edgeApiHost, apiKey, proxy, ids, all, project: projectId, whatIf, verbose }) => {
+  handler: async ({
+    apiHost,
+    edgeApiHost,
+    apiKey,
+    proxy,
+    ids,
+    all,
+    project: projectId,
+    whatIf,
+    verbose,
+    directory
+  }) => {
     if (!all && !ids || all && ids) {
       console.error(`Specify --all or entry ID(s) to publish.`);
       process.exit(1);
@@ -4160,13 +4377,28 @@ var EntryPublishModule = {
       entryIDs: entryIDsArray,
       onlyEntries: true
     });
+    const fileClient = getFileClient({ apiKey, apiHost, fetch: fetch2, projectId });
     await syncEngine({
       source,
       target,
       // Publishing is one-direction operation, so no need to support automatic un-publishing
       mode: "createOrUpdate",
       whatIf,
-      log: createPublishStatusSyncEngineConsoleLogger({ status: "publish" })
+      log: createPublishStatusSyncEngineConsoleLogger({ status: "publish" }),
+      onBeforeCompareObjects: async (sourceObject) => {
+        return replaceLocalUrlsWithRemoteReferences({
+          entity: sourceObject,
+          fileClient
+        });
+      },
+      compareContents: compareCompositionsOrEntriesWithoutAssetUrls,
+      onBeforeWriteObject: async (sourceObject) => {
+        return uploadFilesForCompositionOrEntry({
+          entity: sourceObject,
+          directory,
+          fileClient
+        });
+      }
     });
   }
 };
@@ -4257,16 +4489,21 @@ var EntryPullModule = {
       allowEmptySource: allowEmptySource ?? true,
       log: createSyncEngineConsoleLogger({ diffMode }),
       onBeforeCompareObjects: async (sourceObject, targetObject) => {
-        return replaceRemoteUrlsWithLocalReferences(sourceObject, targetObject, {
+        return replaceRemoteUrlsWithLocalReferences({
+          sourceEntity: sourceObject,
+          targetEntity: targetObject,
           directory,
           fileClient
         });
       },
+      compareContents: compareCompositionsOrEntriesWithoutAssetUrls,
       onBeforeWriteObject: async (sourceObject) => {
-        return extractAndDownloadUniformFilesForObject(sourceObject, {
+        await downloadFilesForCompositionOrEntry({
+          entity: sourceObject.object,
           directory,
           fileClient
         });
+        return sourceObject;
       }
     });
   }
@@ -4346,12 +4583,15 @@ var EntryPushModule = {
       allowEmptySource,
       log: createSyncEngineConsoleLogger({ diffMode }),
       onBeforeCompareObjects: async (sourceObject) => {
-        return swapOutUniformFileUrlsForTargetProject(sourceObject, {
+        return replaceLocalUrlsWithRemoteReferences({
+          entity: sourceObject,
           fileClient
         });
       },
+      compareContents: compareCompositionsOrEntriesWithoutAssetUrls,
       onBeforeWriteObject: async (sourceObject) => {
-        return extractAndUploadUniformFilesForObject(sourceObject, {
+        return uploadFilesForCompositionOrEntry({
+          entity: sourceObject,
           directory,
           fileClient
         });
@@ -4610,7 +4850,18 @@ var EntryPatternPublishModule = {
       )
     )
   ),
-  handler: async ({ apiHost, edgeApiHost, apiKey, proxy, ids, all, whatIf, project: projectId, verbose }) => {
+  handler: async ({
+    apiHost,
+    edgeApiHost,
+    apiKey,
+    proxy,
+    ids,
+    all,
+    whatIf,
+    project: projectId,
+    verbose,
+    directory
+  }) => {
     if (!all && !ids || all && ids) {
       console.error(`Specify --all or entry pattern ID(s) to publish.`);
       process.exit(1);
@@ -4630,13 +4881,28 @@ var EntryPatternPublishModule = {
       entryIDs: entryIDsArray,
       onlyPatterns: true
     });
+    const fileClient = getFileClient({ apiKey, apiHost, fetch: fetch2, projectId });
     await syncEngine({
       source,
       target,
       // Publishing is one-direction operation, so no need to support automatic un-publishing
       mode: "createOrUpdate",
       whatIf,
-      log: createPublishStatusSyncEngineConsoleLogger({ status: "publish" })
+      log: createPublishStatusSyncEngineConsoleLogger({ status: "publish" }),
+      onBeforeCompareObjects: async (sourceObject) => {
+        return replaceLocalUrlsWithRemoteReferences({
+          entity: sourceObject,
+          fileClient
+        });
+      },
+      compareContents: compareCompositionsOrEntriesWithoutAssetUrls,
+      onBeforeWriteObject: async (sourceObject) => {
+        return uploadFilesForCompositionOrEntry({
+          entity: sourceObject,
+          directory,
+          fileClient
+        });
+      }
     });
   }
 };
@@ -4727,16 +4993,21 @@ var EntryPatternPullModule = {
       allowEmptySource: allowEmptySource ?? true,
       log: createSyncEngineConsoleLogger({ diffMode }),
       onBeforeCompareObjects: async (sourceObject, targetObject) => {
-        return replaceRemoteUrlsWithLocalReferences(sourceObject, targetObject, {
+        return replaceRemoteUrlsWithLocalReferences({
+          sourceEntity: sourceObject,
+          targetEntity: targetObject,
           directory,
           fileClient
         });
       },
+      compareContents: compareCompositionsOrEntriesWithoutAssetUrls,
       onBeforeWriteObject: async (sourceObject) => {
-        return extractAndDownloadUniformFilesForObject(sourceObject, {
+        await downloadFilesForCompositionOrEntry({
+          entity: sourceObject.object,
           directory,
           fileClient
         });
+        return sourceObject;
       }
     });
   }
@@ -4821,12 +5092,15 @@ var EntryPatternPushModule = {
       allowEmptySource,
       log: createSyncEngineConsoleLogger({ diffMode }),
       onBeforeCompareObjects: async (sourceObject) => {
-        return swapOutUniformFileUrlsForTargetProject(sourceObject, {
+        return replaceLocalUrlsWithRemoteReferences({
+          entity: sourceObject,
           fileClient
         });
       },
+      compareContents: compareCompositionsOrEntriesWithoutAssetUrls,
       onBeforeWriteObject: async (sourceObject) => {
-        return extractAndUploadUniformFilesForObject(sourceObject, {
+        return uploadFilesForCompositionOrEntry({
+          entity: sourceObject,
           directory,
           fileClient
         });
@@ -8182,7 +8456,7 @@ import { PostHog } from "posthog-node";
 // package.json
 var package_default = {
   name: "@uniformdev/cli",
-  version: "20.1.0",
+  version: "20.2.0",
   description: "Uniform command line interface tool",
   license: "SEE LICENSE IN LICENSE.txt",
   main: "./cli.js",
@@ -8215,6 +8489,7 @@ var package_default = {
     "@uniformdev/files": "workspace:*",
     "@uniformdev/project-map": "workspace:*",
     "@uniformdev/redirect": "workspace:*",
+    "@uniformdev/richtext": "workspace:*",
     "call-bind": "^1.0.2",
     colorette: "2.0.20",
     cosmiconfig: "9.0.0",
@@ -8222,11 +8497,11 @@ var package_default = {
     diff: "^5.0.0",
     dotenv: "^16.0.3",
     execa: "5.1.1",
-    "file-type": "^19.6.0",
+    "file-type": "^20.0.0",
     "fs-jetpack": "5.1.0",
     graphql: "16.9.0",
     "graphql-request": "6.1.0",
-    "image-size": "^1.0.2",
+    "image-size": "^1.2.0",
     inquirer: "9.2.17",
     "isomorphic-git": "1.25.2",
     "js-yaml": "^4.1.0",
@@ -8572,7 +8847,7 @@ ${err.message}`);
 // src/projects/cloneStarter.ts
 import crypto2 from "crypto";
 import fs3 from "fs";
-import fsj2 from "fs-jetpack";
+import fsj5 from "fs-jetpack";
 import * as git from "isomorphic-git";
 import * as http from "isomorphic-git/http/node/index.js";
 import os from "os";
@@ -8603,7 +8878,7 @@ async function cloneStarter({
     throw new Error(`"${targetDir}" is not empty`);
   }
   const starterDir = path.join(cloneDir, ...pathSegments);
-  fsj2.copy(starterDir, targetDir, { overwrite: true });
+  fsj5.copy(starterDir, targetDir, { overwrite: true });
   if (dotEnvFile) {
     fs3.writeFileSync(path.resolve(targetDir, ".env"), dotEnvFile, "utf-8");
   }
@@ -10448,7 +10723,8 @@ var SyncPushModule = {
             ...otherParams,
             patternType: "component",
             onlyPatterns: true,
-            all: true
+            all: true,
+            directory: getPushFilename("componentPattern", config2)
           }),
           {
             text: "publishing component patterns...",
@@ -10471,7 +10747,8 @@ var SyncPushModule = {
             ...otherParams,
             all: true,
             onlyPatterns: true,
-            patternType: "composition"
+            patternType: "composition",
+            directory: getPushFilename("compositionPattern", config2)
           }),
           {
             text: "publishing composition patterns...",
@@ -10493,7 +10770,8 @@ var SyncPushModule = {
           CompositionPublishModule.handler({
             ...otherParams,
             all: true,
-            onlyCompositions: true
+            onlyCompositions: true,
+            directory: getPushFilename("composition", config2)
           }),
           {
             text: "publishing compositions...",
@@ -10511,30 +10789,44 @@ var SyncPushModule = {
     }
     if (config2.entitiesConfig?.entry && config2.entitiesConfig?.entry?.push?.disabled !== true && config2.entitiesConfig?.entry?.publish) {
       try {
-        await spinPromise(EntryPublishModule.handler({ ...otherParams, all: true }), {
-          text: "publishing entries...",
-          successText: "published entries",
-          failText(error) {
-            return `publishing entries
+        await spinPromise(
+          EntryPublishModule.handler({
+            ...otherParams,
+            all: true,
+            directory: getPushFilename("entry", config2)
+          }),
+          {
+            text: "publishing entries...",
+            successText: "published entries",
+            failText(error) {
+              return `publishing entries
 
  ${error.stack ?? error.message}`;
+            }
           }
-        });
+        );
       } catch {
         process.exit(1);
       }
     }
     if (config2.entitiesConfig?.entryPattern && config2.entitiesConfig?.entryPattern?.push?.disabled !== true && config2.entitiesConfig?.entryPattern?.publish) {
       try {
-        await spinPromise(EntryPatternPublishModule.handler({ ...otherParams, all: true }), {
-          text: "publishing entry patterns...",
-          successText: "published entry patterns",
-          failText(error) {
-            return `publishing entry patterns
+        await spinPromise(
+          EntryPatternPublishModule.handler({
+            ...otherParams,
+            all: true,
+            directory: getPushFilename("entryPattern", config2)
+          }),
+          {
+            text: "publishing entry patterns...",
+            successText: "published entry patterns",
+            failText(error) {
+              return `publishing entry patterns
 
  ${error.stack ?? error.message}`;
+            }
           }
-        });
+        );
       } catch {
         process.exit(1);
       }
@@ -10583,14 +10875,14 @@ import { existsSync as existsSync4, promises as fs5 } from "fs";
 import { get as getHttp } from "http";
 import { get as getHttps } from "https";
 import { tmpdir } from "os";
-import { join as join3 } from "path";
+import { join as join7 } from "path";
 import registryUrl from "registry-url";
 import { URL as URL2 } from "url";
 var compareVersions = (a, b) => a.localeCompare(b, "en-US", { numeric: true });
 var encode = (value) => encodeURIComponent(value).replace(/^%40/, "@");
 var getFile = async (details, distTag) => {
   const rootDir = tmpdir();
-  const subDir = join3(rootDir, "update-check");
+  const subDir = join7(rootDir, "update-check");
   if (!existsSync4(subDir)) {
     await fs5.mkdir(subDir);
   }
@@ -10598,7 +10890,7 @@ var getFile = async (details, distTag) => {
   if (details.scope) {
     name = `${details.scope}-${name}`;
   }
-  return join3(subDir, name);
+  return join7(subDir, name);
 };
 var evaluateCache = async (file, time, interval) => {
   if (existsSync4(file)) {
@@ -10753,7 +11045,7 @@ var checkForUpdateMiddleware = async ({ verbose }) => {
 
 // src/middleware/checkLocalDepsVersionsMiddleware.ts
 import { magenta, red as red5 } from "colorette";
-import { join as join4 } from "path";
+import { join as join8 } from "path";
 
 // src/fs.ts
 import { promises as fs6 } from "fs";
@@ -10792,7 +11084,7 @@ var checkLocalDepsVersions = async (args) => {
   try {
     let isOutside = false;
     let warning = `${magenta("Warning:")} Installed Uniform packages should be the same version`;
-    const localPackages = await tryReadJSON(join4(process.cwd(), "package.json"));
+    const localPackages = await tryReadJSON(join8(process.cwd(), "package.json"));
     if (!localPackages) return;
     let firstVersion;
     const allDependencies = {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@uniformdev/cli",
-  "version": "20.1.0",
+  "version": "20.2.0",
   "description": "Uniform command line interface tool",
   "license": "SEE LICENSE IN LICENSE.txt",
   "main": "./cli.js",
@@ -27,12 +27,13 @@
   },
   "dependencies": {
     "@thi.ng/mime": "^2.2.23",
-    "@uniformdev/assets": "20.1.0",
-    "@uniformdev/canvas": "20.1.0",
-    "@uniformdev/context": "20.1.0",
-    "@uniformdev/files": "20.1.0",
-    "@uniformdev/project-map": "20.1.0",
-    "@uniformdev/redirect": "20.1.0",
+    "@uniformdev/assets": "20.2.0",
+    "@uniformdev/canvas": "20.2.0",
+    "@uniformdev/context": "20.2.0",
+    "@uniformdev/files": "20.2.0",
+    "@uniformdev/project-map": "20.2.0",
+    "@uniformdev/redirect": "20.2.0",
+    "@uniformdev/richtext": "20.2.0",
     "call-bind": "^1.0.2",
     "colorette": "2.0.20",
     "cosmiconfig": "9.0.0",
@@ -40,11 +41,11 @@
     "diff": "^5.0.0",
     "dotenv": "^16.0.3",
     "execa": "5.1.1",
-    "file-type": "^19.6.0",
+    "file-type": "^20.0.0",
     "fs-jetpack": "5.1.0",
     "graphql": "16.9.0",
     "graphql-request": "6.1.0",
-    "image-size": "^1.0.2",
+    "image-size": "^1.2.0",
     "inquirer": "9.2.17",
     "isomorphic-git": "1.25.2",
     "js-yaml": "^4.1.0",
@@ -78,5 +79,5 @@
   "publishConfig": {
     "access": "public"
   },
-  "gitHead": "343f488ce579a33e39d95c88bc7184763404ab07"
+  "gitHead": "7ca16ddb9f03fffe82a0252d53790fcaa12ebd3c"
 }