@uniformdev/cli 19.42.0 → 19.42.1-alpha.7

package/dist/index.mjs

@@ -411,7 +411,9 @@ async function syncEngine({
   whatIf = false,
   // eslint-disable-next-line @typescript-eslint/no-empty-function
   log = () => {
-  }
+  },
+  onBeforeCompareObjects,
+  onBeforeWriteObject
 }) {
   var _a, _b;
   const targetItems = /* @__PURE__ */ new Map();
@@ -445,18 +447,20 @@ async function syncEngine({
   }
   const actions = [];
   let sourceHasItems = false;
-  for await (const sourceObject of source.objects) {
+  for await (let sourceObject of source.objects) {
     sourceHasItems = true;
     const ids = Array.isArray(sourceObject.id) ? sourceObject.id : [sourceObject.id];
     const targetObject = targetItems.get(ids[0]);
     const invalidTargetObjects = ids.map((i) => targetItems.get(i)).filter((o) => (o == null ? void 0 : o.object) !== (targetObject == null ? void 0 : targetObject.object));
     if (targetObject && invalidTargetObjects.length == 0) {
+      sourceObject = onBeforeCompareObjects ? await onBeforeCompareObjects(sourceObject, targetObject) : sourceObject;
       if (!compareContents(sourceObject, targetObject)) {
         if (mode === "createOrUpdate" || mode === "mirror") {
           const process2 = async (sourceObject2, targetObject2) => {
             if (!whatIf) {
               try {
-                await target.writeObject(sourceObject2, targetObject2);
+                const finalSourceObject = onBeforeWriteObject ? await onBeforeWriteObject(sourceObject2, targetObject2) : sourceObject2;
+                await target.writeObject(finalSourceObject, targetObject2);
               } catch (e) {
                 throw new SyncEngineError(e, sourceObject2);
               }
@@ -478,7 +482,8 @@ async function syncEngine({
       const process2 = async (sourceObject2, id) => {
         if (!whatIf) {
           try {
-            await target.writeObject(sourceObject2);
+            const finalSourceObject = onBeforeWriteObject ? await onBeforeWriteObject(sourceObject2) : sourceObject2;
+            await target.writeObject(finalSourceObject);
           } catch (e) {
             throw new SyncEngineError(e, sourceObject2);
           }
@@ -1456,6 +1461,180 @@ var CompositionPublishModule = {
 
 // src/commands/canvas/commands/composition/pull.ts
 import { UncachedCanvasClient as UncachedCanvasClient10 } from "@uniformdev/canvas";
+
+// src/files/index.ts
+import { preferredType } from "@thi.ng/mime";
+import { FILE_READY_STATE, getFileNameFromUrl } from "@uniformdev/files-sdk";
+import { createHash } from "crypto";
+import fsj from "fs-jetpack";
+import sizeOf from "image-size";
+import PQueue from "p-queue";
+import { join as join2 } from "path";
+var FILES_DIRECTORY_NAME = "files";
+var urlToHash = (url) => {
+  const hash = createHash("sha256");
+  hash.update(url);
+  return hash.digest("hex");
+};
+var urlToFileName = (url) => {
+  const fileName = urlToHash(url);
+  const fileNameChunks = url.split(".");
+  const fileExtension = fileNameChunks.length > 1 ? fileNameChunks.at(-1) : "";
+  return `${fileName}${fileExtension ? `.${fileExtension}` : ""}`;
+};
+var extractAndDownloadUniformFilesForObject = async (object, options) => {
+  const objectAsString = JSON.stringify(object);
+  const uniformFileUrlMatches = objectAsString.matchAll(
+    /"(https:\/\/(.*)?img\.uniform\.(rocks|global)\/(.*?))"/g
+  );
+  if (uniformFileUrlMatches) {
+    const fileDownloadQueue = new PQueue({ concurrency: 10 });
+    for (const match of uniformFileUrlMatches) {
+      const url = match[1];
+      fileDownloadQueue.add(async () => {
+        try {
+          const fileName = urlToFileName(url);
+          const fileAlreadyExists = await fsj.existsAsync(
+            join2(options.directory, FILES_DIRECTORY_NAME, fileName)
+          );
+          if (fileAlreadyExists) {
+            return;
+          }
+          const response = await fetch(url);
+          if (!response.ok) {
+            return;
+          }
+          const fileBuffer = await response.arrayBuffer();
+          await fsj.writeAsync(
+            join2(options.directory, FILES_DIRECTORY_NAME, fileName),
+            Buffer.from(fileBuffer)
+          );
+        } catch {
+          console.warn(`Failed to download file ${url}`);
+        }
+      });
+    }
+    await fileDownloadQueue.onIdle();
+  }
+  return object;
+};
+var extractAndUploadUniformFilesForObject = async (object, options) => {
+  let objectAsString = JSON.stringify(object);
+  const uniformFileUrlMatches = objectAsString.matchAll(
+    /"(https:\/\/(.*)?img\.uniform\.(rocks|global)\/(.*?))"/g
+  );
+  if (uniformFileUrlMatches) {
+    const fileUploadQueue = new PQueue({ concurrency: 3 });
+    for (const match of uniformFileUrlMatches) {
+      const url = match[1];
+      const hash = urlToHash(url);
+      fileUploadQueue.add(async () => {
+        try {
+          const fileAlreadyExistsChecks = await Promise.all([
+            options.fileClient.getFile({ projectId: options.projectId, url }).catch(() => null),
+            options.fileClient.getFile({ projectId: options.projectId, sourceId: hash }).catch(() => null)
+          ]);
+          if (fileAlreadyExistsChecks.some((check) => check !== null)) {
+            return;
+          }
+          const localFileName = urlToFileName(url);
+          const fileExistsLocally = await fsj.existsAsync(
+            join2(options.directory, FILES_DIRECTORY_NAME, localFileName)
+          );
+          if (!fileExistsLocally) {
+            console.warn(`Skipping file ${url} as we couldn't find a local copy`);
+            return;
+          }
+          const fileBuffer = await fsj.readAsync(
+            join2(options.directory, FILES_DIRECTORY_NAME, localFileName),
+            "buffer"
+          );
+          if (!fileBuffer) {
+            console.warn(`Skipping file ${url} as we couldn't read it`);
+            return;
+          }
+          const fileName = getFileNameFromUrl(url);
+          const { width, height } = (() => {
+            try {
+              return sizeOf(fileBuffer);
+            } catch {
+              return {
+                width: void 0,
+                height: void 0
+              };
+            }
+          })();
+          const { id, method, uploadUrl } = await options.fileClient.createNewProjectFile({
+            name: fileName,
+            mediaType: preferredType(url.split(".").at(-1) ?? ""),
+            size: fileBuffer.length,
+            width,
+            height,
+            projectId: options.projectId,
+            sourceId: hash
+          });
+          const uploadResponse = await fetch(uploadUrl, {
+            method,
+            body: fileBuffer
+          });
+          if (!uploadResponse.ok) {
+            console.warn(`Failed to upload file ${url}`);
+            return;
+          }
+          const checkForFile = async () => {
+            const file = await options.fileClient.getFile({ id });
+            if (!file || file.state !== FILE_READY_STATE) {
+              await new Promise((resolve) => setTimeout(resolve, 500));
+              return checkForFile();
+            }
+            return file.url;
+          };
+          const abortTimeout = setTimeout(() => {
+            throw new Error(`Failed to upload file ${url}`);
+          }, 1e4);
+          const uploadedFileUrl = await checkForFile();
+          clearTimeout(abortTimeout);
+          objectAsString = objectAsString.replaceAll(`"${url}"`, `"${uploadedFileUrl}"`);
+        } catch {
+          console.warn(`Failed to upload file ${url}`);
+        }
+      });
+    }
+    await fileUploadQueue.onIdle();
+  }
+  return JSON.parse(objectAsString);
+};
+var swapOutUniformFileUrlsForTargetProject = async (object, options) => {
+  let objectAsString = JSON.stringify(object);
+  const uniformFileUrlMatches = objectAsString.matchAll(
+    /"(https:\/\/(.*)?img\.uniform\.(rocks|global)\/(.*?))"/g
+  );
+  if (uniformFileUrlMatches) {
+    const fileUrlReplacementQueue = new PQueue({ concurrency: 3 });
+    for (const match of uniformFileUrlMatches) {
+      const url = match[1];
+      const hash = urlToHash(url);
+      fileUrlReplacementQueue.add(async () => {
+        try {
+          const fileAlreadyExistsChecks = await Promise.all([
+            options.fileClient.getFile({ projectId: options.projectId, url }).catch(() => null),
+            options.fileClient.getFile({ projectId: options.projectId, sourceId: hash }).catch(() => null)
+          ]);
+          const file = fileAlreadyExistsChecks.find((check) => check !== null);
+          if (!file) {
+            return;
+          }
+          objectAsString = objectAsString.replaceAll(`"${url}"`, `"${file.url}"`);
+        } catch {
+        }
+      });
+    }
+    await fileUrlReplacementQueue.onIdle();
+  }
+  return JSON.parse(objectAsString);
+};
+
+// src/commands/canvas/commands/composition/pull.ts
 var CompositionPullModule = {
   command: "pull <directory>",
   describe: "Pulls all compositions to local files in a directory",
@@ -1542,13 +1721,19 @@ var CompositionPullModule = {
       target,
       mode,
       whatIf,
-      log: createSyncEngineConsoleLogger({ diffMode })
+      log: createSyncEngineConsoleLogger({ diffMode }),
+      onBeforeWriteObject: async (sourceObject) => {
+        return extractAndDownloadUniformFilesForObject(sourceObject, {
+          directory
+        });
+      }
     });
   }
 };
 
 // src/commands/canvas/commands/composition/push.ts
 import { UncachedCanvasClient as UncachedCanvasClient11 } from "@uniformdev/canvas";
+import { FileClient as FileClient2 } from "@uniformdev/files-sdk";
 var CompositionPushModule = {
   command: "push <directory>",
   describe: "Pushes all compositions from files in a directory to Uniform Canvas",
@@ -1618,12 +1803,26 @@ var CompositionPushModule = {
       });
     }
     const target = createComponentInstanceEngineDataSource({ client, state, onlyCompositions, onlyPatterns });
+    const fileClient = new FileClient2({ apiKey, apiHost, fetch: fetch3, projectId });
     await syncEngine({
       source,
       target,
       mode,
       whatIf,
-      log: createSyncEngineConsoleLogger({ diffMode })
+      log: createSyncEngineConsoleLogger({ diffMode }),
+      onBeforeCompareObjects: async (sourceObject) => {
+        return swapOutUniformFileUrlsForTargetProject(sourceObject, {
+          fileClient,
+          projectId
+        });
+      },
+      onBeforeWriteObject: async (sourceObject) => {
+        return extractAndUploadUniformFilesForObject(sourceObject, {
+          directory,
+          fileClient,
+          projectId
+        });
+      }
     });
   }
 };
@@ -3791,8 +3990,10 @@ var package_default = {
     format: 'prettier --write "src/**/*.{js,ts,tsx}"'
   },
   dependencies: {
+    "@thi.ng/mime": "^2.2.23",
     "@uniformdev/canvas": "workspace:*",
     "@uniformdev/context": "workspace:*",
+    "@uniformdev/files-sdk": "workspace:*",
     "@uniformdev/project-map": "workspace:*",
     "@uniformdev/redirect": "workspace:*",
     colorette: "2.0.20",
@@ -3805,6 +4006,7 @@ var package_default = {
     graphql: "16.7.1",
     "graphql-request": "6.1.0",
     "https-proxy-agent": "^7.0.0",
+    "image-size": "^1.0.2",
     inquirer: "9.2.9",
     "isomorphic-git": "1.24.5",
     "isomorphic-unfetch": "^3.1.0",
@@ -3813,6 +4015,7 @@ var package_default = {
     "lodash.isequalwith": "^4.4.0",
     open: "9.1.0",
     ora: "6.3.1",
+    "p-queue": "7.3.4",
     "posthog-node": "3.1.1",
     slugify: "1.6.6",
     "update-check": "^1.5.4",
@@ -4150,7 +4353,7 @@ ${err.message}`);
 // src/projects/cloneStarter.ts
 import crypto2 from "crypto";
 import fs3 from "fs";
-import fsj from "fs-jetpack";
+import fsj2 from "fs-jetpack";
 import * as git from "isomorphic-git";
 import * as http from "isomorphic-git/http/node/index.js";
 import os from "os";
@@ -4181,7 +4384,7 @@ async function cloneStarter({
     throw new Error(`"${targetDir}" is not empty`);
   }
   const starterDir = path.join(cloneDir, ...pathSegments);
-  fsj.copy(starterDir, targetDir, { overwrite: true });
+  fsj2.copy(starterDir, targetDir, { overwrite: true });
   if (dotEnvFile) {
     fs3.writeFileSync(path.resolve(targetDir, ".env"), dotEnvFile, "utf-8");
   }
@@ -6085,7 +6288,7 @@ async function checkForUpdateMiddleware() {
 
 // src/middleware/checkLocalDepsVersionsMiddleware.ts
 import { magenta, red as red6 } from "colorette";
-import { join as join2 } from "path";
+import { join as join3 } from "path";
 
 // src/fs.ts
 import { promises as fs5 } from "fs";
@@ -6124,7 +6327,7 @@ var checkLocalDepsVersions = async (args) => {
   try {
     let isOutside = false;
     let warning = `${magenta("Warning:")} Installed Uniform packages should be the same version`;
-    const localPackages = await tryReadJSON(join2(process.cwd(), "package.json"));
+    const localPackages = await tryReadJSON(join3(process.cwd(), "package.json"));
     if (!localPackages)
       return;
     let firstVersion;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@uniformdev/cli",
-  "version": "19.42.0",
+  "version": "19.42.1-alpha.7+1380aadd7",
   "description": "Uniform command line interface tool",
   "license": "SEE LICENSE IN LICENSE.txt",
   "main": "./cli.js",
@@ -16,10 +16,12 @@
     "format": "prettier --write \"src/**/*.{js,ts,tsx}\""
   },
   "dependencies": {
-    "@uniformdev/canvas": "19.42.0",
-    "@uniformdev/context": "19.42.0",
-    "@uniformdev/project-map": "19.42.0",
-    "@uniformdev/redirect": "19.42.0",
+    "@thi.ng/mime": "^2.2.23",
+    "@uniformdev/canvas": "19.42.1-alpha.7+1380aadd7",
+    "@uniformdev/context": "19.42.1-alpha.7+1380aadd7",
+    "@uniformdev/files-sdk": "19.37.0",
+    "@uniformdev/project-map": "19.42.1-alpha.7+1380aadd7",
+    "@uniformdev/redirect": "19.42.1-alpha.7+1380aadd7",
     "colorette": "2.0.20",
     "cosmiconfig": "8.2.0",
     "cosmiconfig-typescript-loader": "5.0.0",
@@ -30,6 +32,7 @@
     "graphql": "16.7.1",
     "graphql-request": "6.1.0",
     "https-proxy-agent": "^7.0.0",
+    "image-size": "^1.0.2",
     "inquirer": "9.2.9",
     "isomorphic-git": "1.24.5",
     "isomorphic-unfetch": "^3.1.0",
@@ -38,6 +41,7 @@
     "lodash.isequalwith": "^4.4.0",
     "open": "9.1.0",
     "ora": "6.3.1",
+    "p-queue": "7.3.4",
     "posthog-node": "3.1.1",
     "slugify": "1.6.6",
     "update-check": "^1.5.4",
@@ -62,5 +66,5 @@
   "publishConfig": {
     "access": "public"
   },
-  "gitHead": "33ed87d09bfb50c8557c6bcf4eb232c76f7c99d8"
+  "gitHead": "1380aadd7c71f6f120f933a40e73e113518e3716"
 }