@getguru/import-engine 0.1.1 → 0.1.2

package/docx-import-cli.js

@@ -114320,7 +114320,7 @@ var import_jszip5 = __toESM(require_lib3());
 var import_url_join6 = __toESM(require_url_join());
 
 // packages/import-engine/package.json
-var version = "0.1.1";
+var version = "0.1.2";
 
 // src/docx-import/detectors.js
 function isGoogleSitesDoc(div) {
@@ -114344,7 +114344,7 @@ var showError;
 var updateLoader;
 var showReport;
 var docLinkIndex = /* @__PURE__ */ new Map();
-var toolVersion = true ? `${version}+${"e19020c"}` : version;
+var toolVersion = true ? `${version}+${"4d61d22"}` : version;
 var pendingUpload = null;
 function confirmPendingUpload() {
   const upload = pendingUpload;
@@ -115155,6 +115155,91 @@ function setSubtract(a, b) {
     return !b.includes(value);
   });
 }
+var UUID_RE2 = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+var EXTERNAL_MANIFEST_MAX_BYTES = 1048576;
+var EXTERNAL_MAX_ENTRIES = 500;
+var EXTERNAL_NAME_RE = /^[\p{L}\p{N}._-]{1,128}$/u;
+var EXTERNAL_REFUSED = "the server won't fetch this image; the card will link to the original URL";
+function checkExternalImages(zip, task, report) {
+  var state = { enabled: false, names: /* @__PURE__ */ new Set(), referenced: [] };
+  report.externalImages = state;
+  var collectionFile = zip.file("collection.yaml");
+  var manifestFile = zip.file("resources/external.json");
+  function readCollection() {
+    if (!collectionFile) {
+      return Promise.resolve();
+    }
+    return collectionFile.async("string").then(function(yamlString) {
+      try {
+        var collection = yaml.load(yamlString);
+        state.enabled = !!collection && typeof collection == "object" && collection.FetchExternalImages === true;
+      } catch (e) {
+      }
+    }, function() {
+    });
+  }
+  function readManifest() {
+    if (!manifestFile) {
+      return Promise.resolve();
+    }
+    return manifestFile.async("string").then(function(text2) {
+      var size2 = manifestFile._data && manifestFile._data.uncompressedSize || text2.length;
+      var dropped = "The server will ignore the whole manifest, so no external images will be fetched";
+      if (size2 > EXTERNAL_MANIFEST_MAX_BYTES) {
+        report.error(`resources/external.json is larger than ${EXTERNAL_MANIFEST_MAX_BYTES} bytes. ${dropped}.`);
+        return;
+      }
+      var manifest;
+      try {
+        manifest = JSON.parse(text2);
+      } catch (e) {
+        report.error(`resources/external.json is not valid JSON (${e.message}). ${dropped}.`);
+        return;
+      }
+      if (!manifest || typeof manifest != "object" || Array.isArray(manifest)) {
+        report.error(`resources/external.json must be a JSON object mapping file names to URLs. ${dropped}.`);
+        return;
+      }
+      var keys2 = Object.keys(manifest);
+      if (keys2.some(function(key) {
+        return typeof manifest[key] != "string";
+      })) {
+        report.error(`resources/external.json must map every file name to a URL string. ${dropped}.`);
+        return;
+      }
+      keys2.forEach(function(name, index) {
+        state.names.add(name);
+        var url = manifest[name];
+        var label = `resources/external.json entry "${escapeQuotes(name)}"`;
+        if (index >= EXTERNAL_MAX_ENTRIES) {
+          report.warn(`${label} is past the first ${EXTERNAL_MAX_ENTRIES} entries: ${EXTERNAL_REFUSED}.`);
+        } else if (!EXTERNAL_NAME_RE.test(name) || name.includes("..")) {
+          report.warn(`${label} has an invalid file name: ${EXTERNAL_REFUSED}.`);
+        } else if (url.trim() === "") {
+          report.warn(`${label} has a blank URL: ${EXTERNAL_REFUSED}.`);
+        } else if (!/^https:\/\//i.test(url.trim())) {
+          report.warn(`${label} is not an https URL: ${EXTERNAL_REFUSED}.`);
+        }
+        if (zip.file("resources/" + name)) {
+          report.info(`${label} also exists as resources/${escapeQuotes(name)} in the zip, so the server skips the fetch.`);
+        }
+      });
+    }, function() {
+    });
+  }
+  return readCollection().then(readManifest).then(function() {
+    if (manifestFile && !state.enabled) {
+      report.warn(`resources/external.json is included but collection.yaml doesn't set FetchExternalImages: true, so the server ignores it.`);
+    }
+    if (state.enabled && !manifestFile) {
+      report.warn(`collection.yaml sets FetchExternalImages: true but the zip has no resources/external.json.`);
+    }
+    if (state.enabled) {
+      report.syncError(`If this is a sync, FetchExternalImages is ignored (external images are only fetched on imports).`);
+    }
+    report.next();
+  });
+}
 function escapeQuotes(value) {
   value = value || "";
   return value.replace(/"/g, `\\"`);
@@ -115189,11 +115274,18 @@ function checkCardYaml(zip, task, report) {
   return zip.file(task.filename).async("string").then(function(yamlString) {
     try {
       var card = yaml.load(yamlString);
-      var possibleKeys = ["ExternalId", "ExternalUrl", "Title", "Tags", "externalLastUpdated"];
+      var possibleKeys = ["ExternalId", "ExternalUrl", "Title", "Tags", "externalLastUpdated", "ID"];
       var otherKeys = setSubtract(Object.keys(card), possibleKeys);
       otherKeys.forEach(function(key) {
         report.warn(`Unexpected property "${key}" in the card's yaml.`);
       });
+      if (card.ID !== void 0 && card.ID !== null) {
+        if (typeof card.ID != "string" || !UUID_RE2.test(card.ID)) {
+          report.error(`The card's ID property must be a UUID string (e.g. ID: "123e4567-e89b-12d3-a456-426614174000"), but it was: ${escapeHtml3(String(card.ID))}`);
+        } else {
+          report.info(`The card has an ID, so it updates the existing card with that ID in the same collection if one exists, otherwise a new card is created.`);
+        }
+      }
       if (!card.Title) {
         report.error(`A "Title" property is required in the card's yaml.`);
       } else {
@@ -115258,7 +115350,17 @@ function checkCardHtml(zip, task, report) {
             report.resourceReferences.push(reference2);
           }
         } else {
-          report.error(`The referenced resource "${reference2}" does not exist in the zip.`);
+          var ext = report.externalImages || { enabled: false, names: /* @__PURE__ */ new Set(), referenced: [] };
+          var extName = reference2.slice("resources/".length);
+          if (ext.names.has(extName) && ext.enabled) {
+            if (!ext.referenced.includes(extName)) {
+              ext.referenced.push(extName);
+            }
+          } else if (ext.names.has(extName)) {
+            report.error(`The referenced resource "${reference2}" is listed in resources/external.json but collection.yaml doesn't set FetchExternalImages: true, so it won't be fetched.`);
+          } else {
+            report.error(`The referenced resource "${reference2}" does not exist in the zip.`);
+          }
         }
       } else if (reference2.startsWith("cards/")) {
         if (zip.file(reference2 + ".yaml")) {
@@ -115515,7 +115617,7 @@ function checkCollectionYaml(zip, task, report) {
     }
     try {
       var collection = yaml.load(yamlString);
-      var possibleKeys = ["Items", "Tags", "Title", "Version"];
+      var possibleKeys = ["Items", "Tags", "Title", "Version", "FetchExternalImages", "ReuseFolders"];
       var otherKeys = setSubtract(Object.keys(collection), possibleKeys);
       otherKeys.forEach(function(key) {
         report.warn(`Unexpected property "${key}" in collection.yaml.`);
@@ -115532,6 +115634,14 @@ function checkCollectionYaml(zip, task, report) {
         }
       }
       var isV2 = collection.Version == 2;
+      ["FetchExternalImages", "ReuseFolders"].forEach(function(key) {
+        if (collection[key] !== void 0 && collection[key] !== null && typeof collection[key] != "boolean") {
+          report.warn(`The collection's ${key} property must be true or false (${key} = ${escapeHtml3(String(collection[key]))}).`);
+        }
+      });
+      if (collection.ReuseFolders === true && !isV2) {
+        report.warn(`ReuseFolders is ignored unless the collection has "Version: 2" (it applies only to Version 2 non-sync imports).`);
+      }
       if (collection.Items) {
         if (!Array.isArray(collection.Items)) {
           report.error(`The collection's Items list is optional, but if provided it must be a list.`);
@@ -115635,7 +115745,16 @@ function checkResource(zip, task, report) {
   } else {
     report.resources += 1;
   }
-  if (!report.resourceReferences.includes(task.filename)) {
+  if (task.filename == "resources/external.json") {
+    var ext = report.externalImages;
+    if (ext) {
+      ext.names.forEach(function(name) {
+        if (!ext.referenced.includes(name) && !report.resourceReferences.includes("resources/" + name)) {
+          report.warn(`resources/external.json entry "${escapeQuotes(name)}" isn't referenced by any cards.`);
+        }
+      });
+    }
+  } else if (!report.resourceReferences.includes(task.filename)) {
     report.warn(`Resource ${task.filename} is included in the zip but isn't referenced by any cards.`);
   }
   if (zip.file(task.filename)._data.uncompressedSize > 2e9) {
@@ -115732,6 +115851,7 @@ function checkZip(zip) {
       collectionRoots: [],
       tags: [],
       resourceReferences: [],
+      externalImages: { enabled: false, names: /* @__PURE__ */ new Set(), referenced: [] },
       cardLinks: 0,
       boardLinks: 0,
       checkCount: 0,
@@ -115805,6 +115925,7 @@ function checkZip(zip) {
         info(`Unexpected file ${fullPath}`);
       }
     }
+    tasks.push({ func: checkExternalImages, context: "resources/external.json", checks: 3, priority: 0 });
     tasks.push({ func: checkFolderTree, checks: 5, priority: 6 });
     tasks.push({ func: checkStats, checks: 3, priority: 10 });
     tasks.sort(function(a, b) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@getguru/import-engine",
-  "version": "0.1.1",
+  "version": "0.1.2",
   "description": "Command-line engine that turns Notion, Confluence, Google, Word, PDF, Markdown and spreadsheet exports into Guru import zips.",
   "bin": {
     "guru-import-engine": "docx-import-cli.js"