@getguru/import-engine 0.1.0 → 0.1.2

package/README.md

@@ -5,7 +5,7 @@ Google Sites, Google Docs, Word, PDF, Slab, GitHub wiki, Markdown, spreadsheet)
 that Guru can import. It is the same pipeline as the webapp's "Import content" screen,
 built from `guru-labs`.
 
-Requires Node >=22.
+Requires Node >=22.12.
 
 ## Usage
 

package/docx-import-cli.js

@@ -1,5 +1,4 @@
 #!/usr/bin/env node
-"use strict";
 var __create = Object.create;
 var __defProp = Object.defineProperty;
 var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
@@ -114321,7 +114320,7 @@ var import_jszip5 = __toESM(require_lib3());
 var import_url_join6 = __toESM(require_url_join());
 
 // packages/import-engine/package.json
-var version = "0.1.0";
+var version = "0.1.2";
 
 // src/docx-import/detectors.js
 function isGoogleSitesDoc(div) {
@@ -114345,7 +114344,7 @@ var showError;
 var updateLoader;
 var showReport;
 var docLinkIndex = /* @__PURE__ */ new Map();
-var toolVersion = true ? `${version}+${"66b3d49"}` : version;
+var toolVersion = true ? `${version}+${"4d61d22"}` : version;
 var pendingUpload = null;
 function confirmPendingUpload() {
   const upload = pendingUpload;
@@ -115156,6 +115155,91 @@ function setSubtract(a, b) {
     return !b.includes(value);
   });
 }
+var UUID_RE2 = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+var EXTERNAL_MANIFEST_MAX_BYTES = 1048576;
+var EXTERNAL_MAX_ENTRIES = 500;
+var EXTERNAL_NAME_RE = /^[\p{L}\p{N}._-]{1,128}$/u;
+var EXTERNAL_REFUSED = "the server won't fetch this image; the card will link to the original URL";
+function checkExternalImages(zip, task, report) {
+  var state = { enabled: false, names: /* @__PURE__ */ new Set(), referenced: [] };
+  report.externalImages = state;
+  var collectionFile = zip.file("collection.yaml");
+  var manifestFile = zip.file("resources/external.json");
+  function readCollection() {
+    if (!collectionFile) {
+      return Promise.resolve();
+    }
+    return collectionFile.async("string").then(function(yamlString) {
+      try {
+        var collection = yaml.load(yamlString);
+        state.enabled = !!collection && typeof collection == "object" && collection.FetchExternalImages === true;
+      } catch (e) {
+      }
+    }, function() {
+    });
+  }
+  function readManifest() {
+    if (!manifestFile) {
+      return Promise.resolve();
+    }
+    return manifestFile.async("string").then(function(text2) {
+      var size2 = manifestFile._data && manifestFile._data.uncompressedSize || text2.length;
+      var dropped = "The server will ignore the whole manifest, so no external images will be fetched";
+      if (size2 > EXTERNAL_MANIFEST_MAX_BYTES) {
+        report.error(`resources/external.json is larger than ${EXTERNAL_MANIFEST_MAX_BYTES} bytes. ${dropped}.`);
+        return;
+      }
+      var manifest;
+      try {
+        manifest = JSON.parse(text2);
+      } catch (e) {
+        report.error(`resources/external.json is not valid JSON (${e.message}). ${dropped}.`);
+        return;
+      }
+      if (!manifest || typeof manifest != "object" || Array.isArray(manifest)) {
+        report.error(`resources/external.json must be a JSON object mapping file names to URLs. ${dropped}.`);
+        return;
+      }
+      var keys2 = Object.keys(manifest);
+      if (keys2.some(function(key) {
+        return typeof manifest[key] != "string";
+      })) {
+        report.error(`resources/external.json must map every file name to a URL string. ${dropped}.`);
+        return;
+      }
+      keys2.forEach(function(name, index) {
+        state.names.add(name);
+        var url = manifest[name];
+        var label = `resources/external.json entry "${escapeQuotes(name)}"`;
+        if (index >= EXTERNAL_MAX_ENTRIES) {
+          report.warn(`${label} is past the first ${EXTERNAL_MAX_ENTRIES} entries: ${EXTERNAL_REFUSED}.`);
+        } else if (!EXTERNAL_NAME_RE.test(name) || name.includes("..")) {
+          report.warn(`${label} has an invalid file name: ${EXTERNAL_REFUSED}.`);
+        } else if (url.trim() === "") {
+          report.warn(`${label} has a blank URL: ${EXTERNAL_REFUSED}.`);
+        } else if (!/^https:\/\//i.test(url.trim())) {
+          report.warn(`${label} is not an https URL: ${EXTERNAL_REFUSED}.`);
+        }
+        if (zip.file("resources/" + name)) {
+          report.info(`${label} also exists as resources/${escapeQuotes(name)} in the zip, so the server skips the fetch.`);
+        }
+      });
+    }, function() {
+    });
+  }
+  return readCollection().then(readManifest).then(function() {
+    if (manifestFile && !state.enabled) {
+      report.warn(`resources/external.json is included but collection.yaml doesn't set FetchExternalImages: true, so the server ignores it.`);
+    }
+    if (state.enabled && !manifestFile) {
+      report.warn(`collection.yaml sets FetchExternalImages: true but the zip has no resources/external.json.`);
+    }
+    if (state.enabled) {
+      report.syncError(`If this is a sync, FetchExternalImages is ignored (external images are only fetched on imports).`);
+    }
+    report.next();
+  });
+}
 function escapeQuotes(value) {
   value = value || "";
   return value.replace(/"/g, `\\"`);
@@ -115190,11 +115274,18 @@ function checkCardYaml(zip, task, report) {
   return zip.file(task.filename).async("string").then(function(yamlString) {
     try {
       var card = yaml.load(yamlString);
-      var possibleKeys = ["ExternalId", "ExternalUrl", "Title", "Tags", "externalLastUpdated"];
+      var possibleKeys = ["ExternalId", "ExternalUrl", "Title", "Tags", "externalLastUpdated", "ID"];
       var otherKeys = setSubtract(Object.keys(card), possibleKeys);
       otherKeys.forEach(function(key) {
         report.warn(`Unexpected property "${key}" in the card's yaml.`);
       });
+      if (card.ID !== void 0 && card.ID !== null) {
+        if (typeof card.ID != "string" || !UUID_RE2.test(card.ID)) {
+          report.error(`The card's ID property must be a UUID string (e.g. ID: "123e4567-e89b-12d3-a456-426614174000"), but it was: ${escapeHtml3(String(card.ID))}`);
+        } else {
+          report.info(`The card has an ID, so it updates the existing card with that ID in the same collection if one exists, otherwise a new card is created.`);
+        }
+      }
       if (!card.Title) {
         report.error(`A "Title" property is required in the card's yaml.`);
       } else {
@@ -115259,7 +115350,17 @@ function checkCardHtml(zip, task, report) {
             report.resourceReferences.push(reference2);
           }
         } else {
-          report.error(`The referenced resource "${reference2}" does not exist in the zip.`);
+          var ext = report.externalImages || { enabled: false, names: /* @__PURE__ */ new Set(), referenced: [] };
+          var extName = reference2.slice("resources/".length);
+          if (ext.names.has(extName) && ext.enabled) {
+            if (!ext.referenced.includes(extName)) {
+              ext.referenced.push(extName);
+            }
+          } else if (ext.names.has(extName)) {
+            report.error(`The referenced resource "${reference2}" is listed in resources/external.json but collection.yaml doesn't set FetchExternalImages: true, so it won't be fetched.`);
+          } else {
+            report.error(`The referenced resource "${reference2}" does not exist in the zip.`);
+          }
         }
       } else if (reference2.startsWith("cards/")) {
         if (zip.file(reference2 + ".yaml")) {
@@ -115342,7 +115443,7 @@ function checkBoardYaml(zip, task, report) {
         }
       }
       if (board.Items) {
-        let checkCard2 = function(card) {
+        let checkCard = function(card) {
           var possibleKeys = ["Type", "ID"];
           var otherKeys = setSubtract(Object.keys(card), possibleKeys);
           otherKeys.forEach(function(key) {
@@ -115358,7 +115459,7 @@ function checkBoardYaml(zip, task, report) {
           if (!zip.files[`cards/${card.ID}.yaml`]) {
             report.error(`This board references a card (ID = "${card.ID}") that does not seem to exist in this zip, we expect cards/${card.ID}.yaml would also exist.`);
           }
-        }, checkSection2 = function(section) {
+        }, checkSection = function(section) {
           var possibleKeys = ["Type", "Items", "Title"];
           var otherKeys = setSubtract(Object.keys(section), possibleKeys);
           otherKeys.forEach(function(key) {
@@ -115377,13 +115478,12 @@ function checkBoardYaml(zip, task, report) {
                 } else if (item.Type != "card") {
                   report.error(`Section items must have Type = "card" but one has Type = "${item.Type}".`);
                 } else {
-                  checkCard2(item);
+                  checkCard(item);
                 }
               });
             }
           }
         };
-        var checkCard = checkCard2, checkSection = checkSection2;
         var cardIds = [];
         if (!Array.isArray(board.Items)) {
           report.error(`The board's Items property is optional, but if it's provided it must be a list of objects.`);
@@ -115397,9 +115497,9 @@ function checkBoardYaml(zip, task, report) {
               report.error(`Every item on the board needs a "Type" property.`);
             } else {
               if (item.Type == "card") {
-                checkCard2(item);
+                checkCard(item);
               } else if (item.Type == "section") {
-                checkSection2(item);
+                checkSection(item);
               } else {
                 report.error(`A board item has an unexpected Type value, we expect either "card" or "section" but got "${item.Type}" instead.`);
               }
@@ -115517,7 +115617,7 @@ function checkCollectionYaml(zip, task, report) {
     }
     try {
       var collection = yaml.load(yamlString);
-      var possibleKeys = ["Items", "Tags", "Title", "Version"];
+      var possibleKeys = ["Items", "Tags", "Title", "Version", "FetchExternalImages", "ReuseFolders"];
       var otherKeys = setSubtract(Object.keys(collection), possibleKeys);
       otherKeys.forEach(function(key) {
         report.warn(`Unexpected property "${key}" in collection.yaml.`);
@@ -115534,6 +115634,14 @@ function checkCollectionYaml(zip, task, report) {
         }
       }
       var isV2 = collection.Version == 2;
+      ["FetchExternalImages", "ReuseFolders"].forEach(function(key) {
+        if (collection[key] !== void 0 && collection[key] !== null && typeof collection[key] != "boolean") {
+          report.warn(`The collection's ${key} property must be true or false (${key} = ${escapeHtml3(String(collection[key]))}).`);
+        }
+      });
+      if (collection.ReuseFolders === true && !isV2) {
+        report.warn(`ReuseFolders is ignored unless the collection has "Version: 2" (it applies only to Version 2 non-sync imports).`);
+      }
       if (collection.Items) {
         if (!Array.isArray(collection.Items)) {
           report.error(`The collection's Items list is optional, but if provided it must be a list.`);
@@ -115637,7 +115745,16 @@ function checkResource(zip, task, report) {
   } else {
     report.resources += 1;
   }
-  if (!report.resourceReferences.includes(task.filename)) {
+  if (task.filename == "resources/external.json") {
+    var ext = report.externalImages;
+    if (ext) {
+      ext.names.forEach(function(name) {
+        if (!ext.referenced.includes(name) && !report.resourceReferences.includes("resources/" + name)) {
+          report.warn(`resources/external.json entry "${escapeQuotes(name)}" isn't referenced by any cards.`);
+        }
+      });
+    }
+  } else if (!report.resourceReferences.includes(task.filename)) {
     report.warn(`Resource ${task.filename} is included in the zip but isn't referenced by any cards.`);
   }
   if (zip.file(task.filename)._data.uncompressedSize > 2e9) {
@@ -115734,6 +115851,7 @@ function checkZip(zip) {
       collectionRoots: [],
       tags: [],
       resourceReferences: [],
+      externalImages: { enabled: false, names: /* @__PURE__ */ new Set(), referenced: [] },
       cardLinks: 0,
       boardLinks: 0,
       checkCount: 0,
@@ -115807,6 +115925,7 @@ function checkZip(zip) {
         info(`Unexpected file ${fullPath}`);
       }
     }
+    tasks.push({ func: checkExternalImages, context: "resources/external.json", checks: 3, priority: 0 });
     tasks.push({ func: checkFolderTree, checks: 5, priority: 6 });
     tasks.push({ func: checkStats, checks: 3, priority: 10 });
     tasks.sort(function(a, b) {

package/package.json

@@ -1,9 +1,9 @@
 {
   "name": "@getguru/import-engine",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "description": "Command-line engine that turns Notion, Confluence, Google, Word, PDF, Markdown and spreadsheet exports into Guru import zips.",
   "bin": {
-    "guru-import-engine": "./docx-import-cli.js"
+    "guru-import-engine": "docx-import-cli.js"
   },
   "files": [
     "docx-import-cli.js",
@@ -11,10 +11,10 @@
     "LICENSE.md"
   ],
   "dependencies": {
-    "jsdom": "26.1.0"
+    "jsdom": "28.1.0"
   },
   "engines": {
-    "node": ">=22"
+    "node": ">=22.12"
   },
   "publishConfig": {
     "access": "public"