@getguru/import-engine 0.1.1 → 0.1.3

package/docx-import-cli.js

@@ -98755,6 +98755,34 @@ var importType = typeArgIndex !== -1 ? process.argv[typeArgIndex + 1] : "";
 var dom = new import_jsdom.JSDOM("", {
   url: `http://localhost:8000/docx-import/index.html?type=${importType}&embed=`
 });
+var UNUSED_INPUT_EVENTS = /* @__PURE__ */ new Set([
+  "focus",
+  "focusin",
+  "keydown",
+  "keyup",
+  "mouseover",
+  "mousedown",
+  "mouseup",
+  "click",
+  "mouseout"
+]);
+var addWindowListener = dom.window.addEventListener;
+dom.window.addEventListener = function(type2, ...rest2) {
+  if (UNUSED_INPUT_EVENTS.has(type2)) {
+    return void 0;
+  }
+  return addWindowListener.call(this, type2, ...rest2);
+};
+var windowProperties = require("jsdom/lib/jsdom/living/window-properties.js");
+for (const hook of ["elementAttached", "elementDetached", "elementAttributeModified"]) {
+  const track = windowProperties[hook];
+  windowProperties[hook] = function(element, ...rest2) {
+    if (!element._ownerDocument._defaultView) {
+      return void 0;
+    }
+    return track.call(this, element, ...rest2);
+  };
+}
 var ALLOWED_GLOBALS = [
   "window",
   "document",
@@ -102636,6 +102664,52 @@ function markLinkRecorded(doc, link2) {
 function isLinkRecorded(doc, link2) {
   return !!doc.recordedLinks && doc.recordedLinks.has(link2);
 }
+var RECORDED_LINK_ATTRIBUTE = "data-guru-import-recorded-link";
+function serializeDoctype(doctype) {
+  if (!doctype) {
+    return "";
+  }
+  const quote = (id) => id.includes('"') ? `'${id}'` : `"${id}"`;
+  if (doctype.publicId) {
+    return `<!DOCTYPE ${doctype.name} PUBLIC ${quote(doctype.publicId)}${doctype.systemId ? " " + quote(doctype.systemId) : ""}>`;
+  }
+  if (doctype.systemId) {
+    return `<!DOCTYPE ${doctype.name} SYSTEM ${quote(doctype.systemId)}>`;
+  }
+  return `<!DOCTYPE ${doctype.name}>`;
+}
+function serializeDocTree(div, doc) {
+  for (const element of div.querySelectorAll("pre, textarea, listing")) {
+    const first2 = element.firstChild;
+    if (first2 && first2.nodeType === 3 && first2.data.startsWith("\n")) {
+      first2.data = "\n" + first2.data;
+    }
+  }
+  if (doc.recordedLinks) {
+    for (const element of div.querySelectorAll("*")) {
+      if (doc.recordedLinks.has(element)) {
+        element.setAttribute(RECORDED_LINK_ATTRIBUTE, "");
+      }
+    }
+  }
+  return flattenString(serializeDoctype(div.ownerDocument.doctype) + div.outerHTML);
+}
+function flattenString(string) {
+  string.charCodeAt(0);
+  return string;
+}
+function parseDocTree(source, doc) {
+  const div = new DOMParser().parseFromString(source, "text/html").documentElement;
+  const hadRecordedLinks = !!doc.recordedLinks;
+  delete doc.recordedLinks;
+  if (hadRecordedLinks) {
+    for (const element of div.querySelectorAll(`[${RECORDED_LINK_ATTRIBUTE}]`)) {
+      element.removeAttribute(RECORDED_LINK_ATTRIBUTE);
+      markLinkRecorded(doc, element);
+    }
+  }
+  return div;
+}
 function safeDecode(string) {
   try {
     return decodeURIComponent(string);
@@ -111340,11 +111414,17 @@ function buildSlabPostSlugIndex(allDocs, zip) {
   }
   return index;
 }
+function getSlabPostSlugIndex(zip, allDocs) {
+  if (!zip.slabPostSlugIndex) {
+    zip.slabPostSlugIndex = buildSlabPostSlugIndex(allDocs, zip);
+  }
+  return zip.slabPostSlugIndex;
+}
 function resolveSlabPostLinks(div, doc, allDocs) {
   if (!doc.zip?.isSlabExport) {
     return;
   }
-  const index = buildSlabPostSlugIndex(allDocs, doc.zip);
+  const index = getSlabPostSlugIndex(doc.zip, allDocs);
   Array.from(div.querySelectorAll("a[href]")).forEach((link2) => {
     const href = link2.getAttribute("href");
     if (!href || SLAB_TOPIC_URL_PATTERN.test(href)) {
@@ -114320,7 +114400,7 @@ var import_jszip5 = __toESM(require_lib3());
 var import_url_join6 = __toESM(require_url_join());
 
 // packages/import-engine/package.json
-var version = "0.1.1";
+var version = "0.1.3";
 
 // src/docx-import/detectors.js
 function isGoogleSitesDoc(div) {
@@ -114344,7 +114424,7 @@ var showError;
 var updateLoader;
 var showReport;
 var docLinkIndex = /* @__PURE__ */ new Map();
-var toolVersion = true ? `${version}+${"e19020c"}` : version;
+var toolVersion = true ? `${version}+${"8671a17"}` : version;
 var pendingUpload = null;
 function confirmPendingUpload() {
   const upload = pendingUpload;
@@ -114508,6 +114588,21 @@ async function processImage(doc, image2, imageIndex, newZip, resourceLookup = {}
     }
   });
 }
+var resourceLookupSizes = /* @__PURE__ */ new WeakMap();
+function getResourceLookupSize(resourceLookup) {
+  if (!resourceLookupSizes.has(resourceLookup)) {
+    resourceLookupSizes.set(resourceLookup, Object.keys(resourceLookup).length);
+  }
+  return resourceLookupSizes.get(resourceLookup);
+}
+function addToResourceLookup(resourceLookup, unsafeResourceName, safeResourceName) {
+  const size2 = getResourceLookupSize(resourceLookup);
+  const isNewKey = !Object.hasOwn(resourceLookup, unsafeResourceName);
+  resourceLookup[unsafeResourceName] = safeResourceName;
+  if (isNewKey && Object.hasOwn(resourceLookup, unsafeResourceName)) {
+    resourceLookupSizes.set(resourceLookup, size2 + 1);
+  }
+}
 async function resolveResource(element, doc, newZip, resourceLookup = {}) {
   return new Promise(async function(resolve, reject2) {
     try {
@@ -114536,10 +114631,10 @@ async function resolveResource(element, doc, newZip, resourceLookup = {}) {
         } else {
           safeResourceName = sanitizeFilename(unsafeResourceName);
           if (safeResourceName !== unsafeResourceName) {
-            const nextIndex = Object.keys(resourceLookup).length;
+            const nextIndex = getResourceLookupSize(resourceLookup);
             safeResourceName = `file${nextIndex}_${safeResourceName}`;
           }
-          resourceLookup[unsafeResourceName] = safeResourceName;
+          addToResourceLookup(resourceLookup, unsafeResourceName, safeResourceName);
         }
         const resourcePath = "resources/" + safeResourceName;
         const fileData = await (import_jszip5.default.support.uint8array ? file.async("uint8array") : file.async("base64"));
@@ -114774,7 +114869,7 @@ async function preProcessDoc(doc, docTitleLookup = {}) {
       doc.zip.source = source;
     }
   }
-  doc.htmlContent = div;
+  doc.htmlSource = serializeDocTree(div, doc);
   updateLoader({
     step: 1,
     docIndex: doc.index,
@@ -114805,7 +114900,8 @@ async function postProcessDoc(newZip, doc, resourceLookup = {}, externalLookup =
     writeCardToZip(newZip, doc, "");
     return;
   }
-  const div = doc.htmlContent;
+  const div = parseDocTree(doc.htmlSource, doc);
+  delete doc.htmlSource;
   updateLoader({
     step: 1,
     docIndex: doc.index,
@@ -114874,6 +114970,14 @@ async function postProcessDoc(newZip, doc, resourceLookup = {}, externalLookup =
   }
   writeCardToZip(newZip, doc, cleanUpHtml(div, doc));
 }
+function releaseDocContent(doc, buffersReadFromZip) {
+  delete doc.htmlSource;
+  delete doc.originalContent;
+  delete doc.recordedLinks;
+  if (buffersReadFromZip.has(doc)) {
+    delete doc.buffer;
+  }
+}
 function makeErrorHandler(progress) {
   return function handleError(event) {
     const message = event.reason?.message || event.message || event.error?.message || "Unknown error";
@@ -114957,6 +115061,7 @@ async function runProcessDocs(params, progress) {
       zipToCheck.docxAnchorIndex = buildDocxAnchorIndex(docs, zipToCheck);
     }
   }
+  const buffersReadFromZip = /* @__PURE__ */ new WeakSet();
   await doAll(docs, async function(doc, docIndex, nextDoc) {
     doc.index = docIndex;
     try {
@@ -114965,6 +115070,7 @@ async function runProcessDocs(params, progress) {
       } else if (doc.zip) {
         const buffer = await doc.zip.file(doc.filename).async("arraybuffer");
         doc.buffer = buffer;
+        buffersReadFromZip.add(doc);
         await preProcessDoc(doc, docTitleLookup);
       } else {
         await preProcessDoc(doc);
@@ -114972,6 +115078,7 @@ async function runProcessDocs(params, progress) {
     } catch (err) {
       recordFailure(doc, err);
     } finally {
+      delete doc.div;
       nextDoc();
     }
   });
@@ -114981,6 +115088,9 @@ async function runProcessDocs(params, progress) {
     if (zipToCheck.source === "confluence") {
       zipToCheck.confluencePageIndex = buildConfluencePageIndex(docs, zipToCheck);
     }
+    if (zipToCheck.isSlabExport) {
+      zipToCheck.slabPostSlugIndex = buildSlabPostSlugIndex(docs, zipToCheck);
+    }
   }
   await doAll(docs, async function(doc, docIndex, nextDoc) {
     try {
@@ -114988,6 +115098,7 @@ async function runProcessDocs(params, progress) {
     } catch (err) {
       recordFailure(doc, err);
     } finally {
+      releaseDocContent(doc, buffersReadFromZip);
       nextDoc();
     }
   });
@@ -115155,6 +115266,91 @@ function setSubtract(a, b) {
     return !b.includes(value);
   });
 }
+var UUID_RE2 = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+var EXTERNAL_MANIFEST_MAX_BYTES = 1048576;
+var EXTERNAL_MAX_ENTRIES = 500;
+var EXTERNAL_NAME_RE = /^[\p{L}\p{N}._-]{1,128}$/u;
+var EXTERNAL_REFUSED = "the server won't fetch this image; the card will link to the original URL";
+function checkExternalImages(zip, task, report) {
+  var state = { enabled: false, names: /* @__PURE__ */ new Set(), referenced: [] };
+  report.externalImages = state;
+  var collectionFile = zip.file("collection.yaml");
+  var manifestFile = zip.file("resources/external.json");
+  function readCollection() {
+    if (!collectionFile) {
+      return Promise.resolve();
+    }
+    return collectionFile.async("string").then(function(yamlString) {
+      try {
+        var collection = yaml.load(yamlString);
+        state.enabled = !!collection && typeof collection == "object" && collection.FetchExternalImages === true;
+      } catch (e) {
+      }
+    }, function() {
+    });
+  }
+  function readManifest() {
+    if (!manifestFile) {
+      return Promise.resolve();
+    }
+    return manifestFile.async("string").then(function(text2) {
+      var size2 = manifestFile._data && manifestFile._data.uncompressedSize || text2.length;
+      var dropped = "The server will ignore the whole manifest, so no external images will be fetched";
+      if (size2 > EXTERNAL_MANIFEST_MAX_BYTES) {
+        report.error(`resources/external.json is larger than ${EXTERNAL_MANIFEST_MAX_BYTES} bytes. ${dropped}.`);
+        return;
+      }
+      var manifest;
+      try {
+        manifest = JSON.parse(text2);
+      } catch (e) {
+        report.error(`resources/external.json is not valid JSON (${e.message}). ${dropped}.`);
+        return;
+      }
+      if (!manifest || typeof manifest != "object" || Array.isArray(manifest)) {
+        report.error(`resources/external.json must be a JSON object mapping file names to URLs. ${dropped}.`);
+        return;
+      }
+      var keys2 = Object.keys(manifest);
+      if (keys2.some(function(key) {
+        return typeof manifest[key] != "string";
+      })) {
+        report.error(`resources/external.json must map every file name to a URL string. ${dropped}.`);
+        return;
+      }
+      keys2.forEach(function(name, index) {
+        state.names.add(name);
+        var url = manifest[name];
+        var label = `resources/external.json entry "${escapeQuotes(name)}"`;
+        if (index >= EXTERNAL_MAX_ENTRIES) {
+          report.warn(`${label} is past the first ${EXTERNAL_MAX_ENTRIES} entries: ${EXTERNAL_REFUSED}.`);
+        } else if (!EXTERNAL_NAME_RE.test(name) || name.includes("..")) {
+          report.warn(`${label} has an invalid file name: ${EXTERNAL_REFUSED}.`);
+        } else if (url.trim() === "") {
+          report.warn(`${label} has a blank URL: ${EXTERNAL_REFUSED}.`);
+        } else if (!/^https:\/\//i.test(url.trim())) {
+          report.warn(`${label} is not an https URL: ${EXTERNAL_REFUSED}.`);
+        }
+        if (zip.file("resources/" + name)) {
+          report.info(`${label} also exists as resources/${escapeQuotes(name)} in the zip, so the server skips the fetch.`);
+        }
+      });
+    }, function() {
+    });
+  }
+  return readCollection().then(readManifest).then(function() {
+    if (manifestFile && !state.enabled) {
+      report.warn(`resources/external.json is included but collection.yaml doesn't set FetchExternalImages: true, so the server ignores it.`);
+    }
+    if (state.enabled && !manifestFile) {
+      report.warn(`collection.yaml sets FetchExternalImages: true but the zip has no resources/external.json.`);
+    }
+    if (state.enabled) {
+      report.syncError(`If this is a sync, FetchExternalImages is ignored (external images are only fetched on imports).`);
+    }
+    report.next();
+  });
+}
 function escapeQuotes(value) {
   value = value || "";
   return value.replace(/"/g, `\\"`);
@@ -115189,11 +115385,18 @@ function checkCardYaml(zip, task, report) {
   return zip.file(task.filename).async("string").then(function(yamlString) {
     try {
       var card = yaml.load(yamlString);
-      var possibleKeys = ["ExternalId", "ExternalUrl", "Title", "Tags", "externalLastUpdated"];
+      var possibleKeys = ["ExternalId", "ExternalUrl", "Title", "Tags", "externalLastUpdated", "ID"];
       var otherKeys = setSubtract(Object.keys(card), possibleKeys);
       otherKeys.forEach(function(key) {
         report.warn(`Unexpected property "${key}" in the card's yaml.`);
       });
+      if (card.ID !== void 0 && card.ID !== null) {
+        if (typeof card.ID != "string" || !UUID_RE2.test(card.ID)) {
+          report.error(`The card's ID property must be a UUID string (e.g. ID: "123e4567-e89b-12d3-a456-426614174000"), but it was: ${escapeHtml3(String(card.ID))}`);
+        } else {
+          report.info(`The card has an ID, so it updates the existing card with that ID in the same collection if one exists, otherwise a new card is created.`);
+        }
+      }
       if (!card.Title) {
         report.error(`A "Title" property is required in the card's yaml.`);
       } else {
@@ -115258,7 +115461,17 @@ function checkCardHtml(zip, task, report) {
             report.resourceReferences.push(reference2);
           }
         } else {
-          report.error(`The referenced resource "${reference2}" does not exist in the zip.`);
+          var ext = report.externalImages || { enabled: false, names: /* @__PURE__ */ new Set(), referenced: [] };
+          var extName = reference2.slice("resources/".length);
+          if (ext.names.has(extName) && ext.enabled) {
+            if (!ext.referenced.includes(extName)) {
+              ext.referenced.push(extName);
+            }
+          } else if (ext.names.has(extName)) {
+            report.error(`The referenced resource "${reference2}" is listed in resources/external.json but collection.yaml doesn't set FetchExternalImages: true, so it won't be fetched.`);
+          } else {
+            report.error(`The referenced resource "${reference2}" does not exist in the zip.`);
+          }
         }
       } else if (reference2.startsWith("cards/")) {
         if (zip.file(reference2 + ".yaml")) {
@@ -115515,7 +115728,7 @@ function checkCollectionYaml(zip, task, report) {
     }
     try {
       var collection = yaml.load(yamlString);
-      var possibleKeys = ["Items", "Tags", "Title", "Version"];
+      var possibleKeys = ["Items", "Tags", "Title", "Version", "FetchExternalImages", "ReuseFolders"];
       var otherKeys = setSubtract(Object.keys(collection), possibleKeys);
       otherKeys.forEach(function(key) {
         report.warn(`Unexpected property "${key}" in collection.yaml.`);
@@ -115532,6 +115745,14 @@ function checkCollectionYaml(zip, task, report) {
         }
       }
       var isV2 = collection.Version == 2;
+      ["FetchExternalImages", "ReuseFolders"].forEach(function(key) {
+        if (collection[key] !== void 0 && collection[key] !== null && typeof collection[key] != "boolean") {
+          report.warn(`The collection's ${key} property must be true or false (${key} = ${escapeHtml3(String(collection[key]))}).`);
+        }
+      });
+      if (collection.ReuseFolders === true && !isV2) {
+        report.warn(`ReuseFolders is ignored unless the collection has "Version: 2" (it applies only to Version 2 non-sync imports).`);
+      }
       if (collection.Items) {
         if (!Array.isArray(collection.Items)) {
           report.error(`The collection's Items list is optional, but if provided it must be a list.`);
@@ -115635,7 +115856,16 @@ function checkResource(zip, task, report) {
   } else {
     report.resources += 1;
   }
-  if (!report.resourceReferences.includes(task.filename)) {
+  if (task.filename == "resources/external.json") {
+    var ext = report.externalImages;
+    if (ext) {
+      ext.names.forEach(function(name) {
+        if (!ext.referenced.includes(name) && !report.resourceReferences.includes("resources/" + name)) {
+          report.warn(`resources/external.json entry "${escapeQuotes(name)}" isn't referenced by any cards.`);
+        }
+      });
+    }
+  } else if (!report.resourceReferences.includes(task.filename)) {
     report.warn(`Resource ${task.filename} is included in the zip but isn't referenced by any cards.`);
   }
   if (zip.file(task.filename)._data.uncompressedSize > 2e9) {
@@ -115732,6 +115962,7 @@ function checkZip(zip) {
       collectionRoots: [],
       tags: [],
       resourceReferences: [],
+      externalImages: { enabled: false, names: /* @__PURE__ */ new Set(), referenced: [] },
       cardLinks: 0,
       boardLinks: 0,
       checkCount: 0,
@@ -115805,6 +116036,7 @@ function checkZip(zip) {
         info(`Unexpected file ${fullPath}`);
       }
     }
+    tasks.push({ func: checkExternalImages, context: "resources/external.json", checks: 3, priority: 0 });
     tasks.push({ func: checkFolderTree, checks: 5, priority: 6 });
     tasks.push({ func: checkStats, checks: 3, priority: 10 });
     tasks.sort(function(a, b) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@getguru/import-engine",
-  "version": "0.1.1",
+  "version": "0.1.3",
   "description": "Command-line engine that turns Notion, Confluence, Google, Word, PDF, Markdown and spreadsheet exports into Guru import zips.",
   "bin": {
     "guru-import-engine": "docx-import-cli.js"