@opentermsarchive/engine 11.0.0 → 11.0.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opentermsarchive/engine",
-  "version": "11.0.0",
+  "version": "11.0.2",
   "description": "Tracks and makes visible changes to the terms of online services",
   "homepage": "https://opentermsarchive.org",
   "bugs": {
@@ -84,7 +84,7 @@
     "iconv-lite": "^0.7.0",
     "joplin-turndown-plugin-gfm": "^1.0.12",
     "js-yaml": "^4.1.0",
-    "jsdom": "^27.0.0",
+    "jsdom": "^29.0.1",
     "json-source-map": "^0.6.1",
     "mime": "^4.1.0",
     "mocha": "^11.7.2",
@@ -116,5 +116,10 @@
   },
   "engines": {
     "node": ">=20 <=24"
+  },
+  "overrides": {
+    "jsdom": {
+      "lru-cache": "11.2.7"
+    }
   }
 }

package/src/archivist/services/sourceDocument.js

@@ -1,3 +1,7 @@
+import path from 'path';
+
+import mime from 'mime';
+
 export default class SourceDocument {
   /**
    * Represents a source document containing web content and metadata for extraction.
@@ -21,7 +25,7 @@ export default class SourceDocument {
     this.filters = filters;
     this.content = content;
     this.mimeType = mimeType;
-    this.id = new URL(location).pathname.split('/').filter(Boolean).join('-');
+    this.id = SourceDocument.generateId(location);
   }
 
   get cssSelectors() {
@@ -60,6 +64,20 @@ export default class SourceDocument {
     return [selector];
   }
 
+  static generateId(location) {
+    const ILLEGAL_CHARACTERS = /[\\:"<>|*?]/g; // Characters forbidden in filenames for cross-platform compatibility; see https://github.com/actions/toolkit/blob/main/packages/artifact/src/internal/upload/path-and-artifact-name-validation.ts
+
+    const pathname = decodeURIComponent(new URL(location).pathname);
+    const extension = path.extname(pathname);
+    const pathnameWithoutExtension = mime.getType(extension) ? pathname.slice(0, -extension.length) : pathname; // Remove file extension when it corresponds to a known MIME type, as the extension is not part of the document's identity but a web server implementation detail
+
+    return pathnameWithoutExtension
+      .split('/')
+      .filter(Boolean)
+      .join('-')
+      .replace(ILLEGAL_CHARACTERS, '_');
+  }
+
   toPersistence() {
     return {
       fetch: this.location,

package/src/archivist/services/sourceDocument.test.js

@@ -157,6 +157,63 @@ describe('SourceDocument', () => {
     });
   });
 
+  describe('#generateId', () => {
+    it('generates ID from URL pathname', () => {
+      expect(new SourceDocument({ location: 'https://example.com/legal/terms' }).id).to.equal('legal-terms');
+    });
+
+    it('returns empty string for root URL', () => {
+      expect(new SourceDocument({ location: 'https://example.com/' }).id).to.equal('');
+    });
+
+    it('decodes URL-encoded characters', () => {
+      expect(new SourceDocument({ location: 'https://example.com/terms%20of%20service' }).id).to.equal('terms of service');
+    });
+
+    it('removes known file extension from URL pathname', () => {
+      expect(new SourceDocument({ location: 'https://example.com/en.html' }).id).to.equal('en');
+    });
+
+    it('removes only the last file extension', () => {
+      expect(new SourceDocument({ location: 'https://example.com/terms.backup.html' }).id).to.equal('terms.backup');
+    });
+
+    it('keeps unknown extension in URL pathname', () => {
+      expect(new SourceDocument({ location: 'https://example.com/terms.of.service' }).id).to.equal('terms.of.service');
+    });
+    it('decodes URL-encoded characters before replacing illegal ones', () => {
+      expect(new SourceDocument({ location: 'https://example.com/terms%3Aof%3Aservice' }).id).to.equal('terms_of_service');
+    });
+
+    context('replaces characters that are illegal in filenames for cross-platform compatibility', () => {
+      const ILLEGAL_CHARACTERS_IN_URL_PATHNAME = {
+        ':': 'colon',
+        '"': 'double quote',
+        '<': 'less than',
+        '>': 'greater than',
+        '|': 'vertical bar',
+        '*': 'asterisk',
+      };
+
+      for (const [ character, name ] of Object.entries(ILLEGAL_CHARACTERS_IN_URL_PATHNAME)) {
+        it(`replaces ${name} "${character}"`, () => {
+          expect(new SourceDocument({ location: `https://example.com/before${character}after` }).id).to.equal('before_after');
+        });
+      }
+
+      const ILLEGAL_CHARACTERS_ENCODED_IN_URL = {
+        '%5C': 'backslash',
+        '%3F': 'question mark',
+      };
+
+      for (const [ encoded, name ] of Object.entries(ILLEGAL_CHARACTERS_ENCODED_IN_URL)) {
+        it(`replaces ${name} decoded from "${encoded}"`, () => {
+          expect(new SourceDocument({ location: `https://example.com/before${encoded}after` }).id).to.equal('before_after');
+        });
+      }
+    });
+  });
+
   describe('#toPersistence', () => {
     it('converts basic source document declarations into JSON representation', () => {
       const result = new SourceDocument({