@xapp/arachne-utils 1.2.2 → 1.3.3

package/lib/constants.d.ts

@@ -0,0 +1,5 @@
+/*! Copyright (c) 2024, XAPP AI */
+export declare const IMAGE_EXTENSIONS: string[];
+export declare const VIDEO_EXTENSIONS: string[];
+export declare const AUDIO_EXTENSIONS: string[];
+export declare const COMMON_DOCUMENT_EXTENSIONS: string[];

package/lib/constants.js

@@ -0,0 +1,12 @@
+"use strict";
+/*! Copyright (c) 2024, XAPP AI */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.COMMON_DOCUMENT_EXTENSIONS = exports.AUDIO_EXTENSIONS = exports.VIDEO_EXTENSIONS = exports.IMAGE_EXTENSIONS = void 0;
+// common image extensions
+exports.IMAGE_EXTENSIONS = ['.jpg', '.jpeg', '.png', '.gif', '.svg', '.webp'];
+// common video extensions
+exports.VIDEO_EXTENSIONS = ['.mp4', '.webm', '.ogg'];
+// common audio extensions
+exports.AUDIO_EXTENSIONS = ['.mp3', '.wav', '.ogg'];
+exports.COMMON_DOCUMENT_EXTENSIONS = ['.pdf', '.doc', '.docx', '.xls', '.xlsx', '.ppt', '.pptx'];
+//# sourceMappingURL=constants.js.map

package/lib/constants.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"constants.js","sourceRoot":"","sources":["../src/constants.ts"],"names":[],"mappings":";AAAA,kCAAkC;;;AAElC,0BAA0B;AACb,QAAA,gBAAgB,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;AAEnF,0BAA0B;AACb,QAAA,gBAAgB,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;AAE1D,0BAA0B;AACb,QAAA,gBAAgB,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;AAE5C,QAAA,0BAA0B,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC"}

package/lib/hasExtension.d.ts

@@ -0,0 +1,10 @@
+/*! Copyright (c) 2024, XAPP AI */
+/**
+ * Determines if the provided URL has a certain file extension.  This is useful to filter out undesirable URLs to follow such as .pdf, .jpg, .png or other file types that don't have hyperlinks themselves.
+ *
+ * It assumes that a URL without an extension is HTML.
+ *
+ * @param url
+ * @param extension
+ */
+export declare function hasExtension(url: string | URL, extension: string | string[]): boolean;

package/lib/hasExtension.js

@@ -0,0 +1,53 @@
+"use strict";
+/*! Copyright (c) 2024, XAPP AI */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.hasExtension = void 0;
+/**
+ * Determines if the provided URL has a certain file extension.  This is useful to filter out undesirable URLs to follow such as .pdf, .jpg, .png or other file types that don't have hyperlinks themselves.
+ *
+ * It assumes that a URL without an extension is HTML.
+ *
+ * @param url
+ * @param extension
+ */
+function hasExtension(url, extension) {
+    if (!url) {
+        return false;
+    }
+    let testUrl;
+    try {
+        testUrl = new URL(url);
+    }
+    catch (e) {
+        try {
+            // URL here doesn't really matter, we just need to make sure it parses
+            testUrl = new URL(`https://example.com${url}`);
+        }
+        catch (e) {
+            console.error(`⚠️ Error parsing URL: ${url}`);
+            return false;
+        }
+    }
+    const path = testUrl.pathname;
+    const extensionIndex = path.lastIndexOf('.');
+    const foundExtension = extensionIndex > 0 ? path.substring(extensionIndex) : '.html';
+    const testExtensions = [];
+    if (typeof extension === 'string') {
+        testExtensions.push(extension);
+    }
+    else {
+        testExtensions.push(...extension);
+    }
+    let hasExtension = false;
+    testExtensions.forEach((ext) => {
+        // lowercase and ensure we start with a period
+        ext = ext.toLowerCase();
+        ext = ext.startsWith('.') ? ext : `.${ext}`;
+        if (ext === foundExtension.toLowerCase()) {
+            hasExtension = true;
+        }
+    });
+    return hasExtension;
+}
+exports.hasExtension = hasExtension;
+//# sourceMappingURL=hasExtension.js.map

package/lib/hasExtension.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"hasExtension.js","sourceRoot":"","sources":["../src/hasExtension.ts"],"names":[],"mappings":";AAAA,kCAAkC;;;AAElC;;;;;;;GAOG;AACH,SAAgB,YAAY,CAAC,GAAiB,EAAE,SAA4B;IAExE,IAAI,CAAC,GAAG,EAAE,CAAC;QACP,OAAO,KAAK,CAAC;IACjB,CAAC;IAED,IAAI,OAAY,CAAC;IAEjB,IAAI,CAAC;QACD,OAAO,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IAC3B,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,IAAI,CAAC;YACD,sEAAsE;YACtE,OAAO,GAAG,IAAI,GAAG,CAAC,sBAAsB,GAAG,EAAE,CAAC,CAAC;QACnD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,OAAO,CAAC,KAAK,CAAC,yBAAyB,GAAG,EAAE,CAAC,CAAC;YAE9C,OAAO,KAAK,CAAC;QACjB,CAAC;IACL,CAAC;IAED,MAAM,IAAI,GAAG,OAAO,CAAC,QAAQ,CAAC;IAE9B,MAAM,cAAc,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IAE7C,MAAM,cAAc,GAAG,cAAc,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;IAErF,MAAM,cAAc,GAAa,EAAE,CAAC;IAEpC,IAAI,OAAO,SAAS,KAAK,QAAQ,EAAE,CAAC;QAChC,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACnC,CAAC;SAAM,CAAC;QACJ,cAAc,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;IACtC,CAAC;IAED,IAAI,YAAY,GAAY,KAAK,CAAC;IAElC,cAAc,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;QAC3B,8CAA8C;QAC9C,GAAG,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC;QACxB,GAAG,GAAG,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,EAAE,CAAC;QAE5C,IAAI,GAAG,KAAK,cAAc,CAAC,WAAW,EAAE,EAAE,CAAC;YACvC,YAAY,GAAG,IAAI,CAAC;QACxB,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,OAAO,YAAY,CAAC;AAExB,CAAC;AAjDD,oCAiDC"}

package/lib/index.d.ts

@@ -1,5 +1,7 @@
 /*! Copyright (c) 2020, XAPP AI */
 export * from "./ArachneURLPattern";
+export * from "./constants";
 export * from "./normalizeURL";
 export * from "./promiseWithTimeout";
 export * from "./hallucinationDetector";
+export * from "./hasExtension";

package/lib/index.js

@@ -16,7 +16,9 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 __exportStar(require("./ArachneURLPattern"), exports);
+__exportStar(require("./constants"), exports);
 __exportStar(require("./normalizeURL"), exports);
 __exportStar(require("./promiseWithTimeout"), exports);
 __exportStar(require("./hallucinationDetector"), exports);
+__exportStar(require("./hasExtension"), exports);
 //# sourceMappingURL=index.js.map

package/lib/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAAA,kCAAkC;;;;;;;;;;;;;;;;AAElC,sDAAoC;AACpC,iDAA+B;AAC/B,uDAAqC;AACrC,0DAAwC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAAA,kCAAkC;;;;;;;;;;;;;;;;AAElC,sDAAoC;AACpC,8CAA4B;AAC5B,iDAA+B;AAC/B,uDAAqC;AACrC,0DAAwC;AACxC,iDAA+B"}

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@xapp/arachne-utils",
-    "version": "1.2.2",
+    "version": "1.3.3",
     "types": "lib/index",
     "main": "lib/index",
     "files": [
@@ -11,13 +11,13 @@
         "access": "public"
     },
     "engines": {
-        "node": "^14 || ^16 || ^18"
+        "node": "^14 || ^16 || ^18 || ^20.0.0"
     },
     "devDependencies": {
-        "@microsoft/api-extractor": "7.39.1",
+        "@microsoft/api-extractor": "7.39.4",
         "@types/chai": "4.3.11",
         "@types/mocha": "10.0.6",
-        "@types/node": "18.19.8",
+        "@types/node": "20.11.7",
         "@types/sinon": "17.0.3",
         "@types/sinon-chai": "3.2.12",
         "@xapp/config": "0.2.3",
@@ -38,5 +38,5 @@
         "clean": "rm -rf ./lib/*",
         "test": "mocha --recursive -r ts-node/register \"./src/**/*.test.ts\""
     },
-    "gitHead": "3b4279e8e24beba12e3f8e29275f83ef1ce02086"
+    "gitHead": "14a5571867b6172a795541e8a34437ffd43e6384"
 }