@xapp/arachne-utils 1.8.2 → 1.8.3

package/lib/cleanForCrawl.d.ts

@@ -0,0 +1,12 @@
+/*! Copyright (c) 2024, XAPP AI */
+/**
+ * Prepare a URL for crawling.
+ *
+ * * Ensures there is no trailing slash
+ * * Ensures there is no hash
+ * * Ensures there is a scheme, defaults to https://
+ *
+ * @param url
+ * @returns
+ */
+export declare const cleanForCrawl: (url: string) => string;

package/lib/cleanForCrawl.js

@@ -0,0 +1,38 @@
+"use strict";
+/*! Copyright (c) 2024, XAPP AI */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.cleanForCrawl = void 0;
+/**
+ * Prepare a URL for crawling.
+ *
+ * * Ensures there is no trailing slash
+ * * Ensures there is no hash
+ * * Ensures there is a scheme, defaults to https://
+ *
+ * @param url
+ * @returns
+ */
+const cleanForCrawl = (url) => {
+    // First make sure they have a scheme
+    if (!url.startsWith("http")) {
+        // make sure it doesn't have // in the beginning
+        if (url.startsWith("//")) {
+            url = `https:${url}`;
+        }
+        else {
+            url = `https://${url}`;
+        }
+    }
+    // Remove trailing slash
+    if (url.endsWith("/")) {
+        url = url.slice(0, -1);
+    }
+    // Remove hash
+    const hashIndex = url.indexOf("#");
+    if (hashIndex > -1) {
+        url = url.slice(0, hashIndex);
+    }
+    return url;
+};
+exports.cleanForCrawl = cleanForCrawl;
+//# sourceMappingURL=cleanForCrawl.js.map

package/lib/cleanForCrawl.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cleanForCrawl.js","sourceRoot":"","sources":["../src/cleanForCrawl.ts"],"names":[],"mappings":";AAAA,kCAAkC;;;AAElC;;;;;;;;;GASG;AACI,MAAM,aAAa,GAAG,CAAC,GAAW,EAAU,EAAE;IACjD,qCAAqC;IACrC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;QAC1B,gDAAgD;QAChD,IAAI,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YACvB,GAAG,GAAG,SAAS,GAAG,EAAE,CAAC;QACzB,CAAC;aAAM,CAAC;YACJ,GAAG,GAAG,WAAW,GAAG,EAAE,CAAC;QAC3B,CAAC;IACL,CAAC;IAED,wBAAwB;IACxB,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QACpB,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC3B,CAAC;IAED,cAAc;IACd,MAAM,SAAS,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACnC,IAAI,SAAS,GAAG,CAAC,CAAC,EAAE,CAAC;QACjB,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;IAClC,CAAC;IAED,OAAO,GAAG,CAAC;AACf,CAAC,CAAA;AAvBY,QAAA,aAAa,iBAuBzB"}

package/lib/hashWebsite.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Returns a hash for a website
+ *
+ * @param url
+ * @throws {Error} if the URL is invalid
+ * @returns
+ */
+export declare function hashWebsite(url: string): string;

package/lib/hashWebsite.js

@@ -0,0 +1,44 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.hashWebsite = hashWebsite;
+/*! Copyright (c) 2024, XAPP AI */
+const normalizeURL_1 = require("./normalizeURL");
+const crypto = __importStar(require("crypto"));
+/**
+ * Returns a hash for a website
+ *
+ * @param url
+ * @throws {Error} if the URL is invalid
+ * @returns
+ */
+function hashWebsite(url) {
+    const normalizedUrl = (0, normalizeURL_1.normalizeURL)(url, {
+        stripProtocol: true,
+        stripWWW: true,
+    });
+    return crypto.createHash('sha256').update(normalizedUrl).digest('hex');
+}
+//# sourceMappingURL=hashWebsite.js.map

package/lib/hashWebsite.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"hashWebsite.js","sourceRoot":"","sources":["../src/hashWebsite.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AAWA,kCAMC;AAjBD,kCAAkC;AAClC,iDAA8C;AAC9C,+CAAiC;AAEjC;;;;;;GAMG;AACH,SAAgB,WAAW,CAAC,GAAW;IACnC,MAAM,aAAa,GAAG,IAAA,2BAAY,EAAC,GAAG,EAAE;QACpC,aAAa,EAAE,IAAI;QACnB,QAAQ,EAAE,IAAI;KACjB,CAAC,CAAC;IACH,OAAO,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC3E,CAAC"}

package/lib/index.d.ts

@@ -1,8 +1,11 @@
 /*! Copyright (c) 2020, XAPP AI */
 export * from "./ArachneURLPattern";
+export * from "./cleanForCrawl";
 export * from "./constants";
 export * from "./generateFollowThroughPattern";
 export * from "./hallucinationDetector";
 export * from "./hasExtension";
+export * from "./hashWebsite";
+export * from "./isValidURL";
 export * from "./normalizeURL";
 export * from "./promiseWithTimeout";

package/lib/index.js

@@ -16,10 +16,13 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 __exportStar(require("./ArachneURLPattern"), exports);
+__exportStar(require("./cleanForCrawl"), exports);
 __exportStar(require("./constants"), exports);
 __exportStar(require("./generateFollowThroughPattern"), exports);
 __exportStar(require("./hallucinationDetector"), exports);
 __exportStar(require("./hasExtension"), exports);
+__exportStar(require("./hashWebsite"), exports);
+__exportStar(require("./isValidURL"), exports);
 __exportStar(require("./normalizeURL"), exports);
 __exportStar(require("./promiseWithTimeout"), exports);
 //# sourceMappingURL=index.js.map

package/lib/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAAA,kCAAkC;;;;;;;;;;;;;;;;AAElC,sDAAoC;AACpC,8CAA4B;AAC5B,iEAA+C;AAC/C,0DAAwC;AACxC,iDAA+B;AAC/B,iDAA+B;AAC/B,uDAAqC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAAA,kCAAkC;;;;;;;;;;;;;;;;AAElC,sDAAoC;AACpC,kDAAgC;AAChC,8CAA4B;AAC5B,iEAA+C;AAC/C,0DAAwC;AACxC,iDAA+B;AAC/B,gDAA8B;AAC9B,+CAA6B;AAC7B,iDAA+B;AAC/B,uDAAqC"}

package/lib/isValidURL.d.ts

@@ -0,0 +1,9 @@
+/*! Copyright (c) 2024, XAPP AI */
+/**
+ * Determines if the URL is a valid URL.
+ *
+ * It attempts to create a new URL object from the string and returns true if it is successful.
+ * @param url
+ * @returns
+ */
+export declare const isValidURL: (url: string) => boolean;

package/lib/isValidURL.js

@@ -0,0 +1,22 @@
+"use strict";
+/*! Copyright (c) 2024, XAPP AI */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.isValidURL = void 0;
+/**
+ * Determines if the URL is a valid URL.
+ *
+ * It attempts to create a new URL object from the string and returns true if it is successful.
+ * @param url
+ * @returns
+ */
+const isValidURL = (url) => {
+    try {
+        new URL(url);
+        return true;
+    }
+    catch (e) {
+        return false;
+    }
+};
+exports.isValidURL = isValidURL;
+//# sourceMappingURL=isValidURL.js.map

package/lib/isValidURL.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"isValidURL.js","sourceRoot":"","sources":["../src/isValidURL.ts"],"names":[],"mappings":";AAAA,kCAAkC;;;AAElC;;;;;;GAMG;AACI,MAAM,UAAU,GAAG,CAAC,GAAW,EAAW,EAAE;IAC/C,IAAI,CAAC;QACD,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QACb,OAAO,IAAI,CAAC;IAChB,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,OAAO,KAAK,CAAC;IACjB,CAAC;AACL,CAAC,CAAA;AAPY,QAAA,UAAU,cAOtB"}

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@xapp/arachne-utils",
-    "version": "1.8.2",
+    "version": "1.8.3",
     "types": "lib/index",
     "main": "lib/index",
     "files": [
@@ -38,5 +38,5 @@
         "clean": "rm -rf ./lib/*",
         "test": "mocha --recursive -r ts-node/register \"./src/**/*.test.ts\""
     },
-    "gitHead": "c56c8143f976a7111003f219a9611673ba6f08ba"
+    "gitHead": "f506e88fbc093e8db03beaafadc4057f3647aece"
 }