@xapp/arachne-utils 1.8.0 → 1.8.3

package/lib/cleanForCrawl.d.ts

@@ -0,0 +1,12 @@
+/*! Copyright (c) 2024, XAPP AI */
+/**
+ * Prepare a URL for crawling.
+ *
+ * * Ensures there is no trailing slash
+ * * Ensures there is no hash
+ * * Ensures there is a scheme, defaults to https://
+ *
+ * @param url
+ * @returns
+ */
+export declare const cleanForCrawl: (url: string) => string;

package/lib/cleanForCrawl.js

@@ -0,0 +1,38 @@
+"use strict";
+/*! Copyright (c) 2024, XAPP AI */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.cleanForCrawl = void 0;
+/**
+ * Prepare a URL for crawling.
+ *
+ * * Ensures there is no trailing slash
+ * * Ensures there is no hash
+ * * Ensures there is a scheme, defaults to https://
+ *
+ * @param url
+ * @returns
+ */
+const cleanForCrawl = (url) => {
+    // First make sure they have a scheme
+    if (!url.startsWith("http")) {
+        // make sure it doesn't have // in the beginning
+        if (url.startsWith("//")) {
+            url = `https:${url}`;
+        }
+        else {
+            url = `https://${url}`;
+        }
+    }
+    // Remove trailing slash
+    if (url.endsWith("/")) {
+        url = url.slice(0, -1);
+    }
+    // Remove hash
+    const hashIndex = url.indexOf("#");
+    if (hashIndex > -1) {
+        url = url.slice(0, hashIndex);
+    }
+    return url;
+};
+exports.cleanForCrawl = cleanForCrawl;
+//# sourceMappingURL=cleanForCrawl.js.map

package/lib/cleanForCrawl.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cleanForCrawl.js","sourceRoot":"","sources":["../src/cleanForCrawl.ts"],"names":[],"mappings":";AAAA,kCAAkC;;;AAElC;;;;;;;;;GASG;AACI,MAAM,aAAa,GAAG,CAAC,GAAW,EAAU,EAAE;IACjD,qCAAqC;IACrC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;QAC1B,gDAAgD;QAChD,IAAI,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YACvB,GAAG,GAAG,SAAS,GAAG,EAAE,CAAC;QACzB,CAAC;aAAM,CAAC;YACJ,GAAG,GAAG,WAAW,GAAG,EAAE,CAAC;QAC3B,CAAC;IACL,CAAC;IAED,wBAAwB;IACxB,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QACpB,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC3B,CAAC;IAED,cAAc;IACd,MAAM,SAAS,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACnC,IAAI,SAAS,GAAG,CAAC,CAAC,EAAE,CAAC;QACjB,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;IAClC,CAAC;IAED,OAAO,GAAG,CAAC;AACf,CAAC,CAAA;AAvBY,QAAA,aAAa,iBAuBzB"}

package/lib/generateFollowThroughPattern.js

@@ -1,7 +1,7 @@
 "use strict";
 /*! Copyright (c) 2024, XAPP AI */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.generateFollowThroughPattern = void 0;
+exports.generateFollowThroughPattern = generateFollowThroughPattern;
 /**
  * Generates a URL pattern that will enable the crawler to
  * follow through sub paths for the provided URL.
@@ -56,5 +56,4 @@ function generateFollowThroughPattern(urlString) {
     }
     return pattern;
 }
-exports.generateFollowThroughPattern = generateFollowThroughPattern;
 //# sourceMappingURL=generateFollowThroughPattern.js.map

package/lib/generateFollowThroughPattern.js.map

@@ -1 +1 @@
-{"version":3,"file":"generateFollowThroughPattern.js","sourceRoot":"","sources":["../src/generateFollowThroughPattern.ts"],"names":[],"mappings":";AAAA,kCAAkC;;;AAElC;;;;;;;;;;;;;;;GAeG;AACH,SAAgB,4BAA4B,CAAC,SAAiB;IAE1D,IAAI,CAAC,SAAS,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CAAC,gEAAgE,CAAC,CAAC;IACtF,CAAC;IACD,0BAA0B;IAC1B,IAAI,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC1B,SAAS,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACvC,CAAC;IAED,IAAI,GAAQ,CAAC;IAEb,IAAI,CAAC;QACD,GAAG,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;IAC7B,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,MAAM,IAAI,KAAK,CAAC,oDAAoD,SAAS,aAAa,CAAC,CAAC;IAChG,CAAC;IAED,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC;IAC9B,IAAI,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;IACpB,iCAAiC;IACjC,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;QAC1B,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IACpC,CAAC;IAED,IAAI,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC;IACxB,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QACrB,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC7B,CAAC;IAED,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,gBAAgB,EAAE,kBAAkB,CAAC,CAAC,CAAC,8BAA8B;IAGzF,IAAI,OAAe,CAAC;IAEpB,MAAM,eAAe,GAAG,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC;IAE/E,+CAA+C;IAC/C,IAAI,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAClE,oBAAoB;QACpB,OAAO,GAAG,GAAG,eAAe,cAAc,IAAI,GAAG,IAAI,MAAM,CAAC;IAChE,CAAC;SAAM,CAAC;QACJ,OAAO,GAAG,GAAG,eAAe,KAAK,IAAI,GAAG,IAAI,MAAM,CAAC;IACvD,CAAC;IAED,OAAO,OAAO,CAAC;AACnB,CAAC;AA9CD,oEA8CC"}
+{"version":3,"file":"generateFollowThroughPattern.js","sourceRoot":"","sources":["../src/generateFollowThroughPattern.ts"],"names":[],"mappings":";AAAA,kCAAkC;;AAkBlC,oEA8CC;AA9DD;;;;;;;;;;;;;;;GAeG;AACH,SAAgB,4BAA4B,CAAC,SAAiB;IAE1D,IAAI,CAAC,SAAS,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CAAC,gEAAgE,CAAC,CAAC;IACtF,CAAC;IACD,0BAA0B;IAC1B,IAAI,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC1B,SAAS,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACvC,CAAC;IAED,IAAI,GAAQ,CAAC;IAEb,IAAI,CAAC;QACD,GAAG,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;IAC7B,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,MAAM,IAAI,KAAK,CAAC,oDAAoD,SAAS,aAAa,CAAC,CAAC;IAChG,CAAC;IAED,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC;IAC9B,IAAI,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;IACpB,iCAAiC;IACjC,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;QAC1B,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IACpC,CAAC;IAED,IAAI,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC;IACxB,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QACrB,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC7B,CAAC;IAED,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,gBAAgB,EAAE,kBAAkB,CAAC,CAAC,CAAC,8BAA8B;IAGzF,IAAI,OAAe,CAAC;IAEpB,MAAM,eAAe,GAAG,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC;IAE/E,+CAA+C;IAC/C,IAAI,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAClE,oBAAoB;QACpB,OAAO,GAAG,GAAG,eAAe,cAAc,IAAI,GAAG,IAAI,MAAM,CAAC;IAChE,CAAC;SAAM,CAAC;QACJ,OAAO,GAAG,GAAG,eAAe,KAAK,IAAI,GAAG,IAAI,MAAM,CAAC;IACvD,CAAC;IAED,OAAO,OAAO,CAAC;AACnB,CAAC"}

package/lib/hallucinationDetector.js

@@ -1,7 +1,7 @@
 "use strict";
 /*! Copyright (c) 2024, XAPP AI */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.isHallucinatedPhoneNumber = void 0;
+exports.isHallucinatedPhoneNumber = isHallucinatedPhoneNumber;
 /**
  * Looks for common hallucinated phone numbers.
  * @param number
@@ -28,5 +28,4 @@ function isHallucinatedPhoneNumber(number) {
     ];
     return hallucinatedPatterns.some(pattern => pattern.test(digits));
 }
-exports.isHallucinatedPhoneNumber = isHallucinatedPhoneNumber;
 //# sourceMappingURL=hallucinationDetector.js.map

package/lib/hallucinationDetector.js.map

@@ -1 +1 @@
-{"version":3,"file":"hallucinationDetector.js","sourceRoot":"","sources":["../src/hallucinationDetector.ts"],"names":[],"mappings":";AAAA,kCAAkC;;;AAElC;;;;GAIG;AACH,SAAgB,yBAAyB,CAAC,MAAc;IACpD,6CAA6C;IAC7C,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IAEzC,iCAAiC;IACjC,MAAM,oBAAoB,GAAG;QACzB,YAAY,EAAE,gCAAgC;QAC9C,cAAc,EAAE,yDAAyD;QACzE,kBAAkB;QAClB,WAAW;QACX,sCAAsC,EAAE,uBAAuB;QAC/D,4CAA4C,EAAE,sBAAsB;QACpE,+DAA+D,EAAE,qBAAqB;QACtF,qDAAqD,EAAE,oBAAoB;QAC3E,uBAAuB,EAAE,uBAAuB;QAChD,yBAAyB,EAAE,wBAAwB;QACnD,kBAAkB,EAAE,oBAAoB;QACxC,kBAAkB,EAAE,wBAAwB;QAC5C,WAAW,EAAE,sCAAsC;KACtD,CAAC;IAEF,OAAO,oBAAoB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;AACtE,CAAC;AAtBD,8DAsBC"}
+{"version":3,"file":"hallucinationDetector.js","sourceRoot":"","sources":["../src/hallucinationDetector.ts"],"names":[],"mappings":";AAAA,kCAAkC;;AAOlC,8DAsBC;AA3BD;;;;GAIG;AACH,SAAgB,yBAAyB,CAAC,MAAc;IACpD,6CAA6C;IAC7C,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IAEzC,iCAAiC;IACjC,MAAM,oBAAoB,GAAG;QACzB,YAAY,EAAE,gCAAgC;QAC9C,cAAc,EAAE,yDAAyD;QACzE,kBAAkB;QAClB,WAAW;QACX,sCAAsC,EAAE,uBAAuB;QAC/D,4CAA4C,EAAE,sBAAsB;QACpE,+DAA+D,EAAE,qBAAqB;QACtF,qDAAqD,EAAE,oBAAoB;QAC3E,uBAAuB,EAAE,uBAAuB;QAChD,yBAAyB,EAAE,wBAAwB;QACnD,kBAAkB,EAAE,oBAAoB;QACxC,kBAAkB,EAAE,wBAAwB;QAC5C,WAAW,EAAE,sCAAsC;KACtD,CAAC;IAEF,OAAO,oBAAoB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;AACtE,CAAC"}

package/lib/hasExtension.js

@@ -1,7 +1,7 @@
 "use strict";
 /*! Copyright (c) 2024, XAPP AI */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.hasExtension = void 0;
+exports.hasExtension = hasExtension;
 /**
  * Determines if the provided URL has a certain file extension.  This is useful to filter out undesirable URLs to follow such as .pdf, .jpg, .png or other file types that don't have hyperlinks themselves.
  *
@@ -49,5 +49,4 @@ function hasExtension(url, extension) {
     });
     return hasExtension;
 }
-exports.hasExtension = hasExtension;
 //# sourceMappingURL=hasExtension.js.map

package/lib/hasExtension.js.map

@@ -1 +1 @@
-{"version":3,"file":"hasExtension.js","sourceRoot":"","sources":["../src/hasExtension.ts"],"names":[],"mappings":";AAAA,kCAAkC;;;AAElC;;;;;;;GAOG;AACH,SAAgB,YAAY,CAAC,GAAiB,EAAE,SAA4B;IAExE,IAAI,CAAC,GAAG,EAAE,CAAC;QACP,OAAO,KAAK,CAAC;IACjB,CAAC;IAED,IAAI,OAAY,CAAC;IAEjB,IAAI,CAAC;QACD,OAAO,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IAC3B,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,IAAI,CAAC;YACD,sEAAsE;YACtE,OAAO,GAAG,IAAI,GAAG,CAAC,sBAAsB,GAAG,EAAE,CAAC,CAAC;QACnD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,OAAO,CAAC,KAAK,CAAC,yBAAyB,GAAG,EAAE,CAAC,CAAC;YAE9C,OAAO,KAAK,CAAC;QACjB,CAAC;IACL,CAAC;IAED,MAAM,IAAI,GAAG,OAAO,CAAC,QAAQ,CAAC;IAE9B,MAAM,cAAc,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IAE7C,MAAM,cAAc,GAAG,cAAc,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;IAErF,MAAM,cAAc,GAAa,EAAE,CAAC;IAEpC,IAAI,OAAO,SAAS,KAAK,QAAQ,EAAE,CAAC;QAChC,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACnC,CAAC;SAAM,CAAC;QACJ,cAAc,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;IACtC,CAAC;IAED,IAAI,YAAY,GAAY,KAAK,CAAC;IAElC,cAAc,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;QAC3B,8CAA8C;QAC9C,GAAG,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC;QACxB,GAAG,GAAG,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,EAAE,CAAC;QAE5C,IAAI,GAAG,KAAK,cAAc,CAAC,WAAW,EAAE,EAAE,CAAC;YACvC,YAAY,GAAG,IAAI,CAAC;QACxB,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,OAAO,YAAY,CAAC;AAExB,CAAC;AAjDD,oCAiDC"}
+{"version":3,"file":"hasExtension.js","sourceRoot":"","sources":["../src/hasExtension.ts"],"names":[],"mappings":";AAAA,kCAAkC;;AAUlC,oCAiDC;AAzDD;;;;;;;GAOG;AACH,SAAgB,YAAY,CAAC,GAAiB,EAAE,SAA4B;IAExE,IAAI,CAAC,GAAG,EAAE,CAAC;QACP,OAAO,KAAK,CAAC;IACjB,CAAC;IAED,IAAI,OAAY,CAAC;IAEjB,IAAI,CAAC;QACD,OAAO,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IAC3B,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,IAAI,CAAC;YACD,sEAAsE;YACtE,OAAO,GAAG,IAAI,GAAG,CAAC,sBAAsB,GAAG,EAAE,CAAC,CAAC;QACnD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,OAAO,CAAC,KAAK,CAAC,yBAAyB,GAAG,EAAE,CAAC,CAAC;YAE9C,OAAO,KAAK,CAAC;QACjB,CAAC;IACL,CAAC;IAED,MAAM,IAAI,GAAG,OAAO,CAAC,QAAQ,CAAC;IAE9B,MAAM,cAAc,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IAE7C,MAAM,cAAc,GAAG,cAAc,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;IAErF,MAAM,cAAc,GAAa,EAAE,CAAC;IAEpC,IAAI,OAAO,SAAS,KAAK,QAAQ,EAAE,CAAC;QAChC,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACnC,CAAC;SAAM,CAAC;QACJ,cAAc,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;IACtC,CAAC;IAED,IAAI,YAAY,GAAY,KAAK,CAAC;IAElC,cAAc,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;QAC3B,8CAA8C;QAC9C,GAAG,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC;QACxB,GAAG,GAAG,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,EAAE,CAAC;QAE5C,IAAI,GAAG,KAAK,cAAc,CAAC,WAAW,EAAE,EAAE,CAAC;YACvC,YAAY,GAAG,IAAI,CAAC;QACxB,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,OAAO,YAAY,CAAC;AAExB,CAAC"}

package/lib/hashWebsite.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Returns a hash for a website
+ *
+ * @param url
+ * @throws {Error} if the URL is invalid
+ * @returns
+ */
+export declare function hashWebsite(url: string): string;

package/lib/hashWebsite.js

@@ -0,0 +1,44 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.hashWebsite = hashWebsite;
+/*! Copyright (c) 2024, XAPP AI */
+const normalizeURL_1 = require("./normalizeURL");
+const crypto = __importStar(require("crypto"));
+/**
+ * Returns a hash for a website
+ *
+ * @param url
+ * @throws {Error} if the URL is invalid
+ * @returns
+ */
+function hashWebsite(url) {
+    const normalizedUrl = (0, normalizeURL_1.normalizeURL)(url, {
+        stripProtocol: true,
+        stripWWW: true,
+    });
+    return crypto.createHash('sha256').update(normalizedUrl).digest('hex');
+}
+//# sourceMappingURL=hashWebsite.js.map

package/lib/hashWebsite.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"hashWebsite.js","sourceRoot":"","sources":["../src/hashWebsite.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AAWA,kCAMC;AAjBD,kCAAkC;AAClC,iDAA8C;AAC9C,+CAAiC;AAEjC;;;;;;GAMG;AACH,SAAgB,WAAW,CAAC,GAAW;IACnC,MAAM,aAAa,GAAG,IAAA,2BAAY,EAAC,GAAG,EAAE;QACpC,aAAa,EAAE,IAAI;QACnB,QAAQ,EAAE,IAAI;KACjB,CAAC,CAAC;IACH,OAAO,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC3E,CAAC"}

package/lib/index.d.ts

@@ -1,8 +1,11 @@
 /*! Copyright (c) 2020, XAPP AI */
 export * from "./ArachneURLPattern";
+export * from "./cleanForCrawl";
 export * from "./constants";
 export * from "./generateFollowThroughPattern";
 export * from "./hallucinationDetector";
 export * from "./hasExtension";
+export * from "./hashWebsite";
+export * from "./isValidURL";
 export * from "./normalizeURL";
 export * from "./promiseWithTimeout";

package/lib/index.js

@@ -16,10 +16,13 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 __exportStar(require("./ArachneURLPattern"), exports);
+__exportStar(require("./cleanForCrawl"), exports);
 __exportStar(require("./constants"), exports);
 __exportStar(require("./generateFollowThroughPattern"), exports);
 __exportStar(require("./hallucinationDetector"), exports);
 __exportStar(require("./hasExtension"), exports);
+__exportStar(require("./hashWebsite"), exports);
+__exportStar(require("./isValidURL"), exports);
 __exportStar(require("./normalizeURL"), exports);
 __exportStar(require("./promiseWithTimeout"), exports);
 //# sourceMappingURL=index.js.map

package/lib/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAAA,kCAAkC;;;;;;;;;;;;;;;;AAElC,sDAAoC;AACpC,8CAA4B;AAC5B,iEAA+C;AAC/C,0DAAwC;AACxC,iDAA+B;AAC/B,iDAA+B;AAC/B,uDAAqC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAAA,kCAAkC;;;;;;;;;;;;;;;;AAElC,sDAAoC;AACpC,kDAAgC;AAChC,8CAA4B;AAC5B,iEAA+C;AAC/C,0DAAwC;AACxC,iDAA+B;AAC/B,gDAA8B;AAC9B,+CAA6B;AAC7B,iDAA+B;AAC/B,uDAAqC"}

package/lib/isValidURL.d.ts

@@ -0,0 +1,9 @@
+/*! Copyright (c) 2024, XAPP AI */
+/**
+ * Determines if the URL is a valid URL.
+ *
+ * It attempts to create a new URL object from the string and returns true if it is successful.
+ * @param url
+ * @returns
+ */
+export declare const isValidURL: (url: string) => boolean;

package/lib/isValidURL.js

@@ -0,0 +1,22 @@
+"use strict";
+/*! Copyright (c) 2024, XAPP AI */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.isValidURL = void 0;
+/**
+ * Determines if the URL is a valid URL.
+ *
+ * It attempts to create a new URL object from the string and returns true if it is successful.
+ * @param url
+ * @returns
+ */
+const isValidURL = (url) => {
+    try {
+        new URL(url);
+        return true;
+    }
+    catch (e) {
+        return false;
+    }
+};
+exports.isValidURL = isValidURL;
+//# sourceMappingURL=isValidURL.js.map

package/lib/isValidURL.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"isValidURL.js","sourceRoot":"","sources":["../src/isValidURL.ts"],"names":[],"mappings":";AAAA,kCAAkC;;;AAElC;;;;;;GAMG;AACI,MAAM,UAAU,GAAG,CAAC,GAAW,EAAW,EAAE;IAC/C,IAAI,CAAC;QACD,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QACb,OAAO,IAAI,CAAC;IAChB,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,OAAO,KAAK,CAAC;IACjB,CAAC;AACL,CAAC,CAAA;AAPY,QAAA,UAAU,cAOtB"}

package/lib/normalizeURL.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.normalizeURL = void 0;
+exports.normalizeURL = normalizeURL;
 /*! Copyright (c) 2020, XAPP AI */
 const normalize_url_1 = __importDefault(require("normalize-url"));
 /**
@@ -19,5 +19,4 @@ const normalize_url_1 = __importDefault(require("normalize-url"));
 function normalizeURL(url, options = { stripHash: true }) {
     return (0, normalize_url_1.default)(url, options);
 }
-exports.normalizeURL = normalizeURL;
 //# sourceMappingURL=normalizeURL.js.map

package/lib/normalizeURL.js.map

@@ -1 +1 @@
-{"version":3,"file":"normalizeURL.js","sourceRoot":"","sources":["../src/normalizeURL.ts"],"names":[],"mappings":";;;;;;AAAA,kCAAkC;AAClC,kEAAuD;AAEvD;;;;;;;;;GASG;AACH,SAAgB,YAAY,CAAC,GAAW,EAAE,UAAmB,EAAE,SAAS,EAAE,IAAI,EAAE;IAC5E,OAAO,IAAA,uBAAa,EAAC,GAAG,EAAE,OAAO,CAAC,CAAC;AACvC,CAAC;AAFD,oCAEC"}
+{"version":3,"file":"normalizeURL.js","sourceRoot":"","sources":["../src/normalizeURL.ts"],"names":[],"mappings":";;;;;AAaA,oCAEC;AAfD,kCAAkC;AAClC,kEAAuD;AAEvD;;;;;;;;;GASG;AACH,SAAgB,YAAY,CAAC,GAAW,EAAE,UAAmB,EAAE,SAAS,EAAE,IAAI,EAAE;IAC5E,OAAO,IAAA,uBAAa,EAAC,GAAG,EAAE,OAAO,CAAC,CAAC;AACvC,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@xapp/arachne-utils",
-    "version": "1.8.0",
+    "version": "1.8.3",
     "types": "lib/index",
     "main": "lib/index",
     "files": [
@@ -14,20 +14,20 @@
         "node": "^14 || ^16 || ^18 || ^20.0.0"
     },
     "devDependencies": {
-        "@microsoft/api-extractor": "7.43.8",
-        "@types/chai": "4.3.16",
-        "@types/mocha": "10.0.6",
-        "@types/node": "20.12.12",
+        "@microsoft/api-extractor": "7.47.4",
+        "@types/chai": "4.3.17",
+        "@types/mocha": "10.0.7",
+        "@types/node": "20.14.13",
         "@types/sinon": "17.0.3",
         "@types/sinon-chai": "3.2.12",
         "@xapp/config": "0.2.3",
-        "chai": "4.4.1",
-        "mocha": "10.4.0",
+        "chai": "4.5.0",
+        "mocha": "10.7.0",
         "sinon": "18.0.0",
         "sinon-chai": "3.7.0",
         "ts-node": "10.9.2",
-        "tslib": "2.6.2",
-        "typescript": "5.4.5"
+        "tslib": "2.6.3",
+        "typescript": "5.5.4"
     },
     "dependencies": {
         "normalize-url": "6.1.0"
@@ -38,5 +38,5 @@
         "clean": "rm -rf ./lib/*",
         "test": "mocha --recursive -r ts-node/register \"./src/**/*.test.ts\""
     },
-    "gitHead": "21872d37222311f6edc36d87d03d9735a12ba2d1"
+    "gitHead": "f506e88fbc093e8db03beaafadc4057f3647aece"
 }