@xapp/arachne-utils 1.7.0 → 1.7.1

package/lib/generateFollowThroughPattern.d.ts

@@ -0,0 +1,18 @@
+/*! Copyright (c) 2024, XAPP AI */
+/**
+ * Generates a URL pattern that will enable the crawler to
+ * follow through sub paths for the provided URL.
+ *
+ * For example:
+ *
+ * www.example.com/faqs
+ *
+ * will become
+ *
+ * https://[(www.)?]example.com/faqs[.+]
+ *
+ * NOTE: This will throw an error if the URL was not a proper URL.
+ *
+ * @param urlString
+ */
+export declare function generateFollowThroughPattern(urlString: string): string;

package/lib/generateFollowThroughPattern.js

@@ -0,0 +1,59 @@
+"use strict";
+/*! Copyright (c) 2024, XAPP AI */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.generateFollowThroughPattern = void 0;
+/**
+ * Generates a URL pattern that will enable the crawler to
+ * follow through sub paths for the provided URL.
+ *
+ * For example:
+ *
+ * www.example.com/faqs
+ *
+ * will become
+ *
+ * https://[(www.)?]example.com/faqs[.+]
+ *
+ * NOTE: This will throw an error if the URL was not a proper URL.
+ *
+ * @param urlString
+ */
+function generateFollowThroughPattern(urlString) {
+    if (!urlString) {
+        throw new Error(`Unabled to generate follow through patterns, URL not provided.`);
+    }
+    // Remove the trailing "/"
+    if (urlString.endsWith("/")) {
+        urlString = urlString.slice(0, -1);
+    }
+    let url;
+    try {
+        url = new URL(urlString);
+    }
+    catch (e) {
+        throw new Error(`Unabled to generate follow through patterns, URL ${urlString} is invalid`);
+    }
+    const protocol = url.protocol;
+    let host = url.host;
+    // Remove the www. if it is there
+    if (host.startsWith("www.")) {
+        host = host.replace("www.", "");
+    }
+    let path = url.pathname;
+    if (path.endsWith("/")) {
+        path = path.slice(0, -1);
+    }
+    let pattern;
+    const protocolPattern = protocol.startsWith("http") ? "http[(s)?]:" : protocol;
+    // Either starts with www or is just domain.tld
+    if (url.host.startsWith("www.") || url.host.split(".").length === 2) {
+        // make www optional
+        pattern = `${protocolPattern}//[(www.)?]${host}${path}[.+]`;
+    }
+    else {
+        pattern = `${protocolPattern}//${host}${path}[.+]`;
+    }
+    return pattern;
+}
+exports.generateFollowThroughPattern = generateFollowThroughPattern;
+//# sourceMappingURL=generateFollowThroughPattern.js.map

package/lib/generateFollowThroughPattern.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"generateFollowThroughPattern.js","sourceRoot":"","sources":["../src/generateFollowThroughPattern.ts"],"names":[],"mappings":";AAAA,kCAAkC;;;AAElC;;;;;;;;;;;;;;;GAeG;AACH,SAAgB,4BAA4B,CAAC,SAAiB;IAE1D,IAAI,CAAC,SAAS,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CAAC,gEAAgE,CAAC,CAAC;IACtF,CAAC;IACD,0BAA0B;IAC1B,IAAI,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC1B,SAAS,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACvC,CAAC;IAED,IAAI,GAAQ,CAAC;IAEb,IAAI,CAAC;QACD,GAAG,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;IAC7B,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,MAAM,IAAI,KAAK,CAAC,oDAAoD,SAAS,aAAa,CAAC,CAAC;IAChG,CAAC;IAED,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC;IAC9B,IAAI,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;IACpB,iCAAiC;IACjC,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;QAC1B,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IACpC,CAAC;IAED,IAAI,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC;IACxB,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QACrB,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC7B,CAAC;IAED,IAAI,OAAe,CAAC;IAEpB,MAAM,eAAe,GAAG,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC;IAE/E,+CAA+C;IAC/C,IAAI,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAClE,oBAAoB;QACpB,OAAO,GAAG,GAAG,eAAe,cAAc,IAAI,GAAG,IAAI,MAAM,CAAC;IAChE,CAAC;SAAM,CAAC;QACJ,OAAO,GAAG,GAAG,eAAe,KAAK,IAAI,GAAG,IAAI,MAAM,CAAC;IACvD,CAAC;IAED,OAAO,OAAO,CAAC;AACnB,CAAC;AA3CD,oEA2CC"}

package/lib/index.d.ts

@@ -1,7 +1,8 @@
 /*! Copyright (c) 2020, XAPP AI */
 export * from "./ArachneURLPattern";
 export * from "./constants";
-export * from "./normalizeURL";
-export * from "./promiseWithTimeout";
+export * from "./generateFollowThroughPattern";
 export * from "./hallucinationDetector";
 export * from "./hasExtension";
+export * from "./normalizeURL";
+export * from "./promiseWithTimeout";

package/lib/index.js

@@ -17,8 +17,9 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
 Object.defineProperty(exports, "__esModule", { value: true });
 __exportStar(require("./ArachneURLPattern"), exports);
 __exportStar(require("./constants"), exports);
-__exportStar(require("./normalizeURL"), exports);
-__exportStar(require("./promiseWithTimeout"), exports);
+__exportStar(require("./generateFollowThroughPattern"), exports);
 __exportStar(require("./hallucinationDetector"), exports);
 __exportStar(require("./hasExtension"), exports);
+__exportStar(require("./normalizeURL"), exports);
+__exportStar(require("./promiseWithTimeout"), exports);
 //# sourceMappingURL=index.js.map

package/lib/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAAA,kCAAkC;;;;;;;;;;;;;;;;AAElC,sDAAoC;AACpC,8CAA4B;AAC5B,iDAA+B;AAC/B,uDAAqC;AACrC,0DAAwC;AACxC,iDAA+B"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAAA,kCAAkC;;;;;;;;;;;;;;;;AAElC,sDAAoC;AACpC,8CAA4B;AAC5B,iEAA+C;AAC/C,0DAAwC;AACxC,iDAA+B;AAC/B,iDAA+B;AAC/B,uDAAqC"}

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@xapp/arachne-utils",
-    "version": "1.7.0",
+    "version": "1.7.1",
     "types": "lib/index",
     "main": "lib/index",
     "files": [
@@ -38,5 +38,5 @@
         "clean": "rm -rf ./lib/*",
         "test": "mocha --recursive -r ts-node/register \"./src/**/*.test.ts\""
     },
-    "gitHead": "2d24174076a3a351f2a70f2bf5da88904cf80570"
+    "gitHead": "e47078cc574afec8f4be9f05055d52b7a0171fd1"
 }