@promptbook/website-crawler 0.85.0-11 → 0.85.0-13

package/esm/typings/src/collection/constructors/createCollectionFromDirectory.d.ts

@@ -6,7 +6,7 @@ import type { PipelineCollection } from '../PipelineCollection';
 /**
  * Options for `createCollectionFromDirectory` function
  *
- * Note: `rootDirname` is not needed because it is the folder in which `.book.md` file is located
+ * Note: `rootDirname` is not needed because it is the folder in which `.book` or `.book.md` file is located
  *       This is not same as `path` which is the first argument of `createCollectionFromDirectory` - it can be a subfolder
  */
 type CreatePipelineCollectionFromDirectoryOptions = Omit<PrepareAndScrapeOptions, 'rootDirname'> & {
@@ -51,13 +51,13 @@ type CreatePipelineCollectionFromDirectoryOptions = Omit<PrepareAndScrapeOptions
  *
  * Note: Works only in Node.js environment because it reads the file system
  *
- * @param path - path to the directory with pipelines
+ * @param rootPath - path to the directory with pipelines
  * @param tools - Execution tools to be used for pipeline preparation if needed - If not provided, `$provideExecutionToolsForNode` will be used
  * @param options - Options for the collection creation
  * @returns PipelineCollection
  * @public exported from `@promptbook/node`
  */
-export declare function createCollectionFromDirectory(path: string_dirname, tools?: Pick<ExecutionTools, 'llm' | 'fs' | 'scrapers'>, options?: CreatePipelineCollectionFromDirectoryOptions): Promise<PipelineCollection>;
+export declare function createCollectionFromDirectory(rootPath: string_dirname, tools?: Pick<ExecutionTools, 'llm' | 'fs' | 'scrapers'>, options?: CreatePipelineCollectionFromDirectoryOptions): Promise<PipelineCollection>;
 export {};
 /**
  * TODO: [🖇] What about symlinks? Maybe option isSymlinksFollowed

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@promptbook/website-crawler",
-    "version": "0.85.0-11",
+    "version": "0.85.0-13",
     "description": "It's time for a paradigm shift. The future of software in plain English, French or Latin",
     "private": false,
     "sideEffects": false,
@@ -47,7 +47,7 @@
     "module": "./esm/index.es.js",
     "typings": "./esm/typings/src/_packages/website-crawler.index.d.ts",
     "peerDependencies": {
-        "@promptbook/core": "0.85.0-11"
+        "@promptbook/core": "0.85.0-13"
     },
     "dependencies": {
         "@mozilla/readability": "0.5.0",

package/umd/index.umd.js

@@ -25,7 +25,7 @@
      * @generated
      * @see https://github.com/webgptorg/promptbook
      */
-    var PROMPTBOOK_ENGINE_VERSION = '0.85.0-10';
+    var PROMPTBOOK_ENGINE_VERSION = '0.85.0-12';
     /**
      * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
      * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -1785,57 +1785,6 @@
         return true;
     }
 
-    /**
-     * Checks if an URL is reserved for private networks or localhost.
-     *
-     * Note: There are two simmilar functions:
-     * - `isUrlOnPrivateNetwork` which tests full URL
-     * - `isHostnameOnPrivateNetwork` *(this one)* which tests just hostname
-     *
-     * @public exported from `@promptbook/utils`
-     */
-    function isHostnameOnPrivateNetwork(hostname) {
-        if (hostname === 'example.com' ||
-            hostname === 'localhost' ||
-            hostname.endsWith('.localhost') ||
-            hostname.endsWith('.local') ||
-            hostname.endsWith('.test') ||
-            hostname === '127.0.0.1' ||
-            hostname === '::1') {
-            return true;
-        }
-        if (hostname.includes(':')) {
-            // IPv6
-            var ipParts = hostname.split(':');
-            return ipParts[0] === 'fc00' || ipParts[0] === 'fd00' || ipParts[0] === 'fe80';
-        }
-        else {
-            // IPv4
-            var ipParts = hostname.split('.').map(function (part) { return Number.parseInt(part, 10); });
-            return (ipParts[0] === 10 ||
-                (ipParts[0] === 172 && ipParts[1] >= 16 && ipParts[1] <= 31) ||
-                (ipParts[0] === 192 && ipParts[1] === 168));
-        }
-    }
-
-    /**
-     * Checks if an IP address or hostname is reserved for private networks or localhost.
-     *
-     * Note: There are two simmilar functions:
-     * - `isUrlOnPrivateNetwork` *(this one)* which tests full URL
-     * - `isHostnameOnPrivateNetwork` which tests just hostname
-     *
-     * @param {string} ipAddress - The IP address to check.
-     * @returns {boolean} Returns true if the IP address is reserved for private networks or localhost, otherwise false.
-     * @public exported from `@promptbook/utils`
-     */
-    function isUrlOnPrivateNetwork(url) {
-        if (typeof url === 'string') {
-            url = new URL(url);
-        }
-        return isHostnameOnPrivateNetwork(url.hostname);
-    }
-
     /**
      * Tests if given string is valid pipeline URL URL.
      *
@@ -1849,16 +1798,19 @@
         if (!isValidUrl(url)) {
             return false;
         }
-        if (!url.startsWith('https://')) {
+        if (!url.startsWith('https://') && !url.startsWith('http://') /* <- Note: [👣] */) {
             return false;
         }
         if (url.includes('#')) {
             // TODO: [🐠]
             return false;
         }
+        /*
+        Note: [👣][🧠] Is it secure to allow pipeline URLs on private and unsecured networks?
         if (isUrlOnPrivateNetwork(url)) {
             return false;
         }
+        */
         return true;
     }
     /**