@promptbook/website-crawler 0.77.1 → 0.78.2

package/umd/index.umd.js

@@ -22,7 +22,7 @@
      *
      * @see https://github.com/webgptorg/promptbook
      */
-    var PROMPTBOOK_ENGINE_VERSION = '0.77.0';
+    var PROMPTBOOK_ENGINE_VERSION = '0.78.1';
     /**
      * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
      * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -202,164 +202,25 @@
     }
 
     /**
-     * This error type indicates that the error should not happen and its last check before crashing with some other error
+     * Name for the Promptbook
+     *
+     * TODO: [🗽] Unite branding and make single place for it
      *
      * @public exported from `@promptbook/core`
      */
-    var UnexpectedError = /** @class */ (function (_super) {
-        __extends(UnexpectedError, _super);
-        function UnexpectedError(message) {
-            var _this = _super.call(this, spaceTrim.spaceTrim(function (block) { return "\n                    ".concat(block(message), "\n\n                    Note: This error should not happen.\n                    It's probbably a bug in the pipeline collection\n\n                    Please report issue:\n                    https://github.com/webgptorg/promptbook/issues\n\n                    Or contact us on me@pavolhejny.com\n\n                "); })) || this;
-            _this.name = 'UnexpectedError';
-            Object.setPrototypeOf(_this, UnexpectedError.prototype);
-            return _this;
-        }
-        return UnexpectedError;
-    }(Error));
-
+    var NAME = "Promptbook";
     /**
-     * Checks if the value is [🚉] serializable as JSON
-     * If not, throws an UnexpectedError with a rich error message and tracking
-     *
-     * - Almost all primitives are serializable BUT:
-     * - `undefined` is not serializable
-     * - `NaN` is not serializable
-     * - Objects and arrays are serializable if all their properties are serializable
-     * - Functions are not serializable
-     * - Circular references are not serializable
-     * - `Date` objects are not serializable
-     * - `Map` and `Set` objects are not serializable
-     * - `RegExp` objects are not serializable
-     * - `Error` objects are not serializable
-     * - `Symbol` objects are not serializable
-     * - And much more...
+     * Email of the responsible person
      *
-     * @throws UnexpectedError if the value is not serializable as JSON
-     * @public exported from `@promptbook/utils`
-     */
-    function checkSerializableAsJson(name, value) {
-        var e_1, _a;
-        if (value === undefined) {
-            throw new UnexpectedError("".concat(name, " is undefined"));
-        }
-        else if (value === null) {
-            return;
-        }
-        else if (typeof value === 'boolean') {
-            return;
-        }
-        else if (typeof value === 'number' && !isNaN(value)) {
-            return;
-        }
-        else if (typeof value === 'string') {
-            return;
-        }
-        else if (typeof value === 'symbol') {
-            throw new UnexpectedError("".concat(name, " is symbol"));
-        }
-        else if (typeof value === 'function') {
-            throw new UnexpectedError("".concat(name, " is function"));
-        }
-        else if (typeof value === 'object' && Array.isArray(value)) {
-            for (var i = 0; i < value.length; i++) {
-                checkSerializableAsJson("".concat(name, "[").concat(i, "]"), value[i]);
-            }
-        }
-        else if (typeof value === 'object') {
-            if (value instanceof Date) {
-                throw new UnexpectedError(spaceTrim__default["default"]("\n                    ".concat(name, " is Date\n\n                    Use `string_date_iso8601` instead\n                ")));
-            }
-            else if (value instanceof Map) {
-                throw new UnexpectedError("".concat(name, " is Map"));
-            }
-            else if (value instanceof Set) {
-                throw new UnexpectedError("".concat(name, " is Set"));
-            }
-            else if (value instanceof RegExp) {
-                throw new UnexpectedError("".concat(name, " is RegExp"));
-            }
-            else if (value instanceof Error) {
-                throw new UnexpectedError(spaceTrim__default["default"]("\n                    ".concat(name, " is unserialized Error\n\n                    Use function `serializeError`\n                ")));
-            }
-            else {
-                try {
-                    for (var _b = __values(Object.entries(value)), _c = _b.next(); !_c.done; _c = _b.next()) {
-                        var _d = __read(_c.value, 2), subName = _d[0], subValue = _d[1];
-                        if (subValue === undefined) {
-                            // Note: undefined in object is serializable - it is just omited
-                            continue;
-                        }
-                        checkSerializableAsJson("".concat(name, ".").concat(subName), subValue);
-                    }
-                }
-                catch (e_1_1) { e_1 = { error: e_1_1 }; }
-                finally {
-                    try {
-                        if (_c && !_c.done && (_a = _b.return)) _a.call(_b);
-                    }
-                    finally { if (e_1) throw e_1.error; }
-                }
-                try {
-                    JSON.stringify(value); // <- TODO: [0]
-                }
-                catch (error) {
-                    if (!(error instanceof Error)) {
-                        throw error;
-                    }
-                    throw new UnexpectedError(spaceTrim__default["default"](function (block) { return "\n                            ".concat(name, " is not serializable\n\n                            ").concat(block(error.toString()), "\n                        "); }));
-                }
-                /*
-                TODO: [0] Is there some more elegant way to check circular references?
-                const seen = new Set();
-                const stack = [{ value }];
-                while (stack.length > 0) {
-                    const { value } = stack.pop()!;
-                    if (typeof value === 'object' && value !== null) {
-                        if (seen.has(value)) {
-                            throw new UnexpectedError(`${name} has circular reference`);
-                        }
-                        seen.add(value);
-                        if (Array.isArray(value)) {
-                            stack.push(...value.map((value) => ({ value })));
-                        } else {
-                            stack.push(...Object.values(value).map((value) => ({ value })));
-                        }
-                    }
-                }
-                */
-                return;
-            }
-        }
-        else {
-            throw new UnexpectedError("".concat(name, " is unknown"));
-        }
-    }
-    /**
-     * TODO: [🧠][🛣] More elegant way to tracking than passing `name`
-     * TODO: [🧠][main] !!! In-memory cache of same values to prevent multiple checks
-     * Note: [🐠] This is how `checkSerializableAsJson` + `isSerializableAsJson` together can just retun true/false or rich error message
+     * @public exported from `@promptbook/core`
      */
-
+    var ADMIN_EMAIL = 'me@pavolhejny.com';
     /**
-     * @@@
-     * @@@
+     * Name of the responsible person for the Promptbook on GitHub
      *
-     * Note: This function mutates the object and returns the original (but mutated-deep-freezed) object
-     *
-     * @param name - Name of the object for debugging purposes
-     * @param objectValue - Object to be deeply frozen
-     * @returns The same object as the input, but deeply frozen
-     * @private this is in comparison to `deepFreeze` a more specific utility and maybe not very good practice to use without specific reason and considerations
-     */
-    function $asDeeplyFrozenSerializableJson(name, objectValue) {
-        checkSerializableAsJson(name, objectValue);
-        return $deepFreeze(objectValue);
-    }
-    /**
-     * TODO: [🧠][🛣] More elegant way to tracking than passing `name`
-     * TODO: [🧠] Is there a way how to meaningfully test this utility
+     * @public exported from `@promptbook/core`
      */
-
+    var ADMIN_GITHUB_NAME = 'hejny';
     // <- TODO: [🧠] Better system for generator warnings - not always "code" and "by `@promptbook/cli`"
     /**
      * The maximum number of iterations for a loops
@@ -411,7 +272,8 @@
      *
      * @public exported from `@promptbook/core`
      */
-    var RESERVED_PARAMETER_NAMES = $asDeeplyFrozenSerializableJson('RESERVED_PARAMETER_NAMES', [
+    var RESERVED_PARAMETER_NAMES = 
+    /* !!!!!! $asDeeplyFrozenSerializableJson('RESERVED_PARAMETER_NAMES', _____ as const); */ [
         'content',
         'context',
         'knowledge',
@@ -421,7 +283,7 @@
         // <- TODO: list here all command names
         // <- TODO: Add more like 'date', 'modelName',...
         // <- TODO: Add [emoji] + instructions ACRY when adding new reserved parameter
-    ]);
+    ];
     /**
      * @@@
      *
@@ -489,6 +351,40 @@
         return NotYetImplementedError;
     }(Error));
 
+    /**
+     * Make error report URL for the given error
+     *
+     * @private !!!!!!
+     */
+    function getErrorReportUrl(error) {
+        var report = {
+            title: "\uD83D\uDC1C Error report from ".concat(NAME),
+            body: spaceTrim__default["default"](function (block) { return "\n\n\n            `".concat(error.name || 'Error', "` has occurred in the [").concat(NAME, "], please look into it @").concat(ADMIN_GITHUB_NAME, ".\n\n            ```\n            ").concat(block(error.message || '(no error message)'), "\n            ```\n\n\n            ## More info:\n\n            - **Promptbook engine version:** ").concat(PROMPTBOOK_ENGINE_VERSION, "\n            - **Book language version:** ").concat(BOOK_LANGUAGE_VERSION, "\n            - **Time:** ").concat(new Date().toISOString(), "\n\n            <details>\n            <summary>Stack trace:</summary>\n\n            ## Stack trace:\n\n            ```stacktrace\n            ").concat(block(error.stack || '(empty)'), "\n            ```\n            </details>\n\n        "); }),
+        };
+        var reportUrl = new URL("https://github.com/webgptorg/promptbook/issues/new");
+        reportUrl.searchParams.set('labels', 'bug');
+        reportUrl.searchParams.set('assignees', ADMIN_GITHUB_NAME);
+        reportUrl.searchParams.set('title', report.title);
+        reportUrl.searchParams.set('body', report.body);
+        return reportUrl;
+    }
+
+    /**
+     * This error type indicates that the error should not happen and its last check before crashing with some other error
+     *
+     * @public exported from `@promptbook/core`
+     */
+    var UnexpectedError = /** @class */ (function (_super) {
+        __extends(UnexpectedError, _super);
+        function UnexpectedError(message) {
+            var _this = _super.call(this, spaceTrim.spaceTrim(function (block) { return "\n                    ".concat(block(message), "\n\n                    Note: This error should not happen.\n                    It's probbably a bug in the pipeline collection\n\n                    Please report issue:\n                    ").concat(block(getErrorReportUrl(new Error(message)).href), "\n\n                    Or contact us on ").concat(ADMIN_EMAIL, "\n\n                "); })) || this;
+            _this.name = 'UnexpectedError';
+            Object.setPrototypeOf(_this, UnexpectedError.prototype);
+            return _this;
+        }
+        return UnexpectedError;
+    }(Error));
+
     /**
      * @@@
      *
@@ -700,32 +596,12 @@
     }(Error));
 
     /**
-     * Tests if given string is valid URL.
-     *
-     * Note: Dataurl are considered perfectly valid.
-     * Note: There are two simmilar functions:
-     * - `isValidUrl` which tests any URL
-     * - `isValidPipelineUrl` *(this one)* which tests just promptbook URL
+     * @@@
      *
-     * @public exported from `@promptbook/utils`
+     * @private for `FileCacheStorage`
      */
-    function isValidUrl(url) {
-        if (typeof url !== 'string') {
-            return false;
-        }
-        try {
-            if (url.startsWith('blob:')) {
-                url = url.replace(/^blob:/, '');
-            }
-            var urlObject = new URL(url /* because fail is handled */);
-            if (!['http:', 'https:', 'data:'].includes(urlObject.protocol)) {
-                return false;
-            }
-            return true;
-        }
-        catch (error) {
-            return false;
-        }
+    function nameToSubfolderPath(name) {
+        return [name.substr(0, 1).toLowerCase(), name.substr(1, 1).toLowerCase()];
     }
 
     var defaultDiacriticsRemovalMap = [
@@ -1093,6 +969,35 @@
         return false;
     }
 
+    /**
+     * Tests if given string is valid URL.
+     *
+     * Note: Dataurl are considered perfectly valid.
+     * Note: There are two simmilar functions:
+     * - `isValidUrl` which tests any URL
+     * - `isValidPipelineUrl` *(this one)* which tests just promptbook URL
+     *
+     * @public exported from `@promptbook/utils`
+     */
+    function isValidUrl(url) {
+        if (typeof url !== 'string') {
+            return false;
+        }
+        try {
+            if (url.startsWith('blob:')) {
+                url = url.replace(/^blob:/, '');
+            }
+            var urlObject = new URL(url /* because fail is handled */);
+            if (!['http:', 'https:', 'data:'].includes(urlObject.protocol)) {
+                return false;
+            }
+            return true;
+        }
+        catch (error) {
+            return false;
+        }
+    }
+
     /**
      * @@@
      *
@@ -1117,15 +1022,6 @@
         return value;
     }
 
-    /**
-     * @@@
-     *
-     * @private for `FileCacheStorage`
-     */
-    function nameToSubfolderPath(name) {
-        return [name.substr(0, 1).toLowerCase(), name.substr(1, 1).toLowerCase()];
-    }
-
     /**
      * Create a filename for intermediate cache for scrapers
      *
@@ -1934,6 +1830,149 @@
         return parameterNames;
     }
 
+    /**
+     * Checks if the value is [🚉] serializable as JSON
+     * If not, throws an UnexpectedError with a rich error message and tracking
+     *
+     * - Almost all primitives are serializable BUT:
+     * - `undefined` is not serializable
+     * - `NaN` is not serializable
+     * - Objects and arrays are serializable if all their properties are serializable
+     * - Functions are not serializable
+     * - Circular references are not serializable
+     * - `Date` objects are not serializable
+     * - `Map` and `Set` objects are not serializable
+     * - `RegExp` objects are not serializable
+     * - `Error` objects are not serializable
+     * - `Symbol` objects are not serializable
+     * - And much more...
+     *
+     * @throws UnexpectedError if the value is not serializable as JSON
+     * @public exported from `@promptbook/utils`
+     */
+    function checkSerializableAsJson(name, value) {
+        var e_1, _a;
+        if (value === undefined) {
+            throw new UnexpectedError("".concat(name, " is undefined"));
+        }
+        else if (value === null) {
+            return;
+        }
+        else if (typeof value === 'boolean') {
+            return;
+        }
+        else if (typeof value === 'number' && !isNaN(value)) {
+            return;
+        }
+        else if (typeof value === 'string') {
+            return;
+        }
+        else if (typeof value === 'symbol') {
+            throw new UnexpectedError("".concat(name, " is symbol"));
+        }
+        else if (typeof value === 'function') {
+            throw new UnexpectedError("".concat(name, " is function"));
+        }
+        else if (typeof value === 'object' && Array.isArray(value)) {
+            for (var i = 0; i < value.length; i++) {
+                checkSerializableAsJson("".concat(name, "[").concat(i, "]"), value[i]);
+            }
+        }
+        else if (typeof value === 'object') {
+            if (value instanceof Date) {
+                throw new UnexpectedError(spaceTrim__default["default"]("\n                    ".concat(name, " is Date\n\n                    Use `string_date_iso8601` instead\n                ")));
+            }
+            else if (value instanceof Map) {
+                throw new UnexpectedError("".concat(name, " is Map"));
+            }
+            else if (value instanceof Set) {
+                throw new UnexpectedError("".concat(name, " is Set"));
+            }
+            else if (value instanceof RegExp) {
+                throw new UnexpectedError("".concat(name, " is RegExp"));
+            }
+            else if (value instanceof Error) {
+                throw new UnexpectedError(spaceTrim__default["default"]("\n                    ".concat(name, " is unserialized Error\n\n                    Use function `serializeError`\n                ")));
+            }
+            else {
+                try {
+                    for (var _b = __values(Object.entries(value)), _c = _b.next(); !_c.done; _c = _b.next()) {
+                        var _d = __read(_c.value, 2), subName = _d[0], subValue = _d[1];
+                        if (subValue === undefined) {
+                            // Note: undefined in object is serializable - it is just omited
+                            continue;
+                        }
+                        checkSerializableAsJson("".concat(name, ".").concat(subName), subValue);
+                    }
+                }
+                catch (e_1_1) { e_1 = { error: e_1_1 }; }
+                finally {
+                    try {
+                        if (_c && !_c.done && (_a = _b.return)) _a.call(_b);
+                    }
+                    finally { if (e_1) throw e_1.error; }
+                }
+                try {
+                    JSON.stringify(value); // <- TODO: [0]
+                }
+                catch (error) {
+                    if (!(error instanceof Error)) {
+                        throw error;
+                    }
+                    throw new UnexpectedError(spaceTrim__default["default"](function (block) { return "\n                            ".concat(name, " is not serializable\n\n                            ").concat(block(error.toString()), "\n                        "); }));
+                }
+                /*
+                TODO: [0] Is there some more elegant way to check circular references?
+                const seen = new Set();
+                const stack = [{ value }];
+                while (stack.length > 0) {
+                    const { value } = stack.pop()!;
+                    if (typeof value === 'object' && value !== null) {
+                        if (seen.has(value)) {
+                            throw new UnexpectedError(`${name} has circular reference`);
+                        }
+                        seen.add(value);
+                        if (Array.isArray(value)) {
+                            stack.push(...value.map((value) => ({ value })));
+                        } else {
+                            stack.push(...Object.values(value).map((value) => ({ value })));
+                        }
+                    }
+                }
+                */
+                return;
+            }
+        }
+        else {
+            throw new UnexpectedError("".concat(name, " is unknown"));
+        }
+    }
+    /**
+     * TODO: [🧠][🛣] More elegant way to tracking than passing `name`
+     * TODO: [🧠][main] !!! In-memory cache of same values to prevent multiple checks
+     * Note: [🐠] This is how `checkSerializableAsJson` + `isSerializableAsJson` together can just retun true/false or rich error message
+     */
+
+    /**
+     * @@@
+     * @@@
+     *
+     * Note: This function mutates the object and returns the original (but mutated-deep-freezed) object
+     *
+     * @param name - Name of the object for debugging purposes
+     * @param objectValue - Object to be deeply frozen
+     * @returns The same object as the input, but deeply frozen
+     * @private this is in comparison to `deepFreeze` a more specific utility and maybe not very good practice to use without specific reason and considerations
+     */
+    function $asDeeplyFrozenSerializableJson(name, objectValue) {
+        checkSerializableAsJson(name, objectValue);
+        return $deepFreeze(objectValue);
+    }
+    /**
+     * TODO: [🧠][🛣] More elegant way to tracking than passing `name`
+     * TODO: [🧠] Is there a way how to meaningfully test this utility
+     */
+
     /**
      * Unprepare just strips the preparation data of the pipeline
      *
@@ -3695,10 +3734,11 @@
      * @param script from which to extract the variables
      * @returns the list of variable names
      * @throws {ParseError} if the script is invalid
-     * @public exported from `@promptbook/utils`
+     * @public exported from `@promptbook/utils` <- Note: [👖] This is usable elsewhere than in Promptbook, so keeping in utils
      */
-    function extractVariables(script) {
+    function extractVariablesFromScript(script) {
         var variables = new Set();
+        var originalScript = script;
         script = "(()=>{".concat(script, "})()");
         try {
             for (var i = 0; i < 100 /* <- TODO: This limit to configuration */; i++)
@@ -3730,7 +3770,9 @@
             if (!(error instanceof Error)) {
                 throw error;
             }
-            throw new ParseError(spaceTrim.spaceTrim(function (block) { return "\n                    Can not extract variables from the script\n\n                    ".concat(block(error.toString()), "}\n                "); }));
+            throw new ParseError(spaceTrim.spaceTrim(function (block) { return "\n                    Can not extract variables from the script\n\n                    ".concat(block(error.toString()), "}\n\n\n                    Found variables:\n\n                    ").concat(Array.from(variables)
+                .map(function (variableName, i) { return "".concat(i + 1, ") ").concat(variableName); })
+                .join('\n'), "\n\n\n                    The script:\n\n                    ```javascript\n                    ").concat(block(originalScript), "\n                    ```\n                "); }));
         }
         return variables;
     }
@@ -3744,7 +3786,7 @@
      * @param task the task with used parameters
      * @returns the set of parameter names
      * @throws {ParseError} if the script is invalid
-     * @public exported from `@promptbook/utils`
+     * @public exported from `@promptbook/core` <- Note: [👖] This utility is so tightly interconnected with the Promptbook that it is not exported as util but in core
      */
     function extractParameterNamesFromTask(task) {
         var e_1, _a, e_2, _b, e_3, _c, e_4, _d;
@@ -3765,7 +3807,7 @@
         }
         if (taskType === 'SCRIPT_TASK') {
             try {
-                for (var _g = __values(extractVariables(content)), _h = _g.next(); !_h.done; _h = _g.next()) {
+                for (var _g = __values(extractVariablesFromScript(content)), _h = _g.next(); !_h.done; _h = _g.next()) {
                     var parameterName = _h.value;
                     parameterNames.add(parameterName);
                 }