@promptbook/website-crawler 0.77.1 → 0.78.2

package/esm/index.es.js

@@ -24,7 +24,7 @@ var BOOK_LANGUAGE_VERSION = '1.0.0';
  *
  * @see https://github.com/webgptorg/promptbook
  */
-var PROMPTBOOK_ENGINE_VERSION = '0.77.0';
+var PROMPTBOOK_ENGINE_VERSION = '0.78.1';
 /**
  * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
  * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -204,164 +204,25 @@ function just(value) {
 }
 
 /**
- * This error type indicates that the error should not happen and its last check before crashing with some other error
+ * Name for the Promptbook
+ *
+ * TODO: [🗽] Unite branding and make single place for it
  *
  * @public exported from `@promptbook/core`
  */
-var UnexpectedError = /** @class */ (function (_super) {
-    __extends(UnexpectedError, _super);
-    function UnexpectedError(message) {
-        var _this = _super.call(this, spaceTrim(function (block) { return "\n                    ".concat(block(message), "\n\n                    Note: This error should not happen.\n                    It's probbably a bug in the pipeline collection\n\n                    Please report issue:\n                    https://github.com/webgptorg/promptbook/issues\n\n                    Or contact us on me@pavolhejny.com\n\n                "); })) || this;
-        _this.name = 'UnexpectedError';
-        Object.setPrototypeOf(_this, UnexpectedError.prototype);
-        return _this;
-    }
-    return UnexpectedError;
-}(Error));
-
+var NAME = "Promptbook";
 /**
- * Checks if the value is [🚉] serializable as JSON
- * If not, throws an UnexpectedError with a rich error message and tracking
- *
- * - Almost all primitives are serializable BUT:
- * - `undefined` is not serializable
- * - `NaN` is not serializable
- * - Objects and arrays are serializable if all their properties are serializable
- * - Functions are not serializable
- * - Circular references are not serializable
- * - `Date` objects are not serializable
- * - `Map` and `Set` objects are not serializable
- * - `RegExp` objects are not serializable
- * - `Error` objects are not serializable
- * - `Symbol` objects are not serializable
- * - And much more...
+ * Email of the responsible person
  *
- * @throws UnexpectedError if the value is not serializable as JSON
- * @public exported from `@promptbook/utils`
- */
-function checkSerializableAsJson(name, value) {
-    var e_1, _a;
-    if (value === undefined) {
-        throw new UnexpectedError("".concat(name, " is undefined"));
-    }
-    else if (value === null) {
-        return;
-    }
-    else if (typeof value === 'boolean') {
-        return;
-    }
-    else if (typeof value === 'number' && !isNaN(value)) {
-        return;
-    }
-    else if (typeof value === 'string') {
-        return;
-    }
-    else if (typeof value === 'symbol') {
-        throw new UnexpectedError("".concat(name, " is symbol"));
-    }
-    else if (typeof value === 'function') {
-        throw new UnexpectedError("".concat(name, " is function"));
-    }
-    else if (typeof value === 'object' && Array.isArray(value)) {
-        for (var i = 0; i < value.length; i++) {
-            checkSerializableAsJson("".concat(name, "[").concat(i, "]"), value[i]);
-        }
-    }
-    else if (typeof value === 'object') {
-        if (value instanceof Date) {
-            throw new UnexpectedError(spaceTrim$1("\n                    ".concat(name, " is Date\n\n                    Use `string_date_iso8601` instead\n                ")));
-        }
-        else if (value instanceof Map) {
-            throw new UnexpectedError("".concat(name, " is Map"));
-        }
-        else if (value instanceof Set) {
-            throw new UnexpectedError("".concat(name, " is Set"));
-        }
-        else if (value instanceof RegExp) {
-            throw new UnexpectedError("".concat(name, " is RegExp"));
-        }
-        else if (value instanceof Error) {
-            throw new UnexpectedError(spaceTrim$1("\n                    ".concat(name, " is unserialized Error\n\n                    Use function `serializeError`\n                ")));
-        }
-        else {
-            try {
-                for (var _b = __values(Object.entries(value)), _c = _b.next(); !_c.done; _c = _b.next()) {
-                    var _d = __read(_c.value, 2), subName = _d[0], subValue = _d[1];
-                    if (subValue === undefined) {
-                        // Note: undefined in object is serializable - it is just omited
-                        continue;
-                    }
-                    checkSerializableAsJson("".concat(name, ".").concat(subName), subValue);
-                }
-            }
-            catch (e_1_1) { e_1 = { error: e_1_1 }; }
-            finally {
-                try {
-                    if (_c && !_c.done && (_a = _b.return)) _a.call(_b);
-                }
-                finally { if (e_1) throw e_1.error; }
-            }
-            try {
-                JSON.stringify(value); // <- TODO: [0]
-            }
-            catch (error) {
-                if (!(error instanceof Error)) {
-                    throw error;
-                }
-                throw new UnexpectedError(spaceTrim$1(function (block) { return "\n                            ".concat(name, " is not serializable\n\n                            ").concat(block(error.toString()), "\n                        "); }));
-            }
-            /*
-            TODO: [0] Is there some more elegant way to check circular references?
-            const seen = new Set();
-            const stack = [{ value }];
-            while (stack.length > 0) {
-                const { value } = stack.pop()!;
-                if (typeof value === 'object' && value !== null) {
-                    if (seen.has(value)) {
-                        throw new UnexpectedError(`${name} has circular reference`);
-                    }
-                    seen.add(value);
-                    if (Array.isArray(value)) {
-                        stack.push(...value.map((value) => ({ value })));
-                    } else {
-                        stack.push(...Object.values(value).map((value) => ({ value })));
-                    }
-                }
-            }
-            */
-            return;
-        }
-    }
-    else {
-        throw new UnexpectedError("".concat(name, " is unknown"));
-    }
-}
-/**
- * TODO: [🧠][🛣] More elegant way to tracking than passing `name`
- * TODO: [🧠][main] !!! In-memory cache of same values to prevent multiple checks
- * Note: [🐠] This is how `checkSerializableAsJson` + `isSerializableAsJson` together can just retun true/false or rich error message
+ * @public exported from `@promptbook/core`
  */
-
+var ADMIN_EMAIL = 'me@pavolhejny.com';
 /**
- * @@@
- * @@@
+ * Name of the responsible person for the Promptbook on GitHub
  *
- * Note: This function mutates the object and returns the original (but mutated-deep-freezed) object
- *
- * @param name - Name of the object for debugging purposes
- * @param objectValue - Object to be deeply frozen
- * @returns The same object as the input, but deeply frozen
- * @private this is in comparison to `deepFreeze` a more specific utility and maybe not very good practice to use without specific reason and considerations
- */
-function $asDeeplyFrozenSerializableJson(name, objectValue) {
-    checkSerializableAsJson(name, objectValue);
-    return $deepFreeze(objectValue);
-}
-/**
- * TODO: [🧠][🛣] More elegant way to tracking than passing `name`
- * TODO: [🧠] Is there a way how to meaningfully test this utility
+ * @public exported from `@promptbook/core`
  */
-
+var ADMIN_GITHUB_NAME = 'hejny';
 // <- TODO: [🧠] Better system for generator warnings - not always "code" and "by `@promptbook/cli`"
 /**
  * The maximum number of iterations for a loops
@@ -413,7 +274,8 @@ var REPLACING_NONCE = 'u$k42k%!V2zo34w7Fu#@QUHYPW';
  *
  * @public exported from `@promptbook/core`
  */
-var RESERVED_PARAMETER_NAMES = $asDeeplyFrozenSerializableJson('RESERVED_PARAMETER_NAMES', [
+var RESERVED_PARAMETER_NAMES = 
+/* !!!!!! $asDeeplyFrozenSerializableJson('RESERVED_PARAMETER_NAMES', _____ as const); */ [
     'content',
     'context',
     'knowledge',
@@ -423,7 +285,7 @@ var RESERVED_PARAMETER_NAMES = $asDeeplyFrozenSerializableJson('RESERVED_PARAMET
     // <- TODO: list here all command names
     // <- TODO: Add more like 'date', 'modelName',...
     // <- TODO: Add [emoji] + instructions ACRY when adding new reserved parameter
-]);
+];
 /**
  * @@@
  *
@@ -491,6 +353,40 @@ var NotYetImplementedError = /** @class */ (function (_super) {
     return NotYetImplementedError;
 }(Error));
 
+/**
+ * Make error report URL for the given error
+ *
+ * @private !!!!!!
+ */
+function getErrorReportUrl(error) {
+    var report = {
+        title: "\uD83D\uDC1C Error report from ".concat(NAME),
+        body: spaceTrim$1(function (block) { return "\n\n\n            `".concat(error.name || 'Error', "` has occurred in the [").concat(NAME, "], please look into it @").concat(ADMIN_GITHUB_NAME, ".\n\n            ```\n            ").concat(block(error.message || '(no error message)'), "\n            ```\n\n\n            ## More info:\n\n            - **Promptbook engine version:** ").concat(PROMPTBOOK_ENGINE_VERSION, "\n            - **Book language version:** ").concat(BOOK_LANGUAGE_VERSION, "\n            - **Time:** ").concat(new Date().toISOString(), "\n\n            <details>\n            <summary>Stack trace:</summary>\n\n            ## Stack trace:\n\n            ```stacktrace\n            ").concat(block(error.stack || '(empty)'), "\n            ```\n            </details>\n\n        "); }),
+    };
+    var reportUrl = new URL("https://github.com/webgptorg/promptbook/issues/new");
+    reportUrl.searchParams.set('labels', 'bug');
+    reportUrl.searchParams.set('assignees', ADMIN_GITHUB_NAME);
+    reportUrl.searchParams.set('title', report.title);
+    reportUrl.searchParams.set('body', report.body);
+    return reportUrl;
+}
+
+/**
+ * This error type indicates that the error should not happen and its last check before crashing with some other error
+ *
+ * @public exported from `@promptbook/core`
+ */
+var UnexpectedError = /** @class */ (function (_super) {
+    __extends(UnexpectedError, _super);
+    function UnexpectedError(message) {
+        var _this = _super.call(this, spaceTrim(function (block) { return "\n                    ".concat(block(message), "\n\n                    Note: This error should not happen.\n                    It's probbably a bug in the pipeline collection\n\n                    Please report issue:\n                    ").concat(block(getErrorReportUrl(new Error(message)).href), "\n\n                    Or contact us on ").concat(ADMIN_EMAIL, "\n\n                "); })) || this;
+        _this.name = 'UnexpectedError';
+        Object.setPrototypeOf(_this, UnexpectedError.prototype);
+        return _this;
+    }
+    return UnexpectedError;
+}(Error));
+
 /**
  * @@@
  *
@@ -702,32 +598,12 @@ var KnowledgeScrapeError = /** @class */ (function (_super) {
 }(Error));
 
 /**
- * Tests if given string is valid URL.
- *
- * Note: Dataurl are considered perfectly valid.
- * Note: There are two simmilar functions:
- * - `isValidUrl` which tests any URL
- * - `isValidPipelineUrl` *(this one)* which tests just promptbook URL
+ * @@@
  *
- * @public exported from `@promptbook/utils`
+ * @private for `FileCacheStorage`
  */
-function isValidUrl(url) {
-    if (typeof url !== 'string') {
-        return false;
-    }
-    try {
-        if (url.startsWith('blob:')) {
-            url = url.replace(/^blob:/, '');
-        }
-        var urlObject = new URL(url /* because fail is handled */);
-        if (!['http:', 'https:', 'data:'].includes(urlObject.protocol)) {
-            return false;
-        }
-        return true;
-    }
-    catch (error) {
-        return false;
-    }
+function nameToSubfolderPath(name) {
+    return [name.substr(0, 1).toLowerCase(), name.substr(1, 1).toLowerCase()];
 }
 
 var defaultDiacriticsRemovalMap = [
@@ -1095,6 +971,35 @@ function isValidFilePath(filename) {
     return false;
 }
 
+/**
+ * Tests if given string is valid URL.
+ *
+ * Note: Dataurl are considered perfectly valid.
+ * Note: There are two simmilar functions:
+ * - `isValidUrl` which tests any URL
+ * - `isValidPipelineUrl` *(this one)* which tests just promptbook URL
+ *
+ * @public exported from `@promptbook/utils`
+ */
+function isValidUrl(url) {
+    if (typeof url !== 'string') {
+        return false;
+    }
+    try {
+        if (url.startsWith('blob:')) {
+            url = url.replace(/^blob:/, '');
+        }
+        var urlObject = new URL(url /* because fail is handled */);
+        if (!['http:', 'https:', 'data:'].includes(urlObject.protocol)) {
+            return false;
+        }
+        return true;
+    }
+    catch (error) {
+        return false;
+    }
+}
+
 /**
  * @@@
  *
@@ -1119,15 +1024,6 @@ function titleToName(value) {
     return value;
 }
 
-/**
- * @@@
- *
- * @private for `FileCacheStorage`
- */
-function nameToSubfolderPath(name) {
-    return [name.substr(0, 1).toLowerCase(), name.substr(1, 1).toLowerCase()];
-}
-
 /**
  * Create a filename for intermediate cache for scrapers
  *
@@ -1936,6 +1832,149 @@ function extractParameterNames(template) {
     return parameterNames;
 }
 
+/**
+ * Checks if the value is [🚉] serializable as JSON
+ * If not, throws an UnexpectedError with a rich error message and tracking
+ *
+ * - Almost all primitives are serializable BUT:
+ * - `undefined` is not serializable
+ * - `NaN` is not serializable
+ * - Objects and arrays are serializable if all their properties are serializable
+ * - Functions are not serializable
+ * - Circular references are not serializable
+ * - `Date` objects are not serializable
+ * - `Map` and `Set` objects are not serializable
+ * - `RegExp` objects are not serializable
+ * - `Error` objects are not serializable
+ * - `Symbol` objects are not serializable
+ * - And much more...
+ *
+ * @throws UnexpectedError if the value is not serializable as JSON
+ * @public exported from `@promptbook/utils`
+ */
+function checkSerializableAsJson(name, value) {
+    var e_1, _a;
+    if (value === undefined) {
+        throw new UnexpectedError("".concat(name, " is undefined"));
+    }
+    else if (value === null) {
+        return;
+    }
+    else if (typeof value === 'boolean') {
+        return;
+    }
+    else if (typeof value === 'number' && !isNaN(value)) {
+        return;
+    }
+    else if (typeof value === 'string') {
+        return;
+    }
+    else if (typeof value === 'symbol') {
+        throw new UnexpectedError("".concat(name, " is symbol"));
+    }
+    else if (typeof value === 'function') {
+        throw new UnexpectedError("".concat(name, " is function"));
+    }
+    else if (typeof value === 'object' && Array.isArray(value)) {
+        for (var i = 0; i < value.length; i++) {
+            checkSerializableAsJson("".concat(name, "[").concat(i, "]"), value[i]);
+        }
+    }
+    else if (typeof value === 'object') {
+        if (value instanceof Date) {
+            throw new UnexpectedError(spaceTrim$1("\n                    ".concat(name, " is Date\n\n                    Use `string_date_iso8601` instead\n                ")));
+        }
+        else if (value instanceof Map) {
+            throw new UnexpectedError("".concat(name, " is Map"));
+        }
+        else if (value instanceof Set) {
+            throw new UnexpectedError("".concat(name, " is Set"));
+        }
+        else if (value instanceof RegExp) {
+            throw new UnexpectedError("".concat(name, " is RegExp"));
+        }
+        else if (value instanceof Error) {
+            throw new UnexpectedError(spaceTrim$1("\n                    ".concat(name, " is unserialized Error\n\n                    Use function `serializeError`\n                ")));
+        }
+        else {
+            try {
+                for (var _b = __values(Object.entries(value)), _c = _b.next(); !_c.done; _c = _b.next()) {
+                    var _d = __read(_c.value, 2), subName = _d[0], subValue = _d[1];
+                    if (subValue === undefined) {
+                        // Note: undefined in object is serializable - it is just omited
+                        continue;
+                    }
+                    checkSerializableAsJson("".concat(name, ".").concat(subName), subValue);
+                }
+            }
+            catch (e_1_1) { e_1 = { error: e_1_1 }; }
+            finally {
+                try {
+                    if (_c && !_c.done && (_a = _b.return)) _a.call(_b);
+                }
+                finally { if (e_1) throw e_1.error; }
+            }
+            try {
+                JSON.stringify(value); // <- TODO: [0]
+            }
+            catch (error) {
+                if (!(error instanceof Error)) {
+                    throw error;
+                }
+                throw new UnexpectedError(spaceTrim$1(function (block) { return "\n                            ".concat(name, " is not serializable\n\n                            ").concat(block(error.toString()), "\n                        "); }));
+            }
+            /*
+            TODO: [0] Is there some more elegant way to check circular references?
+            const seen = new Set();
+            const stack = [{ value }];
+            while (stack.length > 0) {
+                const { value } = stack.pop()!;
+                if (typeof value === 'object' && value !== null) {
+                    if (seen.has(value)) {
+                        throw new UnexpectedError(`${name} has circular reference`);
+                    }
+                    seen.add(value);
+                    if (Array.isArray(value)) {
+                        stack.push(...value.map((value) => ({ value })));
+                    } else {
+                        stack.push(...Object.values(value).map((value) => ({ value })));
+                    }
+                }
+            }
+            */
+            return;
+        }
+    }
+    else {
+        throw new UnexpectedError("".concat(name, " is unknown"));
+    }
+}
+/**
+ * TODO: [🧠][🛣] More elegant way to tracking than passing `name`
+ * TODO: [🧠][main] !!! In-memory cache of same values to prevent multiple checks
+ * Note: [🐠] This is how `checkSerializableAsJson` + `isSerializableAsJson` together can just retun true/false or rich error message
+ */
+
+/**
+ * @@@
+ * @@@
+ *
+ * Note: This function mutates the object and returns the original (but mutated-deep-freezed) object
+ *
+ * @param name - Name of the object for debugging purposes
+ * @param objectValue - Object to be deeply frozen
+ * @returns The same object as the input, but deeply frozen
+ * @private this is in comparison to `deepFreeze` a more specific utility and maybe not very good practice to use without specific reason and considerations
+ */
+function $asDeeplyFrozenSerializableJson(name, objectValue) {
+    checkSerializableAsJson(name, objectValue);
+    return $deepFreeze(objectValue);
+}
+/**
+ * TODO: [🧠][🛣] More elegant way to tracking than passing `name`
+ * TODO: [🧠] Is there a way how to meaningfully test this utility
+ */
+
 /**
  * Unprepare just strips the preparation data of the pipeline
  *
@@ -3697,10 +3736,11 @@ function preparePipeline(pipeline, tools, options) {
  * @param script from which to extract the variables
  * @returns the list of variable names
  * @throws {ParseError} if the script is invalid
- * @public exported from `@promptbook/utils`
+ * @public exported from `@promptbook/utils` <- Note: [👖] This is usable elsewhere than in Promptbook, so keeping in utils
  */
-function extractVariables(script) {
+function extractVariablesFromScript(script) {
     var variables = new Set();
+    var originalScript = script;
     script = "(()=>{".concat(script, "})()");
     try {
         for (var i = 0; i < 100 /* <- TODO: This limit to configuration */; i++)
@@ -3732,7 +3772,9 @@ function extractVariables(script) {
         if (!(error instanceof Error)) {
             throw error;
         }
-        throw new ParseError(spaceTrim(function (block) { return "\n                    Can not extract variables from the script\n\n                    ".concat(block(error.toString()), "}\n                "); }));
+        throw new ParseError(spaceTrim(function (block) { return "\n                    Can not extract variables from the script\n\n                    ".concat(block(error.toString()), "}\n\n\n                    Found variables:\n\n                    ").concat(Array.from(variables)
+            .map(function (variableName, i) { return "".concat(i + 1, ") ").concat(variableName); })
+            .join('\n'), "\n\n\n                    The script:\n\n                    ```javascript\n                    ").concat(block(originalScript), "\n                    ```\n                "); }));
     }
     return variables;
 }
@@ -3746,7 +3788,7 @@ function extractVariables(script) {
  * @param task the task with used parameters
  * @returns the set of parameter names
  * @throws {ParseError} if the script is invalid
- * @public exported from `@promptbook/utils`
+ * @public exported from `@promptbook/core` <- Note: [👖] This utility is so tightly interconnected with the Promptbook that it is not exported as util but in core
  */
 function extractParameterNamesFromTask(task) {
     var e_1, _a, e_2, _b, e_3, _c, e_4, _d;
@@ -3767,7 +3809,7 @@ function extractParameterNamesFromTask(task) {
     }
     if (taskType === 'SCRIPT_TASK') {
         try {
-            for (var _g = __values(extractVariables(content)), _h = _g.next(); !_h.done; _h = _g.next()) {
+            for (var _g = __values(extractVariablesFromScript(content)), _h = _g.next(); !_h.done; _h = _g.next()) {
                 var parameterName = _h.value;
                 parameterNames.add(parameterName);
             }