@promptbook/website-crawler 0.77.0 → 0.78.0-0

package/README.md

@@ -23,6 +23,10 @@
 
 
 
+<blockquote style="color: #ff8811">
+    <b>⚠ Warning:</b> This is a pre-release version of the library. It is not yet ready for production use. Please look at <a href="https://www.npmjs.com/package/@promptbook/core?activeTab=versions">latest stable release</a>.
+</blockquote>
+
 ## 📦 Package `@promptbook/website-crawler`
 
 - Promptbooks are [divided into several](#-packages) packages, all are published from [single monorepo](https://github.com/webgptorg/promptbook).

package/esm/index.es.js

@@ -24,7 +24,7 @@ var BOOK_LANGUAGE_VERSION = '1.0.0';
  *
  * @see https://github.com/webgptorg/promptbook
  */
-var PROMPTBOOK_ENGINE_VERSION = '0.77.0-6';
+var PROMPTBOOK_ENGINE_VERSION = '0.77.1';
 /**
  * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
  * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -702,32 +702,12 @@ var KnowledgeScrapeError = /** @class */ (function (_super) {
 }(Error));
 
 /**
- * Tests if given string is valid URL.
- *
- * Note: Dataurl are considered perfectly valid.
- * Note: There are two simmilar functions:
- * - `isValidUrl` which tests any URL
- * - `isValidPipelineUrl` *(this one)* which tests just promptbook URL
+ * @@@
  *
- * @public exported from `@promptbook/utils`
+ * @private for `FileCacheStorage`
  */
-function isValidUrl(url) {
-    if (typeof url !== 'string') {
-        return false;
-    }
-    try {
-        if (url.startsWith('blob:')) {
-            url = url.replace(/^blob:/, '');
-        }
-        var urlObject = new URL(url /* because fail is handled */);
-        if (!['http:', 'https:', 'data:'].includes(urlObject.protocol)) {
-            return false;
-        }
-        return true;
-    }
-    catch (error) {
-        return false;
-    }
+function nameToSubfolderPath(name) {
+    return [name.substr(0, 1).toLowerCase(), name.substr(1, 1).toLowerCase()];
 }
 
 var defaultDiacriticsRemovalMap = [
@@ -1095,6 +1075,35 @@ function isValidFilePath(filename) {
     return false;
 }
 
+/**
+ * Tests if given string is valid URL.
+ *
+ * Note: Dataurl are considered perfectly valid.
+ * Note: There are two simmilar functions:
+ * - `isValidUrl` which tests any URL
+ * - `isValidPipelineUrl` *(this one)* which tests just promptbook URL
+ *
+ * @public exported from `@promptbook/utils`
+ */
+function isValidUrl(url) {
+    if (typeof url !== 'string') {
+        return false;
+    }
+    try {
+        if (url.startsWith('blob:')) {
+            url = url.replace(/^blob:/, '');
+        }
+        var urlObject = new URL(url /* because fail is handled */);
+        if (!['http:', 'https:', 'data:'].includes(urlObject.protocol)) {
+            return false;
+        }
+        return true;
+    }
+    catch (error) {
+        return false;
+    }
+}
+
 /**
  * @@@
  *
@@ -1119,15 +1128,6 @@ function titleToName(value) {
     return value;
 }
 
-/**
- * @@@
- *
- * @private for `FileCacheStorage`
- */
-function nameToSubfolderPath(name) {
-    return [name.substr(0, 1).toLowerCase(), name.substr(1, 1).toLowerCase()];
-}
-
 /**
  * Create a filename for intermediate cache for scrapers
  *
@@ -2190,7 +2190,7 @@ var LimitReachedError = /** @class */ (function (_super) {
  *
  * @public exported from `@promptbook/core`
  */
-var ERRORS = {
+var PROMPTBOOK_ERRORS = {
     AbstractFormatError: AbstractFormatError,
     CsvFormatError: CsvFormatError,
     CollectionError: CollectionError,
@@ -2208,6 +2208,35 @@ var ERRORS = {
     UnexpectedError: UnexpectedError,
     // TODO: [🪑]> VersionMismatchError,
 };
+/**
+ * Index of all javascript errors
+ *
+ * @private for internal usage
+ */
+var COMMON_JAVASCRIPT_ERRORS = {
+    Error: Error,
+    EvalError: EvalError,
+    RangeError: RangeError,
+    ReferenceError: ReferenceError,
+    SyntaxError: SyntaxError,
+    TypeError: TypeError,
+    URIError: URIError,
+    AggregateError: AggregateError,
+    /*
+  Note: Not widely supported
+  > InternalError,
+  > ModuleError,
+  > HeapError,
+  > WebAssemblyCompileError,
+  > WebAssemblyRuntimeError,
+  */
+};
+/**
+ * Index of all errors
+ *
+ * @private for internal usage
+ */
+var ALL_ERRORS = __assign(__assign({}, PROMPTBOOK_ERRORS), COMMON_JAVASCRIPT_ERRORS);
 /**
  * Note: [💞] Ignore a discrepancy between file name and entity name
  */
@@ -2218,11 +2247,11 @@ var ERRORS = {
  * @public exported from `@promptbook/utils`
  */
 function deserializeError(error) {
-    if (error.name === 'Error') {
-        return new Error(error.message);
+    var ErrorClass = ALL_ERRORS[error.name];
+    if (ErrorClass === undefined) {
+        return new Error("".concat(error.name, ": ").concat(error.message));
     }
-    var CustomError = ERRORS[error.name];
-    return new CustomError(error.message);
+    return new ErrorClass(error.message);
 }
 
 /**
@@ -2296,8 +2325,8 @@ function isPipelinePrepared(pipeline) {
  */
 function serializeError(error) {
     var name = error.name, message = error.message, stack = error.stack;
-    if (!__spreadArray(['Error'], __read(Object.keys(ERRORS)), false).includes(name)) {
-        throw new UnexpectedError(spaceTrim$1(function (block) { return "\n          \n                    Cannot serialize error with name \"".concat(name, "\"\n\n                    ").concat(block(stack || message), "\n                \n                "); }));
+    if (!Object.keys(ALL_ERRORS).includes(name)) {
+        console.error(spaceTrim$1(function (block) { return "\n          \n                    Cannot serialize error with name \"".concat(name, "\"\n\n                    ").concat(block(stack || message), "\n                \n                "); }));
     }
     return {
         name: name,
@@ -3668,9 +3697,9 @@ function preparePipeline(pipeline, tools, options) {
  * @param script from which to extract the variables
  * @returns the list of variable names
  * @throws {ParseError} if the script is invalid
- * @public exported from `@promptbook/utils`
+ * @public exported from `@promptbook/utils` <- Note: [👖] This is usable elsewhere than in Promptbook, so keeping in utils
  */
-function extractVariables(script) {
+function extractVariablesFromScript(script) {
     var variables = new Set();
     script = "(()=>{".concat(script, "})()");
     try {
@@ -3717,7 +3746,7 @@ function extractVariables(script) {
  * @param task the task with used parameters
  * @returns the set of parameter names
  * @throws {ParseError} if the script is invalid
- * @public exported from `@promptbook/utils`
+ * @public exported from `@promptbook/core` <- Note: [👖] This utility is so tightly interconnected with the Promptbook that it is not exported as util but in core
  */
 function extractParameterNamesFromTask(task) {
     var e_1, _a, e_2, _b, e_3, _c, e_4, _d;
@@ -3738,7 +3767,7 @@ function extractParameterNamesFromTask(task) {
     }
     if (taskType === 'SCRIPT_TASK') {
         try {
-            for (var _g = __values(extractVariables(content)), _h = _g.next(); !_h.done; _h = _g.next()) {
+            for (var _g = __values(extractVariablesFromScript(content)), _h = _g.next(); !_h.done; _h = _g.next()) {
                 var parameterName = _h.value;
                 parameterNames.add(parameterName);
             }