@promptbook/website-crawler 0.89.0-3 → 0.89.0-31

package/README.md

@@ -244,6 +244,10 @@ But unlike programming languages, it is designed to be understandable by non-pro
 
 
 
+## 🔒 Security
+
+For information on reporting security vulnerabilities, see our [Security Policy](./SECURITY.md).
+
 ## 📦 Packages _(for developers)_
 
 This library is divided into several packages, all are published from [single monorepo](https://github.com/webgptorg/promptbook).
@@ -300,7 +304,7 @@ The following glossary is used to clarify certain concepts:
 ### General LLM / AI terms
 
 -   **Prompt drift** is a phenomenon where the AI model starts to generate outputs that are not aligned with the original prompt. This can happen due to the model's training data, the prompt's wording, or the model's architecture.
--   **Pipeline, workflow or chain** is a sequence of tasks that are executed in a specific order. In the context of AI, a pipeline can refer to a sequence of AI models that are used to process data.
+-   [**Pipeline, workflow scenario or chain** is a sequence of tasks that are executed in a specific order. In the context of AI, a pipeline can refer to a sequence of AI models that are used to process data.](https://github.com/webgptorg/promptbook/discussions/88)
 -   **Fine-tuning** is a process where a pre-trained AI model is further trained on a specific dataset to improve its performance on a specific task.
 -   **Zero-shot learning** is a machine learning paradigm where a model is trained to perform a task without any labeled examples. Instead, the model is provided with a description of the task and is expected to generate the correct output.
 -   **Few-shot learning** is a machine learning paradigm where a model is trained to perform a task with only a few labeled examples. This is in contrast to traditional machine learning, where models are trained on large datasets.
@@ -308,10 +312,6 @@ The following glossary is used to clarify certain concepts:
 -   **Retrieval-augmented generation** is a machine learning paradigm where a model generates text by retrieving relevant information from a large database of text. This approach combines the benefits of generative models and retrieval models.
 -   **Longtail** refers to non-common or rare events, items, or entities that are not well-represented in the training data of machine learning models. Longtail items are often challenging for models to predict accurately.
 
-
-
-
-
 _Note: This section is not complete dictionary, more list of general AI / LLM terms that has connection with Promptbook_
 
 
@@ -425,6 +425,8 @@ See [TODO.md](./TODO.md)
 
 ## 🖋️ Contributing
 
-We are open to pull requests, feedback, and suggestions.
+You can also ⭐ star the project, [follow us on GitHub](https://github.com/hejny) or [various other social networks](https://www.pavolhejny.com/contact/).We are open to [pull requests, feedback, and suggestions](./CONTRIBUTING.md).
+
+## 📞 Support
 
-You can also ⭐ star the project, [follow us on GitHub](https://github.com/hejny) or [various other social networks](https://www.pavolhejny.com/contact/).
+If you need help or have questions, please check our [Support Resources](./SUPPORT.md).

package/esm/index.es.js

@@ -29,7 +29,7 @@ const BOOK_LANGUAGE_VERSION = '1.0.0';
  * @generated
  * @see https://github.com/webgptorg/promptbook
  */
-const PROMPTBOOK_ENGINE_VERSION = '0.89.0-3';
+const PROMPTBOOK_ENGINE_VERSION = '0.89.0-31';
 /**
  * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
  * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -119,6 +119,7 @@ const ADMIN_EMAIL = 'pavol@ptbk.io';
  * @public exported from `@promptbook/core`
  */
 const ADMIN_GITHUB_NAME = 'hejny';
+//            <- TODO: [🐊] Pick the best claim
 /**
  * When the title is not provided, the default title is used
  *
@@ -151,6 +152,7 @@ const VALUE_STRINGS = {
     infinity: '(infinity; ∞)',
     negativeInfinity: '(negative infinity; -∞)',
     unserializable: '(unserializable value)',
+    circular: '(circular JSON)',
 };
 /**
  * Small number limit
@@ -190,6 +192,7 @@ const DEFAULT_MAX_PARALLEL_COUNT = 5; // <- TODO: [🤹‍♂️]
  */
 const DEFAULT_MAX_EXECUTION_ATTEMPTS = 10; // <- TODO: [🤹‍♂️]
 // <- TODO: [🕝] Make also `BOOKS_DIRNAME_ALTERNATIVES`
+// TODO: Just `.promptbook` in config, hardcode subfolders like `download-cache` or `execution-cache`
 /**
  * Where to store the temporary downloads
  *
@@ -241,7 +244,7 @@ const IS_PIPELINE_LOGIC_VALIDATED = just(
 true);
 /**
  * Note: [💞] Ignore a discrepancy between file name and entity name
- * TODO: [🧠][🧜‍♂️] Maybe join remoteUrl and path into single value
+ * TODO: [🧠][🧜‍♂️] Maybe join remoteServerUrl and path into single value
  */
 
 /**
@@ -1055,6 +1058,54 @@ class ParseError extends Error {
  * TODO: Maybe split `ParseError` and `ApplyError`
  */
 
+/**
+ * This error type indicates that somewhere in the code non-Error object was thrown and it was wrapped into the `WrappedError`
+ *
+ * @public exported from `@promptbook/core`
+ */
+class WrappedError extends Error {
+    constructor(whatWasThrown) {
+        const tag = `[🤮]`;
+        console.error(tag, whatWasThrown);
+        super(spaceTrim(`
+                Non-Error object was thrown
+
+                Note: Look for ${tag} in the console for more details
+                Please report issue on ${ADMIN_EMAIL}
+            `));
+        this.name = 'WrappedError';
+        Object.setPrototypeOf(this, WrappedError.prototype);
+    }
+}
+
+/**
+ * Helper used in catch blocks to assert that the error is an instance of `Error`
+ *
+ * @param whatWasThrown Any object that was thrown
+ * @returns Nothing if the error is an instance of `Error`
+ * @throws `WrappedError` or `UnexpectedError` if the error is not standard
+ *
+ * @private within the repository
+ */
+function assertsError(whatWasThrown) {
+    // Case 1: Handle error which was rethrown as `WrappedError`
+    if (whatWasThrown instanceof WrappedError) {
+        const wrappedError = whatWasThrown;
+        throw wrappedError;
+    }
+    // Case 2: Handle unexpected errors
+    if (whatWasThrown instanceof UnexpectedError) {
+        const unexpectedError = whatWasThrown;
+        throw unexpectedError;
+    }
+    // Case 3: Handle standard errors - keep them up to consumer
+    if (whatWasThrown instanceof Error) {
+        return;
+    }
+    // Case 4: Handle non-standard errors - wrap them into `WrappedError` and throw
+    throw new WrappedError(whatWasThrown);
+}
+
 /**
  * Function isValidJsonString will tell you if the string is valid JSON or not
  *
@@ -1066,9 +1117,7 @@ function isValidJsonString(value /* <- [👨‍⚖️] */) {
         return true;
     }
     catch (error) {
-        if (!(error instanceof Error)) {
-            throw error;
-        }
+        assertsError(error);
         if (error.message.includes('Unexpected token')) {
             return false;
         }
@@ -1394,9 +1443,7 @@ function checkSerializableAsJson(options) {
                 JSON.stringify(value); // <- TODO: [0]
             }
             catch (error) {
-                if (!(error instanceof Error)) {
-                    throw error;
-                }
+                assertsError(error);
                 throw new UnexpectedError(spaceTrim$1((block) => `
                             \`${name}\` is not serializable
 
@@ -2185,7 +2232,7 @@ class PipelineExecutionError extends Error {
     }
 }
 /**
- * TODO: !!!!!! Add id to all errors
+ * TODO: [🧠][🌂] Add id to all errors
  */
 
 /**
@@ -2285,6 +2332,19 @@ class CsvFormatError extends AbstractFormatError {
     }
 }
 
+/**
+ * AuthenticationError is thrown from login function which is dependency of remote server
+ *
+ * @public exported from `@promptbook/core`
+ */
+class AuthenticationError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'AuthenticationError';
+        Object.setPrototypeOf(this, AuthenticationError.prototype);
+    }
+}
+
 /**
  * This error indicates that the pipeline collection cannot be propperly loaded
  *
@@ -2327,6 +2387,19 @@ class LimitReachedError extends Error {
     }
 }
 
+/**
+ * Error thrown when a fetch request fails
+ *
+ * @public exported from `@promptbook/core`
+ */
+class PromptbookFetchError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'PromptbookFetchError';
+        Object.setPrototypeOf(this, PromptbookFetchError.prototype);
+    }
+}
+
 /**
  * Index of all custom errors
  *
@@ -2347,7 +2420,10 @@ const PROMPTBOOK_ERRORS = {
     PipelineExecutionError,
     PipelineLogicError,
     PipelineUrlError,
+    AuthenticationError,
+    PromptbookFetchError,
     UnexpectedError,
+    WrappedError,
     // TODO: [🪑]> VersionMismatchError,
 };
 /**
@@ -2486,8 +2562,8 @@ function createTask(options) {
                 updatedAt = new Date();
                 errors.push(...executionResult.errors);
                 warnings.push(...executionResult.warnings);
-                // <- TODO: !!! Only unique errors and warnings should be added (or filtered)
-                // TODO: [🧠] !!! errors, warning, isSuccessful  are redundant both in `ExecutionTask` and `ExecutionTask.currentValue`
+                // <- TODO: [🌂] Only unique errors and warnings should be added (or filtered)
+                // TODO: [🧠] !! errors, warning, isSuccessful  are redundant both in `ExecutionTask` and `ExecutionTask.currentValue`
                 //            Also maybe move `ExecutionTask.currentValue.usage` -> `ExecutionTask.usage`
                 //            And delete `ExecutionTask.currentValue.preparedPipeline`
                 assertsTaskSuccessful(executionResult);
@@ -2497,6 +2573,7 @@ function createTask(options) {
                 partialResultSubject.next(executionResult);
             }
             catch (error) {
+                assertsError(error);
                 status = 'ERROR';
                 errors.push(error);
                 partialResultSubject.error(error);
@@ -2563,6 +2640,10 @@ function serializeError(error) {
 
                     Cannot serialize error with name "${name}"
 
+                    Authors of Promptbook probably forgot to add this error into the list of errors:
+                    https://github.com/webgptorg/promptbook/blob/main/src/errors/0-index.ts
+
+
                     ${block(stack || message)}
 
                 `));
@@ -2884,14 +2965,15 @@ class MultipleLlmExecutionTools {
                 }
             }
             catch (error) {
-                if (!(error instanceof Error) || error instanceof UnexpectedError) {
+                assertsError(error);
+                if (error instanceof UnexpectedError) {
                     throw error;
                 }
                 errors.push({ llmExecutionTools, error });
             }
         }
         if (errors.length === 1) {
-            throw errors[0];
+            throw errors[0].error;
         }
         else if (errors.length > 1) {
             throw new PipelineExecutionError(
@@ -3225,17 +3307,22 @@ function mimeTypeToExtension(value) {
 /**
  * The built-in `fetch' function with a lightweight error handling wrapper as default fetch function used in Promptbook scrapers
  *
- * @private as default `fetch` function used in Promptbook scrapers
+ * @public exported from `@promptbook/core`
  */
-const scraperFetch = async (url, init) => {
+const promptbookFetch = async (urlOrRequest, init) => {
     try {
-        return await fetch(url, init);
+        return await fetch(urlOrRequest, init);
     }
     catch (error) {
-        if (!(error instanceof Error)) {
-            throw error;
+        assertsError(error);
+        let url;
+        if (typeof urlOrRequest === 'string') {
+            url = urlOrRequest;
+        }
+        else if (urlOrRequest instanceof Request) {
+            url = urlOrRequest.url;
         }
-        throw new KnowledgeScrapeError(spaceTrim$1((block) => `
+        throw new PromptbookFetchError(spaceTrim$1((block) => `
                     Can not fetch "${url}"
 
                     Fetch error:
@@ -3256,7 +3343,7 @@ const scraperFetch = async (url, init) => {
 async function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
     // console.log('!! makeKnowledgeSourceHandler', knowledgeSource);
     var _a;
-    const { fetch = scraperFetch } = tools;
+    const { fetch = promptbookFetch } = tools;
     const { knowledgeSourceContent } = knowledgeSource;
     let { name } = knowledgeSource;
     const { rootDirname = null, 
@@ -3458,9 +3545,7 @@ async function prepareKnowledgePieces(knowledgeSources, tools, options) {
             knowledgePreparedUnflatten[index] = pieces;
         }
         catch (error) {
-            if (!(error instanceof Error)) {
-                throw error;
-            }
+            assertsError(error);
             console.warn(error);
             // <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
         }
@@ -3752,13 +3837,19 @@ function valueToString(value) {
             return value.toISOString();
         }
         else {
-            return JSON.stringify(value);
+            try {
+                return JSON.stringify(value);
+            }
+            catch (error) {
+                if (error instanceof TypeError && error.message.includes('circular structure')) {
+                    return VALUE_STRINGS.circular;
+                }
+                throw error;
+            }
         }
     }
     catch (error) {
-        if (!(error instanceof Error)) {
-            throw error;
-        }
+        assertsError(error);
         console.error(error);
         return VALUE_STRINGS.unserializable;
     }
@@ -3815,9 +3906,7 @@ function extractVariablesFromJavascript(script) {
             }
     }
     catch (error) {
-        if (!(error instanceof Error)) {
-            throw error;
-        }
+        assertsError(error);
         throw new ParseError(spaceTrim((block) => `
                     Can not extract variables from the script
                     ${block(error.stack || error.message)}
@@ -4682,9 +4771,7 @@ async function executeAttempts(options) {
                                 break scripts;
                             }
                             catch (error) {
-                                if (!(error instanceof Error)) {
-                                    throw error;
-                                }
+                                assertsError(error);
                                 if (error instanceof UnexpectedError) {
                                     throw error;
                                 }
@@ -4754,9 +4841,7 @@ async function executeAttempts(options) {
                             break scripts;
                         }
                         catch (error) {
-                            if (!(error instanceof Error)) {
-                                throw error;
-                            }
+                            assertsError(error);
                             if (error instanceof UnexpectedError) {
                                 throw error;
                             }
@@ -5377,9 +5462,7 @@ async function executePipeline(options) {
         await Promise.all(resolving);
     }
     catch (error /* <- Note: [3] */) {
-        if (!(error instanceof Error)) {
-            throw error;
-        }
+        assertsError(error);
         // Note: No need to rethrow UnexpectedError
         // if (error instanceof UnexpectedError) {
         // Note: Count usage, [🧠] Maybe put to separate function executionReportJsonToUsage + DRY [🤹‍♂️]