@promptbook/website-crawler 0.88.0-8 → 0.88.0

package/README.md

@@ -23,10 +23,6 @@
 
 
 
-<blockquote style="color: #ff8811">
-    <b>⚠ Warning:</b> This is a pre-release version of the library. It is not yet ready for production use. Please look at <a href="https://www.npmjs.com/package/@promptbook/core?activeTab=versions">latest stable release</a>.
-</blockquote>
-
 ## 📦 Package `@promptbook/website-crawler`
 
 - Promptbooks are [divided into several](#-packages) packages, all are published from [single monorepo](https://github.com/webgptorg/promptbook).
@@ -70,6 +66,9 @@ This shift is going to happen, whether we are ready for it or not. Our mission i
 
 
 
+
+
+
 ## 🚀 Get started
 
 Take a look at the simple starter kit with books integrated into the **Hello World** sample applications:
@@ -81,6 +80,8 @@ Take a look at the simple starter kit with books integrated into the **Hello Wor
 
 
 
+
+
 ## 💜 The Promptbook Project
 
 Promptbook project is ecosystem of multiple projects and tools, following is a list of most important pieces of the project:
@@ -116,6 +117,14 @@ Promptbook project is ecosystem of multiple projects and tools, following is a l
   </tbody>
 </table>
 
+Hello world examples:
+
+-   [Hello world](https://github.com/webgptorg/hello-world)
+-   [Hello world in Node.js](https://github.com/webgptorg/hello-world-node-js)
+-   [Hello world in Next.js](https://github.com/webgptorg/hello-world-next-js)
+
+
+
 We also have a community of developers and users of **Promptbook**:
 
 -   [Discord community](https://discord.gg/x3QWNaa89N)
@@ -282,16 +291,9 @@ Or you can install them separately:
 
 ## 📚 Dictionary
 
-
-
-
-
-
-### 📚 Dictionary
-
 The following glossary is used to clarify certain concepts:
 
-#### General LLM / AI terms
+### General LLM / AI terms
 
 -   **Prompt drift** is a phenomenon where the AI model starts to generate outputs that are not aligned with the original prompt. This can happen due to the model's training data, the prompt's wording, or the model's architecture.
 -   **Pipeline, workflow or chain** is a sequence of tasks that are executed in a specific order. In the context of AI, a pipeline can refer to a sequence of AI models that are used to process data.
@@ -304,11 +306,11 @@ The following glossary is used to clarify certain concepts:
 
 
 
-_Note: Thos section is not complete dictionary, more list of general AI / LLM terms that has connection with Promptbook_
+_Note: This section is not complete dictionary, more list of general AI / LLM terms that has connection with Promptbook_
 
 
 
-#### 💯 Core concepts
+### 💯 Core concepts
 
 -   [📚 Collection of pipelines](https://github.com/webgptorg/promptbook/discussions/65)
 -   [📯 Pipeline](https://github.com/webgptorg/promptbook/discussions/64)
@@ -321,7 +323,7 @@ _Note: Thos section is not complete dictionary, more list of general AI / LLM te
 -   [🔣 Words not tokens](https://github.com/webgptorg/promptbook/discussions/29)
 -   [☯ Separation of concerns](https://github.com/webgptorg/promptbook/discussions/32)
 
-##### Advanced concepts
+#### Advanced concepts
 
 -   [📚 Knowledge (Retrieval-augmented generation)](https://github.com/webgptorg/promptbook/discussions/41)
 -   [🌏 Remote server](https://github.com/webgptorg/promptbook/discussions/89)
@@ -338,17 +340,9 @@ _Note: Thos section is not complete dictionary, more list of general AI / LLM te
 
 
 
-### Terms specific to Promptbook TypeScript implementation
-
--   Anonymous mode
--   Application mode
-
-
-
-## 🔌 Usage in Typescript / Javascript
+## 🚂 Promptbook Engine
 
--   [Simple usage](./examples/usage/simple-script)
--   [Usage with client and remote server](./examples/usage/remote)
+![Schema of Promptbook Engine](./documents/promptbook-engine.svg)
 
 ## ➕➖ When to use Promptbook?
 
@@ -414,11 +408,11 @@ See [TODO.md](./TODO.md)
 <div style="display: flex; align-items: center; gap: 20px;">
 
   <a href="https://promptbook.studio/">
-    <img src="./design/promptbook-studio-logo.png" alt="Partner 3" height="100">
+    <img src="./design/promptbook-studio-logo.png" alt="Partner 3" height="70">
   </a>
 
   <a href="https://technologickainkubace.org/en/about-technology-incubation/about-the-project/">
-    <img src="./other/partners/CI-Technology-Incubation.png" alt="Technology Incubation" height="100">
+    <img src="./other/partners/CI-Technology-Incubation.png" alt="Technology Incubation" height="70">
   </a>
 
 </div>

package/esm/index.es.js

@@ -7,8 +7,8 @@ import { mkdir, rm } from 'fs/promises';
 import { basename, join, dirname } from 'path';
 import { format } from 'prettier';
 import parserHtml from 'prettier/parser-html';
-import { Subject } from 'rxjs';
 import { randomBytes } from 'crypto';
+import { Subject } from 'rxjs';
 import { forTime } from 'waitasecond';
 import sha256 from 'crypto-js/sha256';
 import { lookup, extension } from 'mime-types';
@@ -29,7 +29,7 @@ const BOOK_LANGUAGE_VERSION = '1.0.0';
  * @generated
  * @see https://github.com/webgptorg/promptbook
  */
-const PROMPTBOOK_ENGINE_VERSION = '0.88.0-8';
+const PROMPTBOOK_ENGINE_VERSION = '0.88.0';
 /**
  * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
  * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -188,7 +188,7 @@ const DEFAULT_MAX_PARALLEL_COUNT = 5; // <- TODO: [🤹‍♂️]
  *
  * @public exported from `@promptbook/core`
  */
-const DEFAULT_MAX_EXECUTION_ATTEMPTS = 3; // <- TODO: [🤹‍♂️]
+const DEFAULT_MAX_EXECUTION_ATTEMPTS = 10; // <- TODO: [🤹‍♂️]
 // <- TODO: [🕝] Make also `BOOKS_DIRNAME_ALTERNATIVES`
 /**
  * Where to store the temporary downloads
@@ -2154,6 +2154,21 @@ class MissingToolsError extends Error {
     }
 }
 
+/**
+ * Generates random token
+ *
+ * Note: This function is cryptographically secure (it uses crypto.randomBytes internally)
+ *
+ * @private internal helper function
+ * @returns secure random token
+ */
+function $randomToken(randomness) {
+    return randomBytes(randomness).toString('hex');
+}
+/**
+ * TODO: Maybe use nanoid instead https://github.com/ai/nanoid
+ */
+
 /**
  * This error indicates errors during the execution of the pipeline
  *
@@ -2161,11 +2176,17 @@ class MissingToolsError extends Error {
  */
 class PipelineExecutionError extends Error {
     constructor(message) {
+        // Added id parameter
         super(message);
         this.name = 'PipelineExecutionError';
+        // TODO: [🐙] DRY - Maybe $randomId
+        this.id = `error-${$randomToken(8 /* <- TODO: To global config + Use Base58 to avoid simmilar char conflicts   */)}`;
         Object.setPrototypeOf(this, PipelineExecutionError.prototype);
     }
 }
+/**
+ * TODO: !!!!!! Add id to all errors
+ */
 
 /**
  * Determine if the pipeline is fully prepared
@@ -2205,18 +2226,33 @@ function isPipelinePrepared(pipeline) {
  */
 
 /**
- * Generates random token
- *
- * Note: This function is cryptographically secure (it uses crypto.randomBytes internally)
- *
- * @private internal helper function
- * @returns secure random token
+ * Recursively converts JSON strings to JSON objects
+
+ * @public exported from `@promptbook/utils`
  */
-function $randomToken(randomness) {
-    return randomBytes(randomness).toString('hex');
+function jsonStringsToJsons(object) {
+    if (object === null) {
+        return object;
+    }
+    if (Array.isArray(object)) {
+        return object.map(jsonStringsToJsons);
+    }
+    if (typeof object !== 'object') {
+        return object;
+    }
+    const newObject = { ...object };
+    for (const [key, value] of Object.entries(object)) {
+        if (typeof value === 'string' && isValidJsonString(value)) {
+            newObject[key] = JSON.parse(value);
+        }
+        else {
+            newObject[key] = jsonStringsToJsons(value);
+        }
+    }
+    return newObject;
 }
 /**
- * TODO: Maybe use nanoid instead https://github.com/ai/nanoid
+ * TODO: Type the return type correctly
  */
 
 /**
@@ -2356,7 +2392,7 @@ const ALL_ERRORS = {
  * @public exported from `@promptbook/utils`
  */
 function deserializeError(error) {
-    const { name, stack } = error;
+    const { name, stack, id } = error; // Added id
     let { message } = error;
     let ErrorClass = ALL_ERRORS[error.name];
     if (ErrorClass === undefined) {
@@ -2371,7 +2407,9 @@ function deserializeError(error) {
                 ${block(stack || '')}
             `);
     }
-    return new ErrorClass(message);
+    const deserializedError = new ErrorClass(message);
+    deserializedError.id = id; // Assign id to the error object
+    return deserializedError;
 }
 
 /**
@@ -2421,17 +2459,19 @@ function assertsTaskSuccessful(executionResult) {
  */
 function createTask(options) {
     const { taskType, taskProcessCallback } = options;
+    // TODO: [🐙] DRY
     const taskId = `${taskType.toLowerCase().substring(0, 4)}-${$randomToken(8 /* <- TODO: To global config + Use Base58 to avoid simmilar char conflicts   */)}`;
     let status = 'RUNNING';
     const createdAt = new Date();
     let updatedAt = createdAt;
     const errors = [];
     const warnings = [];
-    const currentValue = {};
+    let currentValue = {};
     const partialResultSubject = new Subject();
     // <- Note: Not using `BehaviorSubject` because on error we can't access the last value
     const finalResultPromise = /* not await */ taskProcessCallback((newOngoingResult) => {
         Object.assign(currentValue, newOngoingResult);
+        // <- TODO: assign deep
         partialResultSubject.next(newOngoingResult);
     });
     finalResultPromise
@@ -2451,7 +2491,8 @@ function createTask(options) {
                 //            And delete `ExecutionTask.currentValue.preparedPipeline`
                 assertsTaskSuccessful(executionResult);
                 status = 'FINISHED';
-                Object.assign(currentValue, executionResult);
+                currentValue = jsonStringsToJsons(executionResult);
+                // <- TODO: [🧠] Is this a good idea to convert JSON strins to JSONs?
                 partialResultSubject.next(executionResult);
             }
             catch (error) {
@@ -2515,19 +2556,21 @@ function createTask(options) {
  */
 function serializeError(error) {
     const { name, message, stack } = error;
+    const { id } = error;
     if (!Object.keys(ALL_ERRORS).includes(name)) {
         console.error(spaceTrim$1((block) => `
-          
+
                     Cannot serialize error with name "${name}"
 
                     ${block(stack || message)}
-                
+
                 `));
     }
     return {
         name: name,
         message,
         stack,
+        id, // Include id in the serialized object
     };
 }
 
@@ -4325,6 +4368,9 @@ function countCharacters(text) {
     text = text.replace(/\p{Extended_Pictographic}(\u{200D}\p{Extended_Pictographic})*/gu, '-');
     return text.length;
 }
+/**
+ * TODO: [🥴] Implement counting in formats - like JSON, CSV, XML,...
+ */
 
 /**
  * Number of characters per standard line with 11pt Arial font size.
@@ -4356,6 +4402,9 @@ function countLines(text) {
     const lines = text.split('\n');
     return lines.reduce((count, line) => count + Math.ceil(line.length / CHARACTERS_PER_STANDARD_LINE), 0);
 }
+/**
+ * TODO: [🥴] Implement counting in formats - like JSON, CSV, XML,...
+ */
 
 /**
  * Counts number of pages in the text
@@ -4367,6 +4416,9 @@ function countLines(text) {
 function countPages(text) {
     return Math.ceil(countLines(text) / LINES_PER_STANDARD_PAGE);
 }
+/**
+ * TODO: [🥴] Implement counting in formats - like JSON, CSV, XML,...
+ */
 
 /**
  * Counts number of paragraphs in the text
@@ -4376,6 +4428,9 @@ function countPages(text) {
 function countParagraphs(text) {
     return text.split(/\n\s*\n/).filter((paragraph) => paragraph.trim() !== '').length;
 }
+/**
+ * TODO: [🥴] Implement counting in formats - like JSON, CSV, XML,...
+ */
 
 /**
  * Split text into sentences
@@ -4393,6 +4448,9 @@ function splitIntoSentences(text) {
 function countSentences(text) {
     return splitIntoSentences(text).length;
 }
+/**
+ * TODO: [🥴] Implement counting in formats - like JSON, CSV, XML,...
+ */
 
 /**
  * Counts number of words in the text
@@ -4406,6 +4464,9 @@ function countWords(text) {
     text = text.replace(/([a-z])([A-Z])/g, '$1 $2');
     return text.split(/[^a-zа-я0-9]+/i).filter((word) => word.length > 0).length;
 }
+/**
+ * TODO: [🥴] Implement counting in formats - like JSON, CSV, XML,...
+ */
 
 /**
  * Index of all counter functions