@promptbook/website-crawler 0.92.0-24 → 0.92.0-25

package/esm/index.es.js

@@ -29,7 +29,7 @@ const BOOK_LANGUAGE_VERSION = '1.0.0';
  * @generated
  * @see https://github.com/webgptorg/promptbook
  */
-const PROMPTBOOK_ENGINE_VERSION = '0.92.0-24';
+const PROMPTBOOK_ENGINE_VERSION = '0.92.0-25';
 /**
  * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
  * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -366,10 +366,10 @@ function $getGlobalScope() {
 }
 
 /**
- * @@@
+ * Normalizes a text string to SCREAMING_CASE (all uppercase with underscores).
  *
- * @param text @@@
- * @returns @@@
+ * @param text The text string to be converted to SCREAMING_CASE format.
+ * @returns The normalized text in SCREAMING_CASE format.
  * @example 'HELLO_WORLD'
  * @example 'I_LOVE_PROMPTBOOK'
  * @public exported from `@promptbook/utils`
@@ -421,10 +421,10 @@ function normalizeTo_SCREAMING_CASE(text) {
  */
 
 /**
- * @@@
+ * Normalizes a text string to snake_case format.
  *
- * @param text @@@
- * @returns @@@
+ * @param text The text string to be converted to snake_case format.
+ * @returns The normalized text in snake_case format.
  * @example 'hello_world'
  * @example 'i_love_promptbook'
  * @public exported from `@promptbook/utils`
@@ -549,7 +549,8 @@ class KnowledgeScrapeError extends Error {
 }
 
 /**
- * @@@
+ * Converts a name to a properly formatted subfolder path for cache storage.
+ * Handles normalization and path formatting to create consistent cache directory structures.
  *
  * @private for `FileCacheStorage`
  */
@@ -802,10 +803,10 @@ for (let i = 0; i < defaultDiacriticsRemovalMap.length; i++) {
 */
 
 /**
- * @@@
+ * Removes diacritic marks (accents) from characters in a string.
  *
- * @param input @@@
- * @returns @@@
+ * @param input The string containing diacritics to be normalized.
+ * @returns The string with diacritics removed or normalized.
  * @public exported from `@promptbook/utils`
  */
 function removeDiacritics(input) {
@@ -819,10 +820,10 @@ function removeDiacritics(input) {
  */
 
 /**
- * @@@
+ * Converts a given text to kebab-case format.
  *
- * @param text @@@
- * @returns @@@
+ * @param text The text to be converted.
+ * @returns The kebab-case formatted string.
  * @example 'hello-world'
  * @example 'i-love-promptbook'
  * @public exported from `@promptbook/utils`
@@ -964,11 +965,11 @@ function isValidUrl(url) {
 }
 
 /**
- * @@@
+ * Converts a title string into a normalized name.
  *
- * @param value @@@
- * @returns @@@
- * @example @@@
+ * @param value The title string to be converted to a name.
+ * @returns A normalized name derived from the input title.
+ * @example 'Hello World!' -> 'hello-world'
  * @public exported from `@promptbook/utils`
  */
 function titleToName(value) {
@@ -988,9 +989,8 @@ function titleToName(value) {
 }
 
 /**
- * Create a filename for intermediate cache for scrapers
- *
- * Note: It also checks if directory exists and creates it if not
+ * Retrieves an intermediate source for a scraper based on the knowledge source.
+ * Manages the caching and retrieval of intermediate scraper results for optimized performance.
  *
  * @private as internal utility for scrapers
  */
@@ -1597,13 +1597,13 @@ const ORDER_OF_PIPELINE_JSON = [
  */
 const REPLACING_NONCE = 'ptbkauk42kV2dzao34faw7FudQUHYPtW';
 /**
- * @@@
+ * Placeholder value indicating a parameter is missing its value.
  *
  * @private within the repository
  */
 const RESERVED_PARAMETER_MISSING_VALUE = 'MISSING-' + REPLACING_NONCE;
 /**
- * @@@
+ * Placeholder value indicating a parameter is restricted and cannot be used directly.
  *
  * @private within the repository
  */
@@ -2266,8 +2266,8 @@ class PipelineExecutionError extends Error {
  * @public exported from `@promptbook/core`
  */
 function isPipelinePrepared(pipeline) {
-    // Note: Ignoring `pipeline.preparations` @@@
-    // Note: Ignoring `pipeline.knowledgePieces` @@@
+    // Note: Ignoring `pipeline.preparations`
+    // Note: Ignoring `pipeline.knowledgePieces`
     if (pipeline.title === undefined || pipeline.title === '' || pipeline.title === DEFAULT_BOOK_TITLE) {
         // TODO: !!! Comment this out
         console.log('Pipeline is not prepared because title is undefined or empty', pipeline);
@@ -3216,10 +3216,11 @@ async function preparePersona(personaDescription, tools, options) {
  */
 
 /**
- * @@@
+ * Registry for all available scrapers in the system.
+ * Central point for registering and accessing different types of content scrapers.
  *
  * Note: `$` is used to indicate that this interacts with the global scope
- * @singleton Only one instance of each register is created per build, but thare can be more @@@
+ * @singleton Only one instance of each register is created per build, but there can be more than one in different build modules
  * @public exported from `@promptbook/core`
  */
 const $scrapersRegister = new $Register('scraper_constructors');
@@ -3422,7 +3423,9 @@ const promptbookFetch = async (urlOrRequest, init) => {
  */
 
 /**
- * @@@
+ * Factory function that creates a handler for processing knowledge sources.
+ * Provides standardized processing of different types of knowledge sources
+ * across various scraper implementations.
  *
  * @public exported from `@promptbook/core`
  */
@@ -3663,9 +3666,12 @@ TODO: [🧊] This is how it can look in future
  */
 
 /**
- * @@@
+ * Prepares tasks by adding knowledge to the prompt and ensuring all necessary parameters are included.
  *
- * @public exported from `@promptbook/core`
+ * @param tasks Sequence of tasks that are chained together to form a pipeline
+ * @returns A promise that resolves to the prepared tasks.
+ *
+ * @private internal utility of `preparePipeline`
  */
 async function prepareTasks(pipeline, tools, options) {
     const { maxParallelCount = DEFAULT_MAX_PARALLEL_COUNT } = options;
@@ -4428,7 +4434,7 @@ function mapAvailableToExpectedParameters(options) {
         else if (!availableParametersNames.has(parameterName) && expectedParameterNames.has(parameterName)) ;
     }
     if (expectedParameterNames.size === 0) {
-        // Note: [👨‍👨‍👧] Now we can freeze `mappedParameters` to prevent @@@
+        // Note: [👨‍👨‍👧] Now we can freeze `mappedParameters` to prevent accidental modifications after mapping
         Object.freeze(mappedParameters);
         return mappedParameters;
     }
@@ -4459,7 +4465,7 @@ function mapAvailableToExpectedParameters(options) {
     for (let i = 0; i < expectedParameterNames.size; i++) {
         mappedParameters[expectedParameterNamesArray[i]] = availableParameters[availableParametersNamesArray[i]];
     }
-    // Note: [👨‍👨‍👧] Now we can freeze `mappedParameters` to prevent @@@
+    // Note: [👨‍👨‍👧] Now we can freeze `mappedParameters` to prevent accidental modifications after mapping
     Object.freeze(mappedParameters);
     return mappedParameters;
 }
@@ -5212,11 +5218,10 @@ async function executeFormatSubvalues(options) {
                 const highLevelError = new PipelineExecutionError(spaceTrim$1((block) => `
                             ${error.message}
 
-                            This is error in FOREACH command when mapping data
+                            This is error in FOREACH command when mapping ${formatDefinition.formatName} ${subvalueParser.subvalueName} data (${index + 1}/${length})
                             You have probbably passed wrong data to pipeline or wrong data was generated which are processed by FOREACH command
 
                             ${block(pipelineIdentification)}
-                            Subparameter index: ${index}
                         `));
                 if (length > BIG_DATASET_TRESHOLD) {
                     console.error(highLevelError);
@@ -5244,12 +5249,11 @@ async function executeFormatSubvalues(options) {
             catch (error) {
                 if (length > BIG_DATASET_TRESHOLD) {
                     console.error(spaceTrim$1((block) => `
-                              Error in FOREACH command:
+                              ${error.message}
 
-                              ${block(pipelineIdentification)}
+                              This is error in FOREACH command when processing ${formatDefinition.formatName} ${subvalueParser.subvalueName} data (${index + 1}/${length})
 
                               ${block(pipelineIdentification)}
-                              Subparameter index: ${index}
                           `));
                     return FAILED_VALUE_PLACEHOLDER;
                 }
@@ -6208,7 +6212,8 @@ class WebsiteScraper {
  */
 
 /**
- * @@@
+ * Factory function to create an instance of WebsiteScraper.
+ * It bundles the scraper class with its metadata.
  *
  * @public exported from `@promptbook/website-crawler`
  */