@promptbook/website-crawler 0.84.0 → 0.85.0-0

package/README.md

@@ -24,6 +24,10 @@
 
 
 
+<blockquote style="color: #ff8811">
+    <b>⚠ Warning:</b> This is a pre-release version of the library. It is not yet ready for production use. Please look at <a href="https://www.npmjs.com/package/@promptbook/core?activeTab=versions">latest stable release</a>.
+</blockquote>
+
 ## 📦 Package `@promptbook/website-crawler`
 
 - Promptbooks are [divided into several](#-packages) packages, all are published from [single monorepo](https://github.com/webgptorg/promptbook).

package/esm/index.es.js

@@ -7,6 +7,8 @@ import { mkdir, rm } from 'fs/promises';
 import { basename, join, dirname } from 'path';
 import { format } from 'prettier';
 import parserHtml from 'prettier/parser-html';
+import { BehaviorSubject, concat, from } from 'rxjs';
+import { randomBytes } from 'crypto';
 import { forTime } from 'waitasecond';
 import sha256 from 'crypto-js/sha256';
 import { lookup, extension } from 'mime-types';
@@ -27,7 +29,7 @@ var BOOK_LANGUAGE_VERSION = '1.0.0';
  * @generated
  * @see https://github.com/webgptorg/promptbook
  */
-var PROMPTBOOK_ENGINE_VERSION = '0.84.0-21';
+var PROMPTBOOK_ENGINE_VERSION = '0.84.0';
 /**
  * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
  * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -2379,6 +2381,58 @@ var PipelineExecutionError = /** @class */ (function (_super) {
     return PipelineExecutionError;
 }(Error));
 
+/**
+ * Determine if the pipeline is fully prepared
+ *
+ * @see https://github.com/webgptorg/promptbook/discussions/196
+ *
+ * @public exported from `@promptbook/core`
+ */
+function isPipelinePrepared(pipeline) {
+    // Note: Ignoring `pipeline.preparations` @@@
+    // Note: Ignoring `pipeline.knowledgePieces` @@@
+    if (pipeline.title === undefined || pipeline.title === '' || pipeline.title === DEFAULT_BOOK_TITLE) {
+        return false;
+    }
+    if (!pipeline.personas.every(function (persona) { return persona.modelRequirements !== undefined; })) {
+        return false;
+    }
+    if (!pipeline.knowledgeSources.every(function (knowledgeSource) { return knowledgeSource.preparationIds !== undefined; })) {
+        return false;
+    }
+    /*
+    TODO: [🧠][🍫] `tasks` can not be determined if they are fully prepared SO ignoring them
+    > if (!pipeline.tasks.every(({ preparedContent }) => preparedContent === undefined)) {
+    >     return false;
+    > }
+    */
+    return true;
+}
+/**
+ * TODO: [🔃][main] If the pipeline was prepared with different version or different set of models, prepare it once again
+ * TODO: [🐠] Maybe base this on `makeValidator`
+ * TODO: [🧊] Pipeline can be partially prepared, this should return true ONLY if fully prepared
+ * TODO: [🧿] Maybe do same process with same granularity and subfinctions as `preparePipeline`
+ *     - [🏍] ? Is context in each task
+ *     - [♨] Are examples prepared
+ *     - [♨] Are tasks prepared
+ */
+
+/**
+ * Generates random token
+ *
+ * Note: This function is cryptographically secure (it uses crypto.randomBytes internally)
+ *
+ * @private internal helper function
+ * @returns secure random token
+ */
+function $randomToken(randomness) {
+    return randomBytes(randomness).toString('hex');
+}
+/**
+ * TODO: Maybe use nanoid instead https://github.com/ai/nanoid
+ */
+
 /**
  * This error indicates problems parsing the format value
  *
@@ -2548,9 +2602,9 @@ function deserializeError(error) {
  *
  * @param executionResult - The partial result of the Promptbook execution
  * @throws {PipelineExecutionError} If the execution is not successful or if multiple errors occurred
- * @public exported from `@promptbook/core`
+ * @private internal helper function of `asPromise` method of `ExecutionTask`
  */
-function assertsExecutionSuccessful(executionResult) {
+function assertsTaskSuccessful(executionResult) {
     var e_1, _a;
     var isSuccessful = executionResult.isSuccessful, errors = executionResult.errors, warnings = executionResult.warnings;
     try {
@@ -2585,121 +2639,54 @@ function assertsExecutionSuccessful(executionResult) {
     }
 }
 /**
- * TODO: [🐚] This function should be removed OR changed OR be completely rewritten
  * TODO: [🧠] Can this return type be better typed than void
  */
 
 /**
- * Determine if the pipeline is fully prepared
- *
- * @see https://github.com/webgptorg/promptbook/discussions/196
- *
- * @public exported from `@promptbook/core`
- */
-function isPipelinePrepared(pipeline) {
-    // Note: Ignoring `pipeline.preparations` @@@
-    // Note: Ignoring `pipeline.knowledgePieces` @@@
-    if (pipeline.title === undefined || pipeline.title === '' || pipeline.title === DEFAULT_BOOK_TITLE) {
-        return false;
-    }
-    if (!pipeline.personas.every(function (persona) { return persona.modelRequirements !== undefined; })) {
-        return false;
-    }
-    if (!pipeline.knowledgeSources.every(function (knowledgeSource) { return knowledgeSource.preparationIds !== undefined; })) {
-        return false;
-    }
-    /*
-    TODO: [🧠][🍫] `tasks` can not be determined if they are fully prepared SO ignoring them
-    > if (!pipeline.tasks.every(({ preparedContent }) => preparedContent === undefined)) {
-    >     return false;
-    > }
-    */
-    return true;
-}
-/**
- * TODO: [🔃][main] If the pipeline was prepared with different version or different set of models, prepare it once again
- * TODO: [🐠] Maybe base this on `makeValidator`
- * TODO: [🧊] Pipeline can be partially prepared, this should return true ONLY if fully prepared
- * TODO: [🧿] Maybe do same process with same granularity and subfinctions as `preparePipeline`
- *     - [🏍] ? Is context in each task
- *     - [♨] Are examples prepared
- *     - [♨] Are tasks prepared
- */
-
-/**
- * Format either small or big number
+ * Helper to create a new task
  *
- * @public exported from `@promptbook/utils`
+ * @private internal helper function
  */
-function numberToString(value) {
-    if (value === 0) {
-        return '0';
-    }
-    else if (Number.isNaN(value)) {
-        return VALUE_STRINGS.nan;
-    }
-    else if (value === Infinity) {
-        return VALUE_STRINGS.infinity;
-    }
-    else if (value === -Infinity) {
-        return VALUE_STRINGS.negativeInfinity;
-    }
-    for (var exponent = 0; exponent < 15; exponent++) {
-        var factor = Math.pow(10, exponent);
-        var valueRounded = Math.round(value * factor) / factor;
-        if (Math.abs(value - valueRounded) / value < SMALL_NUMBER) {
-            return valueRounded.toFixed(exponent);
-        }
+function createTask(options) {
+    var taskType = options.taskType, taskProcessCallback = options.taskProcessCallback;
+    var taskId = "".concat(taskType.toLowerCase(), "-").concat($randomToken(256 /* <- TODO: !!! To global config */));
+    var resultSubject = new BehaviorSubject({});
+    var finalResultPromise = /* not await */ taskProcessCallback(function (newOngoingResult) {
+        resultSubject.next(newOngoingResult);
+    });
+    function asPromise(options) {
+        return __awaiter(this, void 0, void 0, function () {
+            var _a, isCrashedOnError, finalResult;
+            return __generator(this, function (_b) {
+                switch (_b.label) {
+                    case 0:
+                        _a = (options || {}).isCrashedOnError, isCrashedOnError = _a === void 0 ? true : _a;
+                        return [4 /*yield*/, finalResultPromise];
+                    case 1:
+                        finalResult = _b.sent();
+                        if (isCrashedOnError) {
+                            assertsTaskSuccessful(finalResult);
+                        }
+                        return [2 /*return*/, finalResult];
+                }
+            });
+        });
     }
-    return value.toString();
+    return {
+        taskType: taskType,
+        taskId: taskId,
+        asPromise: asPromise,
+        asObservable: function () {
+            return concat(resultSubject.asObservable(), from(asPromise({
+                isCrashedOnError: true,
+            })));
+        },
+    };
 }
-
 /**
- * Function `valueToString` will convert the given value to string
- * This is useful and used in the `templateParameters` function
- *
- * Note: This function is not just calling `toString` method
- *       It's more complex and can handle this conversion specifically for LLM models
- *       See `VALUE_STRINGS`
- *
- * Note: There are 2 similar functions
- * - `valueToString` converts value to string for LLM models as human-readable string
- * - `asSerializable` converts value to string to preserve full information to be able to convert it back
- *
- * @public exported from `@promptbook/utils`
+ * TODO: Maybe allow to terminate the task and add getter `isFinished` or `status`
+ * TODO: [🐚] Split into more files and make `PrepareTask` & `RemoteTask` + split the function
  */
-function valueToString(value) {
-    try {
-        if (value === '') {
-            return VALUE_STRINGS.empty;
-        }
-        else if (value === null) {
-            return VALUE_STRINGS.null;
-        }
-        else if (value === undefined) {
-            return VALUE_STRINGS.undefined;
-        }
-        else if (typeof value === 'string') {
-            return value;
-        }
-        else if (typeof value === 'number') {
-            return numberToString(value);
-        }
-        else if (value instanceof Date) {
-            return value.toISOString();
-        }
-        else {
-            return JSON.stringify(value);
-        }
-    }
-    catch (error) {
-        if (!(error instanceof Error)) {
-            throw error;
-        }
-        console.error(error);
-        return VALUE_STRINGS.unserializable;
-    }
-}
 
 /**
  * Serializes an error into a [🚉] JSON-serializable object
@@ -3363,10 +3350,9 @@ function preparePersona(personaDescription, tools, options) {
                         return modelName;
                     })
                         .join(',');
-                    return [4 /*yield*/, preparePersonaExecutor({ availableModelNames: availableModelNames, personaDescription: personaDescription })];
+                    return [4 /*yield*/, preparePersonaExecutor({ availableModelNames: availableModelNames, personaDescription: personaDescription }).asPromise()];
                 case 3:
                     result = _d.sent();
-                    assertsExecutionSuccessful(result);
                     outputParameters = result.outputParameters;
                     modelRequirementsRaw = outputParameters.modelRequirements;
                     modelRequirements = JSON.parse(modelRequirementsRaw);
@@ -4049,10 +4035,9 @@ function preparePipeline(pipeline, tools, options) {
                                 var content = _a.content;
                                 return content;
                             }).join('\n\n'),
-                        })];
+                        }).asPromise()];
                 case 2:
                     result = _e.sent();
-                    assertsExecutionSuccessful(result);
                     outputParameters = result.outputParameters;
                     titleRaw = outputParameters.title;
                     if (isVerbose) {
@@ -4124,6 +4109,81 @@ function preparePipeline(pipeline, tools, options) {
  *       @see https://docs.anthropic.com/en/docs/test-and-evaluate/strengthen-guardrails/increase-consistency#specify-the-desired-output-format
  */
 
+/**
+ * Format either small or big number
+ *
+ * @public exported from `@promptbook/utils`
+ */
+function numberToString(value) {
+    if (value === 0) {
+        return '0';
+    }
+    else if (Number.isNaN(value)) {
+        return VALUE_STRINGS.nan;
+    }
+    else if (value === Infinity) {
+        return VALUE_STRINGS.infinity;
+    }
+    else if (value === -Infinity) {
+        return VALUE_STRINGS.negativeInfinity;
+    }
+    for (var exponent = 0; exponent < 15; exponent++) {
+        var factor = Math.pow(10, exponent);
+        var valueRounded = Math.round(value * factor) / factor;
+        if (Math.abs(value - valueRounded) / value < SMALL_NUMBER) {
+            return valueRounded.toFixed(exponent);
+        }
+    }
+    return value.toString();
+}
+
+/**
+ * Function `valueToString` will convert the given value to string
+ * This is useful and used in the `templateParameters` function
+ *
+ * Note: This function is not just calling `toString` method
+ *       It's more complex and can handle this conversion specifically for LLM models
+ *       See `VALUE_STRINGS`
+ *
+ * Note: There are 2 similar functions
+ * - `valueToString` converts value to string for LLM models as human-readable string
+ * - `asSerializable` converts value to string to preserve full information to be able to convert it back
+ *
+ * @public exported from `@promptbook/utils`
+ */
+function valueToString(value) {
+    try {
+        if (value === '') {
+            return VALUE_STRINGS.empty;
+        }
+        else if (value === null) {
+            return VALUE_STRINGS.null;
+        }
+        else if (value === undefined) {
+            return VALUE_STRINGS.undefined;
+        }
+        else if (typeof value === 'string') {
+            return value;
+        }
+        else if (typeof value === 'number') {
+            return numberToString(value);
+        }
+        else if (value instanceof Date) {
+            return value.toISOString();
+        }
+        else {
+            return JSON.stringify(value);
+        }
+    }
+    catch (error) {
+        if (!(error instanceof Error)) {
+            throw error;
+        }
+        console.error(error);
+        return VALUE_STRINGS.unserializable;
+    }
+}
+
 /**
  * Parses the given script and returns the list of all used variables that are not defined in the script
  *
@@ -5568,27 +5628,20 @@ function getReservedParametersForTask(options) {
  */
 function executeTask(options) {
     return __awaiter(this, void 0, void 0, function () {
-        var currentTask, preparedPipeline, parametersToPass, tools, onProgress, $executionReport, pipelineIdentification, maxExecutionAttempts, maxParallelCount, csvSettings, isVerbose, rootDirname, cacheDirname, intermediateFilesStrategy, isAutoInstalled, isNotPreparedWarningSupressed, name, title, priority, usedParameterNames, dependentParameterNames, definedParameters, _a, _b, _c, definedParameterNames, parameters, _loop_1, _d, _e, parameterName, maxAttempts, jokerParameterNames, preparedContent, resultString;
-        var e_1, _f, _g;
-        return __generator(this, function (_h) {
-            switch (_h.label) {
+        var currentTask, preparedPipeline, parametersToPass, tools, onProgress, $executionReport, pipelineIdentification, maxExecutionAttempts, maxParallelCount, csvSettings, isVerbose, rootDirname, cacheDirname, intermediateFilesStrategy, isAutoInstalled, isNotPreparedWarningSupressed, priority, usedParameterNames, dependentParameterNames, definedParameters, _a, _b, _c, definedParameterNames, parameters, _loop_1, _d, _e, parameterName, maxAttempts, jokerParameterNames, preparedContent, resultString;
+        var _f, e_1, _g, _h, _j;
+        return __generator(this, function (_k) {
+            switch (_k.label) {
                 case 0:
                     currentTask = options.currentTask, preparedPipeline = options.preparedPipeline, parametersToPass = options.parametersToPass, tools = options.tools, onProgress = options.onProgress, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification, maxExecutionAttempts = options.maxExecutionAttempts, maxParallelCount = options.maxParallelCount, csvSettings = options.csvSettings, isVerbose = options.isVerbose, rootDirname = options.rootDirname, cacheDirname = options.cacheDirname, intermediateFilesStrategy = options.intermediateFilesStrategy, isAutoInstalled = options.isAutoInstalled, isNotPreparedWarningSupressed = options.isNotPreparedWarningSupressed;
-                    name = "pipeline-executor-frame-".concat(currentTask.name);
-                    title = currentTask.title;
                     priority = preparedPipeline.tasks.length - preparedPipeline.tasks.indexOf(currentTask);
                     return [4 /*yield*/, onProgress({
-                            name: name,
-                            title: title,
-                            isStarted: false,
-                            isDone: false,
-                            taskType: currentTask.taskType,
-                            parameterName: currentTask.resultingParameterName,
-                            parameterValue: null,
-                            // <- [🍸]
+                            outputParameters: (_f = {},
+                                _f[currentTask.resultingParameterName] = '',
+                                _f),
                         })];
                 case 1:
-                    _h.sent();
+                    _k.sent();
                     usedParameterNames = extractParameterNamesFromTask(currentTask);
                     dependentParameterNames = new Set(currentTask.dependentParameterNames);
                     // TODO: [👩🏾‍🤝‍👩🏻] Use here `mapAvailableToExpectedParameters`
@@ -5607,7 +5660,7 @@ function executeTask(options) {
                             pipelineIdentification: pipelineIdentification,
                         })];
                 case 2:
-                    definedParameters = _b.apply(_a, [__assign.apply(void 0, [__assign.apply(void 0, _c.concat([(_h.sent())])), parametersToPass])]);
+                    definedParameters = _b.apply(_a, [__assign.apply(void 0, [__assign.apply(void 0, _c.concat([(_k.sent())])), parametersToPass])]);
                     definedParameterNames = new Set(Object.keys(definedParameters));
                     parameters = {};
                     _loop_1 = function (parameterName) {
@@ -5635,7 +5688,7 @@ function executeTask(options) {
                     catch (e_1_1) { e_1 = { error: e_1_1 }; }
                     finally {
                         try {
-                            if (_e && !_e.done && (_f = _d.return)) _f.call(_d);
+                            if (_e && !_e.done && (_g = _d.return)) _g.call(_d);
                         }
                         finally { if (e_1) throw e_1.error; }
                     }
@@ -5666,24 +5719,19 @@ function executeTask(options) {
                             isNotPreparedWarningSupressed: isNotPreparedWarningSupressed,
                         })];
                 case 3:
-                    resultString = _h.sent();
+                    resultString = _k.sent();
                     return [4 /*yield*/, onProgress({
-                            name: name,
-                            title: title,
-                            isStarted: true,
-                            isDone: true,
-                            taskType: currentTask.taskType,
-                            parameterName: currentTask.resultingParameterName,
-                            parameterValue: resultString,
-                            // <- [🍸]
+                            outputParameters: (_h = {},
+                                _h[currentTask.resultingParameterName] = resultString,
+                                _h),
                         })];
                 case 4:
-                    _h.sent();
-                    return [2 /*return*/, Object.freeze((_g = {},
-                            _g[currentTask.resultingParameterName] = 
+                    _k.sent();
+                    return [2 /*return*/, Object.freeze((_j = {},
+                            _j[currentTask.resultingParameterName] = 
                             // <- Note: [👩‍👩‍👧] No need to detect parameter collision here because pipeline checks logic consistency during construction
                             resultString,
-                            _g))];
+                            _j))];
             }
         });
     });
@@ -5691,9 +5739,6 @@ function executeTask(options) {
 /**
  * TODO: [🤹‍♂️]
  */
-/**
- * TODO: [🐚] Change onProgress to object that represents the running execution, can be subscribed via RxJS to and also awaited
- */
 
 /**
  * @@@
@@ -5955,15 +6000,15 @@ function executePipeline(options) {
                                     return [3 /*break*/, 4];
                                 case 3:
                                     unresovedTasks_1 = unresovedTasks_1.filter(function (task) { return task !== currentTask; });
-                                    work_1 = executeTask(__assign(__assign({}, options), { currentTask: currentTask, preparedPipeline: preparedPipeline, parametersToPass: parametersToPass, tools: tools, onProgress: function (progress) {
+                                    work_1 = executeTask(__assign(__assign({}, options), { currentTask: currentTask, preparedPipeline: preparedPipeline, parametersToPass: parametersToPass, tools: tools, onProgress: function (newOngoingResult) {
                                             if (isReturned) {
-                                                throw new UnexpectedError(spaceTrim(function (block) { return "\n                                        Can not call `onProgress` after pipeline execution is finished\n\n                                        ".concat(block(pipelineIdentification), "\n\n                                        ").concat(block(JSON.stringify(progress, null, 4)
+                                                throw new UnexpectedError(spaceTrim(function (block) { return "\n                                        Can not call `onProgress` after pipeline execution is finished\n\n                                        ".concat(block(pipelineIdentification), "\n\n                                        ").concat(block(JSON.stringify(newOngoingResult, null, 4)
                                                     .split('\n')
                                                     .map(function (line) { return "> ".concat(line); })
                                                     .join('\n')), "\n                                    "); }));
                                             }
                                             if (onProgress) {
-                                                onProgress(progress);
+                                                onProgress(newOngoingResult);
                                             }
                                         }, $executionReport: executionReport, pipelineIdentification: spaceTrim(function (block) { return "\n                            ".concat(block(pipelineIdentification), "\n                            Task name: ").concat(currentTask.name, "\n                            Task title: ").concat(currentTask.title, "\n                        "); }) }))
                                         .then(function (newParametersToPass) {
@@ -6066,9 +6111,6 @@ function executePipeline(options) {
         });
     });
 }
-/**
- * TODO: [🐚] Change onProgress to object that represents the running execution, can be subscribed via RxJS to and also awaited
- */
 
 /**
  * Creates executor function from pipeline and execution tools.
@@ -6100,7 +6142,7 @@ function createPipelineExecutor(options) {
         console.warn(spaceTrim(function (block) { return "\n                    Pipeline is not prepared\n\n                    ".concat(block(pipelineIdentification), "\n\n                    It will be prepared ad-hoc before the first execution and **returned as `preparedPipeline` in `PipelineExecutorResult`**\n                    But it is recommended to prepare the pipeline during collection preparation\n\n                    @see more at https://ptbk.io/prepare-pipeline\n                "); }));
     }
     var runCount = 0;
-    var pipelineExecutor = function (inputParameters, onProgress) { return __awaiter(_this, void 0, void 0, function () {
+    var pipelineExecutorWithCallback = function (inputParameters, onProgress) { return __awaiter(_this, void 0, void 0, function () {
         return __generator(this, function (_a) {
             runCount++;
             return [2 /*return*/, /* not await */ executePipeline({
@@ -6125,11 +6167,23 @@ function createPipelineExecutor(options) {
                 })];
         });
     }); };
+    var pipelineExecutor = function (inputParameters) {
+        return createTask({
+            taskType: 'EXECUTION',
+            taskProcessCallback: function (updateOngoingResult) {
+                var _this = this;
+                return pipelineExecutorWithCallback(inputParameters, function (newOngoingResult) { return __awaiter(_this, void 0, void 0, function () {
+                    return __generator(this, function (_a) {
+                        updateOngoingResult(newOngoingResult);
+                        return [2 /*return*/];
+                    });
+                }); });
+            },
+        });
+    };
+    //        <- TODO: Make types such as there is no need to do `as` for `createTask`
     return pipelineExecutor;
 }
-/**
- * TODO: [🐚] Change onProgress to object that represents the running execution, can be subscribed via RxJS to and also awaited
- */
 
 /**
  * Metadata of the scraper
@@ -6231,10 +6285,9 @@ var MarkdownScraper = /** @class */ (function () {
                         return [4 /*yield*/, source.asText()];
                     case 4:
                         knowledgeContent = _k.sent();
-                        return [4 /*yield*/, prepareKnowledgeFromMarkdownExecutor({ knowledgeContent: knowledgeContent })];
+                        return [4 /*yield*/, prepareKnowledgeFromMarkdownExecutor({ knowledgeContent: knowledgeContent }).asPromise()];
                     case 5:
                         result = _k.sent();
-                        assertsExecutionSuccessful(result);
                         outputParameters = result.outputParameters;
                         knowledgePiecesRaw = outputParameters.knowledgePieces;
                         knowledgeTextPieces = (knowledgePiecesRaw || '').split('\n---\n');
@@ -6257,13 +6310,13 @@ var MarkdownScraper = /** @class */ (function () {
                                             _c.label = 1;
                                         case 1:
                                             _c.trys.push([1, 7, , 8]);
-                                            return [4 /*yield*/, prepareTitleExecutor({ knowledgePieceContent: knowledgePieceContent })];
+                                            return [4 /*yield*/, prepareTitleExecutor({ knowledgePieceContent: knowledgePieceContent }).asPromise()];
                                         case 2:
                                             titleResult = _c.sent();
                                             _a = titleResult.outputParameters.title, titleRaw = _a === void 0 ? 'Untitled' : _a;
                                             title = spaceTrim$1(titleRaw) /* <- TODO: Maybe do in pipeline */;
                                             name = titleToName(title);
-                                            return [4 /*yield*/, prepareKeywordsExecutor({ knowledgePieceContent: knowledgePieceContent })];
+                                            return [4 /*yield*/, prepareKeywordsExecutor({ knowledgePieceContent: knowledgePieceContent }).asPromise()];
                                         case 3:
                                             keywordsResult = _c.sent();
                                             _b = keywordsResult.outputParameters.keywords, keywordsRaw = _b === void 0 ? '' : _b;