@promptbook/cli 0.88.0-1 → 0.88.0-9

package/README.md

@@ -112,8 +112,6 @@ Rest of the documentation is common for **entire promptbook ecosystem**:
 
 During the computer revolution, we have seen [multiple generations of computer languages](https://github.com/webgptorg/promptbook/discussions/180), from the physical rewiring of the vacuum tubes through low-level machine code to the high-level languages like Python or JavaScript. And now, we're on the edge of the **next revolution**!
 
-
-
 It's a revolution of writing software in **plain human language** that is understandable and executable by both humans and machines – and it's going to change everything!
 
 The incredible growth in power of microprocessors and the Moore's Law have been the driving force behind the ever-more powerful languages, and it's been an amazing journey! Similarly, the large language models (like GPT or Claude) are the next big thing in language technology, and they're set to transform the way we interact with computers.
@@ -176,21 +174,16 @@ We also have a community of developers and users of **Promptbook**:
 -   [Landing page `ptbk.io`](https://ptbk.io)
 -   [Github discussions](https://github.com/webgptorg/promptbook/discussions)
 -   [LinkedIn `Promptbook`](https://linkedin.com/company/promptbook)
--   [Facebook `Promptbook`](https://www.facebook.com/61560776453536) 
+-   [Facebook `Promptbook`](https://www.facebook.com/61560776453536)
 
 And **Promptbook.studio** branded socials:
 
-
-
 -   [Instagram `@promptbook.studio`](https://www.instagram.com/promptbook.studio/)
 
 And **Promptujeme** sub-brand:
 
 _/Subbrand for Czech clients/_
 
-
-
-
 -   [Promptujeme.cz](https://www.promptujeme.cz/)
 -   [Facebook `Promptujeme`](https://www.facebook.com/promptujeme/)
 
@@ -208,8 +201,6 @@ _/Sub-brand for images and graphics generated via Promptbook prompting/_
 
 ## 💙 The Book language
 
-
-
 Following is the documentation and blueprint of the [Book language](https://github.com/webgptorg/book).
 
 Book is a language that can be used to write AI applications, agents, workflows, automations, knowledgebases, translators, sheet processors, email automations and more. It allows you to harness the power of AI models in human-like terms, without the need to know the specifics and technicalities of the models.
@@ -259,8 +250,6 @@ Personas can have access to different knowledge, tools and actions. They can als
 
 -   [PERSONA](https://github.com/webgptorg/promptbook/blob/main/documents/commands/PERSONA.md)
 
-
-
 ### **How:** Knowledge, Instruments and Actions
 
 The resources used by the personas are used to do the work.
@@ -336,11 +325,6 @@ Or you can install them separately:
 
 ## 📚 Dictionary
 
-
-
-
-
-
 ### 📚 Dictionary
 
 The following glossary is used to clarify certain concepts:
@@ -356,12 +340,8 @@ The following glossary is used to clarify certain concepts:
 -   **Retrieval-augmented generation** is a machine learning paradigm where a model generates text by retrieving relevant information from a large database of text. This approach combines the benefits of generative models and retrieval models.
 -   **Longtail** refers to non-common or rare events, items, or entities that are not well-represented in the training data of machine learning models. Longtail items are often challenging for models to predict accurately.
 
-
-
 _Note: Thos section is not complete dictionary, more list of general AI / LLM terms that has connection with Promptbook_
 
-
-
 #### 💯 Core concepts
 
 -   [📚 Collection of pipelines](https://github.com/webgptorg/promptbook/discussions/65)
@@ -390,8 +370,6 @@ _Note: Thos section is not complete dictionary, more list of general AI / LLM te
 -   [👮 Agent adversary expectations](https://github.com/webgptorg/promptbook/discussions/39)
 -   [view more](https://github.com/webgptorg/promptbook/discussions/categories/concepts)
 
-
-
 ### Terms specific to Promptbook TypeScript implementation
 
 -   Anonymous mode
@@ -399,10 +377,9 @@ _Note: Thos section is not complete dictionary, more list of general AI / LLM te
 
 
 
-## 🔌 Usage in Typescript / Javascript
+## 🚂 Promptbook Engine
 
--   [Simple usage](./examples/usage/simple-script)
--   [Usage with client and remote server](./examples/usage/remote)
+![Schema of Promptbook Engine](./documents/promptbook-engine.svg)
 
 ## ➕➖ When to use Promptbook?
 

package/esm/index.es.js

@@ -11,7 +11,7 @@ import { spawn } from 'child_process';
 import JSZip from 'jszip';
 import { format } from 'prettier';
 import parserHtml from 'prettier/parser-html';
-import { BehaviorSubject } from 'rxjs';
+import { Subject } from 'rxjs';
 import { randomBytes } from 'crypto';
 import { parse, unparse } from 'papaparse';
 import { SHA256 } from 'crypto-js';
@@ -44,7 +44,7 @@ const BOOK_LANGUAGE_VERSION = '1.0.0';
  * @generated
  * @see https://github.com/webgptorg/promptbook
  */
-const PROMPTBOOK_ENGINE_VERSION = '0.88.0-1';
+const PROMPTBOOK_ENGINE_VERSION = '0.88.0-9';
 /**
  * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
  * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -397,6 +397,30 @@ const $isRunningInWebWorker = new Function(`
  * TODO: [🎺]
  */
 
+/**
+ * Wraps action to handle error console logging and exit process with error code
+ *
+ * @param action Action to be wrapped in error handling
+ * @returns Wrapped action
+ * @private internal helper function for CLI commands
+ */
+function handleActionErrors(action) {
+    return async (...args) => {
+        try {
+            await action(...args);
+            return process.exit(0);
+        }
+        catch (error) {
+            if (!(error instanceof Error)) {
+                throw error;
+            }
+            // console.error(colors.bgRed(error.name));
+            console.error(colors.red(/* error.stack || */ error.message));
+            return process.exit(1);
+        }
+    };
+}
+
 /**
  * Initializes `about` command for Promptbook CLI utilities
  *
@@ -409,7 +433,7 @@ function $initializeAboutCommand(program) {
     makeCommand.description(spaceTrim(`
             Tells about Promptbook CLI and its abilities
         `));
-    makeCommand.action(async () => {
+    makeCommand.action(handleActionErrors(async () => {
         console.info(colors.bold(colors.blue(`Promptbook: ${CLAIM}`)));
         console.info(colors.cyan(`Book language version: ${BOOK_LANGUAGE_VERSION}`));
         console.info(colors.cyan(`Promptbook engine version: ${PROMPTBOOK_ENGINE_VERSION}`));
@@ -436,7 +460,7 @@ function $initializeAboutCommand(program) {
         console.info(colors.gray(`https://github.com/webgptorg/promptbook`));
         console.info(colors.gray(`https://ptbk.io`));
         return process.exit(0);
-    });
+    }));
 }
 /**
  * TODO: [🗽] Unite branding and make single place for it
@@ -459,12 +483,12 @@ function $initializeHelloCommand(program) {
     helloCommand.alias('hi');
     helloCommand.argument('[name]', 'Your name', 'Paul');
     helloCommand.option('-g, --greeting <greeting>', `Greeting`, 'Hello');
-    helloCommand.action(async (name, { greeting }) => {
+    helloCommand.action(handleActionErrors(async (name, { greeting }) => {
         console.info(colors.cyan(`${greeting} ${name}`));
         await forTime(1000);
         console.info(colors.rainbow(`Nice to meet you!`));
         return process.exit(0);
-    });
+    }));
 }
 /**
  * TODO: [🧠][🐣] Make here some easter egg with generated hello greeting via LLM models
@@ -2553,13 +2577,13 @@ function $initializeListModelsCommand(program) {
         `));
     listModelsCommand.alias('models');
     listModelsCommand.alias('llm');
-    listModelsCommand.action(async () => {
+    listModelsCommand.action(handleActionErrors(async () => {
         const llm = await $provideLlmToolsForWizzardOrCli({});
         $sideEffect(llm);
         // <- Note: Providing LLM tools will make a side effect of registering all available LLM tools to show the message
         console.info($registeredLlmToolsMessage());
         return process.exit(0);
-    });
+    }));
 }
 /**
  * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -3085,7 +3109,7 @@ function $initializeListScrapersCommand(program) {
             List all available and configured scrapers and executables
         `));
     listModelsCommand.alias('scrapers');
-    listModelsCommand.action(async () => {
+    listModelsCommand.action(handleActionErrors(async () => {
         const scrapers = await $provideScrapersForNode({});
         const executables = await $provideExecutablesForNode();
         console.info(spaceTrim((block) => `
@@ -3102,7 +3126,7 @@ function $initializeListScrapersCommand(program) {
             .join('\n'))}
                 `));
         return process.exit(0);
-    });
+    }));
 }
 /**
  * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -4064,6 +4088,36 @@ function $randomToken(randomness) {
  * TODO: Maybe use nanoid instead https://github.com/ai/nanoid
  */
 
+/**
+ * Recursively converts JSON strings to JSON objects
+
+ * @public exported from `@promptbook/utils`
+ */
+function jsonStringsToJsons(object) {
+    if (object === null) {
+        return object;
+    }
+    if (Array.isArray(object)) {
+        return object.map(jsonStringsToJsons);
+    }
+    if (typeof object !== 'object') {
+        return object;
+    }
+    const newObject = { ...object };
+    for (const [key, value] of Object.entries(object)) {
+        if (typeof value === 'string' && isValidJsonString(value)) {
+            newObject[key] = JSON.parse(value);
+        }
+        else {
+            newObject[key] = jsonStringsToJsons(value);
+        }
+    }
+    return newObject;
+}
+/**
+ * TODO: Type the return type correctly
+ */
+
 /**
  * This error indicates problems parsing the format value
  *
@@ -4280,21 +4334,43 @@ function assertsTaskSuccessful(executionResult) {
 function createTask(options) {
     const { taskType, taskProcessCallback } = options;
     const taskId = `${taskType.toLowerCase().substring(0, 4)}-${$randomToken(8 /* <- TODO: To global config + Use Base58 to avoid simmilar char conflicts   */)}`;
-    const partialResultSubject = new BehaviorSubject({});
+    let status = 'RUNNING';
+    const createdAt = new Date();
+    let updatedAt = createdAt;
+    const errors = [];
+    const warnings = [];
+    let currentValue = {};
+    const partialResultSubject = new Subject();
+    // <- Note: Not using `BehaviorSubject` because on error we can't access the last value
     const finalResultPromise = /* not await */ taskProcessCallback((newOngoingResult) => {
+        Object.assign(currentValue, newOngoingResult);
+        // <- TODO: assign deep
         partialResultSubject.next(newOngoingResult);
     });
     finalResultPromise
         .catch((error) => {
+        errors.push(error);
         partialResultSubject.error(error);
     })
-        .then((value) => {
-        if (value) {
+        .then((executionResult) => {
+        if (executionResult) {
             try {
-                assertsTaskSuccessful(value);
-                partialResultSubject.next(value);
+                updatedAt = new Date();
+                errors.push(...executionResult.errors);
+                warnings.push(...executionResult.warnings);
+                // <- TODO: !!! Only unique errors and warnings should be added (or filtered)
+                // TODO: [🧠] !!! errors, warning, isSuccessful  are redundant both in `ExecutionTask` and `ExecutionTask.currentValue`
+                //            Also maybe move `ExecutionTask.currentValue.usage` -> `ExecutionTask.usage`
+                //            And delete `ExecutionTask.currentValue.preparedPipeline`
+                assertsTaskSuccessful(executionResult);
+                status = 'FINISHED';
+                currentValue = jsonStringsToJsons(executionResult);
+                // <- TODO: Convert JSON values in string to JSON objects
+                partialResultSubject.next(executionResult);
             }
             catch (error) {
+                status = 'ERROR';
+                errors.push(error);
                 partialResultSubject.error(error);
             }
         }
@@ -4311,12 +4387,33 @@ function createTask(options) {
     return {
         taskType,
         taskId,
+        get status() {
+            return status;
+            // <- Note: [1] Theese must be getters to allow changing the value in the future
+        },
+        get createdAt() {
+            return createdAt;
+            // <- Note: [1]
+        },
+        get updatedAt() {
+            return updatedAt;
+            // <- Note: [1]
+        },
         asPromise,
         asObservable() {
             return partialResultSubject.asObservable();
         },
+        get errors() {
+            return errors;
+            // <- Note: [1]
+        },
+        get warnings() {
+            return warnings;
+            // <- Note: [1]
+        },
         get currentValue() {
-            return partialResultSubject.value;
+            return currentValue;
+            // <- Note: [1]
         },
     };
 }
@@ -5512,7 +5609,7 @@ async function executeAttempts(options) {
                         Last result:
                         ${block($ongoingTaskResult.$resultString === null
                     ? 'null'
-                    : $ongoingTaskResult.$resultString
+                    : spaceTrim$1($ongoingTaskResult.$resultString)
                         .split('\n')
                         .map((line) => `> ${line}`)
                         .join('\n'))}
@@ -11229,7 +11326,7 @@ function $initializeMakeCommand(program) {
             Note: This can be used only with "javascript" or "typescript" format
 
         `), DEFAULT_GET_PIPELINE_COLLECTION_FUNCTION_NAME);
-    makeCommand.action(async (path, { projectName, rootUrl, format, functionName, validation, reload: isCacheReloaded, verbose: isVerbose, output, }) => {
+    makeCommand.action(handleActionErrors(async (path, { projectName, rootUrl, format, functionName, validation, reload: isCacheReloaded, verbose: isVerbose, output, }) => {
         if (!isValidJavascriptName(functionName)) {
             console.error(colors.red(`Function name "${functionName}" is not valid javascript name`));
             return process.exit(1);
@@ -11256,9 +11353,9 @@ function $initializeMakeCommand(program) {
             isVerbose,
             isCacheReloaded,
         }; /* <- TODO: ` satisfies PrepareAndScrapeOptions` */
-        const fs = $provideFilesystemForNode();
+        const fs = $provideFilesystemForNode(prepareAndScrapeOptions);
         const llm = await $provideLlmToolsForWizzardOrCli(prepareAndScrapeOptions);
-        const executables = await $provideExecutablesForNode();
+        const executables = await $provideExecutablesForNode(prepareAndScrapeOptions);
         const tools = {
             llm,
             fs,
@@ -11416,7 +11513,7 @@ function $initializeMakeCommand(program) {
             console.info(colors.cyan(usageToHuman(llm.getTotalUsage())));
         }
         return process.exit(0);
-    });
+    }));
 }
 /**
  * TODO: [🥃][main] !!3 Allow `ptbk make` without configuring any llm tools
@@ -11523,7 +11620,7 @@ function $initializePrettifyCommand(program) {
     'Pipelines to prettify as glob pattern');
     prettifyCommand.option('-i, --ignore <glob>', `Ignore as glob pattern`);
     prettifyCommand.option('-v, --verbose', `Is output verbose`, false);
-    prettifyCommand.action(async (filesGlob, { ignore, verbose: isVerbose }) => {
+    prettifyCommand.action(handleActionErrors(async (filesGlob, { ignore, verbose: isVerbose }) => {
         const filenames = await glob(filesGlob, { ignore });
         //                       <- TODO: [😶]
         for (const filename of filenames) {
@@ -11555,7 +11652,7 @@ function $initializePrettifyCommand(program) {
         }
         console.info(colors.green(`All pipelines are prettified`));
         return process.exit(0);
-    });
+    }));
 }
 /**
  * TODO: [😶] Unite floder listing
@@ -12129,7 +12226,7 @@ function $initializeRunCommand(program) {
     runCommand.option('--no-formfactor', `When set, behavior of the interactive mode is not changed by the formfactor of the pipeline`);
     runCommand.option('-j, --json <json>', `Pass all or some input parameters as JSON record, if used the output is also returned as JSON`);
     runCommand.option('-s, --save-report <path>', `Save report to file`);
-    runCommand.action(async (pipelineSource, options) => {
+    runCommand.action(handleActionErrors(async (pipelineSource, options) => {
         const { reload: isCacheReloaded, interactive: isInteractive, formfactor: isFormfactorUsed, json, verbose: isVerbose, saveReport, } = options;
         if (pipelineSource.includes('-') && normalizeToKebabCase(pipelineSource) === pipelineSource) {
             console.error(colors.red(`""${pipelineSource}" is not a valid command or book. See 'ptbk --help'.`));
@@ -12152,7 +12249,7 @@ function $initializeRunCommand(program) {
         if (isVerbose) {
             console.info(colors.gray('--- Preparing tools ---'));
         }
-        const fs = $provideFilesystemForNode();
+        const fs = $provideFilesystemForNode(prepareAndScrapeOptions);
         let llm;
         try {
             llm = await $provideLlmToolsForWizzardOrCli(prepareAndScrapeOptions);
@@ -12206,7 +12303,7 @@ function $initializeRunCommand(program) {
         if (isVerbose) {
             console.info(colors.gray('--- Getting the tools ---'));
         }
-        const executables = await $provideExecutablesForNode();
+        const executables = await $provideExecutablesForNode(prepareAndScrapeOptions);
         const tools = {
             llm,
             fs,
@@ -12355,7 +12452,7 @@ function $initializeRunCommand(program) {
             console.info(JSON.stringify(outputParameters, null, 4));
         }
         return process.exit(0);
-    });
+    }));
 }
 /**
  * TODO: !!5 Catch and wrap all errors from CLI
@@ -12543,8 +12640,35 @@ function startRemoteServer(options) {
                 .send({ error: serializeError(error) });
         }
     });
+    function exportExecutionTask(executionTask, isFull) {
+        // <- TODO: [🧠] This should be maybe method of `ExecutionTask` itself
+        const { taskType, taskId, status, errors, warnings, createdAt, updatedAt, currentValue } = executionTask;
+        if (isFull) {
+            return {
+                nonce: '✨',
+                taskId,
+                taskType,
+                status,
+                errors: errors.map(serializeError),
+                warnings: warnings.map(serializeError),
+                createdAt,
+                updatedAt,
+                currentValue,
+            };
+        }
+        else {
+            return {
+                nonce: '✨',
+                taskId,
+                taskType,
+                status,
+                createdAt,
+                updatedAt,
+            };
+        }
+    }
     app.get(`${rootPath}/executions`, async (request, response) => {
-        response.send(runningExecutionTasks);
+        response.send(runningExecutionTasks.map((runningExecutionTask) => exportExecutionTask(runningExecutionTask, false)));
     });
     app.get(`${rootPath}/executions/last`, async (request, response) => {
         // TODO: [🤬] Filter only for user
@@ -12552,20 +12676,20 @@ function startRemoteServer(options) {
             response.status(404).send('No execution tasks found');
             return;
         }
-        const lastExecution = runningExecutionTasks[runningExecutionTasks.length - 1];
-        response.send(lastExecution);
+        const lastExecutionTask = runningExecutionTasks[runningExecutionTasks.length - 1];
+        response.send(exportExecutionTask(lastExecutionTask, true));
     });
     app.get(`${rootPath}/executions/:taskId`, async (request, response) => {
         const { taskId } = request.params;
         // TODO: [🤬] Filter only for user
-        const execution = runningExecutionTasks.find((executionTask) => executionTask.taskId === taskId);
-        if (execution === undefined) {
+        const executionTask = runningExecutionTasks.find((executionTask) => executionTask.taskId === taskId);
+        if (executionTask === undefined) {
             response
                 .status(404)
                 .send(`Execution "${taskId}" not found`);
             return;
         }
-        response.send(execution.currentValue);
+        response.send(exportExecutionTask(executionTask, true));
     });
     app.post(`${rootPath}/executions/new`, async (request, response) => {
         try {
@@ -12799,7 +12923,7 @@ function $initializeStartServerCommand(program) {
             Starts a remote server to execute books
         `));
     startServerCommand.alias('server');
-    startServerCommand.action(async (path, { port: portRaw, url: rawUrl, allowAnonymous: isAnonymousModeAllowed, reload: isCacheReloaded, verbose: isVerbose, }) => {
+    startServerCommand.action(handleActionErrors(async (path, { port: portRaw, url: rawUrl, allowAnonymous: isAnonymousModeAllowed, reload: isCacheReloaded, verbose: isVerbose, }) => {
         if (rawUrl && !isValidUrl(rawUrl)) {
             console.error(colors.red(`Invalid URL: ${rawUrl}`));
             return process.exit(1);
@@ -12826,9 +12950,9 @@ function $initializeStartServerCommand(program) {
             isVerbose,
             isCacheReloaded,
         }; /* <- TODO: ` satisfies PrepareAndScrapeOptions` */
-        const fs = $provideFilesystemForNode();
+        const fs = $provideFilesystemForNode(prepareAndScrapeOptions);
         const llm = await $provideLlmToolsForWizzardOrCli(prepareAndScrapeOptions);
-        const executables = await $provideExecutablesForNode();
+        const executables = await $provideExecutablesForNode(prepareAndScrapeOptions);
         const tools = {
             llm,
             fs,
@@ -12845,20 +12969,23 @@ function $initializeStartServerCommand(program) {
             // <- TODO: [🍖] Add `intermediateFilesStrategy`
         });
         // console.log(path, await collection.listPipelines());
-        startRemoteServer({
+        const server = startRemoteServer({
             rootPath,
             port,
             isAnonymousModeAllowed,
             isApplicationModeAllowed: true,
             collection,
             createLlmExecutionTools(options) {
+                const { appId, userId } = options;
+                TODO_USE({ appId, userId });
                 return llm;
             },
         });
+        keepUnused(server);
         // Note: Already logged by `startRemoteServer`
         // console.error(colors.green(`Server started on port ${port}`));
         return await forEver();
-    });
+    }));
 }
 /**
  * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -12885,7 +13012,7 @@ function $initializeTestCommand(program) {
     testCommand.option('--no-prepare', `Do not prepare the pipelines, ideal when no LLM tools or scrapers available`, true);
     testCommand.option('-r, --reload', `Call LLM models even if same prompt with result is in the cache `, false);
     testCommand.option('-v, --verbose', `Is output verbose`, false);
-    testCommand.action(async (filesGlob, { ignore: ignoreRaw = '', validation: isValidated, prepare: isPrepared, reload: isCacheReloaded, verbose: isVerbose, }) => {
+    testCommand.action(handleActionErrors(async (filesGlob, { ignore: ignoreRaw = '', validation: isValidated, prepare: isPrepared, reload: isCacheReloaded, verbose: isVerbose, }) => {
         let tools = undefined;
         if (isPrepared) {
             // TODO: DRY [◽]
@@ -12893,9 +13020,9 @@ function $initializeTestCommand(program) {
                 isVerbose,
                 isCacheReloaded,
             }; /* <- TODO: ` satisfies PrepareAndScrapeOptions` */
-            const fs = $provideFilesystemForNode();
+            const fs = $provideFilesystemForNode(prepareAndScrapeOptions);
             const llm = await $provideLlmToolsForWizzardOrCli(prepareAndScrapeOptions);
-            const executables = await $provideExecutablesForNode();
+            const executables = await $provideExecutablesForNode(prepareAndScrapeOptions);
             tools = {
                 llm,
                 fs,
@@ -12944,7 +13071,7 @@ function $initializeTestCommand(program) {
         }
         console.info(colors.green(`All pipelines are valid`));
         return process.exit(0);
-    });
+    }));
 }
 /**
  * TODO: [😶] Unite floder listing