@promptbook/cli 0.61.0-1 → 0.61.0-10

package/umd/index.umd.js

@@ -149,7 +149,7 @@
     /**
      * The version of the Promptbook library
      */
-    var PROMPTBOOK_VERSION = '0.61.0-0';
+    var PROMPTBOOK_VERSION = '0.61.0-9';
     // TODO: !!!! List here all the versions and annotate + put into script
 
     /**
@@ -215,6 +215,10 @@
      * The maximum number of iterations for a loops which adds characters one by one
      */
     var CHARACTER_LOOP_LIMIT = 100000;
+    /**
+     * The maximum number of (LLM) tasks running in parallel
+     */
+    var MAX_PARALLEL_COUNT = 5;
     /**
      * The name of the builded pipeline collection made by CLI `ptbk make` and for lookup in `createCollectionFromDirectory`
      */
@@ -228,7 +232,172 @@
         // <- TODO: Add [emoji] + instructions ACRY when adding new reserved parameter
     ];
 
-    var PipelineCollection = [{title:"Prepare Knowledge from Markdown",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.ptbk.md",promptbookVersion:"0.61.0-0",parameters:[{name:"content",description:"Markdown document content",isInput:true,isOutput:false},{name:"knowledge",description:"The knowledge JSON object",isInput:false,isOutput:true}],promptTemplates:[{name:"knowledge",title:"Knowledge",dependentParameterNames:["content"],blockType:"PROMPT_TEMPLATE",modelRequirements:{modelVariant:"CHAT",modelName:"claude-3-opus-20240229"},content:"You are experienced data researcher, extract the important knowledge from the document.\n\n# Rules\n\n-   Make pieces of information concise, clear, and easy to understand\n-   One piece of information should be approximately 1 paragraph\n-   Divide the paragraphs by markdown horizontal lines ---\n-   Omit irrelevant information\n-   Group redundant information\n-   Write just extracted information, nothing else\n\n# The document\n\nTake information from this document:\n\n> {content}",resultingParameterName:"knowledge"}],knowledge:[{name:"u",title:"U",content:"E",keywords:["E"],index:[{modelName:"mocked-facked",position:[0.3862088172833116,0.7349284028017404,-0.9438230019966163,-0.9250069104750818,-0.026138914694730175,-0.7268311159817045,-0.07935019253731612,0.6470171268509612,-0.43852570574155214,0.34435062959144824,-0.03450022605546588,-0.5277705379240651,0.00828213142899159,0.15691069644743383,-0.625404064630672,-0.33277847012240613,-0.2936622789266794,0.18401827710902152,0.3055511517178138,0.2943796117307138,0.7709829510346151,-0.4424955841467506,0.6031092188828207,-0.02473045258733153,0.33586695334417804]}],sources:[{title:"Markdown document",href:"#"}]}],sourceFile:"./promptbook-collection/prepare-knowledge-from-markdown.ptbk.md"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-keywords.ptbk.md",promptbookVersion:"0.61.0-0",parameters:[{name:"content",description:"The content",isInput:true,isOutput:false},{name:"keywords",description:"Keywords separated by comma",isInput:false,isOutput:true}],promptTemplates:[{name:"knowledge",title:"Knowledge",dependentParameterNames:["content"],blockType:"PROMPT_TEMPLATE",modelRequirements:{modelVariant:"CHAT",modelName:"claude-3-opus-20240229"},content:"You are experienced data researcher, detect the important keywords in the document.\n\n# Rules\n\n-   Write just keywords separated by comma\n\n# The document\n\nTake information from this document:\n\n> {content}",resultingParameterName:"keywords"}],knowledge:[{name:"s",title:"S",content:"A",keywords:["I"],index:[{modelName:"mocked-facked",position:[0.6409598940537844,-0.599282163088533,-0.7951357406825994,0.5034713531055992,0.9186084716269733,-0.6841151439158257,-0.39762364246107484,-0.7646814627420149,0.9551202443145854,0.36871821752653666,0.27710484531156565,-0.6886362623745139,0.1246419058549284,-0.7957866917804912,-0.4688680561538634,-0.7037555014650994,0.003694510488622793,0.3186841135345695,-0.7081253398806422,0.7052620275090136,-0.9744710956043585,0.6828863755008538,-0.021569809814358187,-0.23961644339743904,-0.5963677915161343]}],sources:[{title:"Markdown document",href:"#"}]}],sourceFile:"./promptbook-collection/prepare-knowledge-keywords.ptbk.md"},{title:"Prepare Title",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-title.ptbk.md",promptbookVersion:"0.61.0-0",parameters:[{name:"content",description:"The content",isInput:true,isOutput:false},{name:"title",description:"The title of the document",isInput:false,isOutput:true}],promptTemplates:[{name:"knowledge",title:"Knowledge",dependentParameterNames:["content"],blockType:"PROMPT_TEMPLATE",expectations:{words:{min:1,max:8}},modelRequirements:{modelVariant:"CHAT",modelName:"claude-3-opus-20240229"},content:"You are experienced content creator, write best title for the document.\n\n# Rules\n\n-   Write just title, nothing else\n-   Title should be concise and clear\n-   Write maximum 5 words for the title\n\n# The document\n\n> {content}",resultingParameterName:"title"}],knowledge:[{name:"c",title:"C",content:"A",keywords:["E"],index:[{modelName:"mocked-facked",position:[0.3053077802556703,0.45834476154202175,0.11951317167907494,-0.8925032092763412,-0.041809804577583876,-0.7087885568549193,0.0373992576888198,0.9495631083993494,-0.6050023289380846,0.14919003109680862,-0.3228347383159025,0.3273383195761892,-0.5509884895794359,0.7137115707360451,-0.3455818427015154,-0.7130289858826453,0.6105746565153107,-0.30829913719966573,0.6287711326338274,-0.8588121879147286,-0.583980568533244,-0.23082011458606733,-0.10987445073290791,-0.1129755192200137,0.4046462365369812]}],sources:[{title:"Markdown document",href:"#"}]}],sourceFile:"./promptbook-collection/prepare-knowledge-title.ptbk.md"}];
+    /**
+     * Function `addUsage` will add multiple usages into one
+     *
+     * Note: If you provide 0 values, it returns void usage
+     */
+    function addUsage() {
+        var usageItems = [];
+        for (var _i = 0; _i < arguments.length; _i++) {
+            usageItems[_i] = arguments[_i];
+        }
+        var initialStructure = {
+            price: { value: 0 },
+            input: {
+                tokensCount: { value: 0 },
+                charactersCount: { value: 0 },
+                wordsCount: { value: 0 },
+                sentencesCount: { value: 0 },
+                linesCount: { value: 0 },
+                paragraphsCount: { value: 0 },
+                pagesCount: { value: 0 },
+            },
+            output: {
+                tokensCount: { value: 0 },
+                charactersCount: { value: 0 },
+                wordsCount: { value: 0 },
+                sentencesCount: { value: 0 },
+                linesCount: { value: 0 },
+                paragraphsCount: { value: 0 },
+                pagesCount: { value: 0 },
+            },
+        };
+        return usageItems.reduce(function (acc, item) {
+            var e_1, _a, e_2, _b;
+            var _c;
+            acc.price.value += ((_c = item.price) === null || _c === void 0 ? void 0 : _c.value) || 0;
+            try {
+                for (var _d = __values(Object.keys(acc.input)), _e = _d.next(); !_e.done; _e = _d.next()) {
+                    var key = _e.value;
+                    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+                    //@ts-ignore
+                    if (item.input[key]) {
+                        // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+                        //@ts-ignore
+                        acc.input[key].value += item.input[key].value || 0;
+                        // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+                        //@ts-ignore
+                        if (item.input[key].isUncertain) {
+                            // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+                            //@ts-ignore
+                            acc.input[key].isUncertain = true;
+                        }
+                    }
+                }
+            }
+            catch (e_1_1) { e_1 = { error: e_1_1 }; }
+            finally {
+                try {
+                    if (_e && !_e.done && (_a = _d.return)) _a.call(_d);
+                }
+                finally { if (e_1) throw e_1.error; }
+            }
+            try {
+                for (var _f = __values(Object.keys(acc.output)), _g = _f.next(); !_g.done; _g = _f.next()) {
+                    var key = _g.value;
+                    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+                    //@ts-ignore
+                    if (item.output[key]) {
+                        // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+                        //@ts-ignore
+                        acc.output[key].value += item.output[key].value || 0;
+                        // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+                        //@ts-ignore
+                        if (item.output[key].isUncertain) {
+                            // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+                            //@ts-ignore
+                            acc.output[key].isUncertain = true;
+                        }
+                    }
+                }
+            }
+            catch (e_2_1) { e_2 = { error: e_2_1 }; }
+            finally {
+                try {
+                    if (_g && !_g.done && (_b = _f.return)) _b.call(_f);
+                }
+                finally { if (e_2) throw e_2.error; }
+            }
+            return acc;
+        }, initialStructure);
+    }
+
+    /**
+     * Async version of Array.forEach
+     *
+     * @param array - Array to iterate over
+     * @param options - Options for the function
+     * @param callbackfunction - Function to call for each item
+     */
+    function forEachAsync(array, options, callbackfunction) {
+        return __awaiter(this, void 0, void 0, function () {
+            var _a, maxParallelCount, index, runningTasks, tasks, _loop_1, _b, _c, item, e_1_1;
+            var e_1, _d;
+            return __generator(this, function (_e) {
+                switch (_e.label) {
+                    case 0:
+                        _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? Infinity : _a;
+                        index = 0;
+                        runningTasks = [];
+                        tasks = [];
+                        _loop_1 = function (item) {
+                            var currentIndex, task;
+                            return __generator(this, function (_f) {
+                                switch (_f.label) {
+                                    case 0:
+                                        currentIndex = index++;
+                                        task = callbackfunction(item, currentIndex, array);
+                                        tasks.push(task);
+                                        runningTasks.push(task);
+                                        /* not await */ Promise.resolve(task).then(function () {
+                                            runningTasks = runningTasks.filter(function (t) { return t !== task; });
+                                        });
+                                        if (!(maxParallelCount < runningTasks.length)) return [3 /*break*/, 2];
+                                        return [4 /*yield*/, Promise.race(runningTasks)];
+                                    case 1:
+                                        _f.sent();
+                                        _f.label = 2;
+                                    case 2: return [2 /*return*/];
+                                }
+                            });
+                        };
+                        _e.label = 1;
+                    case 1:
+                        _e.trys.push([1, 6, 7, 8]);
+                        _b = __values(array), _c = _b.next();
+                        _e.label = 2;
+                    case 2:
+                        if (!!_c.done) return [3 /*break*/, 5];
+                        item = _c.value;
+                        return [5 /*yield**/, _loop_1(item)];
+                    case 3:
+                        _e.sent();
+                        _e.label = 4;
+                    case 4:
+                        _c = _b.next();
+                        return [3 /*break*/, 2];
+                    case 5: return [3 /*break*/, 8];
+                    case 6:
+                        e_1_1 = _e.sent();
+                        e_1 = { error: e_1_1 };
+                        return [3 /*break*/, 8];
+                    case 7:
+                        try {
+                            if (_c && !_c.done && (_d = _b.return)) _d.call(_b);
+                        }
+                        finally { if (e_1) throw e_1.error; }
+                        return [7 /*endfinally*/];
+                    case 8: return [4 /*yield*/, Promise.all(tasks)];
+                    case 9:
+                        _e.sent();
+                        return [2 /*return*/];
+                }
+            });
+        });
+    }
+
+    var PipelineCollection = [{title:"Prepare Knowledge from Markdown",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.ptbk.md",promptbookVersion:"0.61.0-9",parameters:[{name:"content",description:"Markdown document content",isInput:true,isOutput:false},{name:"knowledge",description:"The knowledge JSON object",isInput:false,isOutput:true}],promptTemplates:[{name:"knowledge",title:"Knowledge",dependentParameterNames:["content"],blockType:"PROMPT_TEMPLATE",personaName:null,modelRequirements:{modelVariant:"CHAT",modelName:"claude-3-opus-20240229"},content:"You are experienced data researcher, extract the important knowledge from the document.\n\n# Rules\n\n-   Make pieces of information concise, clear, and easy to understand\n-   One piece of information should be approximately 1 paragraph\n-   Divide the paragraphs by markdown horizontal lines ---\n-   Omit irrelevant information\n-   Group redundant information\n-   Write just extracted information, nothing else\n\n# The document\n\nTake information from this document:\n\n> {content}",resultingParameterName:"knowledge"}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[{id:1,date:"2024-07-29T00:04:31.966Z",promptbookVersion:"0.61.0-9",modelUsage:{price:{value:0},input:{tokensCount:{value:0},charactersCount:{value:0},wordsCount:{value:0},sentencesCount:{value:0},linesCount:{value:0},paragraphsCount:{value:0},pagesCount:{value:0}},output:{tokensCount:{value:0},charactersCount:{value:0},wordsCount:{value:0},sentencesCount:{value:0},linesCount:{value:0},paragraphsCount:{value:0},pagesCount:{value:0}}}}],sourceFile:"./promptbook-collection/prepare-knowledge-from-markdown.ptbk.md"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-keywords.ptbk.md",promptbookVersion:"0.61.0-9",parameters:[{name:"content",description:"The content",isInput:true,isOutput:false},{name:"keywords",description:"Keywords separated by comma",isInput:false,isOutput:true}],promptTemplates:[{name:"knowledge",title:"Knowledge",dependentParameterNames:["content"],blockType:"PROMPT_TEMPLATE",personaName:null,modelRequirements:{modelVariant:"CHAT",modelName:"claude-3-opus-20240229"},content:"You are experienced data researcher, detect the important keywords in the document.\n\n# Rules\n\n-   Write just keywords separated by comma\n\n# The document\n\nTake information from this document:\n\n> {content}",resultingParameterName:"keywords"}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[{id:1,date:"2024-07-29T00:04:31.970Z",promptbookVersion:"0.61.0-9",modelUsage:{price:{value:0},input:{tokensCount:{value:0},charactersCount:{value:0},wordsCount:{value:0},sentencesCount:{value:0},linesCount:{value:0},paragraphsCount:{value:0},pagesCount:{value:0}},output:{tokensCount:{value:0},charactersCount:{value:0},wordsCount:{value:0},sentencesCount:{value:0},linesCount:{value:0},paragraphsCount:{value:0},pagesCount:{value:0}}}}],sourceFile:"./promptbook-collection/prepare-knowledge-keywords.ptbk.md"},{title:"Prepare Title",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-title.ptbk.md",promptbookVersion:"0.61.0-9",parameters:[{name:"content",description:"The content",isInput:true,isOutput:false},{name:"title",description:"The title of the document",isInput:false,isOutput:true}],promptTemplates:[{name:"knowledge",title:"Knowledge",dependentParameterNames:["content"],blockType:"PROMPT_TEMPLATE",expectations:{words:{min:1,max:8}},personaName:null,modelRequirements:{modelVariant:"CHAT",modelName:"claude-3-opus-20240229"},content:"You are experienced content creator, write best title for the document.\n\n# Rules\n\n-   Write just title, nothing else\n-   Title should be concise and clear\n-   Write maximum 5 words for the title\n\n# The document\n\n> {content}",resultingParameterName:"title"}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[{id:1,date:"2024-07-29T00:04:31.976Z",promptbookVersion:"0.61.0-9",modelUsage:{price:{value:0},input:{tokensCount:{value:0},charactersCount:{value:0},wordsCount:{value:0},sentencesCount:{value:0},linesCount:{value:0},paragraphsCount:{value:0},pagesCount:{value:0}},output:{tokensCount:{value:0},charactersCount:{value:0},wordsCount:{value:0},sentencesCount:{value:0},linesCount:{value:0},paragraphsCount:{value:0},pagesCount:{value:0}}}}],sourceFile:"./promptbook-collection/prepare-knowledge-title.ptbk.md"}];
 
     /**
      * Prettify the html code
@@ -1297,6 +1466,14 @@
      * Function normalizes title to name which can be used as identifier
      */
     function titleToName(value) {
+        if (value.startsWith('http://') || value.startsWith('https://')) {
+            // TODO: Maybe check against some list unallowed characters
+            return value;
+        }
+        if (value.startsWith('./') || value.startsWith('../')) {
+            // TODO: Maybe check against some list unallowed characters
+            return value;
+        }
         value = removeEmojis(value);
         value = normalizeToKebabCase(value);
         // TODO: [🧠] Maybe warn or add some padding to short name which are not good identifiers
@@ -1631,97 +1808,6 @@
         return [input];
     }
 
-    /**
-     * Function `addUsage` will add multiple usages into one
-     *
-     * Note: If you provide 0 values, it returns void usage
-     */
-    function addUsage() {
-        var usageItems = [];
-        for (var _i = 0; _i < arguments.length; _i++) {
-            usageItems[_i] = arguments[_i];
-        }
-        var initialStructure = {
-            price: { value: 0 },
-            input: {
-                tokensCount: { value: 0 },
-                charactersCount: { value: 0 },
-                wordsCount: { value: 0 },
-                sentencesCount: { value: 0 },
-                linesCount: { value: 0 },
-                paragraphsCount: { value: 0 },
-                pagesCount: { value: 0 },
-            },
-            output: {
-                tokensCount: { value: 0 },
-                charactersCount: { value: 0 },
-                wordsCount: { value: 0 },
-                sentencesCount: { value: 0 },
-                linesCount: { value: 0 },
-                paragraphsCount: { value: 0 },
-                pagesCount: { value: 0 },
-            },
-        };
-        return usageItems.reduce(function (acc, item) {
-            var e_1, _a, e_2, _b;
-            var _c;
-            acc.price.value += ((_c = item.price) === null || _c === void 0 ? void 0 : _c.value) || 0;
-            try {
-                for (var _d = __values(Object.keys(acc.input)), _e = _d.next(); !_e.done; _e = _d.next()) {
-                    var key = _e.value;
-                    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-                    //@ts-ignore
-                    if (item.input[key]) {
-                        // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-                        //@ts-ignore
-                        acc.input[key].value += item.input[key].value || 0;
-                        // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-                        //@ts-ignore
-                        if (item.input[key].isUncertain) {
-                            // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-                            //@ts-ignore
-                            acc.input[key].isUncertain = true;
-                        }
-                    }
-                }
-            }
-            catch (e_1_1) { e_1 = { error: e_1_1 }; }
-            finally {
-                try {
-                    if (_e && !_e.done && (_a = _d.return)) _a.call(_d);
-                }
-                finally { if (e_1) throw e_1.error; }
-            }
-            try {
-                for (var _f = __values(Object.keys(acc.output)), _g = _f.next(); !_g.done; _g = _f.next()) {
-                    var key = _g.value;
-                    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-                    //@ts-ignore
-                    if (item.output[key]) {
-                        // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-                        //@ts-ignore
-                        acc.output[key].value += item.output[key].value || 0;
-                        // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-                        //@ts-ignore
-                        if (item.output[key].isUncertain) {
-                            // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-                            //@ts-ignore
-                            acc.output[key].isUncertain = true;
-                        }
-                    }
-                }
-            }
-            catch (e_2_1) { e_2 = { error: e_2_1 }; }
-            finally {
-                try {
-                    if (_g && !_g.done && (_b = _f.return)) _b.call(_f);
-                }
-                finally { if (e_2) throw e_2.error; }
-            }
-            return acc;
-        }, initialStructure);
-    }
-
     /**
      * Counts number of characters in the text
      */
@@ -1925,8 +2011,6 @@
         var _b = settings.maxExecutionAttempts, maxExecutionAttempts = _b === void 0 ? 3 : _b;
         // TODO: !!!!! Implement new commands
         validatePipeline(pipeline);
-        // TODO: !!!!! Do here materialization of still unmaterialized or dynamic knowledge
-        // TODO: !!!!! Do here personaToModelRequirements
         var llmTools = joinLlmExecutionTools.apply(void 0, __spreadArray([], __read(arrayableToArray(tools.llm)), false));
         var pipelineExecutor = function (inputParameters, onProgress) { return __awaiter(_this, void 0, void 0, function () {
             function executeSingleTemplate(currentTemplate) {
@@ -2447,53 +2531,74 @@
         return pipelineExecutor;
     }
     /**
+     * TODO: [🪂] Pass maxParallelCount here
      * TODO: [🧠] When not meet expectations in PROMPT_DIALOG, make some way to tell the user
      * TODO: [👧] Strongly type the executors to avoid need of remove nullables whtn noUncheckedIndexedAccess in tsconfig.json
      * Note: CreatePipelineExecutorOptions are just connected to PipelineExecutor so do not extract to types folder
      * TODO: [🧠][3] transparent = (report intermediate parameters) / opaque execution = (report only output parameters) progress reporting mode
      */
 
-    function prepareKnowledgeFromMarkdown(options) {
+    /**
+     * Just marks a place of place where should be something implemented
+     * No side effects.
+     *
+     * Note: It can be usefull suppressing eslint errors of unused variables
+     *
+     * @param value any values
+     * @returns void
+     */
+    function TODO_USE() {
+        var value = [];
+        for (var _i = 0; _i < arguments.length; _i++) {
+            value[_i] = arguments[_i];
+        }
+    }
+    // TODO: !!!! Find ACRY all just(...) and replace with TODO_USE
+
+    /**
+     * @@@
+     */
+    function prepareKnowledgeFromMarkdown(content /* <- TODO: [🖖] (?maybe not) Always the file */, options) {
         return __awaiter(this, void 0, void 0, function () {
-            var content, llmTools, _a, isVerbose, collection, prepareKnowledgeFromMarkdownExecutor, _b, prepareTitleExecutor, _c, prepareKeywordsExecutor, _d, result, outputParameters, knowledgeRaw, knowledgeTextPieces, knowledge;
-            var _e, _f, _g;
+            var llmTools, _a, maxParallelCount, _b, isVerbose, collection, prepareKnowledgeFromMarkdownExecutor, _c, prepareTitleExecutor, _d, prepareKeywordsExecutor, _e, result, outputParameters, knowledgeRaw, knowledgeTextPieces, knowledge;
+            var _f, _g, _h;
             var _this = this;
-            return __generator(this, function (_h) {
-                switch (_h.label) {
+            return __generator(this, function (_j) {
+                switch (_j.label) {
                     case 0:
-                        content = options.content, llmTools = options.llmTools, _a = options.isVerbose, isVerbose = _a === void 0 ? false : _a;
+                        llmTools = options.llmTools, _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a, _b = options.isVerbose, isVerbose = _b === void 0 ? false : _b;
+                        TODO_USE(maxParallelCount); // <- [🪂]
                         collection = createCollectionFromJson.apply(void 0, __spreadArray([], __read(PipelineCollection), false));
-                        _b = createPipelineExecutor;
-                        _e = {};
-                        return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.ptbk.md')];
-                    case 1:
-                        prepareKnowledgeFromMarkdownExecutor = _b.apply(void 0, [(_e.pipeline = _h.sent(),
-                                _e.tools = {
-                                    llm: llmTools,
-                                },
-                                _e)]);
                         _c = createPipelineExecutor;
                         _f = {};
-                        return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-knowledge-title.ptbk.md')];
-                    case 2:
-                        prepareTitleExecutor = _c.apply(void 0, [(_f.pipeline = _h.sent(),
+                        return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.ptbk.md')];
+                    case 1:
+                        prepareKnowledgeFromMarkdownExecutor = _c.apply(void 0, [(_f.pipeline = _j.sent(),
                                 _f.tools = {
                                     llm: llmTools,
                                 },
                                 _f)]);
                         _d = createPipelineExecutor;
                         _g = {};
-                        return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-knowledge-keywords.ptbk.md')];
-                    case 3:
-                        prepareKeywordsExecutor = _d.apply(void 0, [(_g.pipeline = _h.sent(),
+                        return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-knowledge-title.ptbk.md')];
+                    case 2:
+                        prepareTitleExecutor = _d.apply(void 0, [(_g.pipeline = _j.sent(),
                                 _g.tools = {
                                     llm: llmTools,
                                 },
                                 _g)]);
+                        _e = createPipelineExecutor;
+                        _h = {};
+                        return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-knowledge-keywords.ptbk.md')];
+                    case 3:
+                        prepareKeywordsExecutor = _e.apply(void 0, [(_h.pipeline = _j.sent(),
+                                _h.tools = {
+                                    llm: llmTools,
+                                },
+                                _h)]);
                         return [4 /*yield*/, prepareKnowledgeFromMarkdownExecutor({ content: content })];
                     case 4:
-                        result = _h.sent();
-                        // TODO: [0] !!! Aggeregate usage
+                        result = _j.sent();
                         assertsExecutionSuccessful(result);
                         outputParameters = result.outputParameters;
                         knowledgeRaw = outputParameters.knowledge;
@@ -2502,9 +2607,9 @@
                             console.info('knowledgeTextPieces:', knowledgeTextPieces);
                         }
                         return [4 /*yield*/, Promise.all(
-                            // TODO: !!! Do not send all at once but in chunks
+                            // TODO: [🪂] !! Do not send all at once but in chunks
                             knowledgeTextPieces.map(function (knowledgeTextPiece, i) { return __awaiter(_this, void 0, void 0, function () {
-                                var name, title, content, keywords, index, sources, titleResult, _a, titleRaw, keywordsResult, _b, keywordsRaw, embeddingResult, error_1;
+                                var name, title, content, keywords, index, titleResult, _a, titleRaw, keywordsResult, _b, keywordsRaw, embeddingResult, error_1;
                                 return __generator(this, function (_c) {
                                     switch (_c.label) {
                                         case 0:
@@ -2513,12 +2618,6 @@
                                             content = spaceTrim__default["default"](knowledgeTextPiece);
                                             keywords = [];
                                             index = [];
-                                            sources = [
-                                                {
-                                                    title: 'Markdown document' /* <- TODO: !!! Unhardcode */,
-                                                    href: '#' /* <- TODO: !!! Unhardcode */,
-                                                },
-                                            ];
                                             _c.label = 1;
                                         case 1:
                                             _c.trys.push([1, 7, , 8]);
@@ -2553,7 +2652,6 @@
                                             })];
                                         case 5:
                                             embeddingResult = _c.sent();
-                                            // TODO: [0] !!! Aggeregate usage embeddingResult.usage
                                             index.push({
                                                 modelName: embeddingResult.modelName,
                                                 position: embeddingResult.content,
@@ -2571,20 +2669,186 @@
                                                 content: content,
                                                 keywords: keywords,
                                                 index: index,
-                                                sources: sources,
+                                                // <- TODO: [☀] sources,
                                             }];
                                     }
                                 });
                             }); }))];
                     case 5:
-                        knowledge = _h.sent();
+                        knowledge = _j.sent();
                         return [2 /*return*/, knowledge];
                 }
             });
         });
     }
     /**
-     * 11:11
+     * TODO: [🪂] Do it in parallel 11:11
+     * Note: No need to aggregate usage here, it is done by intercepting the llmTools
+     */
+
+    /**
+     * Prepares the knowle
+     *
+     * @see https://github.com/webgptorg/promptbook/discussions/41
+     * @private within the package
+     */
+    function prepareKnowledgePieces(knowledgeSources, options) {
+        return __awaiter(this, void 0, void 0, function () {
+            var _a, maxParallelCount, knowledgePrepared;
+            var _this = this;
+            return __generator(this, function (_b) {
+                switch (_b.label) {
+                    case 0:
+                        _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a;
+                        knowledgePrepared = [];
+                        return [4 /*yield*/, forEachAsync(knowledgeSources, { maxParallelCount: maxParallelCount }, function (knowledgeSource) { return __awaiter(_this, void 0, void 0, function () {
+                                var partialPieces, pieces;
+                                return __generator(this, function (_a) {
+                                    switch (_a.label) {
+                                        case 0: return [4 /*yield*/, prepareKnowledgeFromMarkdown(knowledgeSource.source, // <- TODO: !!!!! Unhardcode markdown, detect which type it is
+                                            options)];
+                                        case 1:
+                                            partialPieces = _a.sent();
+                                            pieces = partialPieces.map(function (partialPiece) { return (__assign(__assign({}, partialPiece), { sources: [
+                                                    {
+                                                        name: knowledgeSource.name,
+                                                        // line, column <- TODO: [☀]
+                                                        // <- TODO: [❎]
+                                                    },
+                                                ] })); });
+                                            knowledgePrepared.push.apply(knowledgePrepared, __spreadArray([], __read(pieces), false));
+                                            return [2 /*return*/];
+                                    }
+                                });
+                            }); })];
+                    case 1:
+                        _b.sent();
+                        return [2 /*return*/, knowledgePrepared];
+                }
+            });
+        });
+    }
+    /*
+    TODO: [🧊] This is how it can look in future
+    > type PrepareKnowledgeKnowledge = {
+    >   /**
+    >    * Unprepared knowledge
+    >    * /
+    >   readonly knowledgeSources: Array<KnowledgeSourceJson>;
+    > };
+    >
+    > export async function prepareKnowledgePieces(
+    >   knowledge: PrepareKnowledgeKnowledge,
+    >   options: PrepareOptions,
+    > ):
+    */
+    /**
+     * TODO: [🧊] In future one preparation can take data from previous preparation and save tokens and time
+     *       Put `knowledgePieces` into `PrepareKnowledgeOptions`
+     * TODO: [🪂] More than max things can run in parallel by acident [1,[2a,2b,_],[3a,3b,_]]
+     * TODO: [🧠][❎] Do here propper M:N mapping
+     *       [x] One source can make multiple pieces
+     *       [ ] One piece can have multiple sources
+     */
+
+    /**
+     * Prepares the persona for the pipeline
+     *
+     * @see https://github.com/webgptorg/promptbook/discussions/22
+     * @private within the package
+     */
+    function preparePersona(personaDescription, options) {
+        return __awaiter(this, void 0, void 0, function () {
+            var llmTools, _a, maxParallelCount, _b, isVerbose;
+            return __generator(this, function (_c) {
+                llmTools = options.llmTools, _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a, _b = options.isVerbose, isVerbose = _b === void 0 ? false : _b;
+                TODO_USE(maxParallelCount); // <- [🪂]
+                TODO_USE(personaDescription); // <- !!!!!
+                TODO_USE(llmTools); // <- !!!!!
+                TODO_USE(isVerbose); // <- !!!!!
+                return [2 /*return*/, {
+                        modelVariant: 'CHAT',
+                        modelName: 'gpt-4',
+                    }];
+            });
+        });
+    }
+    /**
+     * TODO: [🪂] Do it in parallel
+     */
+
+    /**
+     * Simple wrapper `new Date().toISOString()`
+     *
+     * @returns string_date branded type
+     */
+    function $currentDate() {
+        return new Date().toISOString();
+    }
+
+    /**
+     * Prepare pipeline from string (markdown) format to JSON format
+     *
+     * Note: This function does not validate logic of the pipeline
+     * Note: This function acts as part of compilation process
+     */
+    function preparePipeline(pipeline, options) {
+        return __awaiter(this, void 0, void 0, function () {
+            var _a, maxParallelCount, 
+            /*
+            <- TODO: [🧠][0] `promptbookVersion` */
+            knowledgeSources /*
+            <- TODO: [🧊] `knowledgePieces` */, personas /*
+            <- TODO: [🧊] `preparations` */, currentPreparation, preparations, preparedPersonas, knowledgeSourcesPrepared, partialknowledgePiecesPrepared, knowledgePiecesPrepared;
+            var _this = this;
+            return __generator(this, function (_b) {
+                switch (_b.label) {
+                    case 0:
+                        _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a;
+                        knowledgeSources = pipeline.knowledgeSources, personas = pipeline.personas;
+                        currentPreparation = {
+                            id: 1,
+                            date: $currentDate(),
+                            promptbookVersion: PROMPTBOOK_VERSION,
+                            modelUsage: addUsage(),
+                        };
+                        preparations = [
+                            // ...preparations
+                            // <- TODO: [🧊]
+                            currentPreparation,
+                        ];
+                        preparedPersonas = [];
+                        return [4 /*yield*/, forEachAsync(personas, { maxParallelCount: maxParallelCount /* <- TODO: [🪂] When there are subtasks, this maximul limit can be broken */ }, function (persona) { return __awaiter(_this, void 0, void 0, function () {
+                                var modelRequirements, preparedPersona;
+                                return __generator(this, function (_a) {
+                                    switch (_a.label) {
+                                        case 0: return [4 /*yield*/, preparePersona(persona.description, options)];
+                                        case 1:
+                                            modelRequirements = _a.sent();
+                                            preparedPersona = __assign(__assign({}, persona), { modelRequirements: modelRequirements, preparationIds: [/* TODO: [🧊] -> */ currentPreparation.id] });
+                                            preparedPersonas.push(preparedPersona);
+                                            return [2 /*return*/];
+                                    }
+                                });
+                            }); })];
+                    case 1:
+                        _b.sent();
+                        knowledgeSourcesPrepared = knowledgeSources.map(function (source) { return (__assign(__assign({}, source), { preparationIds: [/* TODO: [🧊] -> */ currentPreparation.id] })); });
+                        return [4 /*yield*/, prepareKnowledgePieces(knowledgeSources /* <- TODO: [🧊] {knowledgeSources, knowledgePieces} */, options)];
+                    case 2:
+                        partialknowledgePiecesPrepared = _b.sent();
+                        knowledgePiecesPrepared = partialknowledgePiecesPrepared.map(function (piece) { return (__assign(__assign({}, piece), { preparationIds: [/* TODO: [🧊] -> */ currentPreparation.id] })); });
+                        // ----- /Knowledge preparation -----
+                        return [2 /*return*/, __assign(__assign({}, pipeline), { knowledgeSources: knowledgeSourcesPrepared, knowledgePieces: knowledgePiecesPrepared, personas: preparedPersonas, preparations: preparations })];
+                }
+            });
+        });
+    }
+    /**
+     * TODO: Write tests for `preparePipeline`
+     * TODO: [🏏] Leverage the batch API and build queues @see https://platform.openai.com/docs/guides/batch
+     * TODO: [🧊] In future one preparation can take data from previous preparation and save tokens and time
+     * TODO: [💸] Make utilities `interceptLlmTools` and `costLlmTools` to compute cost and DO put this counting logic in `prepareKnowledge` or `preparePersona`
      */
 
     /**
@@ -2771,6 +3035,7 @@
 
     /**
      * Units of text measurement
+     * @see https://github.com/webgptorg/promptbook/discussions/30
      */
     var EXPECTATION_UNITS = ['CHARACTERS', 'WORDS', 'SENTENCES', 'LINES', 'PARAGRAPHS', 'PAGES'];
     /**
@@ -3091,9 +3356,10 @@
 
     var MODEL_VARIANTS = ['COMPLETION', 'CHAT', 'EMBEDDING' /* <- TODO [🏳] */ /* <- [🤖] */];
     /**
+     * TODO: !!!!! Add and use systemMessage, temprerature, top_k, top_p, presencePenalty, frequencyPenalty, bestOf, n, logitBias, logitBiasType, stop, ... to ModelRequirements
      * TODO: Maybe figure out better word than "variant"
      * TODO: Add here more requirement options like max context size, max tokens, etc.
-     * TODO: [👙][🧠] Just selecting gpt3 or gpt4 level of model
+     * TODO: [💕][🧠] Just selecting gpt3 or gpt4 level of model
      */
 
     /**
@@ -4236,14 +4502,14 @@
      */
 
     /**
-     * Compile promptbook from string (markdown) format to JSON format synchronously
+     * Compile pipeline from string (markdown) format to JSON format synchronously
      *
-     * Note: There are two similar functions:
+     * Note: There are 3 similar functions:
      * - `pipelineStringToJson` **(preferred)** - which propperly compiles the promptbook and use embedding for external knowledge
      * - `pipelineStringToJsonSync` - use only if you need to compile promptbook synchronously and it contains NO external knowledge
+     * - `preparePipeline` - just one step in the compilation process
      *
      * @param pipelineString {Promptbook} in string markdown format (.ptbk.md)
-     * @param options - Options and tools for the compilation
      * @returns {Promptbook} compiled in JSON format (.ptbk.json)
      * @throws {ParsingError} if the promptbook string is not valid
      *
@@ -4259,7 +4525,10 @@
             description: undefined /* <- Note: Putting here placeholder to keep `description` on top at final JSON */,
             parameters: [],
             promptTemplates: [],
-            knowledge: [],
+            knowledgeSources: [],
+            knowledgePieces: [],
+            personas: [],
+            preparations: [],
         };
         // =============================================================
         // Note: 1️⃣ Parsing of the markdown into object
@@ -4527,6 +4796,7 @@
                 postprocessing: postprocessing,
                 expectations: expectAmount,
                 expectFormat: expectFormat,
+                personaName: null,
                 modelRequirements: templateModelRequirements,
                 contentLanguage: blockType === 'SCRIPT' ? language : undefined,
                 content: content,
@@ -4606,11 +4876,12 @@
      */
 
     /**
-     * Compile promptbook from string (markdown) format to JSON format
+     * Compile pipeline from string (markdown) format to JSON format
      *
-     * Note: There are two similar functions:
+     * Note: There are 3 similar functions:
      * - `pipelineStringToJson` **(preferred)** - which propperly compiles the promptbook and use embedding for external knowledge
      * - `pipelineStringToJsonSync` - use only if you need to compile promptbook synchronously and it contains NO external knowledge
+     * - `preparePipeline` - just one step in the compilation process
      *
      * @param pipelineString {Promptbook} in string markdown format (.ptbk.md)
      * @param options - Options and tools for the compilation
@@ -4621,22 +4892,18 @@
      * Note: This function acts as compilation process
      */
     function pipelineStringToJson(pipelineString, options) {
-        if (options === void 0) { options = {}; }
+        if (options === void 0) { options = { llmTools: null }; }
         return __awaiter(this, void 0, void 0, function () {
-            var llmTools, pipelineJson, knowledge;
+            var llmTools, pipelineJson;
             return __generator(this, function (_a) {
                 switch (_a.label) {
                     case 0:
                         llmTools = options.llmTools;
                         pipelineJson = pipelineStringToJsonSync(pipelineString);
-                        if (!llmTools) return [3 /*break*/, 2];
-                        return [4 /*yield*/, prepareKnowledgeFromMarkdown({
-                                content: 'Roses are red, violets are blue, programmers use Promptbook, users too',
-                                llmTools: llmTools,
-                            })];
+                        if (!(llmTools !== null)) return [3 /*break*/, 2];
+                        return [4 /*yield*/, preparePipeline(pipelineJson, { llmTools: llmTools })];
                     case 1:
-                        knowledge = _a.sent();
-                        pipelineJson = __assign(__assign({}, pipelineJson), { knowledge: __spreadArray(__spreadArray([], __read((pipelineJson.knowledge || [])), false), __read(knowledge), false) });
+                        pipelineJson = _a.sent();
                         _a.label = 2;
                     case 2: return [2 /*return*/, pipelineJson];
                 }
@@ -4758,7 +5025,7 @@
      */
     function createCollectionFromDirectory(path$1, options) {
         return __awaiter(this, void 0, void 0, function () {
-            var makedLibraryFilePath, makedLibraryFileExists, _a, _b, isRecursive, _c, isVerbose, _d, isLazyLoaded, _e, isCrashOnError, collection;
+            var makedLibraryFilePath, makedLibraryFileExists, _a, _b, isRecursive, _c, isVerbose, _d, isLazyLoaded, _e, isCrashedOnError, collection;
             var _this = this;
             return __generator(this, function (_f) {
                 switch (_f.label) {
@@ -4779,7 +5046,7 @@
                             colors__default["default"].green("(In future, not implemented yet) Using your prebuild pipeline collection ".concat(makedLibraryFilePath));
                             // TODO: !! Implement;
                         }
-                        _a = options || {}, _b = _a.isRecursive, isRecursive = _b === void 0 ? true : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? false : _c, _d = _a.isLazyLoaded, isLazyLoaded = _d === void 0 ? false : _d, _e = _a.isCrashOnError, isCrashOnError = _e === void 0 ? true : _e;
+                        _a = options || {}, _b = _a.isRecursive, isRecursive = _b === void 0 ? true : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? false : _c, _d = _a.isLazyLoaded, isLazyLoaded = _d === void 0 ? false : _d, _e = _a.isCrashedOnError, isCrashedOnError = _e === void 0 ? true : _e;
                         collection = createCollectionFromPromise(function () { return __awaiter(_this, void 0, void 0, function () {
                             var fileNames, promptbooks, _loop_1, fileNames_1, fileNames_1_1, fileName, e_1_1;
                             var e_1, _a;
@@ -4841,7 +5108,7 @@
                                                                 if (isVerbose) {
                                                                     console.info("Loading ".concat(fileName.split('\\').join('/')));
                                                                 }
-                                                                if (!isCrashOnError) {
+                                                                if (!isCrashedOnError) {
                                                                     // Note: Validate promptbook to check if it is logically correct to not crash on invalid promptbooks
                                                                     //       But be handled in current try-catch block
                                                                     validatePipeline(promptbook);
@@ -4857,7 +5124,7 @@
                                                             throw error_1;
                                                         }
                                                         wrappedErrorMessage = spaceTrim__default["default"](function (block) { return "\n                        Error during loading pipeline from file ".concat(fileName.split('\\').join('/'), ":\n\n                        ").concat(block(error_1.message), "\n\n                    "); });
-                                                        if (isCrashOnError) {
+                                                        if (isCrashedOnError) {
                                                             throw new CollectionError(wrappedErrorMessage);
                                                         }
                                                         // TODO: [🟥] Detect browser / node and make it colorfull