@promptbook/core 0.61.0-0 → 0.61.0-10

package/umd/index.umd.js

@@ -401,6 +401,10 @@
      * The maximum number of iterations for a loops
      */
     var LOOP_LIMIT = 1000;
+    /**
+     * The maximum number of (LLM) tasks running in parallel
+     */
+    var MAX_PARALLEL_COUNT = 5;
     /**
      * The names of the parameters that are reserved for special purposes
      */
@@ -1126,7 +1130,172 @@
         // <- [🩻]
     ];
 
-    var PipelineCollection = [{title:"Prepare Knowledge from Markdown",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.ptbk.md",promptbookVersion:"0.60.0",parameters:[{name:"content",description:"Markdown document content",isInput:true,isOutput:false},{name:"knowledge",description:"The knowledge JSON object",isInput:false,isOutput:true}],promptTemplates:[{name:"knowledge",title:"Knowledge",dependentParameterNames:["content"],blockType:"PROMPT_TEMPLATE",modelRequirements:{modelVariant:"CHAT",modelName:"claude-3-opus-20240229"},content:"You are experienced data researcher, extract the important knowledge from the document.\n\n# Rules\n\n-   Make pieces of information concise, clear, and easy to understand\n-   One piece of information should be approximately 1 paragraph\n-   Divide the paragraphs by markdown horizontal lines ---\n-   Omit irrelevant information\n-   Group redundant information\n-   Write just extracted information, nothing else\n\n# The document\n\nTake information from this document:\n\n> {content}",resultingParameterName:"knowledge"}],knowledge:[{name:"u",title:"U",content:"I",keywords:["E"],index:[{modelName:"mocked-facked",position:[0.32603811001648664,-0.8151600779193648,0.7664764784601905,0.40210873521217305,0.222437572764977,-0.4125301487038615,-0.8231889454816157,0.9109486723694067,0.37789733931561553,-0.556486491106039,-0.5670319200115816,-0.1974522088727535,0.022104221421542558,0.7046092137370166,0.07437773177645424,-0.3496446604425767,-0.05274352103805802,0.8342909224282642,-0.435471568520275,0.5164118712487724,0.7271111824447978,-0.09364881082937826,0.32855248741175425,-0.07703932357732901,0.551079774872838]}],sources:[{title:"Markdown document",href:"#"}]}],sourceFile:"./promptbook-collection/prepare-knowledge-from-markdown.ptbk.md"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-keywords.ptbk.md",promptbookVersion:"0.60.0",parameters:[{name:"content",description:"The content",isInput:true,isOutput:false},{name:"keywords",description:"Keywords separated by comma",isInput:false,isOutput:true}],promptTemplates:[{name:"knowledge",title:"Knowledge",dependentParameterNames:["content"],blockType:"PROMPT_TEMPLATE",modelRequirements:{modelVariant:"CHAT",modelName:"claude-3-opus-20240229"},content:"You are experienced data researcher, detect the important keywords in the document.\n\n# Rules\n\n-   Write just keywords separated by comma\n\n# The document\n\nTake information from this document:\n\n> {content}",resultingParameterName:"keywords"}],knowledge:[{name:"p",title:"P",content:"A",keywords:["L"],index:[{modelName:"mocked-facked",position:[0.5640345774427704,-0.07119174635570813,0.015197097443399699,0.4182203436305234,0.25816168398409545,-0.44950167559000853,-0.9607858932820381,-0.5156415714405949,-0.29710522199486133,0.41071386689679246,-0.6337607469953004,0.3530235806138857,0.4759243122693184,-0.5098890815896411,0.5078673989680667,0.09358488419391398,0.4892768751447085,0.7060415586657078,0.8965591399366888,0.708193527752476,0.1883448814037032,0.3039912919801422,0.7969109648391037,-0.7137054462894104,-0.8820651045662413]}],sources:[{title:"Markdown document",href:"#"}]}],sourceFile:"./promptbook-collection/prepare-knowledge-keywords.ptbk.md"},{title:"Prepare Title",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-title.ptbk.md",promptbookVersion:"0.60.0",parameters:[{name:"content",description:"The content",isInput:true,isOutput:false},{name:"title",description:"The title of the document",isInput:false,isOutput:true}],promptTemplates:[{name:"knowledge",title:"Knowledge",dependentParameterNames:["content"],blockType:"PROMPT_TEMPLATE",expectations:{words:{min:1,max:8}},modelRequirements:{modelVariant:"CHAT",modelName:"claude-3-opus-20240229"},content:"You are experienced content creator, write best title for the document.\n\n# Rules\n\n-   Write just title, nothing else\n-   Title should be concise and clear\n-   Write maximum 5 words for the title\n\n# The document\n\n> {content}",resultingParameterName:"title"}],knowledge:[{name:"e",title:"E",content:"L",keywords:["U"],index:[{modelName:"mocked-facked",position:[-0.2552561443130785,-0.997345757973708,-0.9044970690870002,0.7735304678081456,0.05306206079217235,0.9192301773631262,-0.5775416347531936,0.6084954872241011,0.1936789483087087,-0.45511339549687646,0.1322427518665963,-0.6050338969876923,0.5860487011982332,0.5682123165708153,-0.9204885870353041,0.5555932573463336,-0.00506336761935966,-0.4353010802882027,-0.6359219313303703,-0.9560097721865413,0.2090379465178107,0.6670733187433857,0.9388562958310938,-0.6743173553796935,-0.7145540816323397]}],sources:[{title:"Markdown document",href:"#"}]}],sourceFile:"./promptbook-collection/prepare-knowledge-title.ptbk.md"}];
+    /**
+     * Function `addUsage` will add multiple usages into one
+     *
+     * Note: If you provide 0 values, it returns void usage
+     */
+    function addUsage() {
+        var usageItems = [];
+        for (var _i = 0; _i < arguments.length; _i++) {
+            usageItems[_i] = arguments[_i];
+        }
+        var initialStructure = {
+            price: { value: 0 },
+            input: {
+                tokensCount: { value: 0 },
+                charactersCount: { value: 0 },
+                wordsCount: { value: 0 },
+                sentencesCount: { value: 0 },
+                linesCount: { value: 0 },
+                paragraphsCount: { value: 0 },
+                pagesCount: { value: 0 },
+            },
+            output: {
+                tokensCount: { value: 0 },
+                charactersCount: { value: 0 },
+                wordsCount: { value: 0 },
+                sentencesCount: { value: 0 },
+                linesCount: { value: 0 },
+                paragraphsCount: { value: 0 },
+                pagesCount: { value: 0 },
+            },
+        };
+        return usageItems.reduce(function (acc, item) {
+            var e_1, _a, e_2, _b;
+            var _c;
+            acc.price.value += ((_c = item.price) === null || _c === void 0 ? void 0 : _c.value) || 0;
+            try {
+                for (var _d = __values(Object.keys(acc.input)), _e = _d.next(); !_e.done; _e = _d.next()) {
+                    var key = _e.value;
+                    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+                    //@ts-ignore
+                    if (item.input[key]) {
+                        // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+                        //@ts-ignore
+                        acc.input[key].value += item.input[key].value || 0;
+                        // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+                        //@ts-ignore
+                        if (item.input[key].isUncertain) {
+                            // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+                            //@ts-ignore
+                            acc.input[key].isUncertain = true;
+                        }
+                    }
+                }
+            }
+            catch (e_1_1) { e_1 = { error: e_1_1 }; }
+            finally {
+                try {
+                    if (_e && !_e.done && (_a = _d.return)) _a.call(_d);
+                }
+                finally { if (e_1) throw e_1.error; }
+            }
+            try {
+                for (var _f = __values(Object.keys(acc.output)), _g = _f.next(); !_g.done; _g = _f.next()) {
+                    var key = _g.value;
+                    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+                    //@ts-ignore
+                    if (item.output[key]) {
+                        // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+                        //@ts-ignore
+                        acc.output[key].value += item.output[key].value || 0;
+                        // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+                        //@ts-ignore
+                        if (item.output[key].isUncertain) {
+                            // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+                            //@ts-ignore
+                            acc.output[key].isUncertain = true;
+                        }
+                    }
+                }
+            }
+            catch (e_2_1) { e_2 = { error: e_2_1 }; }
+            finally {
+                try {
+                    if (_g && !_g.done && (_b = _f.return)) _b.call(_f);
+                }
+                finally { if (e_2) throw e_2.error; }
+            }
+            return acc;
+        }, initialStructure);
+    }
+
+    /**
+     * Async version of Array.forEach
+     *
+     * @param array - Array to iterate over
+     * @param options - Options for the function
+     * @param callbackfunction - Function to call for each item
+     */
+    function forEachAsync(array, options, callbackfunction) {
+        return __awaiter(this, void 0, void 0, function () {
+            var _a, maxParallelCount, index, runningTasks, tasks, _loop_1, _b, _c, item, e_1_1;
+            var e_1, _d;
+            return __generator(this, function (_e) {
+                switch (_e.label) {
+                    case 0:
+                        _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? Infinity : _a;
+                        index = 0;
+                        runningTasks = [];
+                        tasks = [];
+                        _loop_1 = function (item) {
+                            var currentIndex, task;
+                            return __generator(this, function (_f) {
+                                switch (_f.label) {
+                                    case 0:
+                                        currentIndex = index++;
+                                        task = callbackfunction(item, currentIndex, array);
+                                        tasks.push(task);
+                                        runningTasks.push(task);
+                                        /* not await */ Promise.resolve(task).then(function () {
+                                            runningTasks = runningTasks.filter(function (t) { return t !== task; });
+                                        });
+                                        if (!(maxParallelCount < runningTasks.length)) return [3 /*break*/, 2];
+                                        return [4 /*yield*/, Promise.race(runningTasks)];
+                                    case 1:
+                                        _f.sent();
+                                        _f.label = 2;
+                                    case 2: return [2 /*return*/];
+                                }
+                            });
+                        };
+                        _e.label = 1;
+                    case 1:
+                        _e.trys.push([1, 6, 7, 8]);
+                        _b = __values(array), _c = _b.next();
+                        _e.label = 2;
+                    case 2:
+                        if (!!_c.done) return [3 /*break*/, 5];
+                        item = _c.value;
+                        return [5 /*yield**/, _loop_1(item)];
+                    case 3:
+                        _e.sent();
+                        _e.label = 4;
+                    case 4:
+                        _c = _b.next();
+                        return [3 /*break*/, 2];
+                    case 5: return [3 /*break*/, 8];
+                    case 6:
+                        e_1_1 = _e.sent();
+                        e_1 = { error: e_1_1 };
+                        return [3 /*break*/, 8];
+                    case 7:
+                        try {
+                            if (_c && !_c.done && (_d = _b.return)) _d.call(_b);
+                        }
+                        finally { if (e_1) throw e_1.error; }
+                        return [7 /*endfinally*/];
+                    case 8: return [4 /*yield*/, Promise.all(tasks)];
+                    case 9:
+                        _e.sent();
+                        return [2 /*return*/];
+                }
+            });
+        });
+    }
+
+    var PipelineCollection = [{title:"Prepare Knowledge from Markdown",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.ptbk.md",promptbookVersion:"0.61.0-9",parameters:[{name:"content",description:"Markdown document content",isInput:true,isOutput:false},{name:"knowledge",description:"The knowledge JSON object",isInput:false,isOutput:true}],promptTemplates:[{name:"knowledge",title:"Knowledge",dependentParameterNames:["content"],blockType:"PROMPT_TEMPLATE",personaName:null,modelRequirements:{modelVariant:"CHAT",modelName:"claude-3-opus-20240229"},content:"You are experienced data researcher, extract the important knowledge from the document.\n\n# Rules\n\n-   Make pieces of information concise, clear, and easy to understand\n-   One piece of information should be approximately 1 paragraph\n-   Divide the paragraphs by markdown horizontal lines ---\n-   Omit irrelevant information\n-   Group redundant information\n-   Write just extracted information, nothing else\n\n# The document\n\nTake information from this document:\n\n> {content}",resultingParameterName:"knowledge"}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[{id:1,date:"2024-07-29T00:04:31.966Z",promptbookVersion:"0.61.0-9",modelUsage:{price:{value:0},input:{tokensCount:{value:0},charactersCount:{value:0},wordsCount:{value:0},sentencesCount:{value:0},linesCount:{value:0},paragraphsCount:{value:0},pagesCount:{value:0}},output:{tokensCount:{value:0},charactersCount:{value:0},wordsCount:{value:0},sentencesCount:{value:0},linesCount:{value:0},paragraphsCount:{value:0},pagesCount:{value:0}}}}],sourceFile:"./promptbook-collection/prepare-knowledge-from-markdown.ptbk.md"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-keywords.ptbk.md",promptbookVersion:"0.61.0-9",parameters:[{name:"content",description:"The content",isInput:true,isOutput:false},{name:"keywords",description:"Keywords separated by comma",isInput:false,isOutput:true}],promptTemplates:[{name:"knowledge",title:"Knowledge",dependentParameterNames:["content"],blockType:"PROMPT_TEMPLATE",personaName:null,modelRequirements:{modelVariant:"CHAT",modelName:"claude-3-opus-20240229"},content:"You are experienced data researcher, detect the important keywords in the document.\n\n# Rules\n\n-   Write just keywords separated by comma\n\n# The document\n\nTake information from this document:\n\n> {content}",resultingParameterName:"keywords"}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[{id:1,date:"2024-07-29T00:04:31.970Z",promptbookVersion:"0.61.0-9",modelUsage:{price:{value:0},input:{tokensCount:{value:0},charactersCount:{value:0},wordsCount:{value:0},sentencesCount:{value:0},linesCount:{value:0},paragraphsCount:{value:0},pagesCount:{value:0}},output:{tokensCount:{value:0},charactersCount:{value:0},wordsCount:{value:0},sentencesCount:{value:0},linesCount:{value:0},paragraphsCount:{value:0},pagesCount:{value:0}}}}],sourceFile:"./promptbook-collection/prepare-knowledge-keywords.ptbk.md"},{title:"Prepare Title",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-title.ptbk.md",promptbookVersion:"0.61.0-9",parameters:[{name:"content",description:"The content",isInput:true,isOutput:false},{name:"title",description:"The title of the document",isInput:false,isOutput:true}],promptTemplates:[{name:"knowledge",title:"Knowledge",dependentParameterNames:["content"],blockType:"PROMPT_TEMPLATE",expectations:{words:{min:1,max:8}},personaName:null,modelRequirements:{modelVariant:"CHAT",modelName:"claude-3-opus-20240229"},content:"You are experienced content creator, write best title for the document.\n\n# Rules\n\n-   Write just title, nothing else\n-   Title should be concise and clear\n-   Write maximum 5 words for the title\n\n# The document\n\n> {content}",resultingParameterName:"title"}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[{id:1,date:"2024-07-29T00:04:31.976Z",promptbookVersion:"0.61.0-9",modelUsage:{price:{value:0},input:{tokensCount:{value:0},charactersCount:{value:0},wordsCount:{value:0},sentencesCount:{value:0},linesCount:{value:0},paragraphsCount:{value:0},pagesCount:{value:0}},output:{tokensCount:{value:0},charactersCount:{value:0},wordsCount:{value:0},sentencesCount:{value:0},linesCount:{value:0},paragraphsCount:{value:0},pagesCount:{value:0}}}}],sourceFile:"./promptbook-collection/prepare-knowledge-title.ptbk.md"}];
 
     var defaultDiacriticsRemovalMap = [
         {
@@ -1453,6 +1622,14 @@
      * Function normalizes title to name which can be used as identifier
      */
     function titleToName(value) {
+        if (value.startsWith('http://') || value.startsWith('https://')) {
+            // TODO: Maybe check against some list unallowed characters
+            return value;
+        }
+        if (value.startsWith('./') || value.startsWith('../')) {
+            // TODO: Maybe check against some list unallowed characters
+            return value;
+        }
         value = removeEmojis(value);
         value = normalizeToKebabCase(value);
         // TODO: [🧠] Maybe warn or add some padding to short name which are not good identifiers
@@ -1790,100 +1967,9 @@
     /**
      * The version of the Promptbook library
      */
-    var PROMPTBOOK_VERSION = '0.60.0';
+    var PROMPTBOOK_VERSION = '0.61.0-9';
     // TODO: !!!! List here all the versions and annotate + put into script
 
-    /**
-     * Function `addUsage` will add multiple usages into one
-     *
-     * Note: If you provide 0 values, it returns void usage
-     */
-    function addUsage() {
-        var usageItems = [];
-        for (var _i = 0; _i < arguments.length; _i++) {
-            usageItems[_i] = arguments[_i];
-        }
-        var initialStructure = {
-            price: { value: 0 },
-            input: {
-                tokensCount: { value: 0 },
-                charactersCount: { value: 0 },
-                wordsCount: { value: 0 },
-                sentencesCount: { value: 0 },
-                linesCount: { value: 0 },
-                paragraphsCount: { value: 0 },
-                pagesCount: { value: 0 },
-            },
-            output: {
-                tokensCount: { value: 0 },
-                charactersCount: { value: 0 },
-                wordsCount: { value: 0 },
-                sentencesCount: { value: 0 },
-                linesCount: { value: 0 },
-                paragraphsCount: { value: 0 },
-                pagesCount: { value: 0 },
-            },
-        };
-        return usageItems.reduce(function (acc, item) {
-            var e_1, _a, e_2, _b;
-            var _c;
-            acc.price.value += ((_c = item.price) === null || _c === void 0 ? void 0 : _c.value) || 0;
-            try {
-                for (var _d = __values(Object.keys(acc.input)), _e = _d.next(); !_e.done; _e = _d.next()) {
-                    var key = _e.value;
-                    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-                    //@ts-ignore
-                    if (item.input[key]) {
-                        // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-                        //@ts-ignore
-                        acc.input[key].value += item.input[key].value || 0;
-                        // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-                        //@ts-ignore
-                        if (item.input[key].isUncertain) {
-                            // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-                            //@ts-ignore
-                            acc.input[key].isUncertain = true;
-                        }
-                    }
-                }
-            }
-            catch (e_1_1) { e_1 = { error: e_1_1 }; }
-            finally {
-                try {
-                    if (_e && !_e.done && (_a = _d.return)) _a.call(_d);
-                }
-                finally { if (e_1) throw e_1.error; }
-            }
-            try {
-                for (var _f = __values(Object.keys(acc.output)), _g = _f.next(); !_g.done; _g = _f.next()) {
-                    var key = _g.value;
-                    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-                    //@ts-ignore
-                    if (item.output[key]) {
-                        // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-                        //@ts-ignore
-                        acc.output[key].value += item.output[key].value || 0;
-                        // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-                        //@ts-ignore
-                        if (item.output[key].isUncertain) {
-                            // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-                            //@ts-ignore
-                            acc.output[key].isUncertain = true;
-                        }
-                    }
-                }
-            }
-            catch (e_2_1) { e_2 = { error: e_2_1 }; }
-            finally {
-                try {
-                    if (_g && !_g.done && (_b = _f.return)) _b.call(_f);
-                }
-                finally { if (e_2) throw e_2.error; }
-            }
-            return acc;
-        }, initialStructure);
-    }
-
     /**
      * Counts number of characters in the text
      */
@@ -2087,8 +2173,6 @@
         var _b = settings.maxExecutionAttempts, maxExecutionAttempts = _b === void 0 ? 3 : _b;
         // TODO: !!!!! Implement new commands
         validatePipeline(pipeline);
-        // TODO: !!!!! Do here materialization of still unmaterialized or dynamic knowledge
-        // TODO: !!!!! Do here personaToModelRequirements
         var llmTools = joinLlmExecutionTools.apply(void 0, __spreadArray([], __read(arrayableToArray(tools.llm)), false));
         var pipelineExecutor = function (inputParameters, onProgress) { return __awaiter(_this, void 0, void 0, function () {
             function executeSingleTemplate(currentTemplate) {
@@ -2609,53 +2693,74 @@
         return pipelineExecutor;
     }
     /**
+     * TODO: [🪂] Pass maxParallelCount here
      * TODO: [🧠] When not meet expectations in PROMPT_DIALOG, make some way to tell the user
      * TODO: [👧] Strongly type the executors to avoid need of remove nullables whtn noUncheckedIndexedAccess in tsconfig.json
      * Note: CreatePipelineExecutorOptions are just connected to PipelineExecutor so do not extract to types folder
      * TODO: [🧠][3] transparent = (report intermediate parameters) / opaque execution = (report only output parameters) progress reporting mode
      */
 
-    function prepareKnowledgeFromMarkdown(options) {
+    /**
+     * Just marks a place of place where should be something implemented
+     * No side effects.
+     *
+     * Note: It can be usefull suppressing eslint errors of unused variables
+     *
+     * @param value any values
+     * @returns void
+     */
+    function TODO_USE() {
+        var value = [];
+        for (var _i = 0; _i < arguments.length; _i++) {
+            value[_i] = arguments[_i];
+        }
+    }
+    // TODO: !!!! Find ACRY all just(...) and replace with TODO_USE
+
+    /**
+     * @@@
+     */
+    function prepareKnowledgeFromMarkdown(content /* <- TODO: [🖖] (?maybe not) Always the file */, options) {
         return __awaiter(this, void 0, void 0, function () {
-            var content, llmTools, _a, isVerbose, collection, prepareKnowledgeFromMarkdownExecutor, _b, prepareTitleExecutor, _c, prepareKeywordsExecutor, _d, result, outputParameters, knowledgeRaw, knowledgeTextPieces, knowledge;
-            var _e, _f, _g;
+            var llmTools, _a, maxParallelCount, _b, isVerbose, collection, prepareKnowledgeFromMarkdownExecutor, _c, prepareTitleExecutor, _d, prepareKeywordsExecutor, _e, result, outputParameters, knowledgeRaw, knowledgeTextPieces, knowledge;
+            var _f, _g, _h;
             var _this = this;
-            return __generator(this, function (_h) {
-                switch (_h.label) {
+            return __generator(this, function (_j) {
+                switch (_j.label) {
                     case 0:
-                        content = options.content, llmTools = options.llmTools, _a = options.isVerbose, isVerbose = _a === void 0 ? false : _a;
+                        llmTools = options.llmTools, _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a, _b = options.isVerbose, isVerbose = _b === void 0 ? false : _b;
+                        TODO_USE(maxParallelCount); // <- [🪂]
                         collection = createCollectionFromJson.apply(void 0, __spreadArray([], __read(PipelineCollection), false));
-                        _b = createPipelineExecutor;
-                        _e = {};
-                        return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.ptbk.md')];
-                    case 1:
-                        prepareKnowledgeFromMarkdownExecutor = _b.apply(void 0, [(_e.pipeline = _h.sent(),
-                                _e.tools = {
-                                    llm: llmTools,
-                                },
-                                _e)]);
                         _c = createPipelineExecutor;
                         _f = {};
-                        return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-knowledge-title.ptbk.md')];
-                    case 2:
-                        prepareTitleExecutor = _c.apply(void 0, [(_f.pipeline = _h.sent(),
+                        return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.ptbk.md')];
+                    case 1:
+                        prepareKnowledgeFromMarkdownExecutor = _c.apply(void 0, [(_f.pipeline = _j.sent(),
                                 _f.tools = {
                                     llm: llmTools,
                                 },
                                 _f)]);
                         _d = createPipelineExecutor;
                         _g = {};
-                        return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-knowledge-keywords.ptbk.md')];
-                    case 3:
-                        prepareKeywordsExecutor = _d.apply(void 0, [(_g.pipeline = _h.sent(),
+                        return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-knowledge-title.ptbk.md')];
+                    case 2:
+                        prepareTitleExecutor = _d.apply(void 0, [(_g.pipeline = _j.sent(),
                                 _g.tools = {
                                     llm: llmTools,
                                 },
                                 _g)]);
+                        _e = createPipelineExecutor;
+                        _h = {};
+                        return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-knowledge-keywords.ptbk.md')];
+                    case 3:
+                        prepareKeywordsExecutor = _e.apply(void 0, [(_h.pipeline = _j.sent(),
+                                _h.tools = {
+                                    llm: llmTools,
+                                },
+                                _h)]);
                         return [4 /*yield*/, prepareKnowledgeFromMarkdownExecutor({ content: content })];
                     case 4:
-                        result = _h.sent();
-                        // TODO: [0] !!! Aggeregate usage
+                        result = _j.sent();
                         assertsExecutionSuccessful(result);
                         outputParameters = result.outputParameters;
                         knowledgeRaw = outputParameters.knowledge;
@@ -2664,9 +2769,9 @@
                             console.info('knowledgeTextPieces:', knowledgeTextPieces);
                         }
                         return [4 /*yield*/, Promise.all(
-                            // TODO: !!! Do not send all at once but in chunks
+                            // TODO: [🪂] !! Do not send all at once but in chunks
                             knowledgeTextPieces.map(function (knowledgeTextPiece, i) { return __awaiter(_this, void 0, void 0, function () {
-                                var name, title, content, keywords, index, sources, titleResult, _a, titleRaw, keywordsResult, _b, keywordsRaw, embeddingResult, error_1;
+                                var name, title, content, keywords, index, titleResult, _a, titleRaw, keywordsResult, _b, keywordsRaw, embeddingResult, error_1;
                                 return __generator(this, function (_c) {
                                     switch (_c.label) {
                                         case 0:
@@ -2675,12 +2780,6 @@
                                             content = spaceTrim__default["default"](knowledgeTextPiece);
                                             keywords = [];
                                             index = [];
-                                            sources = [
-                                                {
-                                                    title: 'Markdown document' /* <- TODO: !!! Unhardcode */,
-                                                    href: '#' /* <- TODO: !!! Unhardcode */,
-                                                },
-                                            ];
                                             _c.label = 1;
                                         case 1:
                                             _c.trys.push([1, 7, , 8]);
@@ -2715,7 +2814,6 @@
                                             })];
                                         case 5:
                                             embeddingResult = _c.sent();
-                                            // TODO: [0] !!! Aggeregate usage embeddingResult.usage
                                             index.push({
                                                 modelName: embeddingResult.modelName,
                                                 position: embeddingResult.content,
@@ -2733,20 +2831,186 @@
                                                 content: content,
                                                 keywords: keywords,
                                                 index: index,
-                                                sources: sources,
+                                                // <- TODO: [☀] sources,
                                             }];
                                     }
                                 });
                             }); }))];
                     case 5:
-                        knowledge = _h.sent();
+                        knowledge = _j.sent();
                         return [2 /*return*/, knowledge];
                 }
             });
         });
     }
     /**
-     * 11:11
+     * TODO: [🪂] Do it in parallel 11:11
+     * Note: No need to aggregate usage here, it is done by intercepting the llmTools
+     */
+
+    /**
+     * Prepares the knowle
+     *
+     * @see https://github.com/webgptorg/promptbook/discussions/41
+     * @private within the package
+     */
+    function prepareKnowledgePieces(knowledgeSources, options) {
+        return __awaiter(this, void 0, void 0, function () {
+            var _a, maxParallelCount, knowledgePrepared;
+            var _this = this;
+            return __generator(this, function (_b) {
+                switch (_b.label) {
+                    case 0:
+                        _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a;
+                        knowledgePrepared = [];
+                        return [4 /*yield*/, forEachAsync(knowledgeSources, { maxParallelCount: maxParallelCount }, function (knowledgeSource) { return __awaiter(_this, void 0, void 0, function () {
+                                var partialPieces, pieces;
+                                return __generator(this, function (_a) {
+                                    switch (_a.label) {
+                                        case 0: return [4 /*yield*/, prepareKnowledgeFromMarkdown(knowledgeSource.source, // <- TODO: !!!!! Unhardcode markdown, detect which type it is
+                                            options)];
+                                        case 1:
+                                            partialPieces = _a.sent();
+                                            pieces = partialPieces.map(function (partialPiece) { return (__assign(__assign({}, partialPiece), { sources: [
+                                                    {
+                                                        name: knowledgeSource.name,
+                                                        // line, column <- TODO: [☀]
+                                                        // <- TODO: [❎]
+                                                    },
+                                                ] })); });
+                                            knowledgePrepared.push.apply(knowledgePrepared, __spreadArray([], __read(pieces), false));
+                                            return [2 /*return*/];
+                                    }
+                                });
+                            }); })];
+                    case 1:
+                        _b.sent();
+                        return [2 /*return*/, knowledgePrepared];
+                }
+            });
+        });
+    }
+    /*
+    TODO: [🧊] This is how it can look in future
+    > type PrepareKnowledgeKnowledge = {
+    >   /**
+    >    * Unprepared knowledge
+    >    * /
+    >   readonly knowledgeSources: Array<KnowledgeSourceJson>;
+    > };
+    >
+    > export async function prepareKnowledgePieces(
+    >   knowledge: PrepareKnowledgeKnowledge,
+    >   options: PrepareOptions,
+    > ):
+    */
+    /**
+     * TODO: [🧊] In future one preparation can take data from previous preparation and save tokens and time
+     *       Put `knowledgePieces` into `PrepareKnowledgeOptions`
+     * TODO: [🪂] More than max things can run in parallel by acident [1,[2a,2b,_],[3a,3b,_]]
+     * TODO: [🧠][❎] Do here propper M:N mapping
+     *       [x] One source can make multiple pieces
+     *       [ ] One piece can have multiple sources
+     */
+
+    /**
+     * Prepares the persona for the pipeline
+     *
+     * @see https://github.com/webgptorg/promptbook/discussions/22
+     * @private within the package
+     */
+    function preparePersona(personaDescription, options) {
+        return __awaiter(this, void 0, void 0, function () {
+            var llmTools, _a, maxParallelCount, _b, isVerbose;
+            return __generator(this, function (_c) {
+                llmTools = options.llmTools, _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a, _b = options.isVerbose, isVerbose = _b === void 0 ? false : _b;
+                TODO_USE(maxParallelCount); // <- [🪂]
+                TODO_USE(personaDescription); // <- !!!!!
+                TODO_USE(llmTools); // <- !!!!!
+                TODO_USE(isVerbose); // <- !!!!!
+                return [2 /*return*/, {
+                        modelVariant: 'CHAT',
+                        modelName: 'gpt-4',
+                    }];
+            });
+        });
+    }
+    /**
+     * TODO: [🪂] Do it in parallel
+     */
+
+    /**
+     * Simple wrapper `new Date().toISOString()`
+     *
+     * @returns string_date branded type
+     */
+    function $currentDate() {
+        return new Date().toISOString();
+    }
+
+    /**
+     * Prepare pipeline from string (markdown) format to JSON format
+     *
+     * Note: This function does not validate logic of the pipeline
+     * Note: This function acts as part of compilation process
+     */
+    function preparePipeline(pipeline, options) {
+        return __awaiter(this, void 0, void 0, function () {
+            var _a, maxParallelCount, 
+            /*
+            <- TODO: [🧠][0] `promptbookVersion` */
+            knowledgeSources /*
+            <- TODO: [🧊] `knowledgePieces` */, personas /*
+            <- TODO: [🧊] `preparations` */, currentPreparation, preparations, preparedPersonas, knowledgeSourcesPrepared, partialknowledgePiecesPrepared, knowledgePiecesPrepared;
+            var _this = this;
+            return __generator(this, function (_b) {
+                switch (_b.label) {
+                    case 0:
+                        _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a;
+                        knowledgeSources = pipeline.knowledgeSources, personas = pipeline.personas;
+                        currentPreparation = {
+                            id: 1,
+                            date: $currentDate(),
+                            promptbookVersion: PROMPTBOOK_VERSION,
+                            modelUsage: addUsage(),
+                        };
+                        preparations = [
+                            // ...preparations
+                            // <- TODO: [🧊]
+                            currentPreparation,
+                        ];
+                        preparedPersonas = [];
+                        return [4 /*yield*/, forEachAsync(personas, { maxParallelCount: maxParallelCount /* <- TODO: [🪂] When there are subtasks, this maximul limit can be broken */ }, function (persona) { return __awaiter(_this, void 0, void 0, function () {
+                                var modelRequirements, preparedPersona;
+                                return __generator(this, function (_a) {
+                                    switch (_a.label) {
+                                        case 0: return [4 /*yield*/, preparePersona(persona.description, options)];
+                                        case 1:
+                                            modelRequirements = _a.sent();
+                                            preparedPersona = __assign(__assign({}, persona), { modelRequirements: modelRequirements, preparationIds: [/* TODO: [🧊] -> */ currentPreparation.id] });
+                                            preparedPersonas.push(preparedPersona);
+                                            return [2 /*return*/];
+                                    }
+                                });
+                            }); })];
+                    case 1:
+                        _b.sent();
+                        knowledgeSourcesPrepared = knowledgeSources.map(function (source) { return (__assign(__assign({}, source), { preparationIds: [/* TODO: [🧊] -> */ currentPreparation.id] })); });
+                        return [4 /*yield*/, prepareKnowledgePieces(knowledgeSources /* <- TODO: [🧊] {knowledgeSources, knowledgePieces} */, options)];
+                    case 2:
+                        partialknowledgePiecesPrepared = _b.sent();
+                        knowledgePiecesPrepared = partialknowledgePiecesPrepared.map(function (piece) { return (__assign(__assign({}, piece), { preparationIds: [/* TODO: [🧊] -> */ currentPreparation.id] })); });
+                        // ----- /Knowledge preparation -----
+                        return [2 /*return*/, __assign(__assign({}, pipeline), { knowledgeSources: knowledgeSourcesPrepared, knowledgePieces: knowledgePiecesPrepared, personas: preparedPersonas, preparations: preparations })];
+                }
+            });
+        });
+    }
+    /**
+     * TODO: Write tests for `preparePipeline`
+     * TODO: [🏏] Leverage the batch API and build queues @see https://platform.openai.com/docs/guides/batch
+     * TODO: [🧊] In future one preparation can take data from previous preparation and save tokens and time
+     * TODO: [💸] Make utilities `interceptLlmTools` and `costLlmTools` to compute cost and DO put this counting logic in `prepareKnowledge` or `preparePersona`
      */
 
     /**
@@ -2916,6 +3180,7 @@
 
     /**
      * Units of text measurement
+     * @see https://github.com/webgptorg/promptbook/discussions/30
      */
     var EXPECTATION_UNITS = ['CHARACTERS', 'WORDS', 'SENTENCES', 'LINES', 'PARAGRAPHS', 'PAGES'];
     /**
@@ -3236,9 +3501,10 @@
 
     var MODEL_VARIANTS = ['COMPLETION', 'CHAT', 'EMBEDDING' /* <- TODO [🏳] */ /* <- [🤖] */];
     /**
+     * TODO: !!!!! Add and use systemMessage, temprerature, top_k, top_p, presencePenalty, frequencyPenalty, bestOf, n, logitBias, logitBiasType, stop, ... to ModelRequirements
      * TODO: Maybe figure out better word than "variant"
      * TODO: Add here more requirement options like max context size, max tokens, etc.
-     * TODO: [👙][🧠] Just selecting gpt3 or gpt4 level of model
+     * TODO: [💕][🧠] Just selecting gpt3 or gpt4 level of model
      */
 
     /**
@@ -4397,14 +4663,14 @@
      */
 
     /**
-     * Compile promptbook from string (markdown) format to JSON format synchronously
+     * Compile pipeline from string (markdown) format to JSON format synchronously
      *
-     * Note: There are two similar functions:
+     * Note: There are 3 similar functions:
      * - `pipelineStringToJson` **(preferred)** - which propperly compiles the promptbook and use embedding for external knowledge
      * - `pipelineStringToJsonSync` - use only if you need to compile promptbook synchronously and it contains NO external knowledge
+     * - `preparePipeline` - just one step in the compilation process
      *
      * @param pipelineString {Promptbook} in string markdown format (.ptbk.md)
-     * @param options - Options and tools for the compilation
      * @returns {Promptbook} compiled in JSON format (.ptbk.json)
      * @throws {ParsingError} if the promptbook string is not valid
      *
@@ -4420,7 +4686,10 @@
             description: undefined /* <- Note: Putting here placeholder to keep `description` on top at final JSON */,
             parameters: [],
             promptTemplates: [],
-            knowledge: [],
+            knowledgeSources: [],
+            knowledgePieces: [],
+            personas: [],
+            preparations: [],
         };
         // =============================================================
         // Note: 1️⃣ Parsing of the markdown into object
@@ -4688,6 +4957,7 @@
                 postprocessing: postprocessing,
                 expectations: expectAmount,
                 expectFormat: expectFormat,
+                personaName: null,
                 modelRequirements: templateModelRequirements,
                 contentLanguage: blockType === 'SCRIPT' ? language : undefined,
                 content: content,
@@ -4767,11 +5037,12 @@
      */
 
     /**
-     * Compile promptbook from string (markdown) format to JSON format
+     * Compile pipeline from string (markdown) format to JSON format
      *
-     * Note: There are two similar functions:
+     * Note: There are 3 similar functions:
      * - `pipelineStringToJson` **(preferred)** - which propperly compiles the promptbook and use embedding for external knowledge
      * - `pipelineStringToJsonSync` - use only if you need to compile promptbook synchronously and it contains NO external knowledge
+     * - `preparePipeline` - just one step in the compilation process
      *
      * @param pipelineString {Promptbook} in string markdown format (.ptbk.md)
      * @param options - Options and tools for the compilation
@@ -4782,22 +5053,18 @@
      * Note: This function acts as compilation process
      */
     function pipelineStringToJson(pipelineString, options) {
-        if (options === void 0) { options = {}; }
+        if (options === void 0) { options = { llmTools: null }; }
         return __awaiter(this, void 0, void 0, function () {
-            var llmTools, pipelineJson, knowledge;
+            var llmTools, pipelineJson;
             return __generator(this, function (_a) {
                 switch (_a.label) {
                     case 0:
                         llmTools = options.llmTools;
                         pipelineJson = pipelineStringToJsonSync(pipelineString);
-                        if (!llmTools) return [3 /*break*/, 2];
-                        return [4 /*yield*/, prepareKnowledgeFromMarkdown({
-                                content: 'Roses are red, violets are blue, programmers use Promptbook, users too',
-                                llmTools: llmTools,
-                            })];
+                        if (!(llmTools !== null)) return [3 /*break*/, 2];
+                        return [4 /*yield*/, preparePipeline(pipelineJson, { llmTools: llmTools })];
                     case 1:
-                        knowledge = _a.sent();
-                        pipelineJson = __assign(__assign({}, pipelineJson), { knowledge: __spreadArray(__spreadArray([], __read((pipelineJson.knowledge || [])), false), __read(knowledge), false) });
+                        pipelineJson = _a.sent();
                         _a.label = 2;
                     case 2: return [2 /*return*/, pipelineJson];
                 }
@@ -4994,6 +5261,20 @@
         return CollectionError;
     }(Error));
 
+    /**
+     * This error type indicates that the version of the pipeline is not matching the expected version
+     */
+    var VersionMismatch = /** @class */ (function (_super) {
+        __extends(VersionMismatch, _super);
+        function VersionMismatch(message, expectedVersion) {
+            var _this = _super.call(this, spaceTrim.spaceTrim(function (block) { return "\n                    ".concat(block(message), "\n\n                    This error indicates that there is error with compatibility\n\n                    Expected version:\n                    ").concat(expectedVersion, "\n\n                    Current version:\n                    ").concat(PROMPTBOOK_VERSION, "\n\n                "); })) || this;
+            _this.name = 'UnexpectedError';
+            Object.setPrototypeOf(_this, VersionMismatch.prototype);
+            return _this;
+        }
+        return VersionMismatch;
+    }(Error));
+
     /**
      * Pretty print an embedding vector for logging
      */
@@ -5082,6 +5363,19 @@
         return SimplePromptInterfaceTools;
     }());
 
+    /**
+     * Unprepare just strips the preparation data of the pipeline
+     */
+    function unpreparePipeline(pipeline) {
+        var personas = pipeline.personas, knowledgeSources = pipeline.knowledgeSources;
+        personas = personas.map(function (persona) { return (__assign(__assign({}, persona), { modelRequirements: undefined, preparationIds: undefined })); });
+        knowledgeSources = knowledgeSources.map(function (knowledgeSource) { return (__assign(__assign({}, knowledgeSource), { preparationIds: undefined })); });
+        return __assign(__assign({}, pipeline), { knowledgeSources: knowledgeSources, knowledgePieces: [], personas: personas, preparations: [] });
+    }
+    /**
+     * TODO: Write tests for `preparePipeline`
+     */
+
     /**
      * Default options for generating an execution report string
      */
@@ -5406,6 +5700,7 @@
     exports.ReferenceError = ReferenceError$1;
     exports.SimplePromptInterfaceTools = SimplePromptInterfaceTools;
     exports.UnexpectedError = UnexpectedError;
+    exports.VersionMismatch = VersionMismatch;
     exports.addUsage = addUsage;
     exports.assertsExecutionSuccessful = assertsExecutionSuccessful;
     exports.checkExpectations = checkExpectations;
@@ -5423,7 +5718,9 @@
     exports.pipelineStringToJson = pipelineStringToJson;
     exports.pipelineStringToJsonSync = pipelineStringToJsonSync;
     exports.prepareKnowledgeFromMarkdown = prepareKnowledgeFromMarkdown;
+    exports.preparePipeline = preparePipeline;
     exports.prettifyPipelineString = prettifyPipelineString;
+    exports.unpreparePipeline = unpreparePipeline;
     exports.usageToWorktime = usageToWorktime;
     exports.validatePipeline = validatePipeline;