@promptbook/core 0.61.0-0 → 0.61.0-10

package/esm/index.es.js

@@ -394,6 +394,10 @@ function promptTemplateParameterJsonToString(promptTemplateParameterJson) {
  * The maximum number of iterations for a loops
  */
 var LOOP_LIMIT = 1000;
+/**
+ * The maximum number of (LLM) tasks running in parallel
+ */
+var MAX_PARALLEL_COUNT = 5;
 /**
  * The names of the parameters that are reserved for special purposes
  */
@@ -1119,7 +1123,172 @@ var BlockTypes = [
     // <- [🩻]
 ];
 
-var PipelineCollection = [{title:"Prepare Knowledge from Markdown",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.ptbk.md",promptbookVersion:"0.60.0",parameters:[{name:"content",description:"Markdown document content",isInput:true,isOutput:false},{name:"knowledge",description:"The knowledge JSON object",isInput:false,isOutput:true}],promptTemplates:[{name:"knowledge",title:"Knowledge",dependentParameterNames:["content"],blockType:"PROMPT_TEMPLATE",modelRequirements:{modelVariant:"CHAT",modelName:"claude-3-opus-20240229"},content:"You are experienced data researcher, extract the important knowledge from the document.\n\n# Rules\n\n-   Make pieces of information concise, clear, and easy to understand\n-   One piece of information should be approximately 1 paragraph\n-   Divide the paragraphs by markdown horizontal lines ---\n-   Omit irrelevant information\n-   Group redundant information\n-   Write just extracted information, nothing else\n\n# The document\n\nTake information from this document:\n\n> {content}",resultingParameterName:"knowledge"}],knowledge:[{name:"u",title:"U",content:"I",keywords:["E"],index:[{modelName:"mocked-facked",position:[0.32603811001648664,-0.8151600779193648,0.7664764784601905,0.40210873521217305,0.222437572764977,-0.4125301487038615,-0.8231889454816157,0.9109486723694067,0.37789733931561553,-0.556486491106039,-0.5670319200115816,-0.1974522088727535,0.022104221421542558,0.7046092137370166,0.07437773177645424,-0.3496446604425767,-0.05274352103805802,0.8342909224282642,-0.435471568520275,0.5164118712487724,0.7271111824447978,-0.09364881082937826,0.32855248741175425,-0.07703932357732901,0.551079774872838]}],sources:[{title:"Markdown document",href:"#"}]}],sourceFile:"./promptbook-collection/prepare-knowledge-from-markdown.ptbk.md"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-keywords.ptbk.md",promptbookVersion:"0.60.0",parameters:[{name:"content",description:"The content",isInput:true,isOutput:false},{name:"keywords",description:"Keywords separated by comma",isInput:false,isOutput:true}],promptTemplates:[{name:"knowledge",title:"Knowledge",dependentParameterNames:["content"],blockType:"PROMPT_TEMPLATE",modelRequirements:{modelVariant:"CHAT",modelName:"claude-3-opus-20240229"},content:"You are experienced data researcher, detect the important keywords in the document.\n\n# Rules\n\n-   Write just keywords separated by comma\n\n# The document\n\nTake information from this document:\n\n> {content}",resultingParameterName:"keywords"}],knowledge:[{name:"p",title:"P",content:"A",keywords:["L"],index:[{modelName:"mocked-facked",position:[0.5640345774427704,-0.07119174635570813,0.015197097443399699,0.4182203436305234,0.25816168398409545,-0.44950167559000853,-0.9607858932820381,-0.5156415714405949,-0.29710522199486133,0.41071386689679246,-0.6337607469953004,0.3530235806138857,0.4759243122693184,-0.5098890815896411,0.5078673989680667,0.09358488419391398,0.4892768751447085,0.7060415586657078,0.8965591399366888,0.708193527752476,0.1883448814037032,0.3039912919801422,0.7969109648391037,-0.7137054462894104,-0.8820651045662413]}],sources:[{title:"Markdown document",href:"#"}]}],sourceFile:"./promptbook-collection/prepare-knowledge-keywords.ptbk.md"},{title:"Prepare Title",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-title.ptbk.md",promptbookVersion:"0.60.0",parameters:[{name:"content",description:"The content",isInput:true,isOutput:false},{name:"title",description:"The title of the document",isInput:false,isOutput:true}],promptTemplates:[{name:"knowledge",title:"Knowledge",dependentParameterNames:["content"],blockType:"PROMPT_TEMPLATE",expectations:{words:{min:1,max:8}},modelRequirements:{modelVariant:"CHAT",modelName:"claude-3-opus-20240229"},content:"You are experienced content creator, write best title for the document.\n\n# Rules\n\n-   Write just title, nothing else\n-   Title should be concise and clear\n-   Write maximum 5 words for the title\n\n# The document\n\n> {content}",resultingParameterName:"title"}],knowledge:[{name:"e",title:"E",content:"L",keywords:["U"],index:[{modelName:"mocked-facked",position:[-0.2552561443130785,-0.997345757973708,-0.9044970690870002,0.7735304678081456,0.05306206079217235,0.9192301773631262,-0.5775416347531936,0.6084954872241011,0.1936789483087087,-0.45511339549687646,0.1322427518665963,-0.6050338969876923,0.5860487011982332,0.5682123165708153,-0.9204885870353041,0.5555932573463336,-0.00506336761935966,-0.4353010802882027,-0.6359219313303703,-0.9560097721865413,0.2090379465178107,0.6670733187433857,0.9388562958310938,-0.6743173553796935,-0.7145540816323397]}],sources:[{title:"Markdown document",href:"#"}]}],sourceFile:"./promptbook-collection/prepare-knowledge-title.ptbk.md"}];
+/**
+ * Function `addUsage` will add multiple usages into one
+ *
+ * Note: If you provide 0 values, it returns void usage
+ */
+function addUsage() {
+    var usageItems = [];
+    for (var _i = 0; _i < arguments.length; _i++) {
+        usageItems[_i] = arguments[_i];
+    }
+    var initialStructure = {
+        price: { value: 0 },
+        input: {
+            tokensCount: { value: 0 },
+            charactersCount: { value: 0 },
+            wordsCount: { value: 0 },
+            sentencesCount: { value: 0 },
+            linesCount: { value: 0 },
+            paragraphsCount: { value: 0 },
+            pagesCount: { value: 0 },
+        },
+        output: {
+            tokensCount: { value: 0 },
+            charactersCount: { value: 0 },
+            wordsCount: { value: 0 },
+            sentencesCount: { value: 0 },
+            linesCount: { value: 0 },
+            paragraphsCount: { value: 0 },
+            pagesCount: { value: 0 },
+        },
+    };
+    return usageItems.reduce(function (acc, item) {
+        var e_1, _a, e_2, _b;
+        var _c;
+        acc.price.value += ((_c = item.price) === null || _c === void 0 ? void 0 : _c.value) || 0;
+        try {
+            for (var _d = __values(Object.keys(acc.input)), _e = _d.next(); !_e.done; _e = _d.next()) {
+                var key = _e.value;
+                // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+                //@ts-ignore
+                if (item.input[key]) {
+                    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+                    //@ts-ignore
+                    acc.input[key].value += item.input[key].value || 0;
+                    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+                    //@ts-ignore
+                    if (item.input[key].isUncertain) {
+                        // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+                        //@ts-ignore
+                        acc.input[key].isUncertain = true;
+                    }
+                }
+            }
+        }
+        catch (e_1_1) { e_1 = { error: e_1_1 }; }
+        finally {
+            try {
+                if (_e && !_e.done && (_a = _d.return)) _a.call(_d);
+            }
+            finally { if (e_1) throw e_1.error; }
+        }
+        try {
+            for (var _f = __values(Object.keys(acc.output)), _g = _f.next(); !_g.done; _g = _f.next()) {
+                var key = _g.value;
+                // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+                //@ts-ignore
+                if (item.output[key]) {
+                    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+                    //@ts-ignore
+                    acc.output[key].value += item.output[key].value || 0;
+                    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+                    //@ts-ignore
+                    if (item.output[key].isUncertain) {
+                        // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+                        //@ts-ignore
+                        acc.output[key].isUncertain = true;
+                    }
+                }
+            }
+        }
+        catch (e_2_1) { e_2 = { error: e_2_1 }; }
+        finally {
+            try {
+                if (_g && !_g.done && (_b = _f.return)) _b.call(_f);
+            }
+            finally { if (e_2) throw e_2.error; }
+        }
+        return acc;
+    }, initialStructure);
+}
+
+/**
+ * Async version of Array.forEach
+ *
+ * @param array - Array to iterate over
+ * @param options - Options for the function
+ * @param callbackfunction - Function to call for each item
+ */
+function forEachAsync(array, options, callbackfunction) {
+    return __awaiter(this, void 0, void 0, function () {
+        var _a, maxParallelCount, index, runningTasks, tasks, _loop_1, _b, _c, item, e_1_1;
+        var e_1, _d;
+        return __generator(this, function (_e) {
+            switch (_e.label) {
+                case 0:
+                    _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? Infinity : _a;
+                    index = 0;
+                    runningTasks = [];
+                    tasks = [];
+                    _loop_1 = function (item) {
+                        var currentIndex, task;
+                        return __generator(this, function (_f) {
+                            switch (_f.label) {
+                                case 0:
+                                    currentIndex = index++;
+                                    task = callbackfunction(item, currentIndex, array);
+                                    tasks.push(task);
+                                    runningTasks.push(task);
+                                    /* not await */ Promise.resolve(task).then(function () {
+                                        runningTasks = runningTasks.filter(function (t) { return t !== task; });
+                                    });
+                                    if (!(maxParallelCount < runningTasks.length)) return [3 /*break*/, 2];
+                                    return [4 /*yield*/, Promise.race(runningTasks)];
+                                case 1:
+                                    _f.sent();
+                                    _f.label = 2;
+                                case 2: return [2 /*return*/];
+                            }
+                        });
+                    };
+                    _e.label = 1;
+                case 1:
+                    _e.trys.push([1, 6, 7, 8]);
+                    _b = __values(array), _c = _b.next();
+                    _e.label = 2;
+                case 2:
+                    if (!!_c.done) return [3 /*break*/, 5];
+                    item = _c.value;
+                    return [5 /*yield**/, _loop_1(item)];
+                case 3:
+                    _e.sent();
+                    _e.label = 4;
+                case 4:
+                    _c = _b.next();
+                    return [3 /*break*/, 2];
+                case 5: return [3 /*break*/, 8];
+                case 6:
+                    e_1_1 = _e.sent();
+                    e_1 = { error: e_1_1 };
+                    return [3 /*break*/, 8];
+                case 7:
+                    try {
+                        if (_c && !_c.done && (_d = _b.return)) _d.call(_b);
+                    }
+                    finally { if (e_1) throw e_1.error; }
+                    return [7 /*endfinally*/];
+                case 8: return [4 /*yield*/, Promise.all(tasks)];
+                case 9:
+                    _e.sent();
+                    return [2 /*return*/];
+            }
+        });
+    });
+}
+
+var PipelineCollection = [{title:"Prepare Knowledge from Markdown",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.ptbk.md",promptbookVersion:"0.61.0-9",parameters:[{name:"content",description:"Markdown document content",isInput:true,isOutput:false},{name:"knowledge",description:"The knowledge JSON object",isInput:false,isOutput:true}],promptTemplates:[{name:"knowledge",title:"Knowledge",dependentParameterNames:["content"],blockType:"PROMPT_TEMPLATE",personaName:null,modelRequirements:{modelVariant:"CHAT",modelName:"claude-3-opus-20240229"},content:"You are experienced data researcher, extract the important knowledge from the document.\n\n# Rules\n\n-   Make pieces of information concise, clear, and easy to understand\n-   One piece of information should be approximately 1 paragraph\n-   Divide the paragraphs by markdown horizontal lines ---\n-   Omit irrelevant information\n-   Group redundant information\n-   Write just extracted information, nothing else\n\n# The document\n\nTake information from this document:\n\n> {content}",resultingParameterName:"knowledge"}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[{id:1,date:"2024-07-29T00:04:31.966Z",promptbookVersion:"0.61.0-9",modelUsage:{price:{value:0},input:{tokensCount:{value:0},charactersCount:{value:0},wordsCount:{value:0},sentencesCount:{value:0},linesCount:{value:0},paragraphsCount:{value:0},pagesCount:{value:0}},output:{tokensCount:{value:0},charactersCount:{value:0},wordsCount:{value:0},sentencesCount:{value:0},linesCount:{value:0},paragraphsCount:{value:0},pagesCount:{value:0}}}}],sourceFile:"./promptbook-collection/prepare-knowledge-from-markdown.ptbk.md"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-keywords.ptbk.md",promptbookVersion:"0.61.0-9",parameters:[{name:"content",description:"The content",isInput:true,isOutput:false},{name:"keywords",description:"Keywords separated by comma",isInput:false,isOutput:true}],promptTemplates:[{name:"knowledge",title:"Knowledge",dependentParameterNames:["content"],blockType:"PROMPT_TEMPLATE",personaName:null,modelRequirements:{modelVariant:"CHAT",modelName:"claude-3-opus-20240229"},content:"You are experienced data researcher, detect the important keywords in the document.\n\n# Rules\n\n-   Write just keywords separated by comma\n\n# The document\n\nTake information from this document:\n\n> {content}",resultingParameterName:"keywords"}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[{id:1,date:"2024-07-29T00:04:31.970Z",promptbookVersion:"0.61.0-9",modelUsage:{price:{value:0},input:{tokensCount:{value:0},charactersCount:{value:0},wordsCount:{value:0},sentencesCount:{value:0},linesCount:{value:0},paragraphsCount:{value:0},pagesCount:{value:0}},output:{tokensCount:{value:0},charactersCount:{value:0},wordsCount:{value:0},sentencesCount:{value:0},linesCount:{value:0},paragraphsCount:{value:0},pagesCount:{value:0}}}}],sourceFile:"./promptbook-collection/prepare-knowledge-keywords.ptbk.md"},{title:"Prepare Title",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-title.ptbk.md",promptbookVersion:"0.61.0-9",parameters:[{name:"content",description:"The content",isInput:true,isOutput:false},{name:"title",description:"The title of the document",isInput:false,isOutput:true}],promptTemplates:[{name:"knowledge",title:"Knowledge",dependentParameterNames:["content"],blockType:"PROMPT_TEMPLATE",expectations:{words:{min:1,max:8}},personaName:null,modelRequirements:{modelVariant:"CHAT",modelName:"claude-3-opus-20240229"},content:"You are experienced content creator, write best title for the document.\n\n# Rules\n\n-   Write just title, nothing else\n-   Title should be concise and clear\n-   Write maximum 5 words for the title\n\n# The document\n\n> {content}",resultingParameterName:"title"}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[{id:1,date:"2024-07-29T00:04:31.976Z",promptbookVersion:"0.61.0-9",modelUsage:{price:{value:0},input:{tokensCount:{value:0},charactersCount:{value:0},wordsCount:{value:0},sentencesCount:{value:0},linesCount:{value:0},paragraphsCount:{value:0},pagesCount:{value:0}},output:{tokensCount:{value:0},charactersCount:{value:0},wordsCount:{value:0},sentencesCount:{value:0},linesCount:{value:0},paragraphsCount:{value:0},pagesCount:{value:0}}}}],sourceFile:"./promptbook-collection/prepare-knowledge-title.ptbk.md"}];
 
 var defaultDiacriticsRemovalMap = [
     {
@@ -1446,6 +1615,14 @@ function removeEmojis(text) {
  * Function normalizes title to name which can be used as identifier
  */
 function titleToName(value) {
+    if (value.startsWith('http://') || value.startsWith('https://')) {
+        // TODO: Maybe check against some list unallowed characters
+        return value;
+    }
+    if (value.startsWith('./') || value.startsWith('../')) {
+        // TODO: Maybe check against some list unallowed characters
+        return value;
+    }
     value = removeEmojis(value);
     value = normalizeToKebabCase(value);
     // TODO: [🧠] Maybe warn or add some padding to short name which are not good identifiers
@@ -1783,100 +1960,9 @@ function arrayableToArray(input) {
 /**
  * The version of the Promptbook library
  */
-var PROMPTBOOK_VERSION = '0.60.0';
+var PROMPTBOOK_VERSION = '0.61.0-9';
 // TODO: !!!! List here all the versions and annotate + put into script
 
-/**
- * Function `addUsage` will add multiple usages into one
- *
- * Note: If you provide 0 values, it returns void usage
- */
-function addUsage() {
-    var usageItems = [];
-    for (var _i = 0; _i < arguments.length; _i++) {
-        usageItems[_i] = arguments[_i];
-    }
-    var initialStructure = {
-        price: { value: 0 },
-        input: {
-            tokensCount: { value: 0 },
-            charactersCount: { value: 0 },
-            wordsCount: { value: 0 },
-            sentencesCount: { value: 0 },
-            linesCount: { value: 0 },
-            paragraphsCount: { value: 0 },
-            pagesCount: { value: 0 },
-        },
-        output: {
-            tokensCount: { value: 0 },
-            charactersCount: { value: 0 },
-            wordsCount: { value: 0 },
-            sentencesCount: { value: 0 },
-            linesCount: { value: 0 },
-            paragraphsCount: { value: 0 },
-            pagesCount: { value: 0 },
-        },
-    };
-    return usageItems.reduce(function (acc, item) {
-        var e_1, _a, e_2, _b;
-        var _c;
-        acc.price.value += ((_c = item.price) === null || _c === void 0 ? void 0 : _c.value) || 0;
-        try {
-            for (var _d = __values(Object.keys(acc.input)), _e = _d.next(); !_e.done; _e = _d.next()) {
-                var key = _e.value;
-                // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-                //@ts-ignore
-                if (item.input[key]) {
-                    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-                    //@ts-ignore
-                    acc.input[key].value += item.input[key].value || 0;
-                    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-                    //@ts-ignore
-                    if (item.input[key].isUncertain) {
-                        // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-                        //@ts-ignore
-                        acc.input[key].isUncertain = true;
-                    }
-                }
-            }
-        }
-        catch (e_1_1) { e_1 = { error: e_1_1 }; }
-        finally {
-            try {
-                if (_e && !_e.done && (_a = _d.return)) _a.call(_d);
-            }
-            finally { if (e_1) throw e_1.error; }
-        }
-        try {
-            for (var _f = __values(Object.keys(acc.output)), _g = _f.next(); !_g.done; _g = _f.next()) {
-                var key = _g.value;
-                // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-                //@ts-ignore
-                if (item.output[key]) {
-                    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-                    //@ts-ignore
-                    acc.output[key].value += item.output[key].value || 0;
-                    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-                    //@ts-ignore
-                    if (item.output[key].isUncertain) {
-                        // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-                        //@ts-ignore
-                        acc.output[key].isUncertain = true;
-                    }
-                }
-            }
-        }
-        catch (e_2_1) { e_2 = { error: e_2_1 }; }
-        finally {
-            try {
-                if (_g && !_g.done && (_b = _f.return)) _b.call(_f);
-            }
-            finally { if (e_2) throw e_2.error; }
-        }
-        return acc;
-    }, initialStructure);
-}
-
 /**
  * Counts number of characters in the text
  */
@@ -2080,8 +2166,6 @@ function createPipelineExecutor(options) {
     var _b = settings.maxExecutionAttempts, maxExecutionAttempts = _b === void 0 ? 3 : _b;
     // TODO: !!!!! Implement new commands
     validatePipeline(pipeline);
-    // TODO: !!!!! Do here materialization of still unmaterialized or dynamic knowledge
-    // TODO: !!!!! Do here personaToModelRequirements
     var llmTools = joinLlmExecutionTools.apply(void 0, __spreadArray([], __read(arrayableToArray(tools.llm)), false));
     var pipelineExecutor = function (inputParameters, onProgress) { return __awaiter(_this, void 0, void 0, function () {
         function executeSingleTemplate(currentTemplate) {
@@ -2602,53 +2686,74 @@ function createPipelineExecutor(options) {
     return pipelineExecutor;
 }
 /**
+ * TODO: [🪂] Pass maxParallelCount here
  * TODO: [🧠] When not meet expectations in PROMPT_DIALOG, make some way to tell the user
  * TODO: [👧] Strongly type the executors to avoid need of remove nullables whtn noUncheckedIndexedAccess in tsconfig.json
  * Note: CreatePipelineExecutorOptions are just connected to PipelineExecutor so do not extract to types folder
  * TODO: [🧠][3] transparent = (report intermediate parameters) / opaque execution = (report only output parameters) progress reporting mode
  */
 
-function prepareKnowledgeFromMarkdown(options) {
+/**
+ * Just marks a place of place where should be something implemented
+ * No side effects.
+ *
+ * Note: It can be usefull suppressing eslint errors of unused variables
+ *
+ * @param value any values
+ * @returns void
+ */
+function TODO_USE() {
+    var value = [];
+    for (var _i = 0; _i < arguments.length; _i++) {
+        value[_i] = arguments[_i];
+    }
+}
+// TODO: !!!! Find ACRY all just(...) and replace with TODO_USE
+
+/**
+ * @@@
+ */
+function prepareKnowledgeFromMarkdown(content /* <- TODO: [🖖] (?maybe not) Always the file */, options) {
     return __awaiter(this, void 0, void 0, function () {
-        var content, llmTools, _a, isVerbose, collection, prepareKnowledgeFromMarkdownExecutor, _b, prepareTitleExecutor, _c, prepareKeywordsExecutor, _d, result, outputParameters, knowledgeRaw, knowledgeTextPieces, knowledge;
-        var _e, _f, _g;
+        var llmTools, _a, maxParallelCount, _b, isVerbose, collection, prepareKnowledgeFromMarkdownExecutor, _c, prepareTitleExecutor, _d, prepareKeywordsExecutor, _e, result, outputParameters, knowledgeRaw, knowledgeTextPieces, knowledge;
+        var _f, _g, _h;
         var _this = this;
-        return __generator(this, function (_h) {
-            switch (_h.label) {
+        return __generator(this, function (_j) {
+            switch (_j.label) {
                 case 0:
-                    content = options.content, llmTools = options.llmTools, _a = options.isVerbose, isVerbose = _a === void 0 ? false : _a;
+                    llmTools = options.llmTools, _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a, _b = options.isVerbose, isVerbose = _b === void 0 ? false : _b;
+                    TODO_USE(maxParallelCount); // <- [🪂]
                     collection = createCollectionFromJson.apply(void 0, __spreadArray([], __read(PipelineCollection), false));
-                    _b = createPipelineExecutor;
-                    _e = {};
-                    return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.ptbk.md')];
-                case 1:
-                    prepareKnowledgeFromMarkdownExecutor = _b.apply(void 0, [(_e.pipeline = _h.sent(),
-                            _e.tools = {
-                                llm: llmTools,
-                            },
-                            _e)]);
                     _c = createPipelineExecutor;
                     _f = {};
-                    return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-knowledge-title.ptbk.md')];
-                case 2:
-                    prepareTitleExecutor = _c.apply(void 0, [(_f.pipeline = _h.sent(),
+                    return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.ptbk.md')];
+                case 1:
+                    prepareKnowledgeFromMarkdownExecutor = _c.apply(void 0, [(_f.pipeline = _j.sent(),
                             _f.tools = {
                                 llm: llmTools,
                             },
                             _f)]);
                     _d = createPipelineExecutor;
                     _g = {};
-                    return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-knowledge-keywords.ptbk.md')];
-                case 3:
-                    prepareKeywordsExecutor = _d.apply(void 0, [(_g.pipeline = _h.sent(),
+                    return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-knowledge-title.ptbk.md')];
+                case 2:
+                    prepareTitleExecutor = _d.apply(void 0, [(_g.pipeline = _j.sent(),
                             _g.tools = {
                                 llm: llmTools,
                             },
                             _g)]);
+                    _e = createPipelineExecutor;
+                    _h = {};
+                    return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-knowledge-keywords.ptbk.md')];
+                case 3:
+                    prepareKeywordsExecutor = _e.apply(void 0, [(_h.pipeline = _j.sent(),
+                            _h.tools = {
+                                llm: llmTools,
+                            },
+                            _h)]);
                     return [4 /*yield*/, prepareKnowledgeFromMarkdownExecutor({ content: content })];
                 case 4:
-                    result = _h.sent();
-                    // TODO: [0] !!! Aggeregate usage
+                    result = _j.sent();
                     assertsExecutionSuccessful(result);
                     outputParameters = result.outputParameters;
                     knowledgeRaw = outputParameters.knowledge;
@@ -2657,9 +2762,9 @@ function prepareKnowledgeFromMarkdown(options) {
                         console.info('knowledgeTextPieces:', knowledgeTextPieces);
                     }
                     return [4 /*yield*/, Promise.all(
-                        // TODO: !!! Do not send all at once but in chunks
+                        // TODO: [🪂] !! Do not send all at once but in chunks
                         knowledgeTextPieces.map(function (knowledgeTextPiece, i) { return __awaiter(_this, void 0, void 0, function () {
-                            var name, title, content, keywords, index, sources, titleResult, _a, titleRaw, keywordsResult, _b, keywordsRaw, embeddingResult, error_1;
+                            var name, title, content, keywords, index, titleResult, _a, titleRaw, keywordsResult, _b, keywordsRaw, embeddingResult, error_1;
                             return __generator(this, function (_c) {
                                 switch (_c.label) {
                                     case 0:
@@ -2668,12 +2773,6 @@ function prepareKnowledgeFromMarkdown(options) {
                                         content = spaceTrim(knowledgeTextPiece);
                                         keywords = [];
                                         index = [];
-                                        sources = [
-                                            {
-                                                title: 'Markdown document' /* <- TODO: !!! Unhardcode */,
-                                                href: '#' /* <- TODO: !!! Unhardcode */,
-                                            },
-                                        ];
                                         _c.label = 1;
                                     case 1:
                                         _c.trys.push([1, 7, , 8]);
@@ -2708,7 +2807,6 @@ function prepareKnowledgeFromMarkdown(options) {
                                         })];
                                     case 5:
                                         embeddingResult = _c.sent();
-                                        // TODO: [0] !!! Aggeregate usage embeddingResult.usage
                                         index.push({
                                             modelName: embeddingResult.modelName,
                                             position: embeddingResult.content,
@@ -2726,20 +2824,186 @@ function prepareKnowledgeFromMarkdown(options) {
                                             content: content,
                                             keywords: keywords,
                                             index: index,
-                                            sources: sources,
+                                            // <- TODO: [☀] sources,
                                         }];
                                 }
                             });
                         }); }))];
                 case 5:
-                    knowledge = _h.sent();
+                    knowledge = _j.sent();
                     return [2 /*return*/, knowledge];
             }
         });
     });
 }
 /**
- * 11:11
+ * TODO: [🪂] Do it in parallel 11:11
+ * Note: No need to aggregate usage here, it is done by intercepting the llmTools
+ */
+
+/**
+ * Prepares the knowle
+ *
+ * @see https://github.com/webgptorg/promptbook/discussions/41
+ * @private within the package
+ */
+function prepareKnowledgePieces(knowledgeSources, options) {
+    return __awaiter(this, void 0, void 0, function () {
+        var _a, maxParallelCount, knowledgePrepared;
+        var _this = this;
+        return __generator(this, function (_b) {
+            switch (_b.label) {
+                case 0:
+                    _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a;
+                    knowledgePrepared = [];
+                    return [4 /*yield*/, forEachAsync(knowledgeSources, { maxParallelCount: maxParallelCount }, function (knowledgeSource) { return __awaiter(_this, void 0, void 0, function () {
+                            var partialPieces, pieces;
+                            return __generator(this, function (_a) {
+                                switch (_a.label) {
+                                    case 0: return [4 /*yield*/, prepareKnowledgeFromMarkdown(knowledgeSource.source, // <- TODO: !!!!! Unhardcode markdown, detect which type it is
+                                        options)];
+                                    case 1:
+                                        partialPieces = _a.sent();
+                                        pieces = partialPieces.map(function (partialPiece) { return (__assign(__assign({}, partialPiece), { sources: [
+                                                {
+                                                    name: knowledgeSource.name,
+                                                    // line, column <- TODO: [☀]
+                                                    // <- TODO: [❎]
+                                                },
+                                            ] })); });
+                                        knowledgePrepared.push.apply(knowledgePrepared, __spreadArray([], __read(pieces), false));
+                                        return [2 /*return*/];
+                                }
+                            });
+                        }); })];
+                case 1:
+                    _b.sent();
+                    return [2 /*return*/, knowledgePrepared];
+            }
+        });
+    });
+}
+/*
+TODO: [🧊] This is how it can look in future
+> type PrepareKnowledgeKnowledge = {
+>   /**
+>    * Unprepared knowledge
+>    * /
+>   readonly knowledgeSources: Array<KnowledgeSourceJson>;
+> };
+>
+> export async function prepareKnowledgePieces(
+>   knowledge: PrepareKnowledgeKnowledge,
+>   options: PrepareOptions,
+> ):
+*/
+/**
+ * TODO: [🧊] In future one preparation can take data from previous preparation and save tokens and time
+ *       Put `knowledgePieces` into `PrepareKnowledgeOptions`
+ * TODO: [🪂] More than max things can run in parallel by acident [1,[2a,2b,_],[3a,3b,_]]
+ * TODO: [🧠][❎] Do here propper M:N mapping
+ *       [x] One source can make multiple pieces
+ *       [ ] One piece can have multiple sources
+ */
+
+/**
+ * Prepares the persona for the pipeline
+ *
+ * @see https://github.com/webgptorg/promptbook/discussions/22
+ * @private within the package
+ */
+function preparePersona(personaDescription, options) {
+    return __awaiter(this, void 0, void 0, function () {
+        var llmTools, _a, maxParallelCount, _b, isVerbose;
+        return __generator(this, function (_c) {
+            llmTools = options.llmTools, _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a, _b = options.isVerbose, isVerbose = _b === void 0 ? false : _b;
+            TODO_USE(maxParallelCount); // <- [🪂]
+            TODO_USE(personaDescription); // <- !!!!!
+            TODO_USE(llmTools); // <- !!!!!
+            TODO_USE(isVerbose); // <- !!!!!
+            return [2 /*return*/, {
+                    modelVariant: 'CHAT',
+                    modelName: 'gpt-4',
+                }];
+        });
+    });
+}
+/**
+ * TODO: [🪂] Do it in parallel
+ */
+
+/**
+ * Simple wrapper `new Date().toISOString()`
+ *
+ * @returns string_date branded type
+ */
+function $currentDate() {
+    return new Date().toISOString();
+}
+
+/**
+ * Prepare pipeline from string (markdown) format to JSON format
+ *
+ * Note: This function does not validate logic of the pipeline
+ * Note: This function acts as part of compilation process
+ */
+function preparePipeline(pipeline, options) {
+    return __awaiter(this, void 0, void 0, function () {
+        var _a, maxParallelCount, 
+        /*
+        <- TODO: [🧠][0] `promptbookVersion` */
+        knowledgeSources /*
+        <- TODO: [🧊] `knowledgePieces` */, personas /*
+        <- TODO: [🧊] `preparations` */, currentPreparation, preparations, preparedPersonas, knowledgeSourcesPrepared, partialknowledgePiecesPrepared, knowledgePiecesPrepared;
+        var _this = this;
+        return __generator(this, function (_b) {
+            switch (_b.label) {
+                case 0:
+                    _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a;
+                    knowledgeSources = pipeline.knowledgeSources, personas = pipeline.personas;
+                    currentPreparation = {
+                        id: 1,
+                        date: $currentDate(),
+                        promptbookVersion: PROMPTBOOK_VERSION,
+                        modelUsage: addUsage(),
+                    };
+                    preparations = [
+                        // ...preparations
+                        // <- TODO: [🧊]
+                        currentPreparation,
+                    ];
+                    preparedPersonas = [];
+                    return [4 /*yield*/, forEachAsync(personas, { maxParallelCount: maxParallelCount /* <- TODO: [🪂] When there are subtasks, this maximul limit can be broken */ }, function (persona) { return __awaiter(_this, void 0, void 0, function () {
+                            var modelRequirements, preparedPersona;
+                            return __generator(this, function (_a) {
+                                switch (_a.label) {
+                                    case 0: return [4 /*yield*/, preparePersona(persona.description, options)];
+                                    case 1:
+                                        modelRequirements = _a.sent();
+                                        preparedPersona = __assign(__assign({}, persona), { modelRequirements: modelRequirements, preparationIds: [/* TODO: [🧊] -> */ currentPreparation.id] });
+                                        preparedPersonas.push(preparedPersona);
+                                        return [2 /*return*/];
+                                }
+                            });
+                        }); })];
+                case 1:
+                    _b.sent();
+                    knowledgeSourcesPrepared = knowledgeSources.map(function (source) { return (__assign(__assign({}, source), { preparationIds: [/* TODO: [🧊] -> */ currentPreparation.id] })); });
+                    return [4 /*yield*/, prepareKnowledgePieces(knowledgeSources /* <- TODO: [🧊] {knowledgeSources, knowledgePieces} */, options)];
+                case 2:
+                    partialknowledgePiecesPrepared = _b.sent();
+                    knowledgePiecesPrepared = partialknowledgePiecesPrepared.map(function (piece) { return (__assign(__assign({}, piece), { preparationIds: [/* TODO: [🧊] -> */ currentPreparation.id] })); });
+                    // ----- /Knowledge preparation -----
+                    return [2 /*return*/, __assign(__assign({}, pipeline), { knowledgeSources: knowledgeSourcesPrepared, knowledgePieces: knowledgePiecesPrepared, personas: preparedPersonas, preparations: preparations })];
+            }
+        });
+    });
+}
+/**
+ * TODO: Write tests for `preparePipeline`
+ * TODO: [🏏] Leverage the batch API and build queues @see https://platform.openai.com/docs/guides/batch
+ * TODO: [🧊] In future one preparation can take data from previous preparation and save tokens and time
+ * TODO: [💸] Make utilities `interceptLlmTools` and `costLlmTools` to compute cost and DO put this counting logic in `prepareKnowledge` or `preparePersona`
  */
 
 /**
@@ -2909,6 +3173,7 @@ var blockCommandParser = {
 
 /**
  * Units of text measurement
+ * @see https://github.com/webgptorg/promptbook/discussions/30
  */
 var EXPECTATION_UNITS = ['CHARACTERS', 'WORDS', 'SENTENCES', 'LINES', 'PARAGRAPHS', 'PAGES'];
 /**
@@ -3229,9 +3494,10 @@ var knowledgeCommandParser = {
 
 var MODEL_VARIANTS = ['COMPLETION', 'CHAT', 'EMBEDDING' /* <- TODO [🏳] */ /* <- [🤖] */];
 /**
+ * TODO: !!!!! Add and use systemMessage, temprerature, top_k, top_p, presencePenalty, frequencyPenalty, bestOf, n, logitBias, logitBiasType, stop, ... to ModelRequirements
  * TODO: Maybe figure out better word than "variant"
  * TODO: Add here more requirement options like max context size, max tokens, etc.
- * TODO: [👙][🧠] Just selecting gpt3 or gpt4 level of model
+ * TODO: [💕][🧠] Just selecting gpt3 or gpt4 level of model
  */
 
 /**
@@ -4390,14 +4656,14 @@ function extractParametersFromPromptTemplate(promptTemplate) {
  */
 
 /**
- * Compile promptbook from string (markdown) format to JSON format synchronously
+ * Compile pipeline from string (markdown) format to JSON format synchronously
  *
- * Note: There are two similar functions:
+ * Note: There are 3 similar functions:
  * - `pipelineStringToJson` **(preferred)** - which propperly compiles the promptbook and use embedding for external knowledge
  * - `pipelineStringToJsonSync` - use only if you need to compile promptbook synchronously and it contains NO external knowledge
+ * - `preparePipeline` - just one step in the compilation process
  *
  * @param pipelineString {Promptbook} in string markdown format (.ptbk.md)
- * @param options - Options and tools for the compilation
  * @returns {Promptbook} compiled in JSON format (.ptbk.json)
  * @throws {ParsingError} if the promptbook string is not valid
  *
@@ -4413,7 +4679,10 @@ function pipelineStringToJsonSync(pipelineString) {
         description: undefined /* <- Note: Putting here placeholder to keep `description` on top at final JSON */,
         parameters: [],
         promptTemplates: [],
-        knowledge: [],
+        knowledgeSources: [],
+        knowledgePieces: [],
+        personas: [],
+        preparations: [],
     };
     // =============================================================
     // Note: 1️⃣ Parsing of the markdown into object
@@ -4681,6 +4950,7 @@ function pipelineStringToJsonSync(pipelineString) {
             postprocessing: postprocessing,
             expectations: expectAmount,
             expectFormat: expectFormat,
+            personaName: null,
             modelRequirements: templateModelRequirements,
             contentLanguage: blockType === 'SCRIPT' ? language : undefined,
             content: content,
@@ -4760,11 +5030,12 @@ function pipelineStringToJsonSync(pipelineString) {
  */
 
 /**
- * Compile promptbook from string (markdown) format to JSON format
+ * Compile pipeline from string (markdown) format to JSON format
  *
- * Note: There are two similar functions:
+ * Note: There are 3 similar functions:
  * - `pipelineStringToJson` **(preferred)** - which propperly compiles the promptbook and use embedding for external knowledge
  * - `pipelineStringToJsonSync` - use only if you need to compile promptbook synchronously and it contains NO external knowledge
+ * - `preparePipeline` - just one step in the compilation process
  *
  * @param pipelineString {Promptbook} in string markdown format (.ptbk.md)
  * @param options - Options and tools for the compilation
@@ -4775,22 +5046,18 @@ function pipelineStringToJsonSync(pipelineString) {
  * Note: This function acts as compilation process
  */
 function pipelineStringToJson(pipelineString, options) {
-    if (options === void 0) { options = {}; }
+    if (options === void 0) { options = { llmTools: null }; }
     return __awaiter(this, void 0, void 0, function () {
-        var llmTools, pipelineJson, knowledge;
+        var llmTools, pipelineJson;
         return __generator(this, function (_a) {
             switch (_a.label) {
                 case 0:
                     llmTools = options.llmTools;
                     pipelineJson = pipelineStringToJsonSync(pipelineString);
-                    if (!llmTools) return [3 /*break*/, 2];
-                    return [4 /*yield*/, prepareKnowledgeFromMarkdown({
-                            content: 'Roses are red, violets are blue, programmers use Promptbook, users too',
-                            llmTools: llmTools,
-                        })];
+                    if (!(llmTools !== null)) return [3 /*break*/, 2];
+                    return [4 /*yield*/, preparePipeline(pipelineJson, { llmTools: llmTools })];
                 case 1:
-                    knowledge = _a.sent();
-                    pipelineJson = __assign(__assign({}, pipelineJson), { knowledge: __spreadArray(__spreadArray([], __read((pipelineJson.knowledge || [])), false), __read(knowledge), false) });
+                    pipelineJson = _a.sent();
                     _a.label = 2;
                 case 2: return [2 /*return*/, pipelineJson];
             }
@@ -4987,6 +5254,20 @@ var CollectionError = /** @class */ (function (_super) {
     return CollectionError;
 }(Error));
 
+/**
+ * This error type indicates that the version of the pipeline is not matching the expected version
+ */
+var VersionMismatch = /** @class */ (function (_super) {
+    __extends(VersionMismatch, _super);
+    function VersionMismatch(message, expectedVersion) {
+        var _this = _super.call(this, spaceTrim$1(function (block) { return "\n                    ".concat(block(message), "\n\n                    This error indicates that there is error with compatibility\n\n                    Expected version:\n                    ").concat(expectedVersion, "\n\n                    Current version:\n                    ").concat(PROMPTBOOK_VERSION, "\n\n                "); })) || this;
+        _this.name = 'UnexpectedError';
+        Object.setPrototypeOf(_this, VersionMismatch.prototype);
+        return _this;
+    }
+    return VersionMismatch;
+}(Error));
+
 /**
  * Pretty print an embedding vector for logging
  */
@@ -5075,6 +5356,19 @@ var SimplePromptInterfaceTools = /** @class */ (function () {
     return SimplePromptInterfaceTools;
 }());
 
+/**
+ * Unprepare just strips the preparation data of the pipeline
+ */
+function unpreparePipeline(pipeline) {
+    var personas = pipeline.personas, knowledgeSources = pipeline.knowledgeSources;
+    personas = personas.map(function (persona) { return (__assign(__assign({}, persona), { modelRequirements: undefined, preparationIds: undefined })); });
+    knowledgeSources = knowledgeSources.map(function (knowledgeSource) { return (__assign(__assign({}, knowledgeSource), { preparationIds: undefined })); });
+    return __assign(__assign({}, pipeline), { knowledgeSources: knowledgeSources, knowledgePieces: [], personas: personas, preparations: [] });
+}
+/**
+ * TODO: Write tests for `preparePipeline`
+ */
+
 /**
  * Default options for generating an execution report string
  */
@@ -5385,5 +5679,5 @@ function executionReportJsonToString(executionReportJson, options) {
  * TODO: [🧠] Allow to filter out some parts of the report by options
  */
 
-export { BlockTypes, CallbackInterfaceTools, CollectionError, ExecutionReportStringOptionsDefaults, ExpectError, NotFoundError, PROMPTBOOK_VERSION, ParsingError, PipelineExecutionError, PipelineLogicError, RESERVED_PARAMETER_NAMES, ReferenceError$1 as ReferenceError, SimplePromptInterfaceTools, UnexpectedError, addUsage, assertsExecutionSuccessful, checkExpectations, collectionToJson, createCollectionFromJson, createCollectionFromPromise, createCollectionFromUrl, createPipelineExecutor, createSubcollection, embeddingVectorToString, executionReportJsonToString, isPassingExpectations, joinLlmExecutionTools, pipelineJsonToString, pipelineStringToJson, pipelineStringToJsonSync, prepareKnowledgeFromMarkdown, prettifyPipelineString, usageToWorktime, validatePipeline };
+export { BlockTypes, CallbackInterfaceTools, CollectionError, ExecutionReportStringOptionsDefaults, ExpectError, NotFoundError, PROMPTBOOK_VERSION, ParsingError, PipelineExecutionError, PipelineLogicError, RESERVED_PARAMETER_NAMES, ReferenceError$1 as ReferenceError, SimplePromptInterfaceTools, UnexpectedError, VersionMismatch, addUsage, assertsExecutionSuccessful, checkExpectations, collectionToJson, createCollectionFromJson, createCollectionFromPromise, createCollectionFromUrl, createPipelineExecutor, createSubcollection, embeddingVectorToString, executionReportJsonToString, isPassingExpectations, joinLlmExecutionTools, pipelineJsonToString, pipelineStringToJson, pipelineStringToJsonSync, prepareKnowledgeFromMarkdown, preparePipeline, prettifyPipelineString, unpreparePipeline, usageToWorktime, validatePipeline };
 //# sourceMappingURL=index.es.js.map