@promptbook/node 0.61.0-1 → 0.61.0-10

package/esm/index.es.js

@@ -128,6 +128,10 @@ function __spreadArray(to, from, pack) {
  * The maximum number of iterations for a loops
  */
 var LOOP_LIMIT = 1000;
+/**
+ * The maximum number of (LLM) tasks running in parallel
+ */
+var MAX_PARALLEL_COUNT = 5;
 /**
  * The name of the builded pipeline collection made by CLI `ptbk make` and for lookup in `createCollectionFromDirectory`
  */
@@ -141,7 +145,172 @@ var RESERVED_PARAMETER_NAMES = [
     // <- TODO: Add [emoji] + instructions ACRY when adding new reserved parameter
 ];
 
-var PipelineCollection = [{title:"Prepare Knowledge from Markdown",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.ptbk.md",promptbookVersion:"0.61.0-0",parameters:[{name:"content",description:"Markdown document content",isInput:true,isOutput:false},{name:"knowledge",description:"The knowledge JSON object",isInput:false,isOutput:true}],promptTemplates:[{name:"knowledge",title:"Knowledge",dependentParameterNames:["content"],blockType:"PROMPT_TEMPLATE",modelRequirements:{modelVariant:"CHAT",modelName:"claude-3-opus-20240229"},content:"You are experienced data researcher, extract the important knowledge from the document.\n\n# Rules\n\n-   Make pieces of information concise, clear, and easy to understand\n-   One piece of information should be approximately 1 paragraph\n-   Divide the paragraphs by markdown horizontal lines ---\n-   Omit irrelevant information\n-   Group redundant information\n-   Write just extracted information, nothing else\n\n# The document\n\nTake information from this document:\n\n> {content}",resultingParameterName:"knowledge"}],knowledge:[{name:"u",title:"U",content:"E",keywords:["E"],index:[{modelName:"mocked-facked",position:[0.3862088172833116,0.7349284028017404,-0.9438230019966163,-0.9250069104750818,-0.026138914694730175,-0.7268311159817045,-0.07935019253731612,0.6470171268509612,-0.43852570574155214,0.34435062959144824,-0.03450022605546588,-0.5277705379240651,0.00828213142899159,0.15691069644743383,-0.625404064630672,-0.33277847012240613,-0.2936622789266794,0.18401827710902152,0.3055511517178138,0.2943796117307138,0.7709829510346151,-0.4424955841467506,0.6031092188828207,-0.02473045258733153,0.33586695334417804]}],sources:[{title:"Markdown document",href:"#"}]}],sourceFile:"./promptbook-collection/prepare-knowledge-from-markdown.ptbk.md"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-keywords.ptbk.md",promptbookVersion:"0.61.0-0",parameters:[{name:"content",description:"The content",isInput:true,isOutput:false},{name:"keywords",description:"Keywords separated by comma",isInput:false,isOutput:true}],promptTemplates:[{name:"knowledge",title:"Knowledge",dependentParameterNames:["content"],blockType:"PROMPT_TEMPLATE",modelRequirements:{modelVariant:"CHAT",modelName:"claude-3-opus-20240229"},content:"You are experienced data researcher, detect the important keywords in the document.\n\n# Rules\n\n-   Write just keywords separated by comma\n\n# The document\n\nTake information from this document:\n\n> {content}",resultingParameterName:"keywords"}],knowledge:[{name:"s",title:"S",content:"A",keywords:["I"],index:[{modelName:"mocked-facked",position:[0.6409598940537844,-0.599282163088533,-0.7951357406825994,0.5034713531055992,0.9186084716269733,-0.6841151439158257,-0.39762364246107484,-0.7646814627420149,0.9551202443145854,0.36871821752653666,0.27710484531156565,-0.6886362623745139,0.1246419058549284,-0.7957866917804912,-0.4688680561538634,-0.7037555014650994,0.003694510488622793,0.3186841135345695,-0.7081253398806422,0.7052620275090136,-0.9744710956043585,0.6828863755008538,-0.021569809814358187,-0.23961644339743904,-0.5963677915161343]}],sources:[{title:"Markdown document",href:"#"}]}],sourceFile:"./promptbook-collection/prepare-knowledge-keywords.ptbk.md"},{title:"Prepare Title",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-title.ptbk.md",promptbookVersion:"0.61.0-0",parameters:[{name:"content",description:"The content",isInput:true,isOutput:false},{name:"title",description:"The title of the document",isInput:false,isOutput:true}],promptTemplates:[{name:"knowledge",title:"Knowledge",dependentParameterNames:["content"],blockType:"PROMPT_TEMPLATE",expectations:{words:{min:1,max:8}},modelRequirements:{modelVariant:"CHAT",modelName:"claude-3-opus-20240229"},content:"You are experienced content creator, write best title for the document.\n\n# Rules\n\n-   Write just title, nothing else\n-   Title should be concise and clear\n-   Write maximum 5 words for the title\n\n# The document\n\n> {content}",resultingParameterName:"title"}],knowledge:[{name:"c",title:"C",content:"A",keywords:["E"],index:[{modelName:"mocked-facked",position:[0.3053077802556703,0.45834476154202175,0.11951317167907494,-0.8925032092763412,-0.041809804577583876,-0.7087885568549193,0.0373992576888198,0.9495631083993494,-0.6050023289380846,0.14919003109680862,-0.3228347383159025,0.3273383195761892,-0.5509884895794359,0.7137115707360451,-0.3455818427015154,-0.7130289858826453,0.6105746565153107,-0.30829913719966573,0.6287711326338274,-0.8588121879147286,-0.583980568533244,-0.23082011458606733,-0.10987445073290791,-0.1129755192200137,0.4046462365369812]}],sources:[{title:"Markdown document",href:"#"}]}],sourceFile:"./promptbook-collection/prepare-knowledge-title.ptbk.md"}];
+/**
+ * Function `addUsage` will add multiple usages into one
+ *
+ * Note: If you provide 0 values, it returns void usage
+ */
+function addUsage() {
+    var usageItems = [];
+    for (var _i = 0; _i < arguments.length; _i++) {
+        usageItems[_i] = arguments[_i];
+    }
+    var initialStructure = {
+        price: { value: 0 },
+        input: {
+            tokensCount: { value: 0 },
+            charactersCount: { value: 0 },
+            wordsCount: { value: 0 },
+            sentencesCount: { value: 0 },
+            linesCount: { value: 0 },
+            paragraphsCount: { value: 0 },
+            pagesCount: { value: 0 },
+        },
+        output: {
+            tokensCount: { value: 0 },
+            charactersCount: { value: 0 },
+            wordsCount: { value: 0 },
+            sentencesCount: { value: 0 },
+            linesCount: { value: 0 },
+            paragraphsCount: { value: 0 },
+            pagesCount: { value: 0 },
+        },
+    };
+    return usageItems.reduce(function (acc, item) {
+        var e_1, _a, e_2, _b;
+        var _c;
+        acc.price.value += ((_c = item.price) === null || _c === void 0 ? void 0 : _c.value) || 0;
+        try {
+            for (var _d = __values(Object.keys(acc.input)), _e = _d.next(); !_e.done; _e = _d.next()) {
+                var key = _e.value;
+                // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+                //@ts-ignore
+                if (item.input[key]) {
+                    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+                    //@ts-ignore
+                    acc.input[key].value += item.input[key].value || 0;
+                    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+                    //@ts-ignore
+                    if (item.input[key].isUncertain) {
+                        // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+                        //@ts-ignore
+                        acc.input[key].isUncertain = true;
+                    }
+                }
+            }
+        }
+        catch (e_1_1) { e_1 = { error: e_1_1 }; }
+        finally {
+            try {
+                if (_e && !_e.done && (_a = _d.return)) _a.call(_d);
+            }
+            finally { if (e_1) throw e_1.error; }
+        }
+        try {
+            for (var _f = __values(Object.keys(acc.output)), _g = _f.next(); !_g.done; _g = _f.next()) {
+                var key = _g.value;
+                // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+                //@ts-ignore
+                if (item.output[key]) {
+                    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+                    //@ts-ignore
+                    acc.output[key].value += item.output[key].value || 0;
+                    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+                    //@ts-ignore
+                    if (item.output[key].isUncertain) {
+                        // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+                        //@ts-ignore
+                        acc.output[key].isUncertain = true;
+                    }
+                }
+            }
+        }
+        catch (e_2_1) { e_2 = { error: e_2_1 }; }
+        finally {
+            try {
+                if (_g && !_g.done && (_b = _f.return)) _b.call(_f);
+            }
+            finally { if (e_2) throw e_2.error; }
+        }
+        return acc;
+    }, initialStructure);
+}
+
+/**
+ * Async version of Array.forEach
+ *
+ * @param array - Array to iterate over
+ * @param options - Options for the function
+ * @param callbackfunction - Function to call for each item
+ */
+function forEachAsync(array, options, callbackfunction) {
+    return __awaiter(this, void 0, void 0, function () {
+        var _a, maxParallelCount, index, runningTasks, tasks, _loop_1, _b, _c, item, e_1_1;
+        var e_1, _d;
+        return __generator(this, function (_e) {
+            switch (_e.label) {
+                case 0:
+                    _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? Infinity : _a;
+                    index = 0;
+                    runningTasks = [];
+                    tasks = [];
+                    _loop_1 = function (item) {
+                        var currentIndex, task;
+                        return __generator(this, function (_f) {
+                            switch (_f.label) {
+                                case 0:
+                                    currentIndex = index++;
+                                    task = callbackfunction(item, currentIndex, array);
+                                    tasks.push(task);
+                                    runningTasks.push(task);
+                                    /* not await */ Promise.resolve(task).then(function () {
+                                        runningTasks = runningTasks.filter(function (t) { return t !== task; });
+                                    });
+                                    if (!(maxParallelCount < runningTasks.length)) return [3 /*break*/, 2];
+                                    return [4 /*yield*/, Promise.race(runningTasks)];
+                                case 1:
+                                    _f.sent();
+                                    _f.label = 2;
+                                case 2: return [2 /*return*/];
+                            }
+                        });
+                    };
+                    _e.label = 1;
+                case 1:
+                    _e.trys.push([1, 6, 7, 8]);
+                    _b = __values(array), _c = _b.next();
+                    _e.label = 2;
+                case 2:
+                    if (!!_c.done) return [3 /*break*/, 5];
+                    item = _c.value;
+                    return [5 /*yield**/, _loop_1(item)];
+                case 3:
+                    _e.sent();
+                    _e.label = 4;
+                case 4:
+                    _c = _b.next();
+                    return [3 /*break*/, 2];
+                case 5: return [3 /*break*/, 8];
+                case 6:
+                    e_1_1 = _e.sent();
+                    e_1 = { error: e_1_1 };
+                    return [3 /*break*/, 8];
+                case 7:
+                    try {
+                        if (_c && !_c.done && (_d = _b.return)) _d.call(_b);
+                    }
+                    finally { if (e_1) throw e_1.error; }
+                    return [7 /*endfinally*/];
+                case 8: return [4 /*yield*/, Promise.all(tasks)];
+                case 9:
+                    _e.sent();
+                    return [2 /*return*/];
+            }
+        });
+    });
+}
+
+var PipelineCollection = [{title:"Prepare Knowledge from Markdown",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.ptbk.md",promptbookVersion:"0.61.0-9",parameters:[{name:"content",description:"Markdown document content",isInput:true,isOutput:false},{name:"knowledge",description:"The knowledge JSON object",isInput:false,isOutput:true}],promptTemplates:[{name:"knowledge",title:"Knowledge",dependentParameterNames:["content"],blockType:"PROMPT_TEMPLATE",personaName:null,modelRequirements:{modelVariant:"CHAT",modelName:"claude-3-opus-20240229"},content:"You are experienced data researcher, extract the important knowledge from the document.\n\n# Rules\n\n-   Make pieces of information concise, clear, and easy to understand\n-   One piece of information should be approximately 1 paragraph\n-   Divide the paragraphs by markdown horizontal lines ---\n-   Omit irrelevant information\n-   Group redundant information\n-   Write just extracted information, nothing else\n\n# The document\n\nTake information from this document:\n\n> {content}",resultingParameterName:"knowledge"}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[{id:1,date:"2024-07-29T00:04:31.966Z",promptbookVersion:"0.61.0-9",modelUsage:{price:{value:0},input:{tokensCount:{value:0},charactersCount:{value:0},wordsCount:{value:0},sentencesCount:{value:0},linesCount:{value:0},paragraphsCount:{value:0},pagesCount:{value:0}},output:{tokensCount:{value:0},charactersCount:{value:0},wordsCount:{value:0},sentencesCount:{value:0},linesCount:{value:0},paragraphsCount:{value:0},pagesCount:{value:0}}}}],sourceFile:"./promptbook-collection/prepare-knowledge-from-markdown.ptbk.md"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-keywords.ptbk.md",promptbookVersion:"0.61.0-9",parameters:[{name:"content",description:"The content",isInput:true,isOutput:false},{name:"keywords",description:"Keywords separated by comma",isInput:false,isOutput:true}],promptTemplates:[{name:"knowledge",title:"Knowledge",dependentParameterNames:["content"],blockType:"PROMPT_TEMPLATE",personaName:null,modelRequirements:{modelVariant:"CHAT",modelName:"claude-3-opus-20240229"},content:"You are experienced data researcher, detect the important keywords in the document.\n\n# Rules\n\n-   Write just keywords separated by comma\n\n# The document\n\nTake information from this document:\n\n> {content}",resultingParameterName:"keywords"}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[{id:1,date:"2024-07-29T00:04:31.970Z",promptbookVersion:"0.61.0-9",modelUsage:{price:{value:0},input:{tokensCount:{value:0},charactersCount:{value:0},wordsCount:{value:0},sentencesCount:{value:0},linesCount:{value:0},paragraphsCount:{value:0},pagesCount:{value:0}},output:{tokensCount:{value:0},charactersCount:{value:0},wordsCount:{value:0},sentencesCount:{value:0},linesCount:{value:0},paragraphsCount:{value:0},pagesCount:{value:0}}}}],sourceFile:"./promptbook-collection/prepare-knowledge-keywords.ptbk.md"},{title:"Prepare Title",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-title.ptbk.md",promptbookVersion:"0.61.0-9",parameters:[{name:"content",description:"The content",isInput:true,isOutput:false},{name:"title",description:"The title of the document",isInput:false,isOutput:true}],promptTemplates:[{name:"knowledge",title:"Knowledge",dependentParameterNames:["content"],blockType:"PROMPT_TEMPLATE",expectations:{words:{min:1,max:8}},personaName:null,modelRequirements:{modelVariant:"CHAT",modelName:"claude-3-opus-20240229"},content:"You are experienced content creator, write best title for the document.\n\n# Rules\n\n-   Write just title, nothing else\n-   Title should be concise and clear\n-   Write maximum 5 words for the title\n\n# The document\n\n> {content}",resultingParameterName:"title"}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[{id:1,date:"2024-07-29T00:04:31.976Z",promptbookVersion:"0.61.0-9",modelUsage:{price:{value:0},input:{tokensCount:{value:0},charactersCount:{value:0},wordsCount:{value:0},sentencesCount:{value:0},linesCount:{value:0},paragraphsCount:{value:0},pagesCount:{value:0}},output:{tokensCount:{value:0},charactersCount:{value:0},wordsCount:{value:0},sentencesCount:{value:0},linesCount:{value:0},paragraphsCount:{value:0},pagesCount:{value:0}}}}],sourceFile:"./promptbook-collection/prepare-knowledge-title.ptbk.md"}];
 
 /**
  * Prettify the html code
@@ -1210,6 +1379,14 @@ function removeEmojis(text) {
  * Function normalizes title to name which can be used as identifier
  */
 function titleToName(value) {
+    if (value.startsWith('http://') || value.startsWith('https://')) {
+        // TODO: Maybe check against some list unallowed characters
+        return value;
+    }
+    if (value.startsWith('./') || value.startsWith('../')) {
+        // TODO: Maybe check against some list unallowed characters
+        return value;
+    }
     value = removeEmojis(value);
     value = normalizeToKebabCase(value);
     // TODO: [🧠] Maybe warn or add some padding to short name which are not good identifiers
@@ -1547,100 +1724,9 @@ function arrayableToArray(input) {
 /**
  * The version of the Promptbook library
  */
-var PROMPTBOOK_VERSION = '0.61.0-0';
+var PROMPTBOOK_VERSION = '0.61.0-9';
 // TODO: !!!! List here all the versions and annotate + put into script
 
-/**
- * Function `addUsage` will add multiple usages into one
- *
- * Note: If you provide 0 values, it returns void usage
- */
-function addUsage() {
-    var usageItems = [];
-    for (var _i = 0; _i < arguments.length; _i++) {
-        usageItems[_i] = arguments[_i];
-    }
-    var initialStructure = {
-        price: { value: 0 },
-        input: {
-            tokensCount: { value: 0 },
-            charactersCount: { value: 0 },
-            wordsCount: { value: 0 },
-            sentencesCount: { value: 0 },
-            linesCount: { value: 0 },
-            paragraphsCount: { value: 0 },
-            pagesCount: { value: 0 },
-        },
-        output: {
-            tokensCount: { value: 0 },
-            charactersCount: { value: 0 },
-            wordsCount: { value: 0 },
-            sentencesCount: { value: 0 },
-            linesCount: { value: 0 },
-            paragraphsCount: { value: 0 },
-            pagesCount: { value: 0 },
-        },
-    };
-    return usageItems.reduce(function (acc, item) {
-        var e_1, _a, e_2, _b;
-        var _c;
-        acc.price.value += ((_c = item.price) === null || _c === void 0 ? void 0 : _c.value) || 0;
-        try {
-            for (var _d = __values(Object.keys(acc.input)), _e = _d.next(); !_e.done; _e = _d.next()) {
-                var key = _e.value;
-                // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-                //@ts-ignore
-                if (item.input[key]) {
-                    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-                    //@ts-ignore
-                    acc.input[key].value += item.input[key].value || 0;
-                    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-                    //@ts-ignore
-                    if (item.input[key].isUncertain) {
-                        // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-                        //@ts-ignore
-                        acc.input[key].isUncertain = true;
-                    }
-                }
-            }
-        }
-        catch (e_1_1) { e_1 = { error: e_1_1 }; }
-        finally {
-            try {
-                if (_e && !_e.done && (_a = _d.return)) _a.call(_d);
-            }
-            finally { if (e_1) throw e_1.error; }
-        }
-        try {
-            for (var _f = __values(Object.keys(acc.output)), _g = _f.next(); !_g.done; _g = _f.next()) {
-                var key = _g.value;
-                // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-                //@ts-ignore
-                if (item.output[key]) {
-                    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-                    //@ts-ignore
-                    acc.output[key].value += item.output[key].value || 0;
-                    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-                    //@ts-ignore
-                    if (item.output[key].isUncertain) {
-                        // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-                        //@ts-ignore
-                        acc.output[key].isUncertain = true;
-                    }
-                }
-            }
-        }
-        catch (e_2_1) { e_2 = { error: e_2_1 }; }
-        finally {
-            try {
-                if (_g && !_g.done && (_b = _f.return)) _b.call(_f);
-            }
-            finally { if (e_2) throw e_2.error; }
-        }
-        return acc;
-    }, initialStructure);
-}
-
 /**
  * Counts number of characters in the text
  */
@@ -1823,8 +1909,6 @@ function createPipelineExecutor(options) {
     var _b = settings.maxExecutionAttempts, maxExecutionAttempts = _b === void 0 ? 3 : _b;
     // TODO: !!!!! Implement new commands
     validatePipeline(pipeline);
-    // TODO: !!!!! Do here materialization of still unmaterialized or dynamic knowledge
-    // TODO: !!!!! Do here personaToModelRequirements
     var llmTools = joinLlmExecutionTools.apply(void 0, __spreadArray([], __read(arrayableToArray(tools.llm)), false));
     var pipelineExecutor = function (inputParameters, onProgress) { return __awaiter(_this, void 0, void 0, function () {
         function executeSingleTemplate(currentTemplate) {
@@ -2345,53 +2429,74 @@ function createPipelineExecutor(options) {
     return pipelineExecutor;
 }
 /**
+ * TODO: [🪂] Pass maxParallelCount here
  * TODO: [🧠] When not meet expectations in PROMPT_DIALOG, make some way to tell the user
  * TODO: [👧] Strongly type the executors to avoid need of remove nullables whtn noUncheckedIndexedAccess in tsconfig.json
  * Note: CreatePipelineExecutorOptions are just connected to PipelineExecutor so do not extract to types folder
  * TODO: [🧠][3] transparent = (report intermediate parameters) / opaque execution = (report only output parameters) progress reporting mode
  */
 
-function prepareKnowledgeFromMarkdown(options) {
+/**
+ * Just marks a place of place where should be something implemented
+ * No side effects.
+ *
+ * Note: It can be usefull suppressing eslint errors of unused variables
+ *
+ * @param value any values
+ * @returns void
+ */
+function TODO_USE() {
+    var value = [];
+    for (var _i = 0; _i < arguments.length; _i++) {
+        value[_i] = arguments[_i];
+    }
+}
+// TODO: !!!! Find ACRY all just(...) and replace with TODO_USE
+
+/**
+ * @@@
+ */
+function prepareKnowledgeFromMarkdown(content /* <- TODO: [🖖] (?maybe not) Always the file */, options) {
     return __awaiter(this, void 0, void 0, function () {
-        var content, llmTools, _a, isVerbose, collection, prepareKnowledgeFromMarkdownExecutor, _b, prepareTitleExecutor, _c, prepareKeywordsExecutor, _d, result, outputParameters, knowledgeRaw, knowledgeTextPieces, knowledge;
-        var _e, _f, _g;
+        var llmTools, _a, maxParallelCount, _b, isVerbose, collection, prepareKnowledgeFromMarkdownExecutor, _c, prepareTitleExecutor, _d, prepareKeywordsExecutor, _e, result, outputParameters, knowledgeRaw, knowledgeTextPieces, knowledge;
+        var _f, _g, _h;
         var _this = this;
-        return __generator(this, function (_h) {
-            switch (_h.label) {
+        return __generator(this, function (_j) {
+            switch (_j.label) {
                 case 0:
-                    content = options.content, llmTools = options.llmTools, _a = options.isVerbose, isVerbose = _a === void 0 ? false : _a;
+                    llmTools = options.llmTools, _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a, _b = options.isVerbose, isVerbose = _b === void 0 ? false : _b;
+                    TODO_USE(maxParallelCount); // <- [🪂]
                     collection = createCollectionFromJson.apply(void 0, __spreadArray([], __read(PipelineCollection), false));
-                    _b = createPipelineExecutor;
-                    _e = {};
-                    return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.ptbk.md')];
-                case 1:
-                    prepareKnowledgeFromMarkdownExecutor = _b.apply(void 0, [(_e.pipeline = _h.sent(),
-                            _e.tools = {
-                                llm: llmTools,
-                            },
-                            _e)]);
                     _c = createPipelineExecutor;
                     _f = {};
-                    return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-knowledge-title.ptbk.md')];
-                case 2:
-                    prepareTitleExecutor = _c.apply(void 0, [(_f.pipeline = _h.sent(),
+                    return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.ptbk.md')];
+                case 1:
+                    prepareKnowledgeFromMarkdownExecutor = _c.apply(void 0, [(_f.pipeline = _j.sent(),
                             _f.tools = {
                                 llm: llmTools,
                             },
                             _f)]);
                     _d = createPipelineExecutor;
                     _g = {};
-                    return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-knowledge-keywords.ptbk.md')];
-                case 3:
-                    prepareKeywordsExecutor = _d.apply(void 0, [(_g.pipeline = _h.sent(),
+                    return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-knowledge-title.ptbk.md')];
+                case 2:
+                    prepareTitleExecutor = _d.apply(void 0, [(_g.pipeline = _j.sent(),
                             _g.tools = {
                                 llm: llmTools,
                             },
                             _g)]);
+                    _e = createPipelineExecutor;
+                    _h = {};
+                    return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-knowledge-keywords.ptbk.md')];
+                case 3:
+                    prepareKeywordsExecutor = _e.apply(void 0, [(_h.pipeline = _j.sent(),
+                            _h.tools = {
+                                llm: llmTools,
+                            },
+                            _h)]);
                     return [4 /*yield*/, prepareKnowledgeFromMarkdownExecutor({ content: content })];
                 case 4:
-                    result = _h.sent();
-                    // TODO: [0] !!! Aggeregate usage
+                    result = _j.sent();
                     assertsExecutionSuccessful(result);
                     outputParameters = result.outputParameters;
                     knowledgeRaw = outputParameters.knowledge;
@@ -2400,9 +2505,9 @@ function prepareKnowledgeFromMarkdown(options) {
                         console.info('knowledgeTextPieces:', knowledgeTextPieces);
                     }
                     return [4 /*yield*/, Promise.all(
-                        // TODO: !!! Do not send all at once but in chunks
+                        // TODO: [🪂] !! Do not send all at once but in chunks
                         knowledgeTextPieces.map(function (knowledgeTextPiece, i) { return __awaiter(_this, void 0, void 0, function () {
-                            var name, title, content, keywords, index, sources, titleResult, _a, titleRaw, keywordsResult, _b, keywordsRaw, embeddingResult, error_1;
+                            var name, title, content, keywords, index, titleResult, _a, titleRaw, keywordsResult, _b, keywordsRaw, embeddingResult, error_1;
                             return __generator(this, function (_c) {
                                 switch (_c.label) {
                                     case 0:
@@ -2411,12 +2516,6 @@ function prepareKnowledgeFromMarkdown(options) {
                                         content = spaceTrim(knowledgeTextPiece);
                                         keywords = [];
                                         index = [];
-                                        sources = [
-                                            {
-                                                title: 'Markdown document' /* <- TODO: !!! Unhardcode */,
-                                                href: '#' /* <- TODO: !!! Unhardcode */,
-                                            },
-                                        ];
                                         _c.label = 1;
                                     case 1:
                                         _c.trys.push([1, 7, , 8]);
@@ -2451,7 +2550,6 @@ function prepareKnowledgeFromMarkdown(options) {
                                         })];
                                     case 5:
                                         embeddingResult = _c.sent();
-                                        // TODO: [0] !!! Aggeregate usage embeddingResult.usage
                                         index.push({
                                             modelName: embeddingResult.modelName,
                                             position: embeddingResult.content,
@@ -2469,20 +2567,186 @@ function prepareKnowledgeFromMarkdown(options) {
                                             content: content,
                                             keywords: keywords,
                                             index: index,
-                                            sources: sources,
+                                            // <- TODO: [☀] sources,
                                         }];
                                 }
                             });
                         }); }))];
                 case 5:
-                    knowledge = _h.sent();
+                    knowledge = _j.sent();
                     return [2 /*return*/, knowledge];
             }
         });
     });
 }
 /**
- * 11:11
+ * TODO: [🪂] Do it in parallel 11:11
+ * Note: No need to aggregate usage here, it is done by intercepting the llmTools
+ */
+
+/**
+ * Prepares the knowle
+ *
+ * @see https://github.com/webgptorg/promptbook/discussions/41
+ * @private within the package
+ */
+function prepareKnowledgePieces(knowledgeSources, options) {
+    return __awaiter(this, void 0, void 0, function () {
+        var _a, maxParallelCount, knowledgePrepared;
+        var _this = this;
+        return __generator(this, function (_b) {
+            switch (_b.label) {
+                case 0:
+                    _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a;
+                    knowledgePrepared = [];
+                    return [4 /*yield*/, forEachAsync(knowledgeSources, { maxParallelCount: maxParallelCount }, function (knowledgeSource) { return __awaiter(_this, void 0, void 0, function () {
+                            var partialPieces, pieces;
+                            return __generator(this, function (_a) {
+                                switch (_a.label) {
+                                    case 0: return [4 /*yield*/, prepareKnowledgeFromMarkdown(knowledgeSource.source, // <- TODO: !!!!! Unhardcode markdown, detect which type it is
+                                        options)];
+                                    case 1:
+                                        partialPieces = _a.sent();
+                                        pieces = partialPieces.map(function (partialPiece) { return (__assign(__assign({}, partialPiece), { sources: [
+                                                {
+                                                    name: knowledgeSource.name,
+                                                    // line, column <- TODO: [☀]
+                                                    // <- TODO: [❎]
+                                                },
+                                            ] })); });
+                                        knowledgePrepared.push.apply(knowledgePrepared, __spreadArray([], __read(pieces), false));
+                                        return [2 /*return*/];
+                                }
+                            });
+                        }); })];
+                case 1:
+                    _b.sent();
+                    return [2 /*return*/, knowledgePrepared];
+            }
+        });
+    });
+}
+/*
+TODO: [🧊] This is how it can look in future
+> type PrepareKnowledgeKnowledge = {
+>   /**
+>    * Unprepared knowledge
+>    * /
+>   readonly knowledgeSources: Array<KnowledgeSourceJson>;
+> };
+>
+> export async function prepareKnowledgePieces(
+>   knowledge: PrepareKnowledgeKnowledge,
+>   options: PrepareOptions,
+> ):
+*/
+/**
+ * TODO: [🧊] In future one preparation can take data from previous preparation and save tokens and time
+ *       Put `knowledgePieces` into `PrepareKnowledgeOptions`
+ * TODO: [🪂] More than max things can run in parallel by acident [1,[2a,2b,_],[3a,3b,_]]
+ * TODO: [🧠][❎] Do here propper M:N mapping
+ *       [x] One source can make multiple pieces
+ *       [ ] One piece can have multiple sources
+ */
+
+/**
+ * Prepares the persona for the pipeline
+ *
+ * @see https://github.com/webgptorg/promptbook/discussions/22
+ * @private within the package
+ */
+function preparePersona(personaDescription, options) {
+    return __awaiter(this, void 0, void 0, function () {
+        var llmTools, _a, maxParallelCount, _b, isVerbose;
+        return __generator(this, function (_c) {
+            llmTools = options.llmTools, _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a, _b = options.isVerbose, isVerbose = _b === void 0 ? false : _b;
+            TODO_USE(maxParallelCount); // <- [🪂]
+            TODO_USE(personaDescription); // <- !!!!!
+            TODO_USE(llmTools); // <- !!!!!
+            TODO_USE(isVerbose); // <- !!!!!
+            return [2 /*return*/, {
+                    modelVariant: 'CHAT',
+                    modelName: 'gpt-4',
+                }];
+        });
+    });
+}
+/**
+ * TODO: [🪂] Do it in parallel
+ */
+
+/**
+ * Simple wrapper `new Date().toISOString()`
+ *
+ * @returns string_date branded type
+ */
+function $currentDate() {
+    return new Date().toISOString();
+}
+
+/**
+ * Prepare pipeline from string (markdown) format to JSON format
+ *
+ * Note: This function does not validate logic of the pipeline
+ * Note: This function acts as part of compilation process
+ */
+function preparePipeline(pipeline, options) {
+    return __awaiter(this, void 0, void 0, function () {
+        var _a, maxParallelCount, 
+        /*
+        <- TODO: [🧠][0] `promptbookVersion` */
+        knowledgeSources /*
+        <- TODO: [🧊] `knowledgePieces` */, personas /*
+        <- TODO: [🧊] `preparations` */, currentPreparation, preparations, preparedPersonas, knowledgeSourcesPrepared, partialknowledgePiecesPrepared, knowledgePiecesPrepared;
+        var _this = this;
+        return __generator(this, function (_b) {
+            switch (_b.label) {
+                case 0:
+                    _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a;
+                    knowledgeSources = pipeline.knowledgeSources, personas = pipeline.personas;
+                    currentPreparation = {
+                        id: 1,
+                        date: $currentDate(),
+                        promptbookVersion: PROMPTBOOK_VERSION,
+                        modelUsage: addUsage(),
+                    };
+                    preparations = [
+                        // ...preparations
+                        // <- TODO: [🧊]
+                        currentPreparation,
+                    ];
+                    preparedPersonas = [];
+                    return [4 /*yield*/, forEachAsync(personas, { maxParallelCount: maxParallelCount /* <- TODO: [🪂] When there are subtasks, this maximul limit can be broken */ }, function (persona) { return __awaiter(_this, void 0, void 0, function () {
+                            var modelRequirements, preparedPersona;
+                            return __generator(this, function (_a) {
+                                switch (_a.label) {
+                                    case 0: return [4 /*yield*/, preparePersona(persona.description, options)];
+                                    case 1:
+                                        modelRequirements = _a.sent();
+                                        preparedPersona = __assign(__assign({}, persona), { modelRequirements: modelRequirements, preparationIds: [/* TODO: [🧊] -> */ currentPreparation.id] });
+                                        preparedPersonas.push(preparedPersona);
+                                        return [2 /*return*/];
+                                }
+                            });
+                        }); })];
+                case 1:
+                    _b.sent();
+                    knowledgeSourcesPrepared = knowledgeSources.map(function (source) { return (__assign(__assign({}, source), { preparationIds: [/* TODO: [🧊] -> */ currentPreparation.id] })); });
+                    return [4 /*yield*/, prepareKnowledgePieces(knowledgeSources /* <- TODO: [🧊] {knowledgeSources, knowledgePieces} */, options)];
+                case 2:
+                    partialknowledgePiecesPrepared = _b.sent();
+                    knowledgePiecesPrepared = partialknowledgePiecesPrepared.map(function (piece) { return (__assign(__assign({}, piece), { preparationIds: [/* TODO: [🧊] -> */ currentPreparation.id] })); });
+                    // ----- /Knowledge preparation -----
+                    return [2 /*return*/, __assign(__assign({}, pipeline), { knowledgeSources: knowledgeSourcesPrepared, knowledgePieces: knowledgePiecesPrepared, personas: preparedPersonas, preparations: preparations })];
+            }
+        });
+    });
+}
+/**
+ * TODO: Write tests for `preparePipeline`
+ * TODO: [🏏] Leverage the batch API and build queues @see https://platform.openai.com/docs/guides/batch
+ * TODO: [🧊] In future one preparation can take data from previous preparation and save tokens and time
+ * TODO: [💸] Make utilities `interceptLlmTools` and `costLlmTools` to compute cost and DO put this counting logic in `prepareKnowledge` or `preparePersona`
  */
 
 /**
@@ -2669,6 +2933,7 @@ var blockCommandParser = {
 
 /**
  * Units of text measurement
+ * @see https://github.com/webgptorg/promptbook/discussions/30
  */
 var EXPECTATION_UNITS = ['CHARACTERS', 'WORDS', 'SENTENCES', 'LINES', 'PARAGRAPHS', 'PAGES'];
 /**
@@ -2989,9 +3254,10 @@ var knowledgeCommandParser = {
 
 var MODEL_VARIANTS = ['COMPLETION', 'CHAT', 'EMBEDDING' /* <- TODO [🏳] */ /* <- [🤖] */];
 /**
+ * TODO: !!!!! Add and use systemMessage, temprerature, top_k, top_p, presencePenalty, frequencyPenalty, bestOf, n, logitBias, logitBiasType, stop, ... to ModelRequirements
  * TODO: Maybe figure out better word than "variant"
  * TODO: Add here more requirement options like max context size, max tokens, etc.
- * TODO: [👙][🧠] Just selecting gpt3 or gpt4 level of model
+ * TODO: [💕][🧠] Just selecting gpt3 or gpt4 level of model
  */
 
 /**
@@ -4134,14 +4400,14 @@ function extractParametersFromPromptTemplate(promptTemplate) {
  */
 
 /**
- * Compile promptbook from string (markdown) format to JSON format synchronously
+ * Compile pipeline from string (markdown) format to JSON format synchronously
  *
- * Note: There are two similar functions:
+ * Note: There are 3 similar functions:
  * - `pipelineStringToJson` **(preferred)** - which propperly compiles the promptbook and use embedding for external knowledge
  * - `pipelineStringToJsonSync` - use only if you need to compile promptbook synchronously and it contains NO external knowledge
+ * - `preparePipeline` - just one step in the compilation process
  *
  * @param pipelineString {Promptbook} in string markdown format (.ptbk.md)
- * @param options - Options and tools for the compilation
  * @returns {Promptbook} compiled in JSON format (.ptbk.json)
  * @throws {ParsingError} if the promptbook string is not valid
  *
@@ -4157,7 +4423,10 @@ function pipelineStringToJsonSync(pipelineString) {
         description: undefined /* <- Note: Putting here placeholder to keep `description` on top at final JSON */,
         parameters: [],
         promptTemplates: [],
-        knowledge: [],
+        knowledgeSources: [],
+        knowledgePieces: [],
+        personas: [],
+        preparations: [],
     };
     // =============================================================
     // Note: 1️⃣ Parsing of the markdown into object
@@ -4425,6 +4694,7 @@ function pipelineStringToJsonSync(pipelineString) {
             postprocessing: postprocessing,
             expectations: expectAmount,
             expectFormat: expectFormat,
+            personaName: null,
             modelRequirements: templateModelRequirements,
             contentLanguage: blockType === 'SCRIPT' ? language : undefined,
             content: content,
@@ -4504,11 +4774,12 @@ function pipelineStringToJsonSync(pipelineString) {
  */
 
 /**
- * Compile promptbook from string (markdown) format to JSON format
+ * Compile pipeline from string (markdown) format to JSON format
  *
- * Note: There are two similar functions:
+ * Note: There are 3 similar functions:
  * - `pipelineStringToJson` **(preferred)** - which propperly compiles the promptbook and use embedding for external knowledge
  * - `pipelineStringToJsonSync` - use only if you need to compile promptbook synchronously and it contains NO external knowledge
+ * - `preparePipeline` - just one step in the compilation process
  *
  * @param pipelineString {Promptbook} in string markdown format (.ptbk.md)
  * @param options - Options and tools for the compilation
@@ -4519,22 +4790,18 @@ function pipelineStringToJsonSync(pipelineString) {
  * Note: This function acts as compilation process
  */
 function pipelineStringToJson(pipelineString, options) {
-    if (options === void 0) { options = {}; }
+    if (options === void 0) { options = { llmTools: null }; }
     return __awaiter(this, void 0, void 0, function () {
-        var llmTools, pipelineJson, knowledge;
+        var llmTools, pipelineJson;
         return __generator(this, function (_a) {
             switch (_a.label) {
                 case 0:
                     llmTools = options.llmTools;
                     pipelineJson = pipelineStringToJsonSync(pipelineString);
-                    if (!llmTools) return [3 /*break*/, 2];
-                    return [4 /*yield*/, prepareKnowledgeFromMarkdown({
-                            content: 'Roses are red, violets are blue, programmers use Promptbook, users too',
-                            llmTools: llmTools,
-                        })];
+                    if (!(llmTools !== null)) return [3 /*break*/, 2];
+                    return [4 /*yield*/, preparePipeline(pipelineJson, { llmTools: llmTools })];
                 case 1:
-                    knowledge = _a.sent();
-                    pipelineJson = __assign(__assign({}, pipelineJson), { knowledge: __spreadArray(__spreadArray([], __read((pipelineJson.knowledge || [])), false), __read(knowledge), false) });
+                    pipelineJson = _a.sent();
                     _a.label = 2;
                 case 2: return [2 /*return*/, pipelineJson];
             }
@@ -4669,7 +4936,7 @@ function createCollectionFromPromise(promptbookSourcesPromiseOrFactory) {
  */
 function createCollectionFromDirectory(path, options) {
     return __awaiter(this, void 0, void 0, function () {
-        var makedLibraryFilePath, makedLibraryFileExists, _a, _b, isRecursive, _c, isVerbose, _d, isLazyLoaded, _e, isCrashOnError, collection;
+        var makedLibraryFilePath, makedLibraryFileExists, _a, _b, isRecursive, _c, isVerbose, _d, isLazyLoaded, _e, isCrashedOnError, collection;
         var _this = this;
         return __generator(this, function (_f) {
             switch (_f.label) {
@@ -4690,7 +4957,7 @@ function createCollectionFromDirectory(path, options) {
                         colors.green("(In future, not implemented yet) Using your prebuild pipeline collection ".concat(makedLibraryFilePath));
                         // TODO: !! Implement;
                     }
-                    _a = options || {}, _b = _a.isRecursive, isRecursive = _b === void 0 ? true : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? false : _c, _d = _a.isLazyLoaded, isLazyLoaded = _d === void 0 ? false : _d, _e = _a.isCrashOnError, isCrashOnError = _e === void 0 ? true : _e;
+                    _a = options || {}, _b = _a.isRecursive, isRecursive = _b === void 0 ? true : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? false : _c, _d = _a.isLazyLoaded, isLazyLoaded = _d === void 0 ? false : _d, _e = _a.isCrashedOnError, isCrashedOnError = _e === void 0 ? true : _e;
                     collection = createCollectionFromPromise(function () { return __awaiter(_this, void 0, void 0, function () {
                         var fileNames, promptbooks, _loop_1, fileNames_1, fileNames_1_1, fileName, e_1_1;
                         var e_1, _a;
@@ -4752,7 +5019,7 @@ function createCollectionFromDirectory(path, options) {
                                                             if (isVerbose) {
                                                                 console.info("Loading ".concat(fileName.split('\\').join('/')));
                                                             }
-                                                            if (!isCrashOnError) {
+                                                            if (!isCrashedOnError) {
                                                                 // Note: Validate promptbook to check if it is logically correct to not crash on invalid promptbooks
                                                                 //       But be handled in current try-catch block
                                                                 validatePipeline(promptbook);
@@ -4768,7 +5035,7 @@ function createCollectionFromDirectory(path, options) {
                                                         throw error_1;
                                                     }
                                                     wrappedErrorMessage = spaceTrim(function (block) { return "\n                        Error during loading pipeline from file ".concat(fileName.split('\\').join('/'), ":\n\n                        ").concat(block(error_1.message), "\n\n                    "); });
-                                                    if (isCrashOnError) {
+                                                    if (isCrashedOnError) {
                                                         throw new CollectionError(wrappedErrorMessage);
                                                     }
                                                     // TODO: [🟥] Detect browser / node and make it colorfull