@promptbook/node 0.61.0-18 → 0.61.0-19

package/esm/index.es.js

@@ -203,200 +203,6 @@ TODO: !!! Just testing false-negative detection of [🟡][🟢][🔵][⚪] leak
  * TODO: [🔼] Export all to core
  */
 
-/**
- * @@@
- */
-function deepClone(objectValue) {
-    return JSON.parse(JSON.stringify(objectValue));
-    /*
-    TODO: [🧠] Is there a better implementation?
-    > const propertyNames = Object.getOwnPropertyNames(objectValue);
-    > for (const propertyName of propertyNames) {
-    >     const value = (objectValue as really_any)[propertyName];
-    >     if (value && typeof value === 'object') {
-    >         deepClone(value);
-    >     }
-    > }
-    > return Object.assign({}, objectValue);
-    */
-}
-/**
- * TODO: [🔼] Export from `@promptbook/utils`
- * TODO: [🧠] Is there a way how to meaningfully test this utility
- */
-
-/**
- * @@@
- *
- * TODO: [🔼] Export with addUsage
- */
-var ZERO_USAGE = deepFreeze({
-    price: { value: 0 },
-    input: {
-        tokensCount: { value: 0 },
-        charactersCount: { value: 0 },
-        wordsCount: { value: 0 },
-        sentencesCount: { value: 0 },
-        linesCount: { value: 0 },
-        paragraphsCount: { value: 0 },
-        pagesCount: { value: 0 },
-    },
-    output: {
-        tokensCount: { value: 0 },
-        charactersCount: { value: 0 },
-        wordsCount: { value: 0 },
-        sentencesCount: { value: 0 },
-        linesCount: { value: 0 },
-        paragraphsCount: { value: 0 },
-        pagesCount: { value: 0 },
-    },
-});
-/**
- * Function `addUsage` will add multiple usages into one
- *
- * Note: If you provide 0 values, it returns ZERO_USAGE
- */
-function addUsage() {
-    var usageItems = [];
-    for (var _i = 0; _i < arguments.length; _i++) {
-        usageItems[_i] = arguments[_i];
-    }
-    return usageItems.reduce(function (acc, item) {
-        var e_1, _a, e_2, _b;
-        var _c;
-        acc.price.value += ((_c = item.price) === null || _c === void 0 ? void 0 : _c.value) || 0;
-        try {
-            for (var _d = __values(Object.keys(acc.input)), _e = _d.next(); !_e.done; _e = _d.next()) {
-                var key = _e.value;
-                // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-                //@ts-ignore
-                if (item.input[key]) {
-                    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-                    //@ts-ignore
-                    acc.input[key].value += item.input[key].value || 0;
-                    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-                    //@ts-ignore
-                    if (item.input[key].isUncertain) {
-                        // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-                        //@ts-ignore
-                        acc.input[key].isUncertain = true;
-                    }
-                }
-            }
-        }
-        catch (e_1_1) { e_1 = { error: e_1_1 }; }
-        finally {
-            try {
-                if (_e && !_e.done && (_a = _d.return)) _a.call(_d);
-            }
-            finally { if (e_1) throw e_1.error; }
-        }
-        try {
-            for (var _f = __values(Object.keys(acc.output)), _g = _f.next(); !_g.done; _g = _f.next()) {
-                var key = _g.value;
-                // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-                //@ts-ignore
-                if (item.output[key]) {
-                    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-                    //@ts-ignore
-                    acc.output[key].value += item.output[key].value || 0;
-                    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-                    //@ts-ignore
-                    if (item.output[key].isUncertain) {
-                        // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-                        //@ts-ignore
-                        acc.output[key].isUncertain = true;
-                    }
-                }
-            }
-        }
-        catch (e_2_1) { e_2 = { error: e_2_1 }; }
-        finally {
-            try {
-                if (_g && !_g.done && (_b = _f.return)) _b.call(_f);
-            }
-            finally { if (e_2) throw e_2.error; }
-        }
-        return acc;
-    }, deepClone(ZERO_USAGE));
-}
-
-/**
- * Async version of Array.forEach
- *
- * @param array - Array to iterate over
- * @param options - Options for the function
- * @param callbackfunction - Function to call for each item
- */
-function forEachAsync(array, options, callbackfunction) {
-    return __awaiter(this, void 0, void 0, function () {
-        var _a, maxParallelCount, index, runningTasks, tasks, _loop_1, _b, _c, item, e_1_1;
-        var e_1, _d;
-        return __generator(this, function (_e) {
-            switch (_e.label) {
-                case 0:
-                    _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? Infinity : _a;
-                    index = 0;
-                    runningTasks = [];
-                    tasks = [];
-                    _loop_1 = function (item) {
-                        var currentIndex, task;
-                        return __generator(this, function (_f) {
-                            switch (_f.label) {
-                                case 0:
-                                    currentIndex = index++;
-                                    task = callbackfunction(item, currentIndex, array);
-                                    tasks.push(task);
-                                    runningTasks.push(task);
-                                    /* not await */ Promise.resolve(task).then(function () {
-                                        runningTasks = runningTasks.filter(function (t) { return t !== task; });
-                                    });
-                                    if (!(maxParallelCount < runningTasks.length)) return [3 /*break*/, 2];
-                                    return [4 /*yield*/, Promise.race(runningTasks)];
-                                case 1:
-                                    _f.sent();
-                                    _f.label = 2;
-                                case 2: return [2 /*return*/];
-                            }
-                        });
-                    };
-                    _e.label = 1;
-                case 1:
-                    _e.trys.push([1, 6, 7, 8]);
-                    _b = __values(array), _c = _b.next();
-                    _e.label = 2;
-                case 2:
-                    if (!!_c.done) return [3 /*break*/, 5];
-                    item = _c.value;
-                    return [5 /*yield**/, _loop_1(item)];
-                case 3:
-                    _e.sent();
-                    _e.label = 4;
-                case 4:
-                    _c = _b.next();
-                    return [3 /*break*/, 2];
-                case 5: return [3 /*break*/, 8];
-                case 6:
-                    e_1_1 = _e.sent();
-                    e_1 = { error: e_1_1 };
-                    return [3 /*break*/, 8];
-                case 7:
-                    try {
-                        if (_c && !_c.done && (_d = _b.return)) _d.call(_b);
-                    }
-                    finally { if (e_1) throw e_1.error; }
-                    return [7 /*endfinally*/];
-                case 8: return [4 /*yield*/, Promise.all(tasks)];
-                case 9:
-                    _e.sent();
-                    return [2 /*return*/];
-            }
-        });
-    });
-}
-
-var PipelineCollection = [{title:"Prepare Knowledge from Markdown",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.ptbk.md",promptbookVersion:"0.61.0-17",parameters:[{name:"content",description:"Markdown document content",isInput:true,isOutput:false},{name:"knowledge",description:"The knowledge JSON object",isInput:false,isOutput:true}],promptTemplates:[{blockType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",modelRequirements:{modelVariant:"CHAT",modelName:"claude-3-opus-20240229"},content:"You are experienced data researcher, extract the important knowledge from the document.\n\n# Rules\n\n-   Make pieces of information concise, clear, and easy to understand\n-   One piece of information should be approximately 1 paragraph\n-   Divide the paragraphs by markdown horizontal lines ---\n-   Omit irrelevant information\n-   Group redundant information\n-   Write just extracted information, nothing else\n\n# The document\n\nTake information from this document:\n\n> {content}",dependentParameterNames:["content"],resultingParameterName:"knowledge"}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[{id:1,promptbookVersion:"0.61.0-17",modelUsage:{price:{value:0},input:{tokensCount:{value:0},charactersCount:{value:0},wordsCount:{value:0},sentencesCount:{value:0},linesCount:{value:0},paragraphsCount:{value:0},pagesCount:{value:0}},output:{tokensCount:{value:0},charactersCount:{value:0},wordsCount:{value:0},sentencesCount:{value:0},linesCount:{value:0},paragraphsCount:{value:0},pagesCount:{value:0}}}}],sourceFile:"./promptbook-collection/prepare-knowledge-from-markdown.ptbk.md"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-keywords.ptbk.md",promptbookVersion:"0.61.0-17",parameters:[{name:"content",description:"The content",isInput:true,isOutput:false},{name:"keywords",description:"Keywords separated by comma",isInput:false,isOutput:true}],promptTemplates:[{blockType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",modelRequirements:{modelVariant:"CHAT",modelName:"claude-3-opus-20240229"},content:"You are experienced data researcher, detect the important keywords in the document.\n\n# Rules\n\n-   Write just keywords separated by comma\n\n# The document\n\nTake information from this document:\n\n> {content}",dependentParameterNames:["content"],resultingParameterName:"keywords"}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[{id:1,promptbookVersion:"0.61.0-17",modelUsage:{price:{value:0},input:{tokensCount:{value:0},charactersCount:{value:0},wordsCount:{value:0},sentencesCount:{value:0},linesCount:{value:0},paragraphsCount:{value:0},pagesCount:{value:0}},output:{tokensCount:{value:0},charactersCount:{value:0},wordsCount:{value:0},sentencesCount:{value:0},linesCount:{value:0},paragraphsCount:{value:0},pagesCount:{value:0}}}}],sourceFile:"./promptbook-collection/prepare-knowledge-keywords.ptbk.md"},{title:"Prepare Title",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-title.ptbk.md",promptbookVersion:"0.61.0-17",parameters:[{name:"content",description:"The content",isInput:true,isOutput:false},{name:"title",description:"The title of the document",isInput:false,isOutput:true}],promptTemplates:[{blockType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",modelRequirements:{modelVariant:"CHAT",modelName:"claude-3-opus-20240229"},content:"You are experienced content creator, write best title for the document.\n\n# Rules\n\n-   Write just title, nothing else\n-   Title should be concise and clear\n-   Write maximum 5 words for the title\n\n# The document\n\n> {content}",expectations:{words:{min:1,max:8}},dependentParameterNames:["content"],resultingParameterName:"title"}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[{id:1,promptbookVersion:"0.61.0-17",modelUsage:{price:{value:0},input:{tokensCount:{value:0},charactersCount:{value:0},wordsCount:{value:0},sentencesCount:{value:0},linesCount:{value:0},paragraphsCount:{value:0},pagesCount:{value:0}},output:{tokensCount:{value:0},charactersCount:{value:0},wordsCount:{value:0},sentencesCount:{value:0},linesCount:{value:0},paragraphsCount:{value:0},pagesCount:{value:0}}}}],sourceFile:"./promptbook-collection/prepare-knowledge-title.ptbk.md"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-persona.ptbk.md",promptbookVersion:"0.61.0-17",parameters:[{name:"availableModelNames",description:"List of available model names separated by comma (,)",isInput:true,isOutput:false},{name:"personaDescription",description:"Description of the persona",isInput:true,isOutput:false},{name:"modelRequirements",description:"Specific requirements for the model",isInput:false,isOutput:true}],promptTemplates:[{blockType:"PROMPT_TEMPLATE",name:"make-model-requirements",title:"Make modelRequirements",modelRequirements:{modelVariant:"CHAT",modelName:"gpt-4-turbo"},content:"You are experienced AI engineer, you need to create virtual assistant.\nWrite\n\n## Sample\n\n```json\n{\n\"modelName\": \"gpt-4o\",\n\"systemMessage\": \"You are experienced AI engineer and helpfull assistant.\",\n\"temperature\": 0.7\n}\n```\n\n## Instructions\n\n### Option `modelName`\n\nPick from the following models:\n\n-   {availableModelNames}\n\n### Option `systemMessage`\n\nThe system message is used to communicate instructions or provide context to the model at the beginning of a conversation. It is displayed in a different format compared to user messages, helping the model understand its role in the conversation. The system message typically guides the model's behavior, sets the tone, or specifies desired output from the model. By utilizing the system message effectively, users can steer the model towards generating more accurate and relevant responses.\n\nFor example:\n\n> You are an experienced AI engineer and helpful assistant.\n\n> You are a friendly and knowledgeable chatbot.\n\n### Option `temperature`\n\nThe sampling temperature, between 0 and 1. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. If set to 0, the model will use log probability to automatically increase the temperature until certain thresholds are hit.\n\nYou can pick a value between 0 and 2. For example:\n\n-   `0.1`: Low temperature, extremely conservative and deterministic\n-   `0.5`: Medium temperature, balanced between conservative and creative\n-   `1.0`: High temperature, creative and bit random\n-   `1.5`: Very high temperature, extremely creative and often chaotic and unpredictable\n-   `2.0`: Maximum temperature, completely random and unpredictable, for some extreme creative use cases\n\n# The assistant\n\nTake this description of the persona:\n\n> {personaDescription}",expectFormat:"JSON",dependentParameterNames:["availableModelNames","personaDescription"],resultingParameterName:"modelRequirements"}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[{id:1,promptbookVersion:"0.61.0-17",modelUsage:{price:{value:0},input:{tokensCount:{value:0},charactersCount:{value:0},wordsCount:{value:0},sentencesCount:{value:0},linesCount:{value:0},paragraphsCount:{value:0},pagesCount:{value:0}},output:{tokensCount:{value:0},charactersCount:{value:0},wordsCount:{value:0},sentencesCount:{value:0},linesCount:{value:0},paragraphsCount:{value:0},pagesCount:{value:0}}}}],sourceFile:"./promptbook-collection/prepare-persona.ptbk.md"}];
-
 /**
  * Prettify the html code
  *
@@ -613,32 +419,226 @@ function pipelineJsonToString(pipelineJson) {
             pipelineString += '\n\n';
             pipelineString += "`-> {".concat(resultingParameterName, "}`"); // <- TODO: !!! If the parameter here has description, add it and use promptTemplateParameterJsonToString
         }
-    }
-    catch (e_3_1) { e_3 = { error: e_3_1 }; }
-    finally {
-        try {
-            if (promptTemplates_1_1 && !promptTemplates_1_1.done && (_c = promptTemplates_1.return)) _c.call(promptTemplates_1);
+    }
+    catch (e_3_1) { e_3 = { error: e_3_1 }; }
+    finally {
+        try {
+            if (promptTemplates_1_1 && !promptTemplates_1_1.done && (_c = promptTemplates_1.return)) _c.call(promptTemplates_1);
+        }
+        finally { if (e_3) throw e_3.error; }
+    }
+    return pipelineString;
+}
+/**
+ * @private internal util of pipelineJsonToString
+ */
+function promptTemplateParameterJsonToString(promptTemplateParameterJson) {
+    var name = promptTemplateParameterJson.name, description = promptTemplateParameterJson.description;
+    var parameterString = "{".concat(name, "}");
+    if (description) {
+        parameterString = "".concat(parameterString, " ").concat(description);
+    }
+    return parameterString;
+}
+/**
+ * TODO: !!!! Implement new features and commands into `promptTemplateParameterJsonToString`
+ * TODO: [🧠] Is there a way to auto-detect missing features in pipelineJsonToString
+ * TODO: Escape all
+ */
+
+/**
+ * @@@
+ */
+function deepClone(objectValue) {
+    return JSON.parse(JSON.stringify(objectValue));
+    /*
+    TODO: [🧠] Is there a better implementation?
+    > const propertyNames = Object.getOwnPropertyNames(objectValue);
+    > for (const propertyName of propertyNames) {
+    >     const value = (objectValue as really_any)[propertyName];
+    >     if (value && typeof value === 'object') {
+    >         deepClone(value);
+    >     }
+    > }
+    > return Object.assign({}, objectValue);
+    */
+}
+/**
+ * TODO: [🔼] Export from `@promptbook/utils`
+ * TODO: [🧠] Is there a way how to meaningfully test this utility
+ */
+
+/**
+ * @@@
+ *
+ * TODO: [🔼] Export with addUsage
+ */
+var ZERO_USAGE = deepFreeze({
+    price: { value: 0 },
+    input: {
+        tokensCount: { value: 0 },
+        charactersCount: { value: 0 },
+        wordsCount: { value: 0 },
+        sentencesCount: { value: 0 },
+        linesCount: { value: 0 },
+        paragraphsCount: { value: 0 },
+        pagesCount: { value: 0 },
+    },
+    output: {
+        tokensCount: { value: 0 },
+        charactersCount: { value: 0 },
+        wordsCount: { value: 0 },
+        sentencesCount: { value: 0 },
+        linesCount: { value: 0 },
+        paragraphsCount: { value: 0 },
+        pagesCount: { value: 0 },
+    },
+});
+/**
+ * Function `addUsage` will add multiple usages into one
+ *
+ * Note: If you provide 0 values, it returns ZERO_USAGE
+ */
+function addUsage() {
+    var usageItems = [];
+    for (var _i = 0; _i < arguments.length; _i++) {
+        usageItems[_i] = arguments[_i];
+    }
+    return usageItems.reduce(function (acc, item) {
+        var e_1, _a, e_2, _b;
+        var _c;
+        acc.price.value += ((_c = item.price) === null || _c === void 0 ? void 0 : _c.value) || 0;
+        try {
+            for (var _d = __values(Object.keys(acc.input)), _e = _d.next(); !_e.done; _e = _d.next()) {
+                var key = _e.value;
+                // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+                //@ts-ignore
+                if (item.input[key]) {
+                    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+                    //@ts-ignore
+                    acc.input[key].value += item.input[key].value || 0;
+                    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+                    //@ts-ignore
+                    if (item.input[key].isUncertain) {
+                        // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+                        //@ts-ignore
+                        acc.input[key].isUncertain = true;
+                    }
+                }
+            }
+        }
+        catch (e_1_1) { e_1 = { error: e_1_1 }; }
+        finally {
+            try {
+                if (_e && !_e.done && (_a = _d.return)) _a.call(_d);
+            }
+            finally { if (e_1) throw e_1.error; }
+        }
+        try {
+            for (var _f = __values(Object.keys(acc.output)), _g = _f.next(); !_g.done; _g = _f.next()) {
+                var key = _g.value;
+                // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+                //@ts-ignore
+                if (item.output[key]) {
+                    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+                    //@ts-ignore
+                    acc.output[key].value += item.output[key].value || 0;
+                    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+                    //@ts-ignore
+                    if (item.output[key].isUncertain) {
+                        // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+                        //@ts-ignore
+                        acc.output[key].isUncertain = true;
+                    }
+                }
+            }
+        }
+        catch (e_2_1) { e_2 = { error: e_2_1 }; }
+        finally {
+            try {
+                if (_g && !_g.done && (_b = _f.return)) _b.call(_f);
+            }
+            finally { if (e_2) throw e_2.error; }
         }
-        finally { if (e_3) throw e_3.error; }
-    }
-    return pipelineString;
+        return acc;
+    }, deepClone(ZERO_USAGE));
 }
+
 /**
- * @private internal util of pipelineJsonToString
+ * Async version of Array.forEach
+ *
+ * @param array - Array to iterate over
+ * @param options - Options for the function
+ * @param callbackfunction - Function to call for each item
  */
-function promptTemplateParameterJsonToString(promptTemplateParameterJson) {
-    var name = promptTemplateParameterJson.name, description = promptTemplateParameterJson.description;
-    var parameterString = "{".concat(name, "}");
-    if (description) {
-        parameterString = "".concat(parameterString, " ").concat(description);
-    }
-    return parameterString;
+function forEachAsync(array, options, callbackfunction) {
+    return __awaiter(this, void 0, void 0, function () {
+        var _a, maxParallelCount, index, runningTasks, tasks, _loop_1, _b, _c, item, e_1_1;
+        var e_1, _d;
+        return __generator(this, function (_e) {
+            switch (_e.label) {
+                case 0:
+                    _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? Infinity : _a;
+                    index = 0;
+                    runningTasks = [];
+                    tasks = [];
+                    _loop_1 = function (item) {
+                        var currentIndex, task;
+                        return __generator(this, function (_f) {
+                            switch (_f.label) {
+                                case 0:
+                                    currentIndex = index++;
+                                    task = callbackfunction(item, currentIndex, array);
+                                    tasks.push(task);
+                                    runningTasks.push(task);
+                                    /* not await */ Promise.resolve(task).then(function () {
+                                        runningTasks = runningTasks.filter(function (t) { return t !== task; });
+                                    });
+                                    if (!(maxParallelCount < runningTasks.length)) return [3 /*break*/, 2];
+                                    return [4 /*yield*/, Promise.race(runningTasks)];
+                                case 1:
+                                    _f.sent();
+                                    _f.label = 2;
+                                case 2: return [2 /*return*/];
+                            }
+                        });
+                    };
+                    _e.label = 1;
+                case 1:
+                    _e.trys.push([1, 6, 7, 8]);
+                    _b = __values(array), _c = _b.next();
+                    _e.label = 2;
+                case 2:
+                    if (!!_c.done) return [3 /*break*/, 5];
+                    item = _c.value;
+                    return [5 /*yield**/, _loop_1(item)];
+                case 3:
+                    _e.sent();
+                    _e.label = 4;
+                case 4:
+                    _c = _b.next();
+                    return [3 /*break*/, 2];
+                case 5: return [3 /*break*/, 8];
+                case 6:
+                    e_1_1 = _e.sent();
+                    e_1 = { error: e_1_1 };
+                    return [3 /*break*/, 8];
+                case 7:
+                    try {
+                        if (_c && !_c.done && (_d = _b.return)) _d.call(_b);
+                    }
+                    finally { if (e_1) throw e_1.error; }
+                    return [7 /*endfinally*/];
+                case 8: return [4 /*yield*/, Promise.all(tasks)];
+                case 9:
+                    _e.sent();
+                    return [2 /*return*/];
+            }
+        });
+    });
 }
-/**
- * TODO: !!!! Implement new features and commands into `promptTemplateParameterJsonToString`
- * TODO: [🧠] Is there a way to auto-detect missing features in pipelineJsonToString
- * TODO: Escape all
- */
+
+var PipelineCollection = [{title:"Prepare Knowledge from Markdown",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.ptbk.md",promptbookVersion:"0.61.0-18",parameters:[{name:"content",description:"Markdown document content",isInput:true,isOutput:false},{name:"knowledge",description:"The knowledge JSON object",isInput:false,isOutput:true}],promptTemplates:[{blockType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",modelRequirements:{modelVariant:"CHAT",modelName:"claude-3-opus-20240229"},content:"You are experienced data researcher, extract the important knowledge from the document.\n\n# Rules\n\n-   Make pieces of information concise, clear, and easy to understand\n-   One piece of information should be approximately 1 paragraph\n-   Divide the paragraphs by markdown horizontal lines ---\n-   Omit irrelevant information\n-   Group redundant information\n-   Write just extracted information, nothing else\n\n# The document\n\nTake information from this document:\n\n> {content}",dependentParameterNames:["content"],resultingParameterName:"knowledge"}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[{id:1,promptbookVersion:"0.61.0-18",modelUsage:{price:{value:0},input:{tokensCount:{value:0},charactersCount:{value:0},wordsCount:{value:0},sentencesCount:{value:0},linesCount:{value:0},paragraphsCount:{value:0},pagesCount:{value:0}},output:{tokensCount:{value:0},charactersCount:{value:0},wordsCount:{value:0},sentencesCount:{value:0},linesCount:{value:0},paragraphsCount:{value:0},pagesCount:{value:0}}}}],sourceFile:"./promptbook-collection/prepare-knowledge-from-markdown.ptbk.md"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-keywords.ptbk.md",promptbookVersion:"0.61.0-18",parameters:[{name:"content",description:"The content",isInput:true,isOutput:false},{name:"keywords",description:"Keywords separated by comma",isInput:false,isOutput:true}],promptTemplates:[{blockType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",modelRequirements:{modelVariant:"CHAT",modelName:"claude-3-opus-20240229"},content:"You are experienced data researcher, detect the important keywords in the document.\n\n# Rules\n\n-   Write just keywords separated by comma\n\n# The document\n\nTake information from this document:\n\n> {content}",dependentParameterNames:["content"],resultingParameterName:"keywords"}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[{id:1,promptbookVersion:"0.61.0-18",modelUsage:{price:{value:0},input:{tokensCount:{value:0},charactersCount:{value:0},wordsCount:{value:0},sentencesCount:{value:0},linesCount:{value:0},paragraphsCount:{value:0},pagesCount:{value:0}},output:{tokensCount:{value:0},charactersCount:{value:0},wordsCount:{value:0},sentencesCount:{value:0},linesCount:{value:0},paragraphsCount:{value:0},pagesCount:{value:0}}}}],sourceFile:"./promptbook-collection/prepare-knowledge-keywords.ptbk.md"},{title:"Prepare Title",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-title.ptbk.md",promptbookVersion:"0.61.0-18",parameters:[{name:"content",description:"The content",isInput:true,isOutput:false},{name:"title",description:"The title of the document",isInput:false,isOutput:true}],promptTemplates:[{blockType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",modelRequirements:{modelVariant:"CHAT",modelName:"claude-3-opus-20240229"},content:"You are experienced content creator, write best title for the document.\n\n# Rules\n\n-   Write just title, nothing else\n-   Title should be concise and clear\n-   Write maximum 5 words for the title\n\n# The document\n\n> {content}",expectations:{words:{min:1,max:8}},dependentParameterNames:["content"],resultingParameterName:"title"}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[{id:1,promptbookVersion:"0.61.0-18",modelUsage:{price:{value:0},input:{tokensCount:{value:0},charactersCount:{value:0},wordsCount:{value:0},sentencesCount:{value:0},linesCount:{value:0},paragraphsCount:{value:0},pagesCount:{value:0}},output:{tokensCount:{value:0},charactersCount:{value:0},wordsCount:{value:0},sentencesCount:{value:0},linesCount:{value:0},paragraphsCount:{value:0},pagesCount:{value:0}}}}],sourceFile:"./promptbook-collection/prepare-knowledge-title.ptbk.md"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-persona.ptbk.md",promptbookVersion:"0.61.0-18",parameters:[{name:"availableModelNames",description:"List of available model names separated by comma (,)",isInput:true,isOutput:false},{name:"personaDescription",description:"Description of the persona",isInput:true,isOutput:false},{name:"modelRequirements",description:"Specific requirements for the model",isInput:false,isOutput:true}],promptTemplates:[{blockType:"PROMPT_TEMPLATE",name:"make-model-requirements",title:"Make modelRequirements",modelRequirements:{modelVariant:"CHAT",modelName:"gpt-4-turbo"},content:"You are experienced AI engineer, you need to create virtual assistant.\nWrite\n\n## Sample\n\n```json\n{\n\"modelName\": \"gpt-4o\",\n\"systemMessage\": \"You are experienced AI engineer and helpfull assistant.\",\n\"temperature\": 0.7\n}\n```\n\n## Instructions\n\n### Option `modelName`\n\nPick from the following models:\n\n-   {availableModelNames}\n\n### Option `systemMessage`\n\nThe system message is used to communicate instructions or provide context to the model at the beginning of a conversation. It is displayed in a different format compared to user messages, helping the model understand its role in the conversation. The system message typically guides the model's behavior, sets the tone, or specifies desired output from the model. By utilizing the system message effectively, users can steer the model towards generating more accurate and relevant responses.\n\nFor example:\n\n> You are an experienced AI engineer and helpful assistant.\n\n> You are a friendly and knowledgeable chatbot.\n\n### Option `temperature`\n\nThe sampling temperature, between 0 and 1. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. If set to 0, the model will use log probability to automatically increase the temperature until certain thresholds are hit.\n\nYou can pick a value between 0 and 2. For example:\n\n-   `0.1`: Low temperature, extremely conservative and deterministic\n-   `0.5`: Medium temperature, balanced between conservative and creative\n-   `1.0`: High temperature, creative and bit random\n-   `1.5`: Very high temperature, extremely creative and often chaotic and unpredictable\n-   `2.0`: Maximum temperature, completely random and unpredictable, for some extreme creative use cases\n\n# The assistant\n\nTake this description of the persona:\n\n> {personaDescription}",expectFormat:"JSON",dependentParameterNames:["availableModelNames","personaDescription"],resultingParameterName:"modelRequirements"}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[{id:1,promptbookVersion:"0.61.0-18",modelUsage:{price:{value:0},input:{tokensCount:{value:0},charactersCount:{value:0},wordsCount:{value:0},sentencesCount:{value:0},linesCount:{value:0},paragraphsCount:{value:0},pagesCount:{value:0}},output:{tokensCount:{value:0},charactersCount:{value:0},wordsCount:{value:0},sentencesCount:{value:0},linesCount:{value:0},paragraphsCount:{value:0},pagesCount:{value:0}}}}],sourceFile:"./promptbook-collection/prepare-persona.ptbk.md"}];
 
 /**
  * This error indicates that the promptbook in a markdown format cannot be parsed into a valid promptbook object
@@ -1111,17 +1111,23 @@ var SimplePipelineCollection = /** @class */ (function () {
         try {
             for (var pipelines_1 = __values(pipelines), pipelines_1_1 = pipelines_1.next(); !pipelines_1_1.done; pipelines_1_1 = pipelines_1.next()) {
                 var pipeline = pipelines_1_1.value;
+                // TODO: [👠] DRY
                 if (pipeline.pipelineUrl === undefined) {
                     throw new ReferenceError$1(spaceTrim$1("\n                        Pipeline with name \"".concat(pipeline.title, "\" does not have defined URL\n\n                        File:\n                        ").concat(pipeline.sourceFile || 'Unknown', "\n\n                        Note: Pipelines without URLs are called anonymous pipelines\n                              They can be used as standalone pipelines, but they cannot be referenced by other pipelines\n                              And also they cannot be used in the pipeline collection\n\n                    ")));
                 }
+                // Note: [🐨]
                 validatePipeline(pipeline);
+                // TODO: [🦄] DRY
                 // Note: [🦄]
-                if (this.collection.has(pipeline.pipelineUrl) &&
+                if (
+                // TODO: [🐽]
+                this.collection.has(pipeline.pipelineUrl) &&
                     pipelineJsonToString(unpreparePipeline(pipeline)) !==
                         pipelineJsonToString(unpreparePipeline(this.collection.get(pipeline.pipelineUrl)))) {
                     var existing = this.collection.get(pipeline.pipelineUrl);
                     throw new ReferenceError$1(spaceTrim$1("\n                        Pipeline with URL \"".concat(pipeline.pipelineUrl, "\" is already in the collection\n\n                        Conflicting files:\n                        ").concat(existing.sourceFile || 'Unknown', "\n                        ").concat(pipeline.sourceFile || 'Unknown', "\n\n                        Note: Pipelines with the same URL are not allowed\n                              Only exepction is when the pipelines are identical\n\n                    ")));
                 }
+                // Note: [🧠] Overwrite existing pipeline with the same URL
                 this.collection.set(pipeline.pipelineUrl, pipeline);
             }
         }
@@ -1561,39 +1567,6 @@ function assertsExecutionSuccessful(executionResult) {
  * TODO: [🧠] Can this return type be better typed than void
  */
 
-/**
- * Create difference set of two sets.
- *
- * @deprecated use new javascript set methods instead @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Set
- */
-function difference(a, b, isEqual) {
-    var e_1, _a;
-    if (isEqual === void 0) { isEqual = function (a, b) { return a === b; }; }
-    var diff = new Set();
-    var _loop_1 = function (itemA) {
-        if (!Array.from(b).some(function (itemB) { return isEqual(itemA, itemB); })) {
-            diff.add(itemA);
-        }
-    };
-    try {
-        for (var _b = __values(Array.from(a)), _c = _b.next(); !_c.done; _c = _b.next()) {
-            var itemA = _c.value;
-            _loop_1(itemA);
-        }
-    }
-    catch (e_1_1) { e_1 = { error: e_1_1 }; }
-    finally {
-        try {
-            if (_c && !_c.done && (_a = _b.return)) _a.call(_b);
-        }
-        finally { if (e_1) throw e_1.error; }
-    }
-    return diff;
-}
-/**
- * TODO: [🧠][💯] Maybe also implement symmetricDifference
- */
-
 /**
  * Parses the template and returns the list of all parameter names
  *
@@ -1727,46 +1700,6 @@ function extractParametersFromPromptTemplate(promptTemplate) {
  * TODO: [🔣] If script require contentLanguage
  */
 
-/**
- * Creates a new set with all elements that are present in either set
- *
- * @deprecated use new javascript set methods instead @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Set
- */
-function union() {
-    var e_1, _a, e_2, _b;
-    var sets = [];
-    for (var _i = 0; _i < arguments.length; _i++) {
-        sets[_i] = arguments[_i];
-    }
-    var union = new Set();
-    try {
-        for (var sets_1 = __values(sets), sets_1_1 = sets_1.next(); !sets_1_1.done; sets_1_1 = sets_1.next()) {
-            var set = sets_1_1.value;
-            try {
-                for (var _c = (e_2 = void 0, __values(Array.from(set))), _d = _c.next(); !_d.done; _d = _c.next()) {
-                    var item = _d.value;
-                    union.add(item);
-                }
-            }
-            catch (e_2_1) { e_2 = { error: e_2_1 }; }
-            finally {
-                try {
-                    if (_d && !_d.done && (_b = _c.return)) _b.call(_c);
-                }
-                finally { if (e_2) throw e_2.error; }
-            }
-        }
-    }
-    catch (e_1_1) { e_1 = { error: e_1_1 }; }
-    finally {
-        try {
-            if (sets_1_1 && !sets_1_1.done && (_a = sets_1.return)) _a.call(sets_1);
-        }
-        finally { if (e_1) throw e_1.error; }
-    }
-    return union;
-}
-
 /**
  * This error occurs when some expectation is not met in the execution of the pipeline
  *
@@ -2172,10 +2105,83 @@ function replaceParameters(template, parameters) {
     return replacedTemplate;
 }
 
+/**
+ * Create difference set of two sets.
+ *
+ * @deprecated use new javascript set methods instead @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Set
+ */
+function difference(a, b, isEqual) {
+    var e_1, _a;
+    if (isEqual === void 0) { isEqual = function (a, b) { return a === b; }; }
+    var diff = new Set();
+    var _loop_1 = function (itemA) {
+        if (!Array.from(b).some(function (itemB) { return isEqual(itemA, itemB); })) {
+            diff.add(itemA);
+        }
+    };
+    try {
+        for (var _b = __values(Array.from(a)), _c = _b.next(); !_c.done; _c = _b.next()) {
+            var itemA = _c.value;
+            _loop_1(itemA);
+        }
+    }
+    catch (e_1_1) { e_1 = { error: e_1_1 }; }
+    finally {
+        try {
+            if (_c && !_c.done && (_a = _b.return)) _a.call(_b);
+        }
+        finally { if (e_1) throw e_1.error; }
+    }
+    return diff;
+}
+/**
+ * TODO: [🧠][💯] Maybe also implement symmetricDifference
+ */
+
+/**
+ * Creates a new set with all elements that are present in either set
+ *
+ * @deprecated use new javascript set methods instead @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Set
+ */
+function union() {
+    var e_1, _a, e_2, _b;
+    var sets = [];
+    for (var _i = 0; _i < arguments.length; _i++) {
+        sets[_i] = arguments[_i];
+    }
+    var union = new Set();
+    try {
+        for (var sets_1 = __values(sets), sets_1_1 = sets_1.next(); !sets_1_1.done; sets_1_1 = sets_1.next()) {
+            var set = sets_1_1.value;
+            try {
+                for (var _c = (e_2 = void 0, __values(Array.from(set))), _d = _c.next(); !_d.done; _d = _c.next()) {
+                    var item = _d.value;
+                    union.add(item);
+                }
+            }
+            catch (e_2_1) { e_2 = { error: e_2_1 }; }
+            finally {
+                try {
+                    if (_d && !_d.done && (_b = _c.return)) _b.call(_c);
+                }
+                finally { if (e_2) throw e_2.error; }
+            }
+        }
+    }
+    catch (e_1_1) { e_1 = { error: e_1_1 }; }
+    finally {
+        try {
+            if (sets_1_1 && !sets_1_1.done && (_a = sets_1.return)) _a.call(sets_1);
+        }
+        finally { if (e_1) throw e_1.error; }
+    }
+    return union;
+}
+
 /**
  * The version of the Promptbook library
  */
-var PROMPTBOOK_VERSION = '0.61.0-17';
+var PROMPTBOOK_VERSION = '0.61.0-18';
 // TODO: !!!! List here all the versions and annotate + put into script
 
 /**
@@ -2305,7 +2311,7 @@ function createPipelineExecutor(options) {
         pipeline = rawPipeline;
     }
     else {
-        console.warn(spaceTrim$1("\n                Pipeline is not prepared\n\n                It will be prepared ad-hoc before the first execution\n                But it is recommended to prepare the pipeline during collection preparation\n\n                @see more at https://ptbk.io/prepare-pipeline\n            "));
+        console.warn(spaceTrim$1("\n                Pipeline ".concat(rawPipeline.pipelineUrl || rawPipeline.sourceFile || rawPipeline.title, " is not prepared\n\n                It will be prepared ad-hoc before the first execution\n                But it is recommended to prepare the pipeline during collection preparation\n\n                @see more at https://ptbk.io/prepare-pipeline\n            ")));
     }
     var pipelineExecutor = function (inputParameters, onProgress) { return __awaiter(_this, void 0, void 0, function () {
         function getContextForTemplate(// <- TODO: [🧠][🥜]
@@ -5433,7 +5439,7 @@ function createCollectionFromDirectory(path, options) {
                     }
                     _a = options || {}, _b = _a.isRecursive, isRecursive = _b === void 0 ? true : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? false : _c, _d = _a.isLazyLoaded, isLazyLoaded = _d === void 0 ? false : _d, _e = _a.isCrashedOnError, isCrashedOnError = _e === void 0 ? true : _e;
                     collection = createCollectionFromPromise(function () { return __awaiter(_this, void 0, void 0, function () {
-                        var fileNames, pipelines, _loop_1, fileNames_1, fileNames_1_1, fileName, e_1_1;
+                        var fileNames, collection, _loop_1, fileNames_1, fileNames_1_1, fileName, e_1_1;
                         var e_1, _a;
                         return __generator(this, function (_b) {
                             switch (_b.label) {
@@ -5455,9 +5461,9 @@ function createCollectionFromDirectory(path, options) {
                                         }
                                         return 0;
                                     });
-                                    pipelines = [];
+                                    collection = new Map();
                                     _loop_1 = function (fileName) {
-                                        var sourceFile, pipeline, pipelineString, _c, _d, error_1, wrappedErrorMessage;
+                                        var sourceFile, pipeline, pipelineString, _c, _d, existing, error_1, wrappedErrorMessage;
                                         return __generator(this, function (_e) {
                                             switch (_e.label) {
                                                 case 0:
@@ -5493,7 +5499,8 @@ function createCollectionFromDirectory(path, options) {
                                                 case 7:
                                                     // ---
                                                     if (pipeline !== null) {
-                                                        if (!pipeline.pipelineUrl) {
+                                                        // TODO: [👠] DRY
+                                                        if (pipeline.pipelineUrl === undefined) {
                                                             if (isVerbose) {
                                                                 console.info(colors.red("Can not load pipeline from ".concat(fileName
                                                                     .split('\\')
@@ -5501,16 +5508,30 @@ function createCollectionFromDirectory(path, options) {
                                                             }
                                                         }
                                                         else {
-                                                            if (!isCrashedOnError) {
-                                                                // Note: Validate pipeline to check if it is logically correct to not crash on invalid pipelines
-                                                                //       But be handled in current try-catch block
-                                                                validatePipeline(pipeline);
+                                                            // Note: [🐨] Pipeline is checked multiple times
+                                                            // TODO: Maybe once is enough BUT be sure to check it - better to check it multiple times than not at all
+                                                            validatePipeline(pipeline);
+                                                            if (
+                                                            // TODO: [🐽] comparePipelines(pipeline1,pipeline2): 'IDENTICAL' |'IDENTICAL_UNPREPARED' | 'IDENTICAL_INTERFACE' | 'DIFFERENT'
+                                                            !collection.has(pipeline.pipelineUrl)) {
+                                                                if (isVerbose) {
+                                                                    console.info(colors.gray("Loaded pipeline ".concat(fileName.split('\\').join('/'), "\u2060\u2060\u2060\u2060\u2060\u2060\u2060\u2060\u2060\u2060\u2060\u2060")));
+                                                                }
+                                                                // Note: [🦄] Pipeline with same url uniqueness will be double-checked automatically in SimplePipelineCollection
+                                                                collection.set(pipeline.pipelineUrl, pipeline);
                                                             }
-                                                            if (isVerbose) {
-                                                                console.info(colors.green("Loading ".concat(fileName.split('\\').join('/'))));
+                                                            else if (pipelineJsonToString(unpreparePipeline(pipeline)) ===
+                                                                pipelineJsonToString(unpreparePipeline(collection.get(pipeline.pipelineUrl)))) {
+                                                                if (isVerbose) {
+                                                                    console.info(colors.gray("Skipped pipeline ".concat(fileName
+                                                                        .split('\\')
+                                                                        .join('/'), " \u2013\u2060\u2060\u2060\u2060\u2060\u2060\u2060\u2060\u2060\u2060\u2060\u2060\u2060\u2060\u2060\u2060 identical pipeline in the collection")));
+                                                                }
+                                                            }
+                                                            else {
+                                                                existing = collection.get(pipeline.pipelineUrl);
+                                                                throw new ReferenceError(spaceTrim("\n                                  Pipeline with URL \"".concat(pipeline.pipelineUrl, "\" is already in the collection\n\n                                  Conflicting files:\n                                  ").concat(existing.sourceFile || 'Unknown', "\n                                  ").concat(pipeline.sourceFile || 'Unknown', "\n\n                                  Note: Pipelines with the same URL are not allowed\n                                        Only exepction is when the pipelines are identical\n\n                              ")));
                                                             }
-                                                            // Note: [🦄] Pipeline with same url uniqueness will be checked automatically in SimplePipelineCollection
-                                                            pipelines.push(pipeline);
                                                         }
                                                     }
                                                     return [3 /*break*/, 9];
@@ -5519,7 +5540,7 @@ function createCollectionFromDirectory(path, options) {
                                                     if (!(error_1 instanceof Error)) {
                                                         throw error_1;
                                                     }
-                                                    wrappedErrorMessage = spaceTrim(function (block) { return "\n                        Error during loading pipeline from file ".concat(fileName.split('\\').join('/'), ":\n\n                        ").concat(block(error_1.message), "\n\n                    "); });
+                                                    wrappedErrorMessage = spaceTrim(function (block) { return "\n                        ".concat(error_1.name, " in pipeline ").concat(fileName.split('\\').join('/'), "\u2060:\n\n                        ").concat(block(error_1.message), "\n\n                    "); });
                                                     if (isCrashedOnError) {
                                                         throw new CollectionError(wrappedErrorMessage);
                                                     }
@@ -5556,7 +5577,7 @@ function createCollectionFromDirectory(path, options) {
                                     }
                                     finally { if (e_1) throw e_1.error; }
                                     return [7 /*endfinally*/];
-                                case 9: return [2 /*return*/, pipelines];
+                                case 9: return [2 /*return*/, Array.from(collection.values())];
                             }
                         });
                     }); });