@promptbook/documents 0.89.0-9 โ†’ 0.92.0-10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. package/README.md +9 -7
  2. package/esm/index.es.js +303 -68
  3. package/esm/index.es.js.map +1 -1
  4. package/esm/typings/servers.d.ts +40 -0
  5. package/esm/typings/src/_packages/core.index.d.ts +14 -4
  6. package/esm/typings/src/_packages/deepseek.index.d.ts +2 -0
  7. package/esm/typings/src/_packages/google.index.d.ts +2 -0
  8. package/esm/typings/src/_packages/types.index.d.ts +18 -0
  9. package/esm/typings/src/_packages/utils.index.d.ts +6 -0
  10. package/esm/typings/src/cli/cli-commands/login.d.ts +0 -1
  11. package/esm/typings/src/cli/common/$provideLlmToolsForCli.d.ts +16 -3
  12. package/esm/typings/src/cli/test/ptbk.d.ts +1 -1
  13. package/esm/typings/src/commands/EXPECT/expectCommandParser.d.ts +2 -0
  14. package/esm/typings/src/config.d.ts +10 -19
  15. package/esm/typings/src/conversion/archive/loadArchive.d.ts +2 -2
  16. package/esm/typings/src/errors/0-index.d.ts +7 -4
  17. package/esm/typings/src/errors/PipelineExecutionError.d.ts +1 -1
  18. package/esm/typings/src/errors/WrappedError.d.ts +10 -0
  19. package/esm/typings/src/errors/assertsError.d.ts +11 -0
  20. package/esm/typings/src/execution/CommonToolsOptions.d.ts +4 -0
  21. package/esm/typings/src/execution/PromptbookFetch.d.ts +1 -1
  22. package/esm/typings/src/execution/createPipelineExecutor/getKnowledgeForTask.d.ts +12 -0
  23. package/esm/typings/src/execution/createPipelineExecutor/getReservedParametersForTask.d.ts +5 -0
  24. package/esm/typings/src/formats/csv/utils/csvParse.d.ts +12 -0
  25. package/esm/typings/src/formats/csv/utils/isValidCsvString.d.ts +9 -0
  26. package/esm/typings/src/formats/csv/utils/isValidCsvString.test.d.ts +1 -0
  27. package/esm/typings/src/formats/json/utils/isValidJsonString.d.ts +3 -0
  28. package/esm/typings/src/formats/json/utils/jsonParse.d.ts +11 -0
  29. package/esm/typings/src/formats/xml/utils/isValidXmlString.d.ts +9 -0
  30. package/esm/typings/src/formats/xml/utils/isValidXmlString.test.d.ts +1 -0
  31. package/esm/typings/src/llm-providers/_common/filterModels.d.ts +15 -0
  32. package/esm/typings/src/llm-providers/_common/register/{$provideEnvFilepath.d.ts โ†’ $provideEnvFilename.d.ts} +2 -2
  33. package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsConfigurationFromEnv.d.ts +1 -1
  34. package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground.d.ts +1 -1
  35. package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsForWizzardOrCli.d.ts +11 -2
  36. package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsFromEnv.d.ts +1 -1
  37. package/esm/typings/src/llm-providers/_common/register/LlmToolsMetadata.d.ts +43 -0
  38. package/esm/typings/src/llm-providers/azure-openai/AzureOpenAiExecutionTools.d.ts +4 -0
  39. package/esm/typings/src/llm-providers/deepseek/deepseek-models.d.ts +23 -0
  40. package/esm/typings/src/llm-providers/google/google-models.d.ts +23 -0
  41. package/esm/typings/src/llm-providers/openai/OpenAiExecutionTools.d.ts +4 -0
  42. package/esm/typings/src/personas/preparePersona.d.ts +1 -1
  43. package/esm/typings/src/pipeline/PipelineJson/PersonaJson.d.ts +4 -2
  44. package/esm/typings/src/remote-server/openapi-types.d.ts +626 -0
  45. package/esm/typings/src/remote-server/openapi.d.ts +581 -0
  46. package/esm/typings/src/remote-server/socket-types/_subtypes/Identification.d.ts +7 -1
  47. package/esm/typings/src/remote-server/socket-types/_subtypes/identificationToPromptbookToken.d.ts +11 -0
  48. package/esm/typings/src/remote-server/socket-types/_subtypes/promptbookTokenToIdentification.d.ts +10 -0
  49. package/esm/typings/src/remote-server/startRemoteServer.d.ts +1 -2
  50. package/esm/typings/src/remote-server/types/RemoteServerOptions.d.ts +15 -9
  51. package/esm/typings/src/storage/env-storage/$EnvStorage.d.ts +40 -0
  52. package/esm/typings/src/types/typeAliases.d.ts +26 -0
  53. package/package.json +8 -4
  54. package/umd/index.umd.js +303 -68
  55. package/umd/index.umd.js.map +1 -1
  56. package/esm/typings/src/cli/test/ptbk2.d.ts +0 -5
package/umd/index.umd.js CHANGED
@@ -26,7 +26,7 @@
26
26
  * @generated
27
27
  * @see https://github.com/webgptorg/promptbook
28
28
  */
29
- const PROMPTBOOK_ENGINE_VERSION = '0.89.0-9';
29
+ const PROMPTBOOK_ENGINE_VERSION = '0.92.0-10';
30
30
  /**
31
31
  * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
32
32
  * Note: [๐Ÿ’ž] Ignore a discrepancy between file name and entity name
@@ -89,6 +89,7 @@
89
89
  * @public exported from `@promptbook/core`
90
90
  */
91
91
  const ADMIN_GITHUB_NAME = 'hejny';
92
+ // <- TODO: [๐ŸŠ] Pick the best claim
92
93
  /**
93
94
  * When the title is not provided, the default title is used
94
95
  *
@@ -121,6 +122,7 @@
121
122
  infinity: '(infinity; โˆž)',
122
123
  negativeInfinity: '(negative infinity; -โˆž)',
123
124
  unserializable: '(unserializable value)',
125
+ circular: '(circular JSON)',
124
126
  };
125
127
  /**
126
128
  * Small number limit
@@ -160,7 +162,7 @@
160
162
  */
161
163
  const DEFAULT_MAX_EXECUTION_ATTEMPTS = 10; // <- TODO: [๐Ÿคนโ€โ™‚๏ธ]
162
164
  // <- TODO: [๐Ÿ•] Make also `BOOKS_DIRNAME_ALTERNATIVES`
163
- // TODO: !!!!!! Just .promptbook dir, hardocode others
165
+ // TODO: Just `.promptbook` in config, hardcode subfolders like `download-cache` or `execution-cache`
164
166
  /**
165
167
  * Where to store the temporary downloads
166
168
  *
@@ -1035,7 +1037,7 @@
1035
1037
  * Note: [๐ŸŸข] Code in this file should never be never released in packages that could be imported into browser environment
1036
1038
  */
1037
1039
 
1038
- var PipelineCollection = [{title:"Prepare Knowledge from Markdown",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.book",formfactorName:"GENERIC",parameters:[{name:"knowledgeContent",description:"Markdown document content",isInput:true,isOutput:false},{name:"knowledgePieces",description:"The knowledge JSON object",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, extract the important knowledge from the document.\n\n# Rules\n\n- Make pieces of information concise, clear, and easy to understand\n- One piece of information should be approximately 1 paragraph\n- Divide the paragraphs by markdown horizontal lines ---\n- Omit irrelevant information\n- Group redundant information\n- Write just extracted information, nothing else\n\n# The document\n\nTake information from this document:\n\n> {knowledgeContent}",resultingParameterName:"knowledgePieces",dependentParameterNames:["knowledgeContent"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Knowledge from Markdown\n\n- PIPELINE URL `https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.book`\n- INPUT PARAMETER `{knowledgeContent}` Markdown document content\n- OUTPUT PARAMETER `{knowledgePieces}` The knowledge JSON object\n\n## Knowledge\n\n<!-- TODO: [๐Ÿ†] -FORMAT JSON -->\n\n```markdown\nYou are experienced data researcher, extract the important knowledge from the document.\n\n# Rules\n\n- Make pieces of information concise, clear, and easy to understand\n- One piece of information should be approximately 1 paragraph\n- Divide the paragraphs by markdown horizontal lines ---\n- Omit irrelevant information\n- Group redundant information\n- Write just extracted information, nothing else\n\n# The document\n\nTake information from this document:\n\n> {knowledgeContent}\n```\n\n`-> {knowledgePieces}`\n"}],sourceFile:"./books/prepare-knowledge-from-markdown.book"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-keywords.book",formfactorName:"GENERIC",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"keywords",description:"Keywords separated by comma",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, detect the important keywords in the document.\n\n# Rules\n\n- Write just keywords separated by comma\n\n# The document\n\nTake information from this document:\n\n> {knowledgePieceContent}",resultingParameterName:"keywords",dependentParameterNames:["knowledgePieceContent"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Keywords\n\n- PIPELINE URL `https://promptbook.studio/promptbook/prepare-knowledge-keywords.book`\n- INPUT PARAMETER `{knowledgePieceContent}` The content\n- OUTPUT PARAMETER `{keywords}` Keywords separated by comma\n\n## Knowledge\n\n<!-- TODO: [๐Ÿ†] -FORMAT JSON -->\n\n```markdown\nYou are experienced data researcher, detect the important keywords in the document.\n\n# Rules\n\n- Write just keywords separated by comma\n\n# The document\n\nTake information from this document:\n\n> {knowledgePieceContent}\n```\n\n`-> {keywords}`\n"}],sourceFile:"./books/prepare-knowledge-keywords.book"},{title:"Prepare Knowledge-piece Title",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-title.book",formfactorName:"GENERIC",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"title",description:"The title of the document",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"knowledge",title:"Knowledge",content:"You are experienced content creator, write best title for the document.\n\n# Rules\n\n- Write just title, nothing else\n- Write maximum 5 words for the title\n\n# The document\n\n> {knowledgePieceContent}",resultingParameterName:"title",expectations:{words:{min:1,max:8}},dependentParameterNames:["knowledgePieceContent"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Knowledge-piece Title\n\n- PIPELINE URL `https://promptbook.studio/promptbook/prepare-knowledge-title.book`\n- INPUT PARAMETER `{knowledgePieceContent}` The content\n- OUTPUT PARAMETER `{title}` The title of the document\n\n## Knowledge\n\n- EXPECT MIN 1 WORD\n- EXPECT MAX 8 WORDS\n\n```markdown\nYou are experienced content creator, write best title for the document.\n\n# Rules\n\n- Write just title, nothing else\n- Write maximum 5 words for the title\n\n# The document\n\n> {knowledgePieceContent}\n```\n\n`-> {title}`\n"}],sourceFile:"./books/prepare-knowledge-title.book"},{title:"Prepare Persona",pipelineUrl:"https://promptbook.studio/promptbook/prepare-persona.book",formfactorName:"GENERIC",parameters:[{name:"availableModelNames",description:"List of available model names separated by comma (,)",isInput:true,isOutput:false},{name:"personaDescription",description:"Description of the persona",isInput:true,isOutput:false},{name:"modelRequirements",description:"Specific requirements for the model",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"make-model-requirements",title:"Make modelRequirements",content:"You are experienced AI engineer, you need to create virtual assistant.\nWrite\n\n## Example\n\n```json\n{\n\"modelName\": \"gpt-4o\",\n\"systemMessage\": \"You are experienced AI engineer and helpfull assistant.\",\n\"temperature\": 0.7\n}\n```\n\n## Instructions\n\n- Your output format is JSON object\n- Write just the JSON object, no other text should be present\n- It contains the following keys:\n - `modelName`: The name of the model to use\n - `systemMessage`: The system message to provide context to the model\n - `temperature`: The sampling temperature to use\n\n### Key `modelName`\n\nPick from the following models:\n\n- {availableModelNames}\n\n### Key `systemMessage`\n\nThe system message is used to communicate instructions or provide context to the model at the beginning of a conversation. It is displayed in a different format compared to user messages, helping the model understand its role in the conversation. The system message typically guides the model's behavior, sets the tone, or specifies desired output from the model. By utilizing the system message effectively, users can steer the model towards generating more accurate and relevant responses.\n\nFor example:\n\n> You are an experienced AI engineer and helpful assistant.\n\n> You are a friendly and knowledgeable chatbot.\n\n### Key `temperature`\n\nThe sampling temperature, between 0 and 1. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. If set to 0, the model will use log probability to automatically increase the temperature until certain thresholds are hit.\n\nYou can pick a value between 0 and 2. For example:\n\n- `0.1`: Low temperature, extremely conservative and deterministic\n- `0.5`: Medium temperature, balanced between conservative and creative\n- `1.0`: High temperature, creative and bit random\n- `1.5`: Very high temperature, extremely creative and often chaotic and unpredictable\n- `2.0`: Maximum temperature, completely random and unpredictable, for some extreme creative use cases\n\n# The assistant\n\nTake this description of the persona:\n\n> {personaDescription}",resultingParameterName:"modelRequirements",format:"JSON",dependentParameterNames:["availableModelNames","personaDescription"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Persona\n\n- PIPELINE URL `https://promptbook.studio/promptbook/prepare-persona.book`\n- INPUT PARAMETER `{availableModelNames}` List of available model names separated by comma (,)\n- INPUT PARAMETER `{personaDescription}` Description of the persona\n- OUTPUT PARAMETER `{modelRequirements}` Specific requirements for the model\n\n## Make modelRequirements\n\n- FORMAT JSON\n\n```markdown\nYou are experienced AI engineer, you need to create virtual assistant.\nWrite\n\n## Example\n\n\\`\\`\\`json\n{\n\"modelName\": \"gpt-4o\",\n\"systemMessage\": \"You are experienced AI engineer and helpfull assistant.\",\n\"temperature\": 0.7\n}\n\\`\\`\\`\n\n## Instructions\n\n- Your output format is JSON object\n- Write just the JSON object, no other text should be present\n- It contains the following keys:\n - `modelName`: The name of the model to use\n - `systemMessage`: The system message to provide context to the model\n - `temperature`: The sampling temperature to use\n\n### Key `modelName`\n\nPick from the following models:\n\n- {availableModelNames}\n\n### Key `systemMessage`\n\nThe system message is used to communicate instructions or provide context to the model at the beginning of a conversation. It is displayed in a different format compared to user messages, helping the model understand its role in the conversation. The system message typically guides the model's behavior, sets the tone, or specifies desired output from the model. By utilizing the system message effectively, users can steer the model towards generating more accurate and relevant responses.\n\nFor example:\n\n> You are an experienced AI engineer and helpful assistant.\n\n> You are a friendly and knowledgeable chatbot.\n\n### Key `temperature`\n\nThe sampling temperature, between 0 and 1. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. If set to 0, the model will use log probability to automatically increase the temperature until certain thresholds are hit.\n\nYou can pick a value between 0 and 2. For example:\n\n- `0.1`: Low temperature, extremely conservative and deterministic\n- `0.5`: Medium temperature, balanced between conservative and creative\n- `1.0`: High temperature, creative and bit random\n- `1.5`: Very high temperature, extremely creative and often chaotic and unpredictable\n- `2.0`: Maximum temperature, completely random and unpredictable, for some extreme creative use cases\n\n# The assistant\n\nTake this description of the persona:\n\n> {personaDescription}\n```\n\n`-> {modelRequirements}`\n"}],sourceFile:"./books/prepare-persona.book"},{title:"Prepare Title",pipelineUrl:"https://promptbook.studio/promptbook/prepare-title.book",formfactorName:"GENERIC",parameters:[{name:"book",description:"The book to prepare the title for",isInput:true,isOutput:false},{name:"title",description:"Best title for the book",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"make-title",title:"Make title",content:"Make best title for given text which describes the workflow:\n\n## Rules\n\n- Write just title, nothing else\n- Title should be concise and clear - Write maximum ideally 2 words, maximum 5 words\n- Title starts with emoticon\n- Title should not mention the input and output of the workflow but the main purpose of the workflow\n _For example, not \"โœ Convert Knowledge-piece to title\" but \"โœ Title\"_\n\n## The workflow\n\n> {book}",resultingParameterName:"title",expectations:{words:{min:1,max:8},lines:{min:1,max:1}},dependentParameterNames:["book"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Title\n\n- PIPELINE URL `https://promptbook.studio/promptbook/prepare-title.book`\n- INPUT PARAMETER `{book}` The book to prepare the title for\n- OUTPUT PARAMETER `{title}` Best title for the book\n\n## Make title\n\n- EXPECT MIN 1 Word\n- EXPECT MAX 8 Words\n- EXPECT EXACTLY 1 Line\n\n```markdown\nMake best title for given text which describes the workflow:\n\n## Rules\n\n- Write just title, nothing else\n- Title should be concise and clear - Write maximum ideally 2 words, maximum 5 words\n- Title starts with emoticon\n- Title should not mention the input and output of the workflow but the main purpose of the workflow\n _For example, not \"โœ Convert Knowledge-piece to title\" but \"โœ Title\"_\n\n## The workflow\n\n> {book}\n```\n\n`-> {title}`\n"}],sourceFile:"./books/prepare-title.book"}];
1040
+ var PipelineCollection = [{title:"Prepare Knowledge from Markdown",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.book",formfactorName:"GENERIC",parameters:[{name:"knowledgeContent",description:"Markdown document content",isInput:true,isOutput:false},{name:"knowledgePieces",description:"The knowledge JSON object",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, extract the important knowledge from the document.\n\n# Rules\n\n- Make pieces of information concise, clear, and easy to understand\n- One piece of information should be approximately 1 paragraph\n- Divide the paragraphs by markdown horizontal lines ---\n- Omit irrelevant information\n- Group redundant information\n- Write just extracted information, nothing else\n\n# The document\n\nTake information from this document:\n\n> {knowledgeContent}",resultingParameterName:"knowledgePieces",dependentParameterNames:["knowledgeContent"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Knowledge from Markdown\n\n- PIPELINE URL `https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.book`\n- INPUT PARAMETER `{knowledgeContent}` Markdown document content\n- OUTPUT PARAMETER `{knowledgePieces}` The knowledge JSON object\n\n## Knowledge\n\n<!-- TODO: [๐Ÿ†] -FORMAT JSON -->\n\n```markdown\nYou are experienced data researcher, extract the important knowledge from the document.\n\n# Rules\n\n- Make pieces of information concise, clear, and easy to understand\n- One piece of information should be approximately 1 paragraph\n- Divide the paragraphs by markdown horizontal lines ---\n- Omit irrelevant information\n- Group redundant information\n- Write just extracted information, nothing else\n\n# The document\n\nTake information from this document:\n\n> {knowledgeContent}\n```\n\n`-> {knowledgePieces}`\n"}],sourceFile:"./books/prepare-knowledge-from-markdown.book"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-keywords.book",formfactorName:"GENERIC",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"keywords",description:"Keywords separated by comma",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, detect the important keywords in the document.\n\n# Rules\n\n- Write just keywords separated by comma\n\n# The document\n\nTake information from this document:\n\n> {knowledgePieceContent}",resultingParameterName:"keywords",dependentParameterNames:["knowledgePieceContent"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Keywords\n\n- PIPELINE URL `https://promptbook.studio/promptbook/prepare-knowledge-keywords.book`\n- INPUT PARAMETER `{knowledgePieceContent}` The content\n- OUTPUT PARAMETER `{keywords}` Keywords separated by comma\n\n## Knowledge\n\n<!-- TODO: [๐Ÿ†] -FORMAT JSON -->\n\n```markdown\nYou are experienced data researcher, detect the important keywords in the document.\n\n# Rules\n\n- Write just keywords separated by comma\n\n# The document\n\nTake information from this document:\n\n> {knowledgePieceContent}\n```\n\n`-> {keywords}`\n"}],sourceFile:"./books/prepare-knowledge-keywords.book"},{title:"Prepare Knowledge-piece Title",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-title.book",formfactorName:"GENERIC",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"title",description:"The title of the document",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"knowledge",title:"Knowledge",content:"You are experienced content creator, write best title for the document.\n\n# Rules\n\n- Write just title, nothing else\n- Write maximum 5 words for the title\n\n# The document\n\n> {knowledgePieceContent}",resultingParameterName:"title",expectations:{words:{min:1,max:8}},dependentParameterNames:["knowledgePieceContent"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Knowledge-piece Title\n\n- PIPELINE URL `https://promptbook.studio/promptbook/prepare-knowledge-title.book`\n- INPUT PARAMETER `{knowledgePieceContent}` The content\n- OUTPUT PARAMETER `{title}` The title of the document\n\n## Knowledge\n\n- EXPECT MIN 1 WORD\n- EXPECT MAX 8 WORDS\n\n```markdown\nYou are experienced content creator, write best title for the document.\n\n# Rules\n\n- Write just title, nothing else\n- Write maximum 5 words for the title\n\n# The document\n\n> {knowledgePieceContent}\n```\n\n`-> {title}`\n"}],sourceFile:"./books/prepare-knowledge-title.book"},{title:"Prepare Persona",pipelineUrl:"https://promptbook.studio/promptbook/prepare-persona.book",formfactorName:"GENERIC",parameters:[{name:"availableModels",description:"List of available model names together with their descriptions as JSON",isInput:true,isOutput:false},{name:"personaDescription",description:"Description of the persona",isInput:true,isOutput:false},{name:"modelsRequirements",description:"Specific requirements for the model",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"make-model-requirements",title:"Make modelRequirements",content:"You are an experienced AI engineer, you need to find the best models for virtual assistants:\n\n## Example\n\n```json\n[\n {\n \"modelName\": \"gpt-4o\",\n \"systemMessage\": \"You are experienced AI engineer and helpfull assistant.\",\n \"temperature\": 0.7\n },\n {\n \"modelName\": \"claude-3-5-sonnet\",\n \"systemMessage\": \"You are a friendly and knowledgeable chatbot.\",\n \"temperature\": 0.5\n }\n]\n```\n\n## Instructions\n\n- Your output format is JSON array\n- Sort best-fitting models first\n- Omit any models that are not suitable\n- Write just the JSON, no other text should be present\n- Array contain items with following keys:\n - `modelName`: The name of the model to use\n - `systemMessage`: The system message to provide context to the model\n - `temperature`: The sampling temperature to use\n\n### Key `modelName`\n\nHere are the available models:\n\n```json\n{availableModels}\n```\n\n### Key `systemMessage`\n\nThe system message is used to communicate instructions or provide context to the model at the beginning of a conversation. It is displayed in a different format compared to user messages, helping the model understand its role in the conversation. The system message typically guides the model's behavior, sets the tone, or specifies desired output from the model. By utilizing the system message effectively, users can steer the model towards generating more accurate and relevant responses.\n\nFor example:\n\n> You are an experienced AI engineer and helpful assistant.\n\n> You are a friendly and knowledgeable chatbot.\n\n### Key `temperature`\n\nThe sampling temperature, between 0 and 1. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. If set to 0, the model will use log probability to automatically increase the temperature until certain thresholds are hit.\n\nYou can pick a value between 0 and 2. For example:\n\n- `0.1`: Low temperature, extremely conservative and deterministic\n- `0.5`: Medium temperature, balanced between conservative and creative\n- `1.0`: High temperature, creative and bit random\n- `1.5`: Very high temperature, extremely creative and often chaotic and unpredictable\n- `2.0`: Maximum temperature, completely random and unpredictable, for some extreme creative use cases\n\n# The assistant\n\nTake this description of the persona:\n\n> {personaDescription}",resultingParameterName:"modelsRequirements",format:"JSON",dependentParameterNames:["availableModels","personaDescription"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Persona\n\n- PIPELINE URL `https://promptbook.studio/promptbook/prepare-persona.book`\n- INPUT PARAMETER `{availableModels}` List of available model names together with their descriptions as JSON\n- INPUT PARAMETER `{personaDescription}` Description of the persona\n- OUTPUT PARAMETER `{modelsRequirements}` Specific requirements for the model\n\n## Make modelRequirements\n\n- FORMAT JSON\n\n```markdown\nYou are an experienced AI engineer, you need to find the best models for virtual assistants:\n\n## Example\n\n\\`\\`\\`json\n[\n {\n \"modelName\": \"gpt-4o\",\n \"systemMessage\": \"You are experienced AI engineer and helpfull assistant.\",\n \"temperature\": 0.7\n },\n {\n \"modelName\": \"claude-3-5-sonnet\",\n \"systemMessage\": \"You are a friendly and knowledgeable chatbot.\",\n \"temperature\": 0.5\n }\n]\n\\`\\`\\`\n\n## Instructions\n\n- Your output format is JSON array\n- Sort best-fitting models first\n- Omit any models that are not suitable\n- Write just the JSON, no other text should be present\n- Array contain items with following keys:\n - `modelName`: The name of the model to use\n - `systemMessage`: The system message to provide context to the model\n - `temperature`: The sampling temperature to use\n\n### Key `modelName`\n\nHere are the available models:\n\n\\`\\`\\`json\n{availableModels}\n\\`\\`\\`\n\n### Key `systemMessage`\n\nThe system message is used to communicate instructions or provide context to the model at the beginning of a conversation. It is displayed in a different format compared to user messages, helping the model understand its role in the conversation. The system message typically guides the model's behavior, sets the tone, or specifies desired output from the model. By utilizing the system message effectively, users can steer the model towards generating more accurate and relevant responses.\n\nFor example:\n\n> You are an experienced AI engineer and helpful assistant.\n\n> You are a friendly and knowledgeable chatbot.\n\n### Key `temperature`\n\nThe sampling temperature, between 0 and 1. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. If set to 0, the model will use log probability to automatically increase the temperature until certain thresholds are hit.\n\nYou can pick a value between 0 and 2. For example:\n\n- `0.1`: Low temperature, extremely conservative and deterministic\n- `0.5`: Medium temperature, balanced between conservative and creative\n- `1.0`: High temperature, creative and bit random\n- `1.5`: Very high temperature, extremely creative and often chaotic and unpredictable\n- `2.0`: Maximum temperature, completely random and unpredictable, for some extreme creative use cases\n\n# The assistant\n\nTake this description of the persona:\n\n> {personaDescription}\n```\n\n`-> {modelsRequirements}`\n"}],sourceFile:"./books/prepare-persona.book"},{title:"Prepare Title",pipelineUrl:"https://promptbook.studio/promptbook/prepare-title.book",formfactorName:"GENERIC",parameters:[{name:"book",description:"The book to prepare the title for",isInput:true,isOutput:false},{name:"title",description:"Best title for the book",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"make-title",title:"Make title",content:"Make best title for given text which describes the workflow:\n\n## Rules\n\n- Write just title, nothing else\n- Title should be concise and clear - Write maximum ideally 2 words, maximum 5 words\n- Title starts with emoticon\n- Title should not mention the input and output of the workflow but the main purpose of the workflow\n _For example, not \"โœ Convert Knowledge-piece to title\" but \"โœ Title\"_\n\n## The workflow\n\n> {book}",resultingParameterName:"title",expectations:{words:{min:1,max:8},lines:{min:1,max:1}},dependentParameterNames:["book"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Title\n\n- PIPELINE URL `https://promptbook.studio/promptbook/prepare-title.book`\n- INPUT PARAMETER `{book}` The book to prepare the title for\n- OUTPUT PARAMETER `{title}` Best title for the book\n\n## Make title\n\n- EXPECT MIN 1 Word\n- EXPECT MAX 8 Words\n- EXPECT EXACTLY 1 Line\n\n```markdown\nMake best title for given text which describes the workflow:\n\n## Rules\n\n- Write just title, nothing else\n- Title should be concise and clear - Write maximum ideally 2 words, maximum 5 words\n- Title starts with emoticon\n- Title should not mention the input and output of the workflow but the main purpose of the workflow\n _For example, not \"โœ Convert Knowledge-piece to title\" but \"โœ Title\"_\n\n## The workflow\n\n> {book}\n```\n\n`-> {title}`\n"}],sourceFile:"./books/prepare-title.book"}];
1039
1041
 
1040
1042
  /**
1041
1043
  * Checks if value is valid email
@@ -1068,9 +1070,60 @@
1068
1070
  * TODO: Maybe split `ParseError` and `ApplyError`
1069
1071
  */
1070
1072
 
1073
+ /**
1074
+ * This error type indicates that somewhere in the code non-Error object was thrown and it was wrapped into the `WrappedError`
1075
+ *
1076
+ * @public exported from `@promptbook/core`
1077
+ */
1078
+ class WrappedError extends Error {
1079
+ constructor(whatWasThrown) {
1080
+ const tag = `[๐Ÿคฎ]`;
1081
+ console.error(tag, whatWasThrown);
1082
+ super(spaceTrim.spaceTrim(`
1083
+ Non-Error object was thrown
1084
+
1085
+ Note: Look for ${tag} in the console for more details
1086
+ Please report issue on ${ADMIN_EMAIL}
1087
+ `));
1088
+ this.name = 'WrappedError';
1089
+ Object.setPrototypeOf(this, WrappedError.prototype);
1090
+ }
1091
+ }
1092
+
1093
+ /**
1094
+ * Helper used in catch blocks to assert that the error is an instance of `Error`
1095
+ *
1096
+ * @param whatWasThrown Any object that was thrown
1097
+ * @returns Nothing if the error is an instance of `Error`
1098
+ * @throws `WrappedError` or `UnexpectedError` if the error is not standard
1099
+ *
1100
+ * @private within the repository
1101
+ */
1102
+ function assertsError(whatWasThrown) {
1103
+ // Case 1: Handle error which was rethrown as `WrappedError`
1104
+ if (whatWasThrown instanceof WrappedError) {
1105
+ const wrappedError = whatWasThrown;
1106
+ throw wrappedError;
1107
+ }
1108
+ // Case 2: Handle unexpected errors
1109
+ if (whatWasThrown instanceof UnexpectedError) {
1110
+ const unexpectedError = whatWasThrown;
1111
+ throw unexpectedError;
1112
+ }
1113
+ // Case 3: Handle standard errors - keep them up to consumer
1114
+ if (whatWasThrown instanceof Error) {
1115
+ return;
1116
+ }
1117
+ // Case 4: Handle non-standard errors - wrap them into `WrappedError` and throw
1118
+ throw new WrappedError(whatWasThrown);
1119
+ }
1120
+
1071
1121
  /**
1072
1122
  * Function isValidJsonString will tell you if the string is valid JSON or not
1073
1123
  *
1124
+ * @param value The string to check
1125
+ * @returns True if the string is a valid JSON string, false otherwise
1126
+ *
1074
1127
  * @public exported from `@promptbook/utils`
1075
1128
  */
1076
1129
  function isValidJsonString(value /* <- [๐Ÿ‘จโ€โš–๏ธ] */) {
@@ -1079,9 +1132,7 @@
1079
1132
  return true;
1080
1133
  }
1081
1134
  catch (error) {
1082
- if (!(error instanceof Error)) {
1083
- throw error;
1084
- }
1135
+ assertsError(error);
1085
1136
  if (error.message.includes('Unexpected token')) {
1086
1137
  return false;
1087
1138
  }
@@ -1434,9 +1485,7 @@
1434
1485
  JSON.stringify(value); // <- TODO: [0]
1435
1486
  }
1436
1487
  catch (error) {
1437
- if (!(error instanceof Error)) {
1438
- throw error;
1439
- }
1488
+ assertsError(error);
1440
1489
  throw new UnexpectedError(spaceTrim__default["default"]((block) => `
1441
1490
  \`${name}\` is not serializable
1442
1491
 
@@ -2031,7 +2080,7 @@
2031
2080
  */
2032
2081
  function unpreparePipeline(pipeline) {
2033
2082
  let { personas, knowledgeSources, tasks } = pipeline;
2034
- personas = personas.map((persona) => ({ ...persona, modelRequirements: undefined, preparationIds: undefined }));
2083
+ personas = personas.map((persona) => ({ ...persona, modelsRequirements: undefined, preparationIds: undefined }));
2035
2084
  knowledgeSources = knowledgeSources.map((knowledgeSource) => ({ ...knowledgeSource, preparationIds: undefined }));
2036
2085
  tasks = tasks.map((task) => {
2037
2086
  let { dependentParameterNames } = task;
@@ -2207,7 +2256,7 @@
2207
2256
  }
2208
2257
  }
2209
2258
  /**
2210
- * TODO: !!!!!! Add id to all errors
2259
+ * TODO: [๐Ÿง ][๐ŸŒ‚] Add id to all errors
2211
2260
  */
2212
2261
 
2213
2262
  /**
@@ -2223,7 +2272,7 @@
2223
2272
  if (pipeline.title === undefined || pipeline.title === '' || pipeline.title === DEFAULT_BOOK_TITLE) {
2224
2273
  return false;
2225
2274
  }
2226
- if (!pipeline.personas.every((persona) => persona.modelRequirements !== undefined)) {
2275
+ if (!pipeline.personas.every((persona) => persona.modelsRequirements !== undefined)) {
2227
2276
  return false;
2228
2277
  }
2229
2278
  if (!pipeline.knowledgeSources.every((knowledgeSource) => knowledgeSource.preparationIds !== undefined)) {
@@ -2247,6 +2296,45 @@
2247
2296
  * - [โ™จ] Are tasks prepared
2248
2297
  */
2249
2298
 
2299
+ /**
2300
+ * Converts a JavaScript Object Notation (JSON) string into an object.
2301
+ *
2302
+ * Note: This is wrapper around `JSON.parse()` with better error and type handling
2303
+ *
2304
+ * @public exported from `@promptbook/utils`
2305
+ */
2306
+ function jsonParse(value) {
2307
+ if (value === undefined) {
2308
+ throw new Error(`Can not parse JSON from undefined value.`);
2309
+ }
2310
+ else if (typeof value !== 'string') {
2311
+ console.error('Can not parse JSON from non-string value.', { text: value });
2312
+ throw new Error(spaceTrim__default["default"](`
2313
+ Can not parse JSON from non-string value.
2314
+
2315
+ The value type: ${typeof value}
2316
+ See more in console.
2317
+ `));
2318
+ }
2319
+ try {
2320
+ return JSON.parse(value);
2321
+ }
2322
+ catch (error) {
2323
+ if (!(error instanceof Error)) {
2324
+ throw error;
2325
+ }
2326
+ throw new Error(spaceTrim__default["default"]((block) => `
2327
+ ${block(error.message)}
2328
+
2329
+ The JSON text:
2330
+ ${block(value)}
2331
+ `));
2332
+ }
2333
+ }
2334
+ /**
2335
+ * TODO: !!!! Use in Promptbook.studio
2336
+ */
2337
+
2250
2338
  /**
2251
2339
  * Recursively converts JSON strings to JSON objects
2252
2340
 
@@ -2265,7 +2353,7 @@
2265
2353
  const newObject = { ...object };
2266
2354
  for (const [key, value] of Object.entries(object)) {
2267
2355
  if (typeof value === 'string' && isValidJsonString(value)) {
2268
- newObject[key] = JSON.parse(value);
2356
+ newObject[key] = jsonParse(value);
2269
2357
  }
2270
2358
  else {
2271
2359
  newObject[key] = jsonStringsToJsons(value);
@@ -2418,7 +2506,10 @@
2418
2506
  PipelineExecutionError,
2419
2507
  PipelineLogicError,
2420
2508
  PipelineUrlError,
2509
+ AuthenticationError,
2510
+ PromptbookFetchError,
2421
2511
  UnexpectedError,
2512
+ WrappedError,
2422
2513
  // TODO: [๐Ÿช‘]> VersionMismatchError,
2423
2514
  };
2424
2515
  /**
@@ -2435,8 +2526,6 @@
2435
2526
  TypeError,
2436
2527
  URIError,
2437
2528
  AggregateError,
2438
- AuthenticationError,
2439
- PromptbookFetchError,
2440
2529
  /*
2441
2530
  Note: Not widely supported
2442
2531
  > InternalError,
@@ -2559,8 +2648,8 @@
2559
2648
  updatedAt = new Date();
2560
2649
  errors.push(...executionResult.errors);
2561
2650
  warnings.push(...executionResult.warnings);
2562
- // <- TODO: !!! Only unique errors and warnings should be added (or filtered)
2563
- // TODO: [๐Ÿง ] !!! errors, warning, isSuccessful are redundant both in `ExecutionTask` and `ExecutionTask.currentValue`
2651
+ // <- TODO: [๐ŸŒ‚] Only unique errors and warnings should be added (or filtered)
2652
+ // TODO: [๐Ÿง ] !! errors, warning, isSuccessful are redundant both in `ExecutionTask` and `ExecutionTask.currentValue`
2564
2653
  // Also maybe move `ExecutionTask.currentValue.usage` -> `ExecutionTask.usage`
2565
2654
  // And delete `ExecutionTask.currentValue.preparedPipeline`
2566
2655
  assertsTaskSuccessful(executionResult);
@@ -2570,6 +2659,7 @@
2570
2659
  partialResultSubject.next(executionResult);
2571
2660
  }
2572
2661
  catch (error) {
2662
+ assertsError(error);
2573
2663
  status = 'ERROR';
2574
2664
  errors.push(error);
2575
2665
  partialResultSubject.error(error);
@@ -2961,14 +3051,15 @@
2961
3051
  }
2962
3052
  }
2963
3053
  catch (error) {
2964
- if (!(error instanceof Error) || error instanceof UnexpectedError) {
3054
+ assertsError(error);
3055
+ if (error instanceof UnexpectedError) {
2965
3056
  throw error;
2966
3057
  }
2967
3058
  errors.push({ llmExecutionTools, error });
2968
3059
  }
2969
3060
  }
2970
3061
  if (errors.length === 1) {
2971
- throw errors[0];
3062
+ throw errors[0].error;
2972
3063
  }
2973
3064
  else if (errors.length > 1) {
2974
3065
  throw new PipelineExecutionError(
@@ -3094,27 +3185,48 @@
3094
3185
  pipeline: await collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-persona.book'),
3095
3186
  tools,
3096
3187
  });
3097
- // TODO: [๐Ÿš] Make arrayable LLMs -> single LLM DRY
3098
3188
  const _llms = arrayableToArray(tools.llm);
3099
3189
  const llmTools = _llms.length === 1 ? _llms[0] : joinLlmExecutionTools(..._llms);
3100
- const availableModels = await llmTools.listModels();
3101
- const availableModelNames = availableModels
3190
+ const availableModels = (await llmTools.listModels())
3102
3191
  .filter(({ modelVariant }) => modelVariant === 'CHAT')
3103
- .map(({ modelName }) => modelName)
3104
- .join(',');
3105
- const result = await preparePersonaExecutor({ availableModelNames, personaDescription }).asPromise();
3192
+ .map(({ modelName, modelDescription }) => ({
3193
+ modelName,
3194
+ modelDescription,
3195
+ // <- Note: `modelTitle` and `modelVariant` is not relevant for this task
3196
+ }));
3197
+ const result = await preparePersonaExecutor({
3198
+ availableModels /* <- Note: Passing as JSON */,
3199
+ personaDescription,
3200
+ }).asPromise();
3106
3201
  const { outputParameters } = result;
3107
- const { modelRequirements: modelRequirementsRaw } = outputParameters;
3108
- const modelRequirements = JSON.parse(modelRequirementsRaw);
3202
+ const { modelsRequirements: modelsRequirementsJson } = outputParameters;
3203
+ let modelsRequirementsUnchecked = jsonParse(modelsRequirementsJson);
3109
3204
  if (isVerbose) {
3110
- console.info(`PERSONA ${personaDescription}`, modelRequirements);
3205
+ console.info(`PERSONA ${personaDescription}`, modelsRequirementsUnchecked);
3111
3206
  }
3112
- const { modelName, systemMessage, temperature } = modelRequirements;
3113
- return {
3207
+ if (!Array.isArray(modelsRequirementsUnchecked)) {
3208
+ // <- TODO: Book should have syntax and system to enforce shape of JSON
3209
+ modelsRequirementsUnchecked = [modelsRequirementsUnchecked];
3210
+ /*
3211
+ throw new UnexpectedError(
3212
+ spaceTrim(
3213
+ (block) => `
3214
+ Invalid \`modelsRequirements\`:
3215
+
3216
+ \`\`\`json
3217
+ ${block(JSON.stringify(modelsRequirementsUnchecked, null, 4))}
3218
+ \`\`\`
3219
+ `,
3220
+ ),
3221
+ );
3222
+ */
3223
+ }
3224
+ const modelsRequirements = modelsRequirementsUnchecked.map((modelRequirements) => ({
3114
3225
  modelVariant: 'CHAT',
3115
- modelName,
3116
- systemMessage,
3117
- temperature,
3226
+ ...modelRequirements,
3227
+ }));
3228
+ return {
3229
+ modelsRequirements,
3118
3230
  };
3119
3231
  }
3120
3232
  /**
@@ -3413,9 +3525,7 @@
3413
3525
  return await fetch(urlOrRequest, init);
3414
3526
  }
3415
3527
  catch (error) {
3416
- if (!(error instanceof Error)) {
3417
- throw error;
3418
- }
3528
+ assertsError(error);
3419
3529
  let url;
3420
3530
  if (typeof urlOrRequest === 'string') {
3421
3531
  url = urlOrRequest;
@@ -3544,7 +3654,7 @@
3544
3654
  > },
3545
3655
  */
3546
3656
  async asJson() {
3547
- return JSON.parse(await tools.fs.readFile(filename, 'utf-8'));
3657
+ return jsonParse(await tools.fs.readFile(filename, 'utf-8'));
3548
3658
  },
3549
3659
  async asText() {
3550
3660
  return await tools.fs.readFile(filename, 'utf-8');
@@ -3646,9 +3756,7 @@
3646
3756
  knowledgePreparedUnflatten[index] = pieces;
3647
3757
  }
3648
3758
  catch (error) {
3649
- if (!(error instanceof Error)) {
3650
- throw error;
3651
- }
3759
+ assertsError(error);
3652
3760
  console.warn(error);
3653
3761
  // <- TODO: [๐Ÿฎ] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
3654
3762
  }
@@ -3804,14 +3912,14 @@
3804
3912
  // TODO: [๐Ÿ–Œ][๐Ÿง ] Implement some `mapAsync` function
3805
3913
  const preparedPersonas = new Array(personas.length);
3806
3914
  await forEachAsync(personas, { maxParallelCount /* <- TODO: [๐Ÿช‚] When there are subtasks, this maximul limit can be broken */ }, async (persona, index) => {
3807
- const modelRequirements = await preparePersona(persona.description, { ...tools, llm: llmToolsWithUsage }, {
3915
+ const { modelsRequirements } = await preparePersona(persona.description, { ...tools, llm: llmToolsWithUsage }, {
3808
3916
  rootDirname,
3809
3917
  maxParallelCount /* <- TODO: [๐Ÿช‚] */,
3810
3918
  isVerbose,
3811
3919
  });
3812
3920
  const preparedPersona = {
3813
3921
  ...persona,
3814
- modelRequirements,
3922
+ modelsRequirements,
3815
3923
  preparationIds: [/* TODO: [๐ŸงŠ] -> */ currentPreparation.id],
3816
3924
  // <- TODO: [๐Ÿ™] Make some standard order of json properties
3817
3925
  };
@@ -3940,13 +4048,19 @@
3940
4048
  return value.toISOString();
3941
4049
  }
3942
4050
  else {
3943
- return JSON.stringify(value);
4051
+ try {
4052
+ return JSON.stringify(value);
4053
+ }
4054
+ catch (error) {
4055
+ if (error instanceof TypeError && error.message.includes('circular structure')) {
4056
+ return VALUE_STRINGS.circular;
4057
+ }
4058
+ throw error;
4059
+ }
3944
4060
  }
3945
4061
  }
3946
4062
  catch (error) {
3947
- if (!(error instanceof Error)) {
3948
- throw error;
3949
- }
4063
+ assertsError(error);
3950
4064
  console.error(error);
3951
4065
  return VALUE_STRINGS.unserializable;
3952
4066
  }
@@ -4003,9 +4117,7 @@
4003
4117
  }
4004
4118
  }
4005
4119
  catch (error) {
4006
- if (!(error instanceof Error)) {
4007
- throw error;
4008
- }
4120
+ assertsError(error);
4009
4121
  throw new ParseError(spaceTrim.spaceTrim((block) => `
4010
4122
  Can not extract variables from the script
4011
4123
  ${block(error.stack || error.message)}
@@ -4124,6 +4236,46 @@
4124
4236
  // encoding: 'utf-8',
4125
4237
  });
4126
4238
 
4239
+ /**
4240
+ * Function to check if a string is valid CSV
4241
+ *
4242
+ * @param value The string to check
4243
+ * @returns True if the string is a valid CSV string, false otherwise
4244
+ *
4245
+ * @public exported from `@promptbook/utils`
4246
+ */
4247
+ function isValidCsvString(value) {
4248
+ try {
4249
+ // A simple check for CSV format: at least one comma and no invalid characters
4250
+ if (value.includes(',') && /^[\w\s,"']+$/.test(value)) {
4251
+ return true;
4252
+ }
4253
+ return false;
4254
+ }
4255
+ catch (error) {
4256
+ assertsError(error);
4257
+ return false;
4258
+ }
4259
+ }
4260
+
4261
+ /**
4262
+ * Converts a CSV string into an object
4263
+ *
4264
+ * Note: This is wrapper around `papaparse.parse()` with better autohealing
4265
+ *
4266
+ * @private - for now until `@promptbook/csv` is released
4267
+ */
4268
+ function csvParse(value /* <- TODO: string_csv */, settings, schema /* <- TODO: Make CSV Schemas */) {
4269
+ settings = { ...settings, ...MANDATORY_CSV_SETTINGS };
4270
+ // Note: Autoheal invalid '\n' characters
4271
+ if (settings.newline && !settings.newline.includes('\r') && value.includes('\r')) {
4272
+ console.warn('CSV string contains carriage return characters, but in the CSV settings the `newline` setting does not include them. Autohealing the CSV string.');
4273
+ value = value.replace(/\r\n/g, '\n').replace(/\r/g, '\n');
4274
+ }
4275
+ const csv = papaparse.parse(value, settings);
4276
+ return csv;
4277
+ }
4278
+
4127
4279
  /**
4128
4280
  * Definition for CSV spreadsheet
4129
4281
  *
@@ -4134,7 +4286,7 @@
4134
4286
  formatName: 'CSV',
4135
4287
  aliases: ['SPREADSHEET', 'TABLE'],
4136
4288
  isValid(value, settings, schema) {
4137
- return true;
4289
+ return isValidCsvString(value);
4138
4290
  },
4139
4291
  canBeValid(partialValue, settings, schema) {
4140
4292
  return true;
@@ -4146,8 +4298,7 @@
4146
4298
  {
4147
4299
  subvalueName: 'ROW',
4148
4300
  async mapValues(value, outputParameterName, settings, mapCallback) {
4149
- // TODO: [๐Ÿ‘จ๐Ÿพโ€๐Ÿคโ€๐Ÿ‘จ๐Ÿผ] DRY csv parsing
4150
- const csv = papaparse.parse(value, { ...settings, ...MANDATORY_CSV_SETTINGS });
4301
+ const csv = csvParse(value, settings);
4151
4302
  if (csv.errors.length !== 0) {
4152
4303
  throw new CsvFormatError(spaceTrim__default["default"]((block) => `
4153
4304
  CSV parsing error
@@ -4177,8 +4328,7 @@
4177
4328
  {
4178
4329
  subvalueName: 'CELL',
4179
4330
  async mapValues(value, outputParameterName, settings, mapCallback) {
4180
- // TODO: [๐Ÿ‘จ๐Ÿพโ€๐Ÿคโ€๐Ÿ‘จ๐Ÿผ] DRY csv parsing
4181
- const csv = papaparse.parse(value, { ...settings, ...MANDATORY_CSV_SETTINGS });
4331
+ const csv = csvParse(value, settings);
4182
4332
  if (csv.errors.length !== 0) {
4183
4333
  throw new CsvFormatError(spaceTrim__default["default"]((block) => `
4184
4334
  CSV parsing error
@@ -4288,6 +4438,30 @@
4288
4438
  * TODO: [๐Ÿข] Allow to expect something inside each item of list and other formats
4289
4439
  */
4290
4440
 
4441
+ /**
4442
+ * Function to check if a string is valid XML
4443
+ *
4444
+ * @param value
4445
+ * @returns True if the string is a valid XML string, false otherwise
4446
+ *
4447
+ * @public exported from `@promptbook/utils`
4448
+ */
4449
+ function isValidXmlString(value) {
4450
+ try {
4451
+ const parser = new DOMParser();
4452
+ const parsedDocument = parser.parseFromString(value, 'application/xml');
4453
+ const parserError = parsedDocument.getElementsByTagName('parsererror');
4454
+ if (parserError.length > 0) {
4455
+ return false;
4456
+ }
4457
+ return true;
4458
+ }
4459
+ catch (error) {
4460
+ assertsError(error);
4461
+ return false;
4462
+ }
4463
+ }
4464
+
4291
4465
  /**
4292
4466
  * Definition for XML format
4293
4467
  *
@@ -4297,7 +4471,7 @@
4297
4471
  formatName: 'XML',
4298
4472
  mimeType: 'application/xml',
4299
4473
  isValid(value, settings, schema) {
4300
- return true;
4474
+ return isValidXmlString(value);
4301
4475
  },
4302
4476
  canBeValid(partialValue, settings, schema) {
4303
4477
  return true;
@@ -4870,9 +5044,7 @@
4870
5044
  break scripts;
4871
5045
  }
4872
5046
  catch (error) {
4873
- if (!(error instanceof Error)) {
4874
- throw error;
4875
- }
5047
+ assertsError(error);
4876
5048
  if (error instanceof UnexpectedError) {
4877
5049
  throw error;
4878
5050
  }
@@ -4942,9 +5114,7 @@
4942
5114
  break scripts;
4943
5115
  }
4944
5116
  catch (error) {
4945
- if (!(error instanceof Error)) {
4946
- throw error;
4947
- }
5117
+ assertsError(error);
4948
5118
  if (error instanceof UnexpectedError) {
4949
5119
  throw error;
4950
5120
  }
@@ -5187,13 +5357,79 @@
5187
5357
  /**
5188
5358
  * @@@
5189
5359
  *
5360
+ * Here is the place where RAG (retrieval-augmented generation) happens
5361
+ *
5190
5362
  * @private internal utility of `createPipelineExecutor`
5191
5363
  */
5192
5364
  async function getKnowledgeForTask(options) {
5193
- const { preparedPipeline, task } = options;
5194
- return preparedPipeline.knowledgePieces.map(({ content }) => `- ${content}`).join('\n');
5365
+ const { tools, preparedPipeline, task } = options;
5366
+ const firstKnowlegePiece = preparedPipeline.knowledgePieces[0];
5367
+ const firstKnowlegeIndex = firstKnowlegePiece === null || firstKnowlegePiece === void 0 ? void 0 : firstKnowlegePiece.index[0];
5368
+ // <- TODO: Do not use just first knowledge piece and first index to determine embedding model, use also keyword search
5369
+ if (firstKnowlegePiece === undefined || firstKnowlegeIndex === undefined) {
5370
+ return 'No knowledge pieces found';
5371
+ }
5372
+ // TODO: [๐Ÿš] Make arrayable LLMs -> single LLM DRY
5373
+ const _llms = arrayableToArray(tools.llm);
5374
+ const llmTools = _llms.length === 1 ? _llms[0] : joinLlmExecutionTools(..._llms);
5375
+ const taskEmbeddingPrompt = {
5376
+ title: 'Knowledge Search',
5377
+ modelRequirements: {
5378
+ modelVariant: 'EMBEDDING',
5379
+ modelName: firstKnowlegeIndex.modelName,
5380
+ },
5381
+ content: task.content,
5382
+ parameters: {
5383
+ /* !!!!!!!! */
5384
+ },
5385
+ };
5386
+ const taskEmbeddingResult = await llmTools.callEmbeddingModel(taskEmbeddingPrompt);
5387
+ const knowledgePiecesWithRelevance = preparedPipeline.knowledgePieces.map((knowledgePiece) => {
5388
+ const { index } = knowledgePiece;
5389
+ const knowledgePieceIndex = index.find((i) => i.modelName === firstKnowlegeIndex.modelName);
5390
+ // <- TODO: Do not use just first knowledge piece and first index to determine embedding model
5391
+ if (knowledgePieceIndex === undefined) {
5392
+ return {
5393
+ content: knowledgePiece.content,
5394
+ relevance: 0,
5395
+ };
5396
+ }
5397
+ const relevance = computeCosineSimilarity(knowledgePieceIndex.position, taskEmbeddingResult.content);
5398
+ return {
5399
+ content: knowledgePiece.content,
5400
+ relevance,
5401
+ };
5402
+ });
5403
+ const knowledgePiecesSorted = knowledgePiecesWithRelevance.sort((a, b) => a.relevance - b.relevance);
5404
+ const knowledgePiecesLimited = knowledgePiecesSorted.slice(0, 5);
5405
+ console.log('!!! Embedding', {
5406
+ task,
5407
+ taskEmbeddingPrompt,
5408
+ taskEmbeddingResult,
5409
+ firstKnowlegePiece,
5410
+ firstKnowlegeIndex,
5411
+ knowledgePiecesWithRelevance,
5412
+ knowledgePiecesSorted,
5413
+ knowledgePiecesLimited,
5414
+ });
5415
+ return knowledgePiecesLimited.map(({ content }) => `- ${content}`).join('\n');
5195
5416
  // <- TODO: [๐Ÿง ] Some smart aggregation of knowledge pieces, single-line vs multi-line vs mixed
5196
5417
  }
5418
+ // TODO: !!!!!! Annotate + to new file
5419
+ function computeCosineSimilarity(embeddingVector1, embeddingVector2) {
5420
+ if (embeddingVector1.length !== embeddingVector2.length) {
5421
+ throw new TypeError('Embedding vectors must have the same length');
5422
+ }
5423
+ const dotProduct = embeddingVector1.reduce((sum, value, index) => sum + value * embeddingVector2[index], 0);
5424
+ const magnitude1 = Math.sqrt(embeddingVector1.reduce((sum, value) => sum + value * value, 0));
5425
+ const magnitude2 = Math.sqrt(embeddingVector2.reduce((sum, value) => sum + value * value, 0));
5426
+ return 1 - dotProduct / (magnitude1 * magnitude2);
5427
+ }
5428
+ /**
5429
+ * TODO: !!!! Verify if this is working
5430
+ * TODO: [โ™จ] Implement Better - use keyword search
5431
+ * TODO: [โ™จ] Examples of values
5432
+ */
5197
5433
 
5198
5434
  /**
5199
5435
  * @@@
@@ -5201,9 +5437,9 @@
5201
5437
  * @private internal utility of `createPipelineExecutor`
5202
5438
  */
5203
5439
  async function getReservedParametersForTask(options) {
5204
- const { preparedPipeline, task, pipelineIdentification } = options;
5440
+ const { tools, preparedPipeline, task, pipelineIdentification } = options;
5205
5441
  const context = await getContextForTask(); // <- [๐Ÿ]
5206
- const knowledge = await getKnowledgeForTask({ preparedPipeline, task });
5442
+ const knowledge = await getKnowledgeForTask({ tools, preparedPipeline, task });
5207
5443
  const examples = await getExamplesForTask();
5208
5444
  const currentDate = new Date().toISOString(); // <- TODO: [๐Ÿง ][๐Ÿ’ฉ] Better
5209
5445
  const modelName = RESERVED_PARAMETER_MISSING_VALUE;
@@ -5265,6 +5501,7 @@
5265
5501
  }
5266
5502
  const definedParameters = Object.freeze({
5267
5503
  ...(await getReservedParametersForTask({
5504
+ tools,
5268
5505
  preparedPipeline,
5269
5506
  task: currentTask,
5270
5507
  pipelineIdentification,
@@ -5565,9 +5802,7 @@
5565
5802
  await Promise.all(resolving);
5566
5803
  }
5567
5804
  catch (error /* <- Note: [3] */) {
5568
- if (!(error instanceof Error)) {
5569
- throw error;
5570
- }
5805
+ assertsError(error);
5571
5806
  // Note: No need to rethrow UnexpectedError
5572
5807
  // if (error instanceof UnexpectedError) {
5573
5808
  // Note: Count usage, [๐Ÿง ] Maybe put to separate function executionReportJsonToUsage + DRY [๐Ÿคนโ€โ™‚๏ธ]