@promptbook/documents 0.89.0-9 โ†’ 0.92.0-10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. package/README.md +9 -7
  2. package/esm/index.es.js +303 -68
  3. package/esm/index.es.js.map +1 -1
  4. package/esm/typings/servers.d.ts +40 -0
  5. package/esm/typings/src/_packages/core.index.d.ts +14 -4
  6. package/esm/typings/src/_packages/deepseek.index.d.ts +2 -0
  7. package/esm/typings/src/_packages/google.index.d.ts +2 -0
  8. package/esm/typings/src/_packages/types.index.d.ts +18 -0
  9. package/esm/typings/src/_packages/utils.index.d.ts +6 -0
  10. package/esm/typings/src/cli/cli-commands/login.d.ts +0 -1
  11. package/esm/typings/src/cli/common/$provideLlmToolsForCli.d.ts +16 -3
  12. package/esm/typings/src/cli/test/ptbk.d.ts +1 -1
  13. package/esm/typings/src/commands/EXPECT/expectCommandParser.d.ts +2 -0
  14. package/esm/typings/src/config.d.ts +10 -19
  15. package/esm/typings/src/conversion/archive/loadArchive.d.ts +2 -2
  16. package/esm/typings/src/errors/0-index.d.ts +7 -4
  17. package/esm/typings/src/errors/PipelineExecutionError.d.ts +1 -1
  18. package/esm/typings/src/errors/WrappedError.d.ts +10 -0
  19. package/esm/typings/src/errors/assertsError.d.ts +11 -0
  20. package/esm/typings/src/execution/CommonToolsOptions.d.ts +4 -0
  21. package/esm/typings/src/execution/PromptbookFetch.d.ts +1 -1
  22. package/esm/typings/src/execution/createPipelineExecutor/getKnowledgeForTask.d.ts +12 -0
  23. package/esm/typings/src/execution/createPipelineExecutor/getReservedParametersForTask.d.ts +5 -0
  24. package/esm/typings/src/formats/csv/utils/csvParse.d.ts +12 -0
  25. package/esm/typings/src/formats/csv/utils/isValidCsvString.d.ts +9 -0
  26. package/esm/typings/src/formats/csv/utils/isValidCsvString.test.d.ts +1 -0
  27. package/esm/typings/src/formats/json/utils/isValidJsonString.d.ts +3 -0
  28. package/esm/typings/src/formats/json/utils/jsonParse.d.ts +11 -0
  29. package/esm/typings/src/formats/xml/utils/isValidXmlString.d.ts +9 -0
  30. package/esm/typings/src/formats/xml/utils/isValidXmlString.test.d.ts +1 -0
  31. package/esm/typings/src/llm-providers/_common/filterModels.d.ts +15 -0
  32. package/esm/typings/src/llm-providers/_common/register/{$provideEnvFilepath.d.ts โ†’ $provideEnvFilename.d.ts} +2 -2
  33. package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsConfigurationFromEnv.d.ts +1 -1
  34. package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground.d.ts +1 -1
  35. package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsForWizzardOrCli.d.ts +11 -2
  36. package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsFromEnv.d.ts +1 -1
  37. package/esm/typings/src/llm-providers/_common/register/LlmToolsMetadata.d.ts +43 -0
  38. package/esm/typings/src/llm-providers/azure-openai/AzureOpenAiExecutionTools.d.ts +4 -0
  39. package/esm/typings/src/llm-providers/deepseek/deepseek-models.d.ts +23 -0
  40. package/esm/typings/src/llm-providers/google/google-models.d.ts +23 -0
  41. package/esm/typings/src/llm-providers/openai/OpenAiExecutionTools.d.ts +4 -0
  42. package/esm/typings/src/personas/preparePersona.d.ts +1 -1
  43. package/esm/typings/src/pipeline/PipelineJson/PersonaJson.d.ts +4 -2
  44. package/esm/typings/src/remote-server/openapi-types.d.ts +626 -0
  45. package/esm/typings/src/remote-server/openapi.d.ts +581 -0
  46. package/esm/typings/src/remote-server/socket-types/_subtypes/Identification.d.ts +7 -1
  47. package/esm/typings/src/remote-server/socket-types/_subtypes/identificationToPromptbookToken.d.ts +11 -0
  48. package/esm/typings/src/remote-server/socket-types/_subtypes/promptbookTokenToIdentification.d.ts +10 -0
  49. package/esm/typings/src/remote-server/startRemoteServer.d.ts +1 -2
  50. package/esm/typings/src/remote-server/types/RemoteServerOptions.d.ts +15 -9
  51. package/esm/typings/src/storage/env-storage/$EnvStorage.d.ts +40 -0
  52. package/esm/typings/src/types/typeAliases.d.ts +26 -0
  53. package/package.json +8 -4
  54. package/umd/index.umd.js +303 -68
  55. package/umd/index.umd.js.map +1 -1
  56. package/esm/typings/src/cli/test/ptbk2.d.ts +0 -5
package/esm/index.es.js CHANGED
@@ -28,7 +28,7 @@ const BOOK_LANGUAGE_VERSION = '1.0.0';
28
28
  * @generated
29
29
  * @see https://github.com/webgptorg/promptbook
30
30
  */
31
- const PROMPTBOOK_ENGINE_VERSION = '0.89.0-9';
31
+ const PROMPTBOOK_ENGINE_VERSION = '0.92.0-10';
32
32
  /**
33
33
  * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
34
34
  * Note: [๐Ÿ’ž] Ignore a discrepancy between file name and entity name
@@ -91,6 +91,7 @@ const ADMIN_EMAIL = 'pavol@ptbk.io';
91
91
  * @public exported from `@promptbook/core`
92
92
  */
93
93
  const ADMIN_GITHUB_NAME = 'hejny';
94
+ // <- TODO: [๐ŸŠ] Pick the best claim
94
95
  /**
95
96
  * When the title is not provided, the default title is used
96
97
  *
@@ -123,6 +124,7 @@ const VALUE_STRINGS = {
123
124
  infinity: '(infinity; โˆž)',
124
125
  negativeInfinity: '(negative infinity; -โˆž)',
125
126
  unserializable: '(unserializable value)',
127
+ circular: '(circular JSON)',
126
128
  };
127
129
  /**
128
130
  * Small number limit
@@ -162,7 +164,7 @@ const DEFAULT_MAX_PARALLEL_COUNT = 5; // <- TODO: [๐Ÿคนโ€โ™‚๏ธ]
162
164
  */
163
165
  const DEFAULT_MAX_EXECUTION_ATTEMPTS = 10; // <- TODO: [๐Ÿคนโ€โ™‚๏ธ]
164
166
  // <- TODO: [๐Ÿ•] Make also `BOOKS_DIRNAME_ALTERNATIVES`
165
- // TODO: !!!!!! Just .promptbook dir, hardocode others
167
+ // TODO: Just `.promptbook` in config, hardcode subfolders like `download-cache` or `execution-cache`
166
168
  /**
167
169
  * Where to store the temporary downloads
168
170
  *
@@ -1037,7 +1039,7 @@ async function getScraperIntermediateSource(source, options) {
1037
1039
  * Note: [๐ŸŸข] Code in this file should never be never released in packages that could be imported into browser environment
1038
1040
  */
1039
1041
 
1040
- var PipelineCollection = [{title:"Prepare Knowledge from Markdown",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.book",formfactorName:"GENERIC",parameters:[{name:"knowledgeContent",description:"Markdown document content",isInput:true,isOutput:false},{name:"knowledgePieces",description:"The knowledge JSON object",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, extract the important knowledge from the document.\n\n# Rules\n\n- Make pieces of information concise, clear, and easy to understand\n- One piece of information should be approximately 1 paragraph\n- Divide the paragraphs by markdown horizontal lines ---\n- Omit irrelevant information\n- Group redundant information\n- Write just extracted information, nothing else\n\n# The document\n\nTake information from this document:\n\n> {knowledgeContent}",resultingParameterName:"knowledgePieces",dependentParameterNames:["knowledgeContent"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Knowledge from Markdown\n\n- PIPELINE URL `https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.book`\n- INPUT PARAMETER `{knowledgeContent}` Markdown document content\n- OUTPUT PARAMETER `{knowledgePieces}` The knowledge JSON object\n\n## Knowledge\n\n<!-- TODO: [๐Ÿ†] -FORMAT JSON -->\n\n```markdown\nYou are experienced data researcher, extract the important knowledge from the document.\n\n# Rules\n\n- Make pieces of information concise, clear, and easy to understand\n- One piece of information should be approximately 1 paragraph\n- Divide the paragraphs by markdown horizontal lines ---\n- Omit irrelevant information\n- Group redundant information\n- Write just extracted information, nothing else\n\n# The document\n\nTake information from this document:\n\n> {knowledgeContent}\n```\n\n`-> {knowledgePieces}`\n"}],sourceFile:"./books/prepare-knowledge-from-markdown.book"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-keywords.book",formfactorName:"GENERIC",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"keywords",description:"Keywords separated by comma",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, detect the important keywords in the document.\n\n# Rules\n\n- Write just keywords separated by comma\n\n# The document\n\nTake information from this document:\n\n> {knowledgePieceContent}",resultingParameterName:"keywords",dependentParameterNames:["knowledgePieceContent"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Keywords\n\n- PIPELINE URL `https://promptbook.studio/promptbook/prepare-knowledge-keywords.book`\n- INPUT PARAMETER `{knowledgePieceContent}` The content\n- OUTPUT PARAMETER `{keywords}` Keywords separated by comma\n\n## Knowledge\n\n<!-- TODO: [๐Ÿ†] -FORMAT JSON -->\n\n```markdown\nYou are experienced data researcher, detect the important keywords in the document.\n\n# Rules\n\n- Write just keywords separated by comma\n\n# The document\n\nTake information from this document:\n\n> {knowledgePieceContent}\n```\n\n`-> {keywords}`\n"}],sourceFile:"./books/prepare-knowledge-keywords.book"},{title:"Prepare Knowledge-piece Title",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-title.book",formfactorName:"GENERIC",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"title",description:"The title of the document",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"knowledge",title:"Knowledge",content:"You are experienced content creator, write best title for the document.\n\n# Rules\n\n- Write just title, nothing else\n- Write maximum 5 words for the title\n\n# The document\n\n> {knowledgePieceContent}",resultingParameterName:"title",expectations:{words:{min:1,max:8}},dependentParameterNames:["knowledgePieceContent"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Knowledge-piece Title\n\n- PIPELINE URL `https://promptbook.studio/promptbook/prepare-knowledge-title.book`\n- INPUT PARAMETER `{knowledgePieceContent}` The content\n- OUTPUT PARAMETER `{title}` The title of the document\n\n## Knowledge\n\n- EXPECT MIN 1 WORD\n- EXPECT MAX 8 WORDS\n\n```markdown\nYou are experienced content creator, write best title for the document.\n\n# Rules\n\n- Write just title, nothing else\n- Write maximum 5 words for the title\n\n# The document\n\n> {knowledgePieceContent}\n```\n\n`-> {title}`\n"}],sourceFile:"./books/prepare-knowledge-title.book"},{title:"Prepare Persona",pipelineUrl:"https://promptbook.studio/promptbook/prepare-persona.book",formfactorName:"GENERIC",parameters:[{name:"availableModelNames",description:"List of available model names separated by comma (,)",isInput:true,isOutput:false},{name:"personaDescription",description:"Description of the persona",isInput:true,isOutput:false},{name:"modelRequirements",description:"Specific requirements for the model",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"make-model-requirements",title:"Make modelRequirements",content:"You are experienced AI engineer, you need to create virtual assistant.\nWrite\n\n## Example\n\n```json\n{\n\"modelName\": \"gpt-4o\",\n\"systemMessage\": \"You are experienced AI engineer and helpfull assistant.\",\n\"temperature\": 0.7\n}\n```\n\n## Instructions\n\n- Your output format is JSON object\n- Write just the JSON object, no other text should be present\n- It contains the following keys:\n - `modelName`: The name of the model to use\n - `systemMessage`: The system message to provide context to the model\n - `temperature`: The sampling temperature to use\n\n### Key `modelName`\n\nPick from the following models:\n\n- {availableModelNames}\n\n### Key `systemMessage`\n\nThe system message is used to communicate instructions or provide context to the model at the beginning of a conversation. It is displayed in a different format compared to user messages, helping the model understand its role in the conversation. The system message typically guides the model's behavior, sets the tone, or specifies desired output from the model. By utilizing the system message effectively, users can steer the model towards generating more accurate and relevant responses.\n\nFor example:\n\n> You are an experienced AI engineer and helpful assistant.\n\n> You are a friendly and knowledgeable chatbot.\n\n### Key `temperature`\n\nThe sampling temperature, between 0 and 1. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. If set to 0, the model will use log probability to automatically increase the temperature until certain thresholds are hit.\n\nYou can pick a value between 0 and 2. For example:\n\n- `0.1`: Low temperature, extremely conservative and deterministic\n- `0.5`: Medium temperature, balanced between conservative and creative\n- `1.0`: High temperature, creative and bit random\n- `1.5`: Very high temperature, extremely creative and often chaotic and unpredictable\n- `2.0`: Maximum temperature, completely random and unpredictable, for some extreme creative use cases\n\n# The assistant\n\nTake this description of the persona:\n\n> {personaDescription}",resultingParameterName:"modelRequirements",format:"JSON",dependentParameterNames:["availableModelNames","personaDescription"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Persona\n\n- PIPELINE URL `https://promptbook.studio/promptbook/prepare-persona.book`\n- INPUT PARAMETER `{availableModelNames}` List of available model names separated by comma (,)\n- INPUT PARAMETER `{personaDescription}` Description of the persona\n- OUTPUT PARAMETER `{modelRequirements}` Specific requirements for the model\n\n## Make modelRequirements\n\n- FORMAT JSON\n\n```markdown\nYou are experienced AI engineer, you need to create virtual assistant.\nWrite\n\n## Example\n\n\\`\\`\\`json\n{\n\"modelName\": \"gpt-4o\",\n\"systemMessage\": \"You are experienced AI engineer and helpfull assistant.\",\n\"temperature\": 0.7\n}\n\\`\\`\\`\n\n## Instructions\n\n- Your output format is JSON object\n- Write just the JSON object, no other text should be present\n- It contains the following keys:\n - `modelName`: The name of the model to use\n - `systemMessage`: The system message to provide context to the model\n - `temperature`: The sampling temperature to use\n\n### Key `modelName`\n\nPick from the following models:\n\n- {availableModelNames}\n\n### Key `systemMessage`\n\nThe system message is used to communicate instructions or provide context to the model at the beginning of a conversation. It is displayed in a different format compared to user messages, helping the model understand its role in the conversation. The system message typically guides the model's behavior, sets the tone, or specifies desired output from the model. By utilizing the system message effectively, users can steer the model towards generating more accurate and relevant responses.\n\nFor example:\n\n> You are an experienced AI engineer and helpful assistant.\n\n> You are a friendly and knowledgeable chatbot.\n\n### Key `temperature`\n\nThe sampling temperature, between 0 and 1. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. If set to 0, the model will use log probability to automatically increase the temperature until certain thresholds are hit.\n\nYou can pick a value between 0 and 2. For example:\n\n- `0.1`: Low temperature, extremely conservative and deterministic\n- `0.5`: Medium temperature, balanced between conservative and creative\n- `1.0`: High temperature, creative and bit random\n- `1.5`: Very high temperature, extremely creative and often chaotic and unpredictable\n- `2.0`: Maximum temperature, completely random and unpredictable, for some extreme creative use cases\n\n# The assistant\n\nTake this description of the persona:\n\n> {personaDescription}\n```\n\n`-> {modelRequirements}`\n"}],sourceFile:"./books/prepare-persona.book"},{title:"Prepare Title",pipelineUrl:"https://promptbook.studio/promptbook/prepare-title.book",formfactorName:"GENERIC",parameters:[{name:"book",description:"The book to prepare the title for",isInput:true,isOutput:false},{name:"title",description:"Best title for the book",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"make-title",title:"Make title",content:"Make best title for given text which describes the workflow:\n\n## Rules\n\n- Write just title, nothing else\n- Title should be concise and clear - Write maximum ideally 2 words, maximum 5 words\n- Title starts with emoticon\n- Title should not mention the input and output of the workflow but the main purpose of the workflow\n _For example, not \"โœ Convert Knowledge-piece to title\" but \"โœ Title\"_\n\n## The workflow\n\n> {book}",resultingParameterName:"title",expectations:{words:{min:1,max:8},lines:{min:1,max:1}},dependentParameterNames:["book"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Title\n\n- PIPELINE URL `https://promptbook.studio/promptbook/prepare-title.book`\n- INPUT PARAMETER `{book}` The book to prepare the title for\n- OUTPUT PARAMETER `{title}` Best title for the book\n\n## Make title\n\n- EXPECT MIN 1 Word\n- EXPECT MAX 8 Words\n- EXPECT EXACTLY 1 Line\n\n```markdown\nMake best title for given text which describes the workflow:\n\n## Rules\n\n- Write just title, nothing else\n- Title should be concise and clear - Write maximum ideally 2 words, maximum 5 words\n- Title starts with emoticon\n- Title should not mention the input and output of the workflow but the main purpose of the workflow\n _For example, not \"โœ Convert Knowledge-piece to title\" but \"โœ Title\"_\n\n## The workflow\n\n> {book}\n```\n\n`-> {title}`\n"}],sourceFile:"./books/prepare-title.book"}];
1042
+ var PipelineCollection = [{title:"Prepare Knowledge from Markdown",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.book",formfactorName:"GENERIC",parameters:[{name:"knowledgeContent",description:"Markdown document content",isInput:true,isOutput:false},{name:"knowledgePieces",description:"The knowledge JSON object",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, extract the important knowledge from the document.\n\n# Rules\n\n- Make pieces of information concise, clear, and easy to understand\n- One piece of information should be approximately 1 paragraph\n- Divide the paragraphs by markdown horizontal lines ---\n- Omit irrelevant information\n- Group redundant information\n- Write just extracted information, nothing else\n\n# The document\n\nTake information from this document:\n\n> {knowledgeContent}",resultingParameterName:"knowledgePieces",dependentParameterNames:["knowledgeContent"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Knowledge from Markdown\n\n- PIPELINE URL `https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.book`\n- INPUT PARAMETER `{knowledgeContent}` Markdown document content\n- OUTPUT PARAMETER `{knowledgePieces}` The knowledge JSON object\n\n## Knowledge\n\n<!-- TODO: [๐Ÿ†] -FORMAT JSON -->\n\n```markdown\nYou are experienced data researcher, extract the important knowledge from the document.\n\n# Rules\n\n- Make pieces of information concise, clear, and easy to understand\n- One piece of information should be approximately 1 paragraph\n- Divide the paragraphs by markdown horizontal lines ---\n- Omit irrelevant information\n- Group redundant information\n- Write just extracted information, nothing else\n\n# The document\n\nTake information from this document:\n\n> {knowledgeContent}\n```\n\n`-> {knowledgePieces}`\n"}],sourceFile:"./books/prepare-knowledge-from-markdown.book"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-keywords.book",formfactorName:"GENERIC",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"keywords",description:"Keywords separated by comma",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, detect the important keywords in the document.\n\n# Rules\n\n- Write just keywords separated by comma\n\n# The document\n\nTake information from this document:\n\n> {knowledgePieceContent}",resultingParameterName:"keywords",dependentParameterNames:["knowledgePieceContent"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Keywords\n\n- PIPELINE URL `https://promptbook.studio/promptbook/prepare-knowledge-keywords.book`\n- INPUT PARAMETER `{knowledgePieceContent}` The content\n- OUTPUT PARAMETER `{keywords}` Keywords separated by comma\n\n## Knowledge\n\n<!-- TODO: [๐Ÿ†] -FORMAT JSON -->\n\n```markdown\nYou are experienced data researcher, detect the important keywords in the document.\n\n# Rules\n\n- Write just keywords separated by comma\n\n# The document\n\nTake information from this document:\n\n> {knowledgePieceContent}\n```\n\n`-> {keywords}`\n"}],sourceFile:"./books/prepare-knowledge-keywords.book"},{title:"Prepare Knowledge-piece Title",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-title.book",formfactorName:"GENERIC",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"title",description:"The title of the document",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"knowledge",title:"Knowledge",content:"You are experienced content creator, write best title for the document.\n\n# Rules\n\n- Write just title, nothing else\n- Write maximum 5 words for the title\n\n# The document\n\n> {knowledgePieceContent}",resultingParameterName:"title",expectations:{words:{min:1,max:8}},dependentParameterNames:["knowledgePieceContent"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Knowledge-piece Title\n\n- PIPELINE URL `https://promptbook.studio/promptbook/prepare-knowledge-title.book`\n- INPUT PARAMETER `{knowledgePieceContent}` The content\n- OUTPUT PARAMETER `{title}` The title of the document\n\n## Knowledge\n\n- EXPECT MIN 1 WORD\n- EXPECT MAX 8 WORDS\n\n```markdown\nYou are experienced content creator, write best title for the document.\n\n# Rules\n\n- Write just title, nothing else\n- Write maximum 5 words for the title\n\n# The document\n\n> {knowledgePieceContent}\n```\n\n`-> {title}`\n"}],sourceFile:"./books/prepare-knowledge-title.book"},{title:"Prepare Persona",pipelineUrl:"https://promptbook.studio/promptbook/prepare-persona.book",formfactorName:"GENERIC",parameters:[{name:"availableModels",description:"List of available model names together with their descriptions as JSON",isInput:true,isOutput:false},{name:"personaDescription",description:"Description of the persona",isInput:true,isOutput:false},{name:"modelsRequirements",description:"Specific requirements for the model",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"make-model-requirements",title:"Make modelRequirements",content:"You are an experienced AI engineer, you need to find the best models for virtual assistants:\n\n## Example\n\n```json\n[\n {\n \"modelName\": \"gpt-4o\",\n \"systemMessage\": \"You are experienced AI engineer and helpfull assistant.\",\n \"temperature\": 0.7\n },\n {\n \"modelName\": \"claude-3-5-sonnet\",\n \"systemMessage\": \"You are a friendly and knowledgeable chatbot.\",\n \"temperature\": 0.5\n }\n]\n```\n\n## Instructions\n\n- Your output format is JSON array\n- Sort best-fitting models first\n- Omit any models that are not suitable\n- Write just the JSON, no other text should be present\n- Array contain items with following keys:\n - `modelName`: The name of the model to use\n - `systemMessage`: The system message to provide context to the model\n - `temperature`: The sampling temperature to use\n\n### Key `modelName`\n\nHere are the available models:\n\n```json\n{availableModels}\n```\n\n### Key `systemMessage`\n\nThe system message is used to communicate instructions or provide context to the model at the beginning of a conversation. It is displayed in a different format compared to user messages, helping the model understand its role in the conversation. The system message typically guides the model's behavior, sets the tone, or specifies desired output from the model. By utilizing the system message effectively, users can steer the model towards generating more accurate and relevant responses.\n\nFor example:\n\n> You are an experienced AI engineer and helpful assistant.\n\n> You are a friendly and knowledgeable chatbot.\n\n### Key `temperature`\n\nThe sampling temperature, between 0 and 1. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. If set to 0, the model will use log probability to automatically increase the temperature until certain thresholds are hit.\n\nYou can pick a value between 0 and 2. For example:\n\n- `0.1`: Low temperature, extremely conservative and deterministic\n- `0.5`: Medium temperature, balanced between conservative and creative\n- `1.0`: High temperature, creative and bit random\n- `1.5`: Very high temperature, extremely creative and often chaotic and unpredictable\n- `2.0`: Maximum temperature, completely random and unpredictable, for some extreme creative use cases\n\n# The assistant\n\nTake this description of the persona:\n\n> {personaDescription}",resultingParameterName:"modelsRequirements",format:"JSON",dependentParameterNames:["availableModels","personaDescription"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Persona\n\n- PIPELINE URL `https://promptbook.studio/promptbook/prepare-persona.book`\n- INPUT PARAMETER `{availableModels}` List of available model names together with their descriptions as JSON\n- INPUT PARAMETER `{personaDescription}` Description of the persona\n- OUTPUT PARAMETER `{modelsRequirements}` Specific requirements for the model\n\n## Make modelRequirements\n\n- FORMAT JSON\n\n```markdown\nYou are an experienced AI engineer, you need to find the best models for virtual assistants:\n\n## Example\n\n\\`\\`\\`json\n[\n {\n \"modelName\": \"gpt-4o\",\n \"systemMessage\": \"You are experienced AI engineer and helpfull assistant.\",\n \"temperature\": 0.7\n },\n {\n \"modelName\": \"claude-3-5-sonnet\",\n \"systemMessage\": \"You are a friendly and knowledgeable chatbot.\",\n \"temperature\": 0.5\n }\n]\n\\`\\`\\`\n\n## Instructions\n\n- Your output format is JSON array\n- Sort best-fitting models first\n- Omit any models that are not suitable\n- Write just the JSON, no other text should be present\n- Array contain items with following keys:\n - `modelName`: The name of the model to use\n - `systemMessage`: The system message to provide context to the model\n - `temperature`: The sampling temperature to use\n\n### Key `modelName`\n\nHere are the available models:\n\n\\`\\`\\`json\n{availableModels}\n\\`\\`\\`\n\n### Key `systemMessage`\n\nThe system message is used to communicate instructions or provide context to the model at the beginning of a conversation. It is displayed in a different format compared to user messages, helping the model understand its role in the conversation. The system message typically guides the model's behavior, sets the tone, or specifies desired output from the model. By utilizing the system message effectively, users can steer the model towards generating more accurate and relevant responses.\n\nFor example:\n\n> You are an experienced AI engineer and helpful assistant.\n\n> You are a friendly and knowledgeable chatbot.\n\n### Key `temperature`\n\nThe sampling temperature, between 0 and 1. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. If set to 0, the model will use log probability to automatically increase the temperature until certain thresholds are hit.\n\nYou can pick a value between 0 and 2. For example:\n\n- `0.1`: Low temperature, extremely conservative and deterministic\n- `0.5`: Medium temperature, balanced between conservative and creative\n- `1.0`: High temperature, creative and bit random\n- `1.5`: Very high temperature, extremely creative and often chaotic and unpredictable\n- `2.0`: Maximum temperature, completely random and unpredictable, for some extreme creative use cases\n\n# The assistant\n\nTake this description of the persona:\n\n> {personaDescription}\n```\n\n`-> {modelsRequirements}`\n"}],sourceFile:"./books/prepare-persona.book"},{title:"Prepare Title",pipelineUrl:"https://promptbook.studio/promptbook/prepare-title.book",formfactorName:"GENERIC",parameters:[{name:"book",description:"The book to prepare the title for",isInput:true,isOutput:false},{name:"title",description:"Best title for the book",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"make-title",title:"Make title",content:"Make best title for given text which describes the workflow:\n\n## Rules\n\n- Write just title, nothing else\n- Title should be concise and clear - Write maximum ideally 2 words, maximum 5 words\n- Title starts with emoticon\n- Title should not mention the input and output of the workflow but the main purpose of the workflow\n _For example, not \"โœ Convert Knowledge-piece to title\" but \"โœ Title\"_\n\n## The workflow\n\n> {book}",resultingParameterName:"title",expectations:{words:{min:1,max:8},lines:{min:1,max:1}},dependentParameterNames:["book"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Title\n\n- PIPELINE URL `https://promptbook.studio/promptbook/prepare-title.book`\n- INPUT PARAMETER `{book}` The book to prepare the title for\n- OUTPUT PARAMETER `{title}` Best title for the book\n\n## Make title\n\n- EXPECT MIN 1 Word\n- EXPECT MAX 8 Words\n- EXPECT EXACTLY 1 Line\n\n```markdown\nMake best title for given text which describes the workflow:\n\n## Rules\n\n- Write just title, nothing else\n- Title should be concise and clear - Write maximum ideally 2 words, maximum 5 words\n- Title starts with emoticon\n- Title should not mention the input and output of the workflow but the main purpose of the workflow\n _For example, not \"โœ Convert Knowledge-piece to title\" but \"โœ Title\"_\n\n## The workflow\n\n> {book}\n```\n\n`-> {title}`\n"}],sourceFile:"./books/prepare-title.book"}];
1041
1043
 
1042
1044
  /**
1043
1045
  * Checks if value is valid email
@@ -1070,9 +1072,60 @@ class ParseError extends Error {
1070
1072
  * TODO: Maybe split `ParseError` and `ApplyError`
1071
1073
  */
1072
1074
 
1075
+ /**
1076
+ * This error type indicates that somewhere in the code non-Error object was thrown and it was wrapped into the `WrappedError`
1077
+ *
1078
+ * @public exported from `@promptbook/core`
1079
+ */
1080
+ class WrappedError extends Error {
1081
+ constructor(whatWasThrown) {
1082
+ const tag = `[๐Ÿคฎ]`;
1083
+ console.error(tag, whatWasThrown);
1084
+ super(spaceTrim(`
1085
+ Non-Error object was thrown
1086
+
1087
+ Note: Look for ${tag} in the console for more details
1088
+ Please report issue on ${ADMIN_EMAIL}
1089
+ `));
1090
+ this.name = 'WrappedError';
1091
+ Object.setPrototypeOf(this, WrappedError.prototype);
1092
+ }
1093
+ }
1094
+
1095
+ /**
1096
+ * Helper used in catch blocks to assert that the error is an instance of `Error`
1097
+ *
1098
+ * @param whatWasThrown Any object that was thrown
1099
+ * @returns Nothing if the error is an instance of `Error`
1100
+ * @throws `WrappedError` or `UnexpectedError` if the error is not standard
1101
+ *
1102
+ * @private within the repository
1103
+ */
1104
+ function assertsError(whatWasThrown) {
1105
+ // Case 1: Handle error which was rethrown as `WrappedError`
1106
+ if (whatWasThrown instanceof WrappedError) {
1107
+ const wrappedError = whatWasThrown;
1108
+ throw wrappedError;
1109
+ }
1110
+ // Case 2: Handle unexpected errors
1111
+ if (whatWasThrown instanceof UnexpectedError) {
1112
+ const unexpectedError = whatWasThrown;
1113
+ throw unexpectedError;
1114
+ }
1115
+ // Case 3: Handle standard errors - keep them up to consumer
1116
+ if (whatWasThrown instanceof Error) {
1117
+ return;
1118
+ }
1119
+ // Case 4: Handle non-standard errors - wrap them into `WrappedError` and throw
1120
+ throw new WrappedError(whatWasThrown);
1121
+ }
1122
+
1073
1123
  /**
1074
1124
  * Function isValidJsonString will tell you if the string is valid JSON or not
1075
1125
  *
1126
+ * @param value The string to check
1127
+ * @returns True if the string is a valid JSON string, false otherwise
1128
+ *
1076
1129
  * @public exported from `@promptbook/utils`
1077
1130
  */
1078
1131
  function isValidJsonString(value /* <- [๐Ÿ‘จโ€โš–๏ธ] */) {
@@ -1081,9 +1134,7 @@ function isValidJsonString(value /* <- [๐Ÿ‘จโ€โš–๏ธ] */) {
1081
1134
  return true;
1082
1135
  }
1083
1136
  catch (error) {
1084
- if (!(error instanceof Error)) {
1085
- throw error;
1086
- }
1137
+ assertsError(error);
1087
1138
  if (error.message.includes('Unexpected token')) {
1088
1139
  return false;
1089
1140
  }
@@ -1436,9 +1487,7 @@ function checkSerializableAsJson(options) {
1436
1487
  JSON.stringify(value); // <- TODO: [0]
1437
1488
  }
1438
1489
  catch (error) {
1439
- if (!(error instanceof Error)) {
1440
- throw error;
1441
- }
1490
+ assertsError(error);
1442
1491
  throw new UnexpectedError(spaceTrim$1((block) => `
1443
1492
  \`${name}\` is not serializable
1444
1493
 
@@ -2033,7 +2082,7 @@ function extractParameterNames(template) {
2033
2082
  */
2034
2083
  function unpreparePipeline(pipeline) {
2035
2084
  let { personas, knowledgeSources, tasks } = pipeline;
2036
- personas = personas.map((persona) => ({ ...persona, modelRequirements: undefined, preparationIds: undefined }));
2085
+ personas = personas.map((persona) => ({ ...persona, modelsRequirements: undefined, preparationIds: undefined }));
2037
2086
  knowledgeSources = knowledgeSources.map((knowledgeSource) => ({ ...knowledgeSource, preparationIds: undefined }));
2038
2087
  tasks = tasks.map((task) => {
2039
2088
  let { dependentParameterNames } = task;
@@ -2209,7 +2258,7 @@ class PipelineExecutionError extends Error {
2209
2258
  }
2210
2259
  }
2211
2260
  /**
2212
- * TODO: !!!!!! Add id to all errors
2261
+ * TODO: [๐Ÿง ][๐ŸŒ‚] Add id to all errors
2213
2262
  */
2214
2263
 
2215
2264
  /**
@@ -2225,7 +2274,7 @@ function isPipelinePrepared(pipeline) {
2225
2274
  if (pipeline.title === undefined || pipeline.title === '' || pipeline.title === DEFAULT_BOOK_TITLE) {
2226
2275
  return false;
2227
2276
  }
2228
- if (!pipeline.personas.every((persona) => persona.modelRequirements !== undefined)) {
2277
+ if (!pipeline.personas.every((persona) => persona.modelsRequirements !== undefined)) {
2229
2278
  return false;
2230
2279
  }
2231
2280
  if (!pipeline.knowledgeSources.every((knowledgeSource) => knowledgeSource.preparationIds !== undefined)) {
@@ -2249,6 +2298,45 @@ function isPipelinePrepared(pipeline) {
2249
2298
  * - [โ™จ] Are tasks prepared
2250
2299
  */
2251
2300
 
2301
+ /**
2302
+ * Converts a JavaScript Object Notation (JSON) string into an object.
2303
+ *
2304
+ * Note: This is wrapper around `JSON.parse()` with better error and type handling
2305
+ *
2306
+ * @public exported from `@promptbook/utils`
2307
+ */
2308
+ function jsonParse(value) {
2309
+ if (value === undefined) {
2310
+ throw new Error(`Can not parse JSON from undefined value.`);
2311
+ }
2312
+ else if (typeof value !== 'string') {
2313
+ console.error('Can not parse JSON from non-string value.', { text: value });
2314
+ throw new Error(spaceTrim$1(`
2315
+ Can not parse JSON from non-string value.
2316
+
2317
+ The value type: ${typeof value}
2318
+ See more in console.
2319
+ `));
2320
+ }
2321
+ try {
2322
+ return JSON.parse(value);
2323
+ }
2324
+ catch (error) {
2325
+ if (!(error instanceof Error)) {
2326
+ throw error;
2327
+ }
2328
+ throw new Error(spaceTrim$1((block) => `
2329
+ ${block(error.message)}
2330
+
2331
+ The JSON text:
2332
+ ${block(value)}
2333
+ `));
2334
+ }
2335
+ }
2336
+ /**
2337
+ * TODO: !!!! Use in Promptbook.studio
2338
+ */
2339
+
2252
2340
  /**
2253
2341
  * Recursively converts JSON strings to JSON objects
2254
2342
 
@@ -2267,7 +2355,7 @@ function jsonStringsToJsons(object) {
2267
2355
  const newObject = { ...object };
2268
2356
  for (const [key, value] of Object.entries(object)) {
2269
2357
  if (typeof value === 'string' && isValidJsonString(value)) {
2270
- newObject[key] = JSON.parse(value);
2358
+ newObject[key] = jsonParse(value);
2271
2359
  }
2272
2360
  else {
2273
2361
  newObject[key] = jsonStringsToJsons(value);
@@ -2420,7 +2508,10 @@ const PROMPTBOOK_ERRORS = {
2420
2508
  PipelineExecutionError,
2421
2509
  PipelineLogicError,
2422
2510
  PipelineUrlError,
2511
+ AuthenticationError,
2512
+ PromptbookFetchError,
2423
2513
  UnexpectedError,
2514
+ WrappedError,
2424
2515
  // TODO: [๐Ÿช‘]> VersionMismatchError,
2425
2516
  };
2426
2517
  /**
@@ -2437,8 +2528,6 @@ const COMMON_JAVASCRIPT_ERRORS = {
2437
2528
  TypeError,
2438
2529
  URIError,
2439
2530
  AggregateError,
2440
- AuthenticationError,
2441
- PromptbookFetchError,
2442
2531
  /*
2443
2532
  Note: Not widely supported
2444
2533
  > InternalError,
@@ -2561,8 +2650,8 @@ function createTask(options) {
2561
2650
  updatedAt = new Date();
2562
2651
  errors.push(...executionResult.errors);
2563
2652
  warnings.push(...executionResult.warnings);
2564
- // <- TODO: !!! Only unique errors and warnings should be added (or filtered)
2565
- // TODO: [๐Ÿง ] !!! errors, warning, isSuccessful are redundant both in `ExecutionTask` and `ExecutionTask.currentValue`
2653
+ // <- TODO: [๐ŸŒ‚] Only unique errors and warnings should be added (or filtered)
2654
+ // TODO: [๐Ÿง ] !! errors, warning, isSuccessful are redundant both in `ExecutionTask` and `ExecutionTask.currentValue`
2566
2655
  // Also maybe move `ExecutionTask.currentValue.usage` -> `ExecutionTask.usage`
2567
2656
  // And delete `ExecutionTask.currentValue.preparedPipeline`
2568
2657
  assertsTaskSuccessful(executionResult);
@@ -2572,6 +2661,7 @@ function createTask(options) {
2572
2661
  partialResultSubject.next(executionResult);
2573
2662
  }
2574
2663
  catch (error) {
2664
+ assertsError(error);
2575
2665
  status = 'ERROR';
2576
2666
  errors.push(error);
2577
2667
  partialResultSubject.error(error);
@@ -2963,14 +3053,15 @@ class MultipleLlmExecutionTools {
2963
3053
  }
2964
3054
  }
2965
3055
  catch (error) {
2966
- if (!(error instanceof Error) || error instanceof UnexpectedError) {
3056
+ assertsError(error);
3057
+ if (error instanceof UnexpectedError) {
2967
3058
  throw error;
2968
3059
  }
2969
3060
  errors.push({ llmExecutionTools, error });
2970
3061
  }
2971
3062
  }
2972
3063
  if (errors.length === 1) {
2973
- throw errors[0];
3064
+ throw errors[0].error;
2974
3065
  }
2975
3066
  else if (errors.length > 1) {
2976
3067
  throw new PipelineExecutionError(
@@ -3096,27 +3187,48 @@ async function preparePersona(personaDescription, tools, options) {
3096
3187
  pipeline: await collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-persona.book'),
3097
3188
  tools,
3098
3189
  });
3099
- // TODO: [๐Ÿš] Make arrayable LLMs -> single LLM DRY
3100
3190
  const _llms = arrayableToArray(tools.llm);
3101
3191
  const llmTools = _llms.length === 1 ? _llms[0] : joinLlmExecutionTools(..._llms);
3102
- const availableModels = await llmTools.listModels();
3103
- const availableModelNames = availableModels
3192
+ const availableModels = (await llmTools.listModels())
3104
3193
  .filter(({ modelVariant }) => modelVariant === 'CHAT')
3105
- .map(({ modelName }) => modelName)
3106
- .join(',');
3107
- const result = await preparePersonaExecutor({ availableModelNames, personaDescription }).asPromise();
3194
+ .map(({ modelName, modelDescription }) => ({
3195
+ modelName,
3196
+ modelDescription,
3197
+ // <- Note: `modelTitle` and `modelVariant` is not relevant for this task
3198
+ }));
3199
+ const result = await preparePersonaExecutor({
3200
+ availableModels /* <- Note: Passing as JSON */,
3201
+ personaDescription,
3202
+ }).asPromise();
3108
3203
  const { outputParameters } = result;
3109
- const { modelRequirements: modelRequirementsRaw } = outputParameters;
3110
- const modelRequirements = JSON.parse(modelRequirementsRaw);
3204
+ const { modelsRequirements: modelsRequirementsJson } = outputParameters;
3205
+ let modelsRequirementsUnchecked = jsonParse(modelsRequirementsJson);
3111
3206
  if (isVerbose) {
3112
- console.info(`PERSONA ${personaDescription}`, modelRequirements);
3207
+ console.info(`PERSONA ${personaDescription}`, modelsRequirementsUnchecked);
3113
3208
  }
3114
- const { modelName, systemMessage, temperature } = modelRequirements;
3115
- return {
3209
+ if (!Array.isArray(modelsRequirementsUnchecked)) {
3210
+ // <- TODO: Book should have syntax and system to enforce shape of JSON
3211
+ modelsRequirementsUnchecked = [modelsRequirementsUnchecked];
3212
+ /*
3213
+ throw new UnexpectedError(
3214
+ spaceTrim(
3215
+ (block) => `
3216
+ Invalid \`modelsRequirements\`:
3217
+
3218
+ \`\`\`json
3219
+ ${block(JSON.stringify(modelsRequirementsUnchecked, null, 4))}
3220
+ \`\`\`
3221
+ `,
3222
+ ),
3223
+ );
3224
+ */
3225
+ }
3226
+ const modelsRequirements = modelsRequirementsUnchecked.map((modelRequirements) => ({
3116
3227
  modelVariant: 'CHAT',
3117
- modelName,
3118
- systemMessage,
3119
- temperature,
3228
+ ...modelRequirements,
3229
+ }));
3230
+ return {
3231
+ modelsRequirements,
3120
3232
  };
3121
3233
  }
3122
3234
  /**
@@ -3415,9 +3527,7 @@ const promptbookFetch = async (urlOrRequest, init) => {
3415
3527
  return await fetch(urlOrRequest, init);
3416
3528
  }
3417
3529
  catch (error) {
3418
- if (!(error instanceof Error)) {
3419
- throw error;
3420
- }
3530
+ assertsError(error);
3421
3531
  let url;
3422
3532
  if (typeof urlOrRequest === 'string') {
3423
3533
  url = urlOrRequest;
@@ -3546,7 +3656,7 @@ async function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
3546
3656
  > },
3547
3657
  */
3548
3658
  async asJson() {
3549
- return JSON.parse(await tools.fs.readFile(filename, 'utf-8'));
3659
+ return jsonParse(await tools.fs.readFile(filename, 'utf-8'));
3550
3660
  },
3551
3661
  async asText() {
3552
3662
  return await tools.fs.readFile(filename, 'utf-8');
@@ -3648,9 +3758,7 @@ async function prepareKnowledgePieces(knowledgeSources, tools, options) {
3648
3758
  knowledgePreparedUnflatten[index] = pieces;
3649
3759
  }
3650
3760
  catch (error) {
3651
- if (!(error instanceof Error)) {
3652
- throw error;
3653
- }
3761
+ assertsError(error);
3654
3762
  console.warn(error);
3655
3763
  // <- TODO: [๐Ÿฎ] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
3656
3764
  }
@@ -3806,14 +3914,14 @@ async function preparePipeline(pipeline, tools, options) {
3806
3914
  // TODO: [๐Ÿ–Œ][๐Ÿง ] Implement some `mapAsync` function
3807
3915
  const preparedPersonas = new Array(personas.length);
3808
3916
  await forEachAsync(personas, { maxParallelCount /* <- TODO: [๐Ÿช‚] When there are subtasks, this maximul limit can be broken */ }, async (persona, index) => {
3809
- const modelRequirements = await preparePersona(persona.description, { ...tools, llm: llmToolsWithUsage }, {
3917
+ const { modelsRequirements } = await preparePersona(persona.description, { ...tools, llm: llmToolsWithUsage }, {
3810
3918
  rootDirname,
3811
3919
  maxParallelCount /* <- TODO: [๐Ÿช‚] */,
3812
3920
  isVerbose,
3813
3921
  });
3814
3922
  const preparedPersona = {
3815
3923
  ...persona,
3816
- modelRequirements,
3924
+ modelsRequirements,
3817
3925
  preparationIds: [/* TODO: [๐ŸงŠ] -> */ currentPreparation.id],
3818
3926
  // <- TODO: [๐Ÿ™] Make some standard order of json properties
3819
3927
  };
@@ -3942,13 +4050,19 @@ function valueToString(value) {
3942
4050
  return value.toISOString();
3943
4051
  }
3944
4052
  else {
3945
- return JSON.stringify(value);
4053
+ try {
4054
+ return JSON.stringify(value);
4055
+ }
4056
+ catch (error) {
4057
+ if (error instanceof TypeError && error.message.includes('circular structure')) {
4058
+ return VALUE_STRINGS.circular;
4059
+ }
4060
+ throw error;
4061
+ }
3946
4062
  }
3947
4063
  }
3948
4064
  catch (error) {
3949
- if (!(error instanceof Error)) {
3950
- throw error;
3951
- }
4065
+ assertsError(error);
3952
4066
  console.error(error);
3953
4067
  return VALUE_STRINGS.unserializable;
3954
4068
  }
@@ -4005,9 +4119,7 @@ function extractVariablesFromJavascript(script) {
4005
4119
  }
4006
4120
  }
4007
4121
  catch (error) {
4008
- if (!(error instanceof Error)) {
4009
- throw error;
4010
- }
4122
+ assertsError(error);
4011
4123
  throw new ParseError(spaceTrim((block) => `
4012
4124
  Can not extract variables from the script
4013
4125
  ${block(error.stack || error.message)}
@@ -4126,6 +4238,46 @@ const MANDATORY_CSV_SETTINGS = Object.freeze({
4126
4238
  // encoding: 'utf-8',
4127
4239
  });
4128
4240
 
4241
+ /**
4242
+ * Function to check if a string is valid CSV
4243
+ *
4244
+ * @param value The string to check
4245
+ * @returns True if the string is a valid CSV string, false otherwise
4246
+ *
4247
+ * @public exported from `@promptbook/utils`
4248
+ */
4249
+ function isValidCsvString(value) {
4250
+ try {
4251
+ // A simple check for CSV format: at least one comma and no invalid characters
4252
+ if (value.includes(',') && /^[\w\s,"']+$/.test(value)) {
4253
+ return true;
4254
+ }
4255
+ return false;
4256
+ }
4257
+ catch (error) {
4258
+ assertsError(error);
4259
+ return false;
4260
+ }
4261
+ }
4262
+
4263
+ /**
4264
+ * Converts a CSV string into an object
4265
+ *
4266
+ * Note: This is wrapper around `papaparse.parse()` with better autohealing
4267
+ *
4268
+ * @private - for now until `@promptbook/csv` is released
4269
+ */
4270
+ function csvParse(value /* <- TODO: string_csv */, settings, schema /* <- TODO: Make CSV Schemas */) {
4271
+ settings = { ...settings, ...MANDATORY_CSV_SETTINGS };
4272
+ // Note: Autoheal invalid '\n' characters
4273
+ if (settings.newline && !settings.newline.includes('\r') && value.includes('\r')) {
4274
+ console.warn('CSV string contains carriage return characters, but in the CSV settings the `newline` setting does not include them. Autohealing the CSV string.');
4275
+ value = value.replace(/\r\n/g, '\n').replace(/\r/g, '\n');
4276
+ }
4277
+ const csv = parse(value, settings);
4278
+ return csv;
4279
+ }
4280
+
4129
4281
  /**
4130
4282
  * Definition for CSV spreadsheet
4131
4283
  *
@@ -4136,7 +4288,7 @@ const CsvFormatDefinition = {
4136
4288
  formatName: 'CSV',
4137
4289
  aliases: ['SPREADSHEET', 'TABLE'],
4138
4290
  isValid(value, settings, schema) {
4139
- return true;
4291
+ return isValidCsvString(value);
4140
4292
  },
4141
4293
  canBeValid(partialValue, settings, schema) {
4142
4294
  return true;
@@ -4148,8 +4300,7 @@ const CsvFormatDefinition = {
4148
4300
  {
4149
4301
  subvalueName: 'ROW',
4150
4302
  async mapValues(value, outputParameterName, settings, mapCallback) {
4151
- // TODO: [๐Ÿ‘จ๐Ÿพโ€๐Ÿคโ€๐Ÿ‘จ๐Ÿผ] DRY csv parsing
4152
- const csv = parse(value, { ...settings, ...MANDATORY_CSV_SETTINGS });
4303
+ const csv = csvParse(value, settings);
4153
4304
  if (csv.errors.length !== 0) {
4154
4305
  throw new CsvFormatError(spaceTrim$1((block) => `
4155
4306
  CSV parsing error
@@ -4179,8 +4330,7 @@ const CsvFormatDefinition = {
4179
4330
  {
4180
4331
  subvalueName: 'CELL',
4181
4332
  async mapValues(value, outputParameterName, settings, mapCallback) {
4182
- // TODO: [๐Ÿ‘จ๐Ÿพโ€๐Ÿคโ€๐Ÿ‘จ๐Ÿผ] DRY csv parsing
4183
- const csv = parse(value, { ...settings, ...MANDATORY_CSV_SETTINGS });
4333
+ const csv = csvParse(value, settings);
4184
4334
  if (csv.errors.length !== 0) {
4185
4335
  throw new CsvFormatError(spaceTrim$1((block) => `
4186
4336
  CSV parsing error
@@ -4290,6 +4440,30 @@ const TextFormatDefinition = {
4290
4440
  * TODO: [๐Ÿข] Allow to expect something inside each item of list and other formats
4291
4441
  */
4292
4442
 
4443
+ /**
4444
+ * Function to check if a string is valid XML
4445
+ *
4446
+ * @param value
4447
+ * @returns True if the string is a valid XML string, false otherwise
4448
+ *
4449
+ * @public exported from `@promptbook/utils`
4450
+ */
4451
+ function isValidXmlString(value) {
4452
+ try {
4453
+ const parser = new DOMParser();
4454
+ const parsedDocument = parser.parseFromString(value, 'application/xml');
4455
+ const parserError = parsedDocument.getElementsByTagName('parsererror');
4456
+ if (parserError.length > 0) {
4457
+ return false;
4458
+ }
4459
+ return true;
4460
+ }
4461
+ catch (error) {
4462
+ assertsError(error);
4463
+ return false;
4464
+ }
4465
+ }
4466
+
4293
4467
  /**
4294
4468
  * Definition for XML format
4295
4469
  *
@@ -4299,7 +4473,7 @@ const XmlFormatDefinition = {
4299
4473
  formatName: 'XML',
4300
4474
  mimeType: 'application/xml',
4301
4475
  isValid(value, settings, schema) {
4302
- return true;
4476
+ return isValidXmlString(value);
4303
4477
  },
4304
4478
  canBeValid(partialValue, settings, schema) {
4305
4479
  return true;
@@ -4872,9 +5046,7 @@ async function executeAttempts(options) {
4872
5046
  break scripts;
4873
5047
  }
4874
5048
  catch (error) {
4875
- if (!(error instanceof Error)) {
4876
- throw error;
4877
- }
5049
+ assertsError(error);
4878
5050
  if (error instanceof UnexpectedError) {
4879
5051
  throw error;
4880
5052
  }
@@ -4944,9 +5116,7 @@ async function executeAttempts(options) {
4944
5116
  break scripts;
4945
5117
  }
4946
5118
  catch (error) {
4947
- if (!(error instanceof Error)) {
4948
- throw error;
4949
- }
5119
+ assertsError(error);
4950
5120
  if (error instanceof UnexpectedError) {
4951
5121
  throw error;
4952
5122
  }
@@ -5189,13 +5359,79 @@ async function getExamplesForTask(task) {
5189
5359
  /**
5190
5360
  * @@@
5191
5361
  *
5362
+ * Here is the place where RAG (retrieval-augmented generation) happens
5363
+ *
5192
5364
  * @private internal utility of `createPipelineExecutor`
5193
5365
  */
5194
5366
  async function getKnowledgeForTask(options) {
5195
- const { preparedPipeline, task } = options;
5196
- return preparedPipeline.knowledgePieces.map(({ content }) => `- ${content}`).join('\n');
5367
+ const { tools, preparedPipeline, task } = options;
5368
+ const firstKnowlegePiece = preparedPipeline.knowledgePieces[0];
5369
+ const firstKnowlegeIndex = firstKnowlegePiece === null || firstKnowlegePiece === void 0 ? void 0 : firstKnowlegePiece.index[0];
5370
+ // <- TODO: Do not use just first knowledge piece and first index to determine embedding model, use also keyword search
5371
+ if (firstKnowlegePiece === undefined || firstKnowlegeIndex === undefined) {
5372
+ return 'No knowledge pieces found';
5373
+ }
5374
+ // TODO: [๐Ÿš] Make arrayable LLMs -> single LLM DRY
5375
+ const _llms = arrayableToArray(tools.llm);
5376
+ const llmTools = _llms.length === 1 ? _llms[0] : joinLlmExecutionTools(..._llms);
5377
+ const taskEmbeddingPrompt = {
5378
+ title: 'Knowledge Search',
5379
+ modelRequirements: {
5380
+ modelVariant: 'EMBEDDING',
5381
+ modelName: firstKnowlegeIndex.modelName,
5382
+ },
5383
+ content: task.content,
5384
+ parameters: {
5385
+ /* !!!!!!!! */
5386
+ },
5387
+ };
5388
+ const taskEmbeddingResult = await llmTools.callEmbeddingModel(taskEmbeddingPrompt);
5389
+ const knowledgePiecesWithRelevance = preparedPipeline.knowledgePieces.map((knowledgePiece) => {
5390
+ const { index } = knowledgePiece;
5391
+ const knowledgePieceIndex = index.find((i) => i.modelName === firstKnowlegeIndex.modelName);
5392
+ // <- TODO: Do not use just first knowledge piece and first index to determine embedding model
5393
+ if (knowledgePieceIndex === undefined) {
5394
+ return {
5395
+ content: knowledgePiece.content,
5396
+ relevance: 0,
5397
+ };
5398
+ }
5399
+ const relevance = computeCosineSimilarity(knowledgePieceIndex.position, taskEmbeddingResult.content);
5400
+ return {
5401
+ content: knowledgePiece.content,
5402
+ relevance,
5403
+ };
5404
+ });
5405
+ const knowledgePiecesSorted = knowledgePiecesWithRelevance.sort((a, b) => a.relevance - b.relevance);
5406
+ const knowledgePiecesLimited = knowledgePiecesSorted.slice(0, 5);
5407
+ console.log('!!! Embedding', {
5408
+ task,
5409
+ taskEmbeddingPrompt,
5410
+ taskEmbeddingResult,
5411
+ firstKnowlegePiece,
5412
+ firstKnowlegeIndex,
5413
+ knowledgePiecesWithRelevance,
5414
+ knowledgePiecesSorted,
5415
+ knowledgePiecesLimited,
5416
+ });
5417
+ return knowledgePiecesLimited.map(({ content }) => `- ${content}`).join('\n');
5197
5418
  // <- TODO: [๐Ÿง ] Some smart aggregation of knowledge pieces, single-line vs multi-line vs mixed
5198
5419
  }
5420
+ // TODO: !!!!!! Annotate + to new file
5421
+ function computeCosineSimilarity(embeddingVector1, embeddingVector2) {
5422
+ if (embeddingVector1.length !== embeddingVector2.length) {
5423
+ throw new TypeError('Embedding vectors must have the same length');
5424
+ }
5425
+ const dotProduct = embeddingVector1.reduce((sum, value, index) => sum + value * embeddingVector2[index], 0);
5426
+ const magnitude1 = Math.sqrt(embeddingVector1.reduce((sum, value) => sum + value * value, 0));
5427
+ const magnitude2 = Math.sqrt(embeddingVector2.reduce((sum, value) => sum + value * value, 0));
5428
+ return 1 - dotProduct / (magnitude1 * magnitude2);
5429
+ }
5430
+ /**
5431
+ * TODO: !!!! Verify if this is working
5432
+ * TODO: [โ™จ] Implement Better - use keyword search
5433
+ * TODO: [โ™จ] Examples of values
5434
+ */
5199
5435
 
5200
5436
  /**
5201
5437
  * @@@
@@ -5203,9 +5439,9 @@ async function getKnowledgeForTask(options) {
5203
5439
  * @private internal utility of `createPipelineExecutor`
5204
5440
  */
5205
5441
  async function getReservedParametersForTask(options) {
5206
- const { preparedPipeline, task, pipelineIdentification } = options;
5442
+ const { tools, preparedPipeline, task, pipelineIdentification } = options;
5207
5443
  const context = await getContextForTask(); // <- [๐Ÿ]
5208
- const knowledge = await getKnowledgeForTask({ preparedPipeline, task });
5444
+ const knowledge = await getKnowledgeForTask({ tools, preparedPipeline, task });
5209
5445
  const examples = await getExamplesForTask();
5210
5446
  const currentDate = new Date().toISOString(); // <- TODO: [๐Ÿง ][๐Ÿ’ฉ] Better
5211
5447
  const modelName = RESERVED_PARAMETER_MISSING_VALUE;
@@ -5267,6 +5503,7 @@ async function executeTask(options) {
5267
5503
  }
5268
5504
  const definedParameters = Object.freeze({
5269
5505
  ...(await getReservedParametersForTask({
5506
+ tools,
5270
5507
  preparedPipeline,
5271
5508
  task: currentTask,
5272
5509
  pipelineIdentification,
@@ -5567,9 +5804,7 @@ async function executePipeline(options) {
5567
5804
  await Promise.all(resolving);
5568
5805
  }
5569
5806
  catch (error /* <- Note: [3] */) {
5570
- if (!(error instanceof Error)) {
5571
- throw error;
5572
- }
5807
+ assertsError(error);
5573
5808
  // Note: No need to rethrow UnexpectedError
5574
5809
  // if (error instanceof UnexpectedError) {
5575
5810
  // Note: Count usage, [๐Ÿง ] Maybe put to separate function executionReportJsonToUsage + DRY [๐Ÿคนโ€โ™‚๏ธ]