@promptbook/pdf 0.89.0-9 โ†’ 0.92.0-10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. package/README.md +9 -7
  2. package/esm/index.es.js +303 -68
  3. package/esm/index.es.js.map +1 -1
  4. package/esm/typings/servers.d.ts +40 -0
  5. package/esm/typings/src/_packages/core.index.d.ts +14 -4
  6. package/esm/typings/src/_packages/deepseek.index.d.ts +2 -0
  7. package/esm/typings/src/_packages/google.index.d.ts +2 -0
  8. package/esm/typings/src/_packages/types.index.d.ts +18 -0
  9. package/esm/typings/src/_packages/utils.index.d.ts +6 -0
  10. package/esm/typings/src/cli/cli-commands/login.d.ts +0 -1
  11. package/esm/typings/src/cli/common/$provideLlmToolsForCli.d.ts +16 -3
  12. package/esm/typings/src/cli/test/ptbk.d.ts +1 -1
  13. package/esm/typings/src/commands/EXPECT/expectCommandParser.d.ts +2 -0
  14. package/esm/typings/src/config.d.ts +10 -19
  15. package/esm/typings/src/conversion/archive/loadArchive.d.ts +2 -2
  16. package/esm/typings/src/errors/0-index.d.ts +7 -4
  17. package/esm/typings/src/errors/PipelineExecutionError.d.ts +1 -1
  18. package/esm/typings/src/errors/WrappedError.d.ts +10 -0
  19. package/esm/typings/src/errors/assertsError.d.ts +11 -0
  20. package/esm/typings/src/execution/CommonToolsOptions.d.ts +4 -0
  21. package/esm/typings/src/execution/PromptbookFetch.d.ts +1 -1
  22. package/esm/typings/src/execution/createPipelineExecutor/getKnowledgeForTask.d.ts +12 -0
  23. package/esm/typings/src/execution/createPipelineExecutor/getReservedParametersForTask.d.ts +5 -0
  24. package/esm/typings/src/formats/csv/utils/csvParse.d.ts +12 -0
  25. package/esm/typings/src/formats/csv/utils/isValidCsvString.d.ts +9 -0
  26. package/esm/typings/src/formats/csv/utils/isValidCsvString.test.d.ts +1 -0
  27. package/esm/typings/src/formats/json/utils/isValidJsonString.d.ts +3 -0
  28. package/esm/typings/src/formats/json/utils/jsonParse.d.ts +11 -0
  29. package/esm/typings/src/formats/xml/utils/isValidXmlString.d.ts +9 -0
  30. package/esm/typings/src/formats/xml/utils/isValidXmlString.test.d.ts +1 -0
  31. package/esm/typings/src/llm-providers/_common/filterModels.d.ts +15 -0
  32. package/esm/typings/src/llm-providers/_common/register/{$provideEnvFilepath.d.ts โ†’ $provideEnvFilename.d.ts} +2 -2
  33. package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsConfigurationFromEnv.d.ts +1 -1
  34. package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground.d.ts +1 -1
  35. package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsForWizzardOrCli.d.ts +11 -2
  36. package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsFromEnv.d.ts +1 -1
  37. package/esm/typings/src/llm-providers/_common/register/LlmToolsMetadata.d.ts +43 -0
  38. package/esm/typings/src/llm-providers/azure-openai/AzureOpenAiExecutionTools.d.ts +4 -0
  39. package/esm/typings/src/llm-providers/deepseek/deepseek-models.d.ts +23 -0
  40. package/esm/typings/src/llm-providers/google/google-models.d.ts +23 -0
  41. package/esm/typings/src/llm-providers/openai/OpenAiExecutionTools.d.ts +4 -0
  42. package/esm/typings/src/personas/preparePersona.d.ts +1 -1
  43. package/esm/typings/src/pipeline/PipelineJson/PersonaJson.d.ts +4 -2
  44. package/esm/typings/src/remote-server/openapi-types.d.ts +626 -0
  45. package/esm/typings/src/remote-server/openapi.d.ts +581 -0
  46. package/esm/typings/src/remote-server/socket-types/_subtypes/Identification.d.ts +7 -1
  47. package/esm/typings/src/remote-server/socket-types/_subtypes/identificationToPromptbookToken.d.ts +11 -0
  48. package/esm/typings/src/remote-server/socket-types/_subtypes/promptbookTokenToIdentification.d.ts +10 -0
  49. package/esm/typings/src/remote-server/startRemoteServer.d.ts +1 -2
  50. package/esm/typings/src/remote-server/types/RemoteServerOptions.d.ts +15 -9
  51. package/esm/typings/src/storage/env-storage/$EnvStorage.d.ts +40 -0
  52. package/esm/typings/src/types/typeAliases.d.ts +26 -0
  53. package/package.json +9 -5
  54. package/umd/index.umd.js +303 -68
  55. package/umd/index.umd.js.map +1 -1
  56. package/esm/typings/src/cli/test/ptbk2.d.ts +0 -5
package/esm/index.es.js CHANGED
@@ -26,7 +26,7 @@ const BOOK_LANGUAGE_VERSION = '1.0.0';
26
26
  * @generated
27
27
  * @see https://github.com/webgptorg/promptbook
28
28
  */
29
- const PROMPTBOOK_ENGINE_VERSION = '0.89.0-9';
29
+ const PROMPTBOOK_ENGINE_VERSION = '0.92.0-10';
30
30
  /**
31
31
  * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
32
32
  * Note: [๐Ÿ’ž] Ignore a discrepancy between file name and entity name
@@ -89,6 +89,7 @@ const ADMIN_EMAIL = 'pavol@ptbk.io';
89
89
  * @public exported from `@promptbook/core`
90
90
  */
91
91
  const ADMIN_GITHUB_NAME = 'hejny';
92
+ // <- TODO: [๐ŸŠ] Pick the best claim
92
93
  /**
93
94
  * When the title is not provided, the default title is used
94
95
  *
@@ -121,6 +122,7 @@ const VALUE_STRINGS = {
121
122
  infinity: '(infinity; โˆž)',
122
123
  negativeInfinity: '(negative infinity; -โˆž)',
123
124
  unserializable: '(unserializable value)',
125
+ circular: '(circular JSON)',
124
126
  };
125
127
  /**
126
128
  * Small number limit
@@ -160,7 +162,7 @@ const DEFAULT_MAX_PARALLEL_COUNT = 5; // <- TODO: [๐Ÿคนโ€โ™‚๏ธ]
160
162
  */
161
163
  const DEFAULT_MAX_EXECUTION_ATTEMPTS = 10; // <- TODO: [๐Ÿคนโ€โ™‚๏ธ]
162
164
  // <- TODO: [๐Ÿ•] Make also `BOOKS_DIRNAME_ALTERNATIVES`
163
- // TODO: !!!!!! Just .promptbook dir, hardocode others
165
+ // TODO: Just `.promptbook` in config, hardcode subfolders like `download-cache` or `execution-cache`
164
166
  /**
165
167
  * Where to store the temporary downloads
166
168
  *
@@ -858,7 +860,7 @@ async function getScraperIntermediateSource(source, options) {
858
860
  * Note: [๐ŸŸข] Code in this file should never be never released in packages that could be imported into browser environment
859
861
  */
860
862
 
861
- var PipelineCollection = [{title:"Prepare Knowledge from Markdown",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.book",formfactorName:"GENERIC",parameters:[{name:"knowledgeContent",description:"Markdown document content",isInput:true,isOutput:false},{name:"knowledgePieces",description:"The knowledge JSON object",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, extract the important knowledge from the document.\n\n# Rules\n\n- Make pieces of information concise, clear, and easy to understand\n- One piece of information should be approximately 1 paragraph\n- Divide the paragraphs by markdown horizontal lines ---\n- Omit irrelevant information\n- Group redundant information\n- Write just extracted information, nothing else\n\n# The document\n\nTake information from this document:\n\n> {knowledgeContent}",resultingParameterName:"knowledgePieces",dependentParameterNames:["knowledgeContent"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Knowledge from Markdown\n\n- PIPELINE URL `https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.book`\n- INPUT PARAMETER `{knowledgeContent}` Markdown document content\n- OUTPUT PARAMETER `{knowledgePieces}` The knowledge JSON object\n\n## Knowledge\n\n<!-- TODO: [๐Ÿ†] -FORMAT JSON -->\n\n```markdown\nYou are experienced data researcher, extract the important knowledge from the document.\n\n# Rules\n\n- Make pieces of information concise, clear, and easy to understand\n- One piece of information should be approximately 1 paragraph\n- Divide the paragraphs by markdown horizontal lines ---\n- Omit irrelevant information\n- Group redundant information\n- Write just extracted information, nothing else\n\n# The document\n\nTake information from this document:\n\n> {knowledgeContent}\n```\n\n`-> {knowledgePieces}`\n"}],sourceFile:"./books/prepare-knowledge-from-markdown.book"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-keywords.book",formfactorName:"GENERIC",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"keywords",description:"Keywords separated by comma",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, detect the important keywords in the document.\n\n# Rules\n\n- Write just keywords separated by comma\n\n# The document\n\nTake information from this document:\n\n> {knowledgePieceContent}",resultingParameterName:"keywords",dependentParameterNames:["knowledgePieceContent"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Keywords\n\n- PIPELINE URL `https://promptbook.studio/promptbook/prepare-knowledge-keywords.book`\n- INPUT PARAMETER `{knowledgePieceContent}` The content\n- OUTPUT PARAMETER `{keywords}` Keywords separated by comma\n\n## Knowledge\n\n<!-- TODO: [๐Ÿ†] -FORMAT JSON -->\n\n```markdown\nYou are experienced data researcher, detect the important keywords in the document.\n\n# Rules\n\n- Write just keywords separated by comma\n\n# The document\n\nTake information from this document:\n\n> {knowledgePieceContent}\n```\n\n`-> {keywords}`\n"}],sourceFile:"./books/prepare-knowledge-keywords.book"},{title:"Prepare Knowledge-piece Title",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-title.book",formfactorName:"GENERIC",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"title",description:"The title of the document",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"knowledge",title:"Knowledge",content:"You are experienced content creator, write best title for the document.\n\n# Rules\n\n- Write just title, nothing else\n- Write maximum 5 words for the title\n\n# The document\n\n> {knowledgePieceContent}",resultingParameterName:"title",expectations:{words:{min:1,max:8}},dependentParameterNames:["knowledgePieceContent"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Knowledge-piece Title\n\n- PIPELINE URL `https://promptbook.studio/promptbook/prepare-knowledge-title.book`\n- INPUT PARAMETER `{knowledgePieceContent}` The content\n- OUTPUT PARAMETER `{title}` The title of the document\n\n## Knowledge\n\n- EXPECT MIN 1 WORD\n- EXPECT MAX 8 WORDS\n\n```markdown\nYou are experienced content creator, write best title for the document.\n\n# Rules\n\n- Write just title, nothing else\n- Write maximum 5 words for the title\n\n# The document\n\n> {knowledgePieceContent}\n```\n\n`-> {title}`\n"}],sourceFile:"./books/prepare-knowledge-title.book"},{title:"Prepare Persona",pipelineUrl:"https://promptbook.studio/promptbook/prepare-persona.book",formfactorName:"GENERIC",parameters:[{name:"availableModelNames",description:"List of available model names separated by comma (,)",isInput:true,isOutput:false},{name:"personaDescription",description:"Description of the persona",isInput:true,isOutput:false},{name:"modelRequirements",description:"Specific requirements for the model",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"make-model-requirements",title:"Make modelRequirements",content:"You are experienced AI engineer, you need to create virtual assistant.\nWrite\n\n## Example\n\n```json\n{\n\"modelName\": \"gpt-4o\",\n\"systemMessage\": \"You are experienced AI engineer and helpfull assistant.\",\n\"temperature\": 0.7\n}\n```\n\n## Instructions\n\n- Your output format is JSON object\n- Write just the JSON object, no other text should be present\n- It contains the following keys:\n - `modelName`: The name of the model to use\n - `systemMessage`: The system message to provide context to the model\n - `temperature`: The sampling temperature to use\n\n### Key `modelName`\n\nPick from the following models:\n\n- {availableModelNames}\n\n### Key `systemMessage`\n\nThe system message is used to communicate instructions or provide context to the model at the beginning of a conversation. It is displayed in a different format compared to user messages, helping the model understand its role in the conversation. The system message typically guides the model's behavior, sets the tone, or specifies desired output from the model. By utilizing the system message effectively, users can steer the model towards generating more accurate and relevant responses.\n\nFor example:\n\n> You are an experienced AI engineer and helpful assistant.\n\n> You are a friendly and knowledgeable chatbot.\n\n### Key `temperature`\n\nThe sampling temperature, between 0 and 1. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. If set to 0, the model will use log probability to automatically increase the temperature until certain thresholds are hit.\n\nYou can pick a value between 0 and 2. For example:\n\n- `0.1`: Low temperature, extremely conservative and deterministic\n- `0.5`: Medium temperature, balanced between conservative and creative\n- `1.0`: High temperature, creative and bit random\n- `1.5`: Very high temperature, extremely creative and often chaotic and unpredictable\n- `2.0`: Maximum temperature, completely random and unpredictable, for some extreme creative use cases\n\n# The assistant\n\nTake this description of the persona:\n\n> {personaDescription}",resultingParameterName:"modelRequirements",format:"JSON",dependentParameterNames:["availableModelNames","personaDescription"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Persona\n\n- PIPELINE URL `https://promptbook.studio/promptbook/prepare-persona.book`\n- INPUT PARAMETER `{availableModelNames}` List of available model names separated by comma (,)\n- INPUT PARAMETER `{personaDescription}` Description of the persona\n- OUTPUT PARAMETER `{modelRequirements}` Specific requirements for the model\n\n## Make modelRequirements\n\n- FORMAT JSON\n\n```markdown\nYou are experienced AI engineer, you need to create virtual assistant.\nWrite\n\n## Example\n\n\\`\\`\\`json\n{\n\"modelName\": \"gpt-4o\",\n\"systemMessage\": \"You are experienced AI engineer and helpfull assistant.\",\n\"temperature\": 0.7\n}\n\\`\\`\\`\n\n## Instructions\n\n- Your output format is JSON object\n- Write just the JSON object, no other text should be present\n- It contains the following keys:\n - `modelName`: The name of the model to use\n - `systemMessage`: The system message to provide context to the model\n - `temperature`: The sampling temperature to use\n\n### Key `modelName`\n\nPick from the following models:\n\n- {availableModelNames}\n\n### Key `systemMessage`\n\nThe system message is used to communicate instructions or provide context to the model at the beginning of a conversation. It is displayed in a different format compared to user messages, helping the model understand its role in the conversation. The system message typically guides the model's behavior, sets the tone, or specifies desired output from the model. By utilizing the system message effectively, users can steer the model towards generating more accurate and relevant responses.\n\nFor example:\n\n> You are an experienced AI engineer and helpful assistant.\n\n> You are a friendly and knowledgeable chatbot.\n\n### Key `temperature`\n\nThe sampling temperature, between 0 and 1. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. If set to 0, the model will use log probability to automatically increase the temperature until certain thresholds are hit.\n\nYou can pick a value between 0 and 2. For example:\n\n- `0.1`: Low temperature, extremely conservative and deterministic\n- `0.5`: Medium temperature, balanced between conservative and creative\n- `1.0`: High temperature, creative and bit random\n- `1.5`: Very high temperature, extremely creative and often chaotic and unpredictable\n- `2.0`: Maximum temperature, completely random and unpredictable, for some extreme creative use cases\n\n# The assistant\n\nTake this description of the persona:\n\n> {personaDescription}\n```\n\n`-> {modelRequirements}`\n"}],sourceFile:"./books/prepare-persona.book"},{title:"Prepare Title",pipelineUrl:"https://promptbook.studio/promptbook/prepare-title.book",formfactorName:"GENERIC",parameters:[{name:"book",description:"The book to prepare the title for",isInput:true,isOutput:false},{name:"title",description:"Best title for the book",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"make-title",title:"Make title",content:"Make best title for given text which describes the workflow:\n\n## Rules\n\n- Write just title, nothing else\n- Title should be concise and clear - Write maximum ideally 2 words, maximum 5 words\n- Title starts with emoticon\n- Title should not mention the input and output of the workflow but the main purpose of the workflow\n _For example, not \"โœ Convert Knowledge-piece to title\" but \"โœ Title\"_\n\n## The workflow\n\n> {book}",resultingParameterName:"title",expectations:{words:{min:1,max:8},lines:{min:1,max:1}},dependentParameterNames:["book"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Title\n\n- PIPELINE URL `https://promptbook.studio/promptbook/prepare-title.book`\n- INPUT PARAMETER `{book}` The book to prepare the title for\n- OUTPUT PARAMETER `{title}` Best title for the book\n\n## Make title\n\n- EXPECT MIN 1 Word\n- EXPECT MAX 8 Words\n- EXPECT EXACTLY 1 Line\n\n```markdown\nMake best title for given text which describes the workflow:\n\n## Rules\n\n- Write just title, nothing else\n- Title should be concise and clear - Write maximum ideally 2 words, maximum 5 words\n- Title starts with emoticon\n- Title should not mention the input and output of the workflow but the main purpose of the workflow\n _For example, not \"โœ Convert Knowledge-piece to title\" but \"โœ Title\"_\n\n## The workflow\n\n> {book}\n```\n\n`-> {title}`\n"}],sourceFile:"./books/prepare-title.book"}];
863
+ var PipelineCollection = [{title:"Prepare Knowledge from Markdown",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.book",formfactorName:"GENERIC",parameters:[{name:"knowledgeContent",description:"Markdown document content",isInput:true,isOutput:false},{name:"knowledgePieces",description:"The knowledge JSON object",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, extract the important knowledge from the document.\n\n# Rules\n\n- Make pieces of information concise, clear, and easy to understand\n- One piece of information should be approximately 1 paragraph\n- Divide the paragraphs by markdown horizontal lines ---\n- Omit irrelevant information\n- Group redundant information\n- Write just extracted information, nothing else\n\n# The document\n\nTake information from this document:\n\n> {knowledgeContent}",resultingParameterName:"knowledgePieces",dependentParameterNames:["knowledgeContent"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Knowledge from Markdown\n\n- PIPELINE URL `https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.book`\n- INPUT PARAMETER `{knowledgeContent}` Markdown document content\n- OUTPUT PARAMETER `{knowledgePieces}` The knowledge JSON object\n\n## Knowledge\n\n<!-- TODO: [๐Ÿ†] -FORMAT JSON -->\n\n```markdown\nYou are experienced data researcher, extract the important knowledge from the document.\n\n# Rules\n\n- Make pieces of information concise, clear, and easy to understand\n- One piece of information should be approximately 1 paragraph\n- Divide the paragraphs by markdown horizontal lines ---\n- Omit irrelevant information\n- Group redundant information\n- Write just extracted information, nothing else\n\n# The document\n\nTake information from this document:\n\n> {knowledgeContent}\n```\n\n`-> {knowledgePieces}`\n"}],sourceFile:"./books/prepare-knowledge-from-markdown.book"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-keywords.book",formfactorName:"GENERIC",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"keywords",description:"Keywords separated by comma",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, detect the important keywords in the document.\n\n# Rules\n\n- Write just keywords separated by comma\n\n# The document\n\nTake information from this document:\n\n> {knowledgePieceContent}",resultingParameterName:"keywords",dependentParameterNames:["knowledgePieceContent"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Keywords\n\n- PIPELINE URL `https://promptbook.studio/promptbook/prepare-knowledge-keywords.book`\n- INPUT PARAMETER `{knowledgePieceContent}` The content\n- OUTPUT PARAMETER `{keywords}` Keywords separated by comma\n\n## Knowledge\n\n<!-- TODO: [๐Ÿ†] -FORMAT JSON -->\n\n```markdown\nYou are experienced data researcher, detect the important keywords in the document.\n\n# Rules\n\n- Write just keywords separated by comma\n\n# The document\n\nTake information from this document:\n\n> {knowledgePieceContent}\n```\n\n`-> {keywords}`\n"}],sourceFile:"./books/prepare-knowledge-keywords.book"},{title:"Prepare Knowledge-piece Title",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-title.book",formfactorName:"GENERIC",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"title",description:"The title of the document",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"knowledge",title:"Knowledge",content:"You are experienced content creator, write best title for the document.\n\n# Rules\n\n- Write just title, nothing else\n- Write maximum 5 words for the title\n\n# The document\n\n> {knowledgePieceContent}",resultingParameterName:"title",expectations:{words:{min:1,max:8}},dependentParameterNames:["knowledgePieceContent"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Knowledge-piece Title\n\n- PIPELINE URL `https://promptbook.studio/promptbook/prepare-knowledge-title.book`\n- INPUT PARAMETER `{knowledgePieceContent}` The content\n- OUTPUT PARAMETER `{title}` The title of the document\n\n## Knowledge\n\n- EXPECT MIN 1 WORD\n- EXPECT MAX 8 WORDS\n\n```markdown\nYou are experienced content creator, write best title for the document.\n\n# Rules\n\n- Write just title, nothing else\n- Write maximum 5 words for the title\n\n# The document\n\n> {knowledgePieceContent}\n```\n\n`-> {title}`\n"}],sourceFile:"./books/prepare-knowledge-title.book"},{title:"Prepare Persona",pipelineUrl:"https://promptbook.studio/promptbook/prepare-persona.book",formfactorName:"GENERIC",parameters:[{name:"availableModels",description:"List of available model names together with their descriptions as JSON",isInput:true,isOutput:false},{name:"personaDescription",description:"Description of the persona",isInput:true,isOutput:false},{name:"modelsRequirements",description:"Specific requirements for the model",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"make-model-requirements",title:"Make modelRequirements",content:"You are an experienced AI engineer, you need to find the best models for virtual assistants:\n\n## Example\n\n```json\n[\n {\n \"modelName\": \"gpt-4o\",\n \"systemMessage\": \"You are experienced AI engineer and helpfull assistant.\",\n \"temperature\": 0.7\n },\n {\n \"modelName\": \"claude-3-5-sonnet\",\n \"systemMessage\": \"You are a friendly and knowledgeable chatbot.\",\n \"temperature\": 0.5\n }\n]\n```\n\n## Instructions\n\n- Your output format is JSON array\n- Sort best-fitting models first\n- Omit any models that are not suitable\n- Write just the JSON, no other text should be present\n- Array contain items with following keys:\n - `modelName`: The name of the model to use\n - `systemMessage`: The system message to provide context to the model\n - `temperature`: The sampling temperature to use\n\n### Key `modelName`\n\nHere are the available models:\n\n```json\n{availableModels}\n```\n\n### Key `systemMessage`\n\nThe system message is used to communicate instructions or provide context to the model at the beginning of a conversation. It is displayed in a different format compared to user messages, helping the model understand its role in the conversation. The system message typically guides the model's behavior, sets the tone, or specifies desired output from the model. By utilizing the system message effectively, users can steer the model towards generating more accurate and relevant responses.\n\nFor example:\n\n> You are an experienced AI engineer and helpful assistant.\n\n> You are a friendly and knowledgeable chatbot.\n\n### Key `temperature`\n\nThe sampling temperature, between 0 and 1. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. If set to 0, the model will use log probability to automatically increase the temperature until certain thresholds are hit.\n\nYou can pick a value between 0 and 2. For example:\n\n- `0.1`: Low temperature, extremely conservative and deterministic\n- `0.5`: Medium temperature, balanced between conservative and creative\n- `1.0`: High temperature, creative and bit random\n- `1.5`: Very high temperature, extremely creative and often chaotic and unpredictable\n- `2.0`: Maximum temperature, completely random and unpredictable, for some extreme creative use cases\n\n# The assistant\n\nTake this description of the persona:\n\n> {personaDescription}",resultingParameterName:"modelsRequirements",format:"JSON",dependentParameterNames:["availableModels","personaDescription"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Persona\n\n- PIPELINE URL `https://promptbook.studio/promptbook/prepare-persona.book`\n- INPUT PARAMETER `{availableModels}` List of available model names together with their descriptions as JSON\n- INPUT PARAMETER `{personaDescription}` Description of the persona\n- OUTPUT PARAMETER `{modelsRequirements}` Specific requirements for the model\n\n## Make modelRequirements\n\n- FORMAT JSON\n\n```markdown\nYou are an experienced AI engineer, you need to find the best models for virtual assistants:\n\n## Example\n\n\\`\\`\\`json\n[\n {\n \"modelName\": \"gpt-4o\",\n \"systemMessage\": \"You are experienced AI engineer and helpfull assistant.\",\n \"temperature\": 0.7\n },\n {\n \"modelName\": \"claude-3-5-sonnet\",\n \"systemMessage\": \"You are a friendly and knowledgeable chatbot.\",\n \"temperature\": 0.5\n }\n]\n\\`\\`\\`\n\n## Instructions\n\n- Your output format is JSON array\n- Sort best-fitting models first\n- Omit any models that are not suitable\n- Write just the JSON, no other text should be present\n- Array contain items with following keys:\n - `modelName`: The name of the model to use\n - `systemMessage`: The system message to provide context to the model\n - `temperature`: The sampling temperature to use\n\n### Key `modelName`\n\nHere are the available models:\n\n\\`\\`\\`json\n{availableModels}\n\\`\\`\\`\n\n### Key `systemMessage`\n\nThe system message is used to communicate instructions or provide context to the model at the beginning of a conversation. It is displayed in a different format compared to user messages, helping the model understand its role in the conversation. The system message typically guides the model's behavior, sets the tone, or specifies desired output from the model. By utilizing the system message effectively, users can steer the model towards generating more accurate and relevant responses.\n\nFor example:\n\n> You are an experienced AI engineer and helpful assistant.\n\n> You are a friendly and knowledgeable chatbot.\n\n### Key `temperature`\n\nThe sampling temperature, between 0 and 1. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. If set to 0, the model will use log probability to automatically increase the temperature until certain thresholds are hit.\n\nYou can pick a value between 0 and 2. For example:\n\n- `0.1`: Low temperature, extremely conservative and deterministic\n- `0.5`: Medium temperature, balanced between conservative and creative\n- `1.0`: High temperature, creative and bit random\n- `1.5`: Very high temperature, extremely creative and often chaotic and unpredictable\n- `2.0`: Maximum temperature, completely random and unpredictable, for some extreme creative use cases\n\n# The assistant\n\nTake this description of the persona:\n\n> {personaDescription}\n```\n\n`-> {modelsRequirements}`\n"}],sourceFile:"./books/prepare-persona.book"},{title:"Prepare Title",pipelineUrl:"https://promptbook.studio/promptbook/prepare-title.book",formfactorName:"GENERIC",parameters:[{name:"book",description:"The book to prepare the title for",isInput:true,isOutput:false},{name:"title",description:"Best title for the book",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"make-title",title:"Make title",content:"Make best title for given text which describes the workflow:\n\n## Rules\n\n- Write just title, nothing else\n- Title should be concise and clear - Write maximum ideally 2 words, maximum 5 words\n- Title starts with emoticon\n- Title should not mention the input and output of the workflow but the main purpose of the workflow\n _For example, not \"โœ Convert Knowledge-piece to title\" but \"โœ Title\"_\n\n## The workflow\n\n> {book}",resultingParameterName:"title",expectations:{words:{min:1,max:8},lines:{min:1,max:1}},dependentParameterNames:["book"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Title\n\n- PIPELINE URL `https://promptbook.studio/promptbook/prepare-title.book`\n- INPUT PARAMETER `{book}` The book to prepare the title for\n- OUTPUT PARAMETER `{title}` Best title for the book\n\n## Make title\n\n- EXPECT MIN 1 Word\n- EXPECT MAX 8 Words\n- EXPECT EXACTLY 1 Line\n\n```markdown\nMake best title for given text which describes the workflow:\n\n## Rules\n\n- Write just title, nothing else\n- Title should be concise and clear - Write maximum ideally 2 words, maximum 5 words\n- Title starts with emoticon\n- Title should not mention the input and output of the workflow but the main purpose of the workflow\n _For example, not \"โœ Convert Knowledge-piece to title\" but \"โœ Title\"_\n\n## The workflow\n\n> {book}\n```\n\n`-> {title}`\n"}],sourceFile:"./books/prepare-title.book"}];
862
864
 
863
865
  /**
864
866
  * Checks if value is valid email
@@ -891,9 +893,60 @@ class ParseError extends Error {
891
893
  * TODO: Maybe split `ParseError` and `ApplyError`
892
894
  */
893
895
 
896
+ /**
897
+ * This error type indicates that somewhere in the code non-Error object was thrown and it was wrapped into the `WrappedError`
898
+ *
899
+ * @public exported from `@promptbook/core`
900
+ */
901
+ class WrappedError extends Error {
902
+ constructor(whatWasThrown) {
903
+ const tag = `[๐Ÿคฎ]`;
904
+ console.error(tag, whatWasThrown);
905
+ super(spaceTrim$1(`
906
+ Non-Error object was thrown
907
+
908
+ Note: Look for ${tag} in the console for more details
909
+ Please report issue on ${ADMIN_EMAIL}
910
+ `));
911
+ this.name = 'WrappedError';
912
+ Object.setPrototypeOf(this, WrappedError.prototype);
913
+ }
914
+ }
915
+
916
+ /**
917
+ * Helper used in catch blocks to assert that the error is an instance of `Error`
918
+ *
919
+ * @param whatWasThrown Any object that was thrown
920
+ * @returns Nothing if the error is an instance of `Error`
921
+ * @throws `WrappedError` or `UnexpectedError` if the error is not standard
922
+ *
923
+ * @private within the repository
924
+ */
925
+ function assertsError(whatWasThrown) {
926
+ // Case 1: Handle error which was rethrown as `WrappedError`
927
+ if (whatWasThrown instanceof WrappedError) {
928
+ const wrappedError = whatWasThrown;
929
+ throw wrappedError;
930
+ }
931
+ // Case 2: Handle unexpected errors
932
+ if (whatWasThrown instanceof UnexpectedError) {
933
+ const unexpectedError = whatWasThrown;
934
+ throw unexpectedError;
935
+ }
936
+ // Case 3: Handle standard errors - keep them up to consumer
937
+ if (whatWasThrown instanceof Error) {
938
+ return;
939
+ }
940
+ // Case 4: Handle non-standard errors - wrap them into `WrappedError` and throw
941
+ throw new WrappedError(whatWasThrown);
942
+ }
943
+
894
944
  /**
895
945
  * Function isValidJsonString will tell you if the string is valid JSON or not
896
946
  *
947
+ * @param value The string to check
948
+ * @returns True if the string is a valid JSON string, false otherwise
949
+ *
897
950
  * @public exported from `@promptbook/utils`
898
951
  */
899
952
  function isValidJsonString(value /* <- [๐Ÿ‘จโ€โš–๏ธ] */) {
@@ -902,9 +955,7 @@ function isValidJsonString(value /* <- [๐Ÿ‘จโ€โš–๏ธ] */) {
902
955
  return true;
903
956
  }
904
957
  catch (error) {
905
- if (!(error instanceof Error)) {
906
- throw error;
907
- }
958
+ assertsError(error);
908
959
  if (error.message.includes('Unexpected token')) {
909
960
  return false;
910
961
  }
@@ -1257,9 +1308,7 @@ function checkSerializableAsJson(options) {
1257
1308
  JSON.stringify(value); // <- TODO: [0]
1258
1309
  }
1259
1310
  catch (error) {
1260
- if (!(error instanceof Error)) {
1261
- throw error;
1262
- }
1311
+ assertsError(error);
1263
1312
  throw new UnexpectedError(spaceTrim((block) => `
1264
1313
  \`${name}\` is not serializable
1265
1314
 
@@ -1854,7 +1903,7 @@ function extractParameterNames(template) {
1854
1903
  */
1855
1904
  function unpreparePipeline(pipeline) {
1856
1905
  let { personas, knowledgeSources, tasks } = pipeline;
1857
- personas = personas.map((persona) => ({ ...persona, modelRequirements: undefined, preparationIds: undefined }));
1906
+ personas = personas.map((persona) => ({ ...persona, modelsRequirements: undefined, preparationIds: undefined }));
1858
1907
  knowledgeSources = knowledgeSources.map((knowledgeSource) => ({ ...knowledgeSource, preparationIds: undefined }));
1859
1908
  tasks = tasks.map((task) => {
1860
1909
  let { dependentParameterNames } = task;
@@ -2048,7 +2097,7 @@ class PipelineExecutionError extends Error {
2048
2097
  }
2049
2098
  }
2050
2099
  /**
2051
- * TODO: !!!!!! Add id to all errors
2100
+ * TODO: [๐Ÿง ][๐ŸŒ‚] Add id to all errors
2052
2101
  */
2053
2102
 
2054
2103
  /**
@@ -2064,7 +2113,7 @@ function isPipelinePrepared(pipeline) {
2064
2113
  if (pipeline.title === undefined || pipeline.title === '' || pipeline.title === DEFAULT_BOOK_TITLE) {
2065
2114
  return false;
2066
2115
  }
2067
- if (!pipeline.personas.every((persona) => persona.modelRequirements !== undefined)) {
2116
+ if (!pipeline.personas.every((persona) => persona.modelsRequirements !== undefined)) {
2068
2117
  return false;
2069
2118
  }
2070
2119
  if (!pipeline.knowledgeSources.every((knowledgeSource) => knowledgeSource.preparationIds !== undefined)) {
@@ -2088,6 +2137,45 @@ function isPipelinePrepared(pipeline) {
2088
2137
  * - [โ™จ] Are tasks prepared
2089
2138
  */
2090
2139
 
2140
+ /**
2141
+ * Converts a JavaScript Object Notation (JSON) string into an object.
2142
+ *
2143
+ * Note: This is wrapper around `JSON.parse()` with better error and type handling
2144
+ *
2145
+ * @public exported from `@promptbook/utils`
2146
+ */
2147
+ function jsonParse(value) {
2148
+ if (value === undefined) {
2149
+ throw new Error(`Can not parse JSON from undefined value.`);
2150
+ }
2151
+ else if (typeof value !== 'string') {
2152
+ console.error('Can not parse JSON from non-string value.', { text: value });
2153
+ throw new Error(spaceTrim(`
2154
+ Can not parse JSON from non-string value.
2155
+
2156
+ The value type: ${typeof value}
2157
+ See more in console.
2158
+ `));
2159
+ }
2160
+ try {
2161
+ return JSON.parse(value);
2162
+ }
2163
+ catch (error) {
2164
+ if (!(error instanceof Error)) {
2165
+ throw error;
2166
+ }
2167
+ throw new Error(spaceTrim((block) => `
2168
+ ${block(error.message)}
2169
+
2170
+ The JSON text:
2171
+ ${block(value)}
2172
+ `));
2173
+ }
2174
+ }
2175
+ /**
2176
+ * TODO: !!!! Use in Promptbook.studio
2177
+ */
2178
+
2091
2179
  /**
2092
2180
  * Recursively converts JSON strings to JSON objects
2093
2181
 
@@ -2106,7 +2194,7 @@ function jsonStringsToJsons(object) {
2106
2194
  const newObject = { ...object };
2107
2195
  for (const [key, value] of Object.entries(object)) {
2108
2196
  if (typeof value === 'string' && isValidJsonString(value)) {
2109
- newObject[key] = JSON.parse(value);
2197
+ newObject[key] = jsonParse(value);
2110
2198
  }
2111
2199
  else {
2112
2200
  newObject[key] = jsonStringsToJsons(value);
@@ -2259,7 +2347,10 @@ const PROMPTBOOK_ERRORS = {
2259
2347
  PipelineExecutionError,
2260
2348
  PipelineLogicError,
2261
2349
  PipelineUrlError,
2350
+ AuthenticationError,
2351
+ PromptbookFetchError,
2262
2352
  UnexpectedError,
2353
+ WrappedError,
2263
2354
  // TODO: [๐Ÿช‘]> VersionMismatchError,
2264
2355
  };
2265
2356
  /**
@@ -2276,8 +2367,6 @@ const COMMON_JAVASCRIPT_ERRORS = {
2276
2367
  TypeError,
2277
2368
  URIError,
2278
2369
  AggregateError,
2279
- AuthenticationError,
2280
- PromptbookFetchError,
2281
2370
  /*
2282
2371
  Note: Not widely supported
2283
2372
  > InternalError,
@@ -2400,8 +2489,8 @@ function createTask(options) {
2400
2489
  updatedAt = new Date();
2401
2490
  errors.push(...executionResult.errors);
2402
2491
  warnings.push(...executionResult.warnings);
2403
- // <- TODO: !!! Only unique errors and warnings should be added (or filtered)
2404
- // TODO: [๐Ÿง ] !!! errors, warning, isSuccessful are redundant both in `ExecutionTask` and `ExecutionTask.currentValue`
2492
+ // <- TODO: [๐ŸŒ‚] Only unique errors and warnings should be added (or filtered)
2493
+ // TODO: [๐Ÿง ] !! errors, warning, isSuccessful are redundant both in `ExecutionTask` and `ExecutionTask.currentValue`
2405
2494
  // Also maybe move `ExecutionTask.currentValue.usage` -> `ExecutionTask.usage`
2406
2495
  // And delete `ExecutionTask.currentValue.preparedPipeline`
2407
2496
  assertsTaskSuccessful(executionResult);
@@ -2411,6 +2500,7 @@ function createTask(options) {
2411
2500
  partialResultSubject.next(executionResult);
2412
2501
  }
2413
2502
  catch (error) {
2503
+ assertsError(error);
2414
2504
  status = 'ERROR';
2415
2505
  errors.push(error);
2416
2506
  partialResultSubject.error(error);
@@ -2802,14 +2892,15 @@ class MultipleLlmExecutionTools {
2802
2892
  }
2803
2893
  }
2804
2894
  catch (error) {
2805
- if (!(error instanceof Error) || error instanceof UnexpectedError) {
2895
+ assertsError(error);
2896
+ if (error instanceof UnexpectedError) {
2806
2897
  throw error;
2807
2898
  }
2808
2899
  errors.push({ llmExecutionTools, error });
2809
2900
  }
2810
2901
  }
2811
2902
  if (errors.length === 1) {
2812
- throw errors[0];
2903
+ throw errors[0].error;
2813
2904
  }
2814
2905
  else if (errors.length > 1) {
2815
2906
  throw new PipelineExecutionError(
@@ -2935,27 +3026,48 @@ async function preparePersona(personaDescription, tools, options) {
2935
3026
  pipeline: await collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-persona.book'),
2936
3027
  tools,
2937
3028
  });
2938
- // TODO: [๐Ÿš] Make arrayable LLMs -> single LLM DRY
2939
3029
  const _llms = arrayableToArray(tools.llm);
2940
3030
  const llmTools = _llms.length === 1 ? _llms[0] : joinLlmExecutionTools(..._llms);
2941
- const availableModels = await llmTools.listModels();
2942
- const availableModelNames = availableModels
3031
+ const availableModels = (await llmTools.listModels())
2943
3032
  .filter(({ modelVariant }) => modelVariant === 'CHAT')
2944
- .map(({ modelName }) => modelName)
2945
- .join(',');
2946
- const result = await preparePersonaExecutor({ availableModelNames, personaDescription }).asPromise();
3033
+ .map(({ modelName, modelDescription }) => ({
3034
+ modelName,
3035
+ modelDescription,
3036
+ // <- Note: `modelTitle` and `modelVariant` is not relevant for this task
3037
+ }));
3038
+ const result = await preparePersonaExecutor({
3039
+ availableModels /* <- Note: Passing as JSON */,
3040
+ personaDescription,
3041
+ }).asPromise();
2947
3042
  const { outputParameters } = result;
2948
- const { modelRequirements: modelRequirementsRaw } = outputParameters;
2949
- const modelRequirements = JSON.parse(modelRequirementsRaw);
3043
+ const { modelsRequirements: modelsRequirementsJson } = outputParameters;
3044
+ let modelsRequirementsUnchecked = jsonParse(modelsRequirementsJson);
2950
3045
  if (isVerbose) {
2951
- console.info(`PERSONA ${personaDescription}`, modelRequirements);
3046
+ console.info(`PERSONA ${personaDescription}`, modelsRequirementsUnchecked);
2952
3047
  }
2953
- const { modelName, systemMessage, temperature } = modelRequirements;
2954
- return {
3048
+ if (!Array.isArray(modelsRequirementsUnchecked)) {
3049
+ // <- TODO: Book should have syntax and system to enforce shape of JSON
3050
+ modelsRequirementsUnchecked = [modelsRequirementsUnchecked];
3051
+ /*
3052
+ throw new UnexpectedError(
3053
+ spaceTrim(
3054
+ (block) => `
3055
+ Invalid \`modelsRequirements\`:
3056
+
3057
+ \`\`\`json
3058
+ ${block(JSON.stringify(modelsRequirementsUnchecked, null, 4))}
3059
+ \`\`\`
3060
+ `,
3061
+ ),
3062
+ );
3063
+ */
3064
+ }
3065
+ const modelsRequirements = modelsRequirementsUnchecked.map((modelRequirements) => ({
2955
3066
  modelVariant: 'CHAT',
2956
- modelName,
2957
- systemMessage,
2958
- temperature,
3067
+ ...modelRequirements,
3068
+ }));
3069
+ return {
3070
+ modelsRequirements,
2959
3071
  };
2960
3072
  }
2961
3073
  /**
@@ -3264,9 +3376,7 @@ const promptbookFetch = async (urlOrRequest, init) => {
3264
3376
  return await fetch(urlOrRequest, init);
3265
3377
  }
3266
3378
  catch (error) {
3267
- if (!(error instanceof Error)) {
3268
- throw error;
3269
- }
3379
+ assertsError(error);
3270
3380
  let url;
3271
3381
  if (typeof urlOrRequest === 'string') {
3272
3382
  url = urlOrRequest;
@@ -3395,7 +3505,7 @@ async function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
3395
3505
  > },
3396
3506
  */
3397
3507
  async asJson() {
3398
- return JSON.parse(await tools.fs.readFile(filename, 'utf-8'));
3508
+ return jsonParse(await tools.fs.readFile(filename, 'utf-8'));
3399
3509
  },
3400
3510
  async asText() {
3401
3511
  return await tools.fs.readFile(filename, 'utf-8');
@@ -3497,9 +3607,7 @@ async function prepareKnowledgePieces(knowledgeSources, tools, options) {
3497
3607
  knowledgePreparedUnflatten[index] = pieces;
3498
3608
  }
3499
3609
  catch (error) {
3500
- if (!(error instanceof Error)) {
3501
- throw error;
3502
- }
3610
+ assertsError(error);
3503
3611
  console.warn(error);
3504
3612
  // <- TODO: [๐Ÿฎ] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
3505
3613
  }
@@ -3655,14 +3763,14 @@ async function preparePipeline(pipeline, tools, options) {
3655
3763
  // TODO: [๐Ÿ–Œ][๐Ÿง ] Implement some `mapAsync` function
3656
3764
  const preparedPersonas = new Array(personas.length);
3657
3765
  await forEachAsync(personas, { maxParallelCount /* <- TODO: [๐Ÿช‚] When there are subtasks, this maximul limit can be broken */ }, async (persona, index) => {
3658
- const modelRequirements = await preparePersona(persona.description, { ...tools, llm: llmToolsWithUsage }, {
3766
+ const { modelsRequirements } = await preparePersona(persona.description, { ...tools, llm: llmToolsWithUsage }, {
3659
3767
  rootDirname,
3660
3768
  maxParallelCount /* <- TODO: [๐Ÿช‚] */,
3661
3769
  isVerbose,
3662
3770
  });
3663
3771
  const preparedPersona = {
3664
3772
  ...persona,
3665
- modelRequirements,
3773
+ modelsRequirements,
3666
3774
  preparationIds: [/* TODO: [๐ŸงŠ] -> */ currentPreparation.id],
3667
3775
  // <- TODO: [๐Ÿ™] Make some standard order of json properties
3668
3776
  };
@@ -3791,13 +3899,19 @@ function valueToString(value) {
3791
3899
  return value.toISOString();
3792
3900
  }
3793
3901
  else {
3794
- return JSON.stringify(value);
3902
+ try {
3903
+ return JSON.stringify(value);
3904
+ }
3905
+ catch (error) {
3906
+ if (error instanceof TypeError && error.message.includes('circular structure')) {
3907
+ return VALUE_STRINGS.circular;
3908
+ }
3909
+ throw error;
3910
+ }
3795
3911
  }
3796
3912
  }
3797
3913
  catch (error) {
3798
- if (!(error instanceof Error)) {
3799
- throw error;
3800
- }
3914
+ assertsError(error);
3801
3915
  console.error(error);
3802
3916
  return VALUE_STRINGS.unserializable;
3803
3917
  }
@@ -3854,9 +3968,7 @@ function extractVariablesFromJavascript(script) {
3854
3968
  }
3855
3969
  }
3856
3970
  catch (error) {
3857
- if (!(error instanceof Error)) {
3858
- throw error;
3859
- }
3971
+ assertsError(error);
3860
3972
  throw new ParseError(spaceTrim$1((block) => `
3861
3973
  Can not extract variables from the script
3862
3974
  ${block(error.stack || error.message)}
@@ -3975,6 +4087,46 @@ const MANDATORY_CSV_SETTINGS = Object.freeze({
3975
4087
  // encoding: 'utf-8',
3976
4088
  });
3977
4089
 
4090
+ /**
4091
+ * Function to check if a string is valid CSV
4092
+ *
4093
+ * @param value The string to check
4094
+ * @returns True if the string is a valid CSV string, false otherwise
4095
+ *
4096
+ * @public exported from `@promptbook/utils`
4097
+ */
4098
+ function isValidCsvString(value) {
4099
+ try {
4100
+ // A simple check for CSV format: at least one comma and no invalid characters
4101
+ if (value.includes(',') && /^[\w\s,"']+$/.test(value)) {
4102
+ return true;
4103
+ }
4104
+ return false;
4105
+ }
4106
+ catch (error) {
4107
+ assertsError(error);
4108
+ return false;
4109
+ }
4110
+ }
4111
+
4112
+ /**
4113
+ * Converts a CSV string into an object
4114
+ *
4115
+ * Note: This is wrapper around `papaparse.parse()` with better autohealing
4116
+ *
4117
+ * @private - for now until `@promptbook/csv` is released
4118
+ */
4119
+ function csvParse(value /* <- TODO: string_csv */, settings, schema /* <- TODO: Make CSV Schemas */) {
4120
+ settings = { ...settings, ...MANDATORY_CSV_SETTINGS };
4121
+ // Note: Autoheal invalid '\n' characters
4122
+ if (settings.newline && !settings.newline.includes('\r') && value.includes('\r')) {
4123
+ console.warn('CSV string contains carriage return characters, but in the CSV settings the `newline` setting does not include them. Autohealing the CSV string.');
4124
+ value = value.replace(/\r\n/g, '\n').replace(/\r/g, '\n');
4125
+ }
4126
+ const csv = parse(value, settings);
4127
+ return csv;
4128
+ }
4129
+
3978
4130
  /**
3979
4131
  * Definition for CSV spreadsheet
3980
4132
  *
@@ -3985,7 +4137,7 @@ const CsvFormatDefinition = {
3985
4137
  formatName: 'CSV',
3986
4138
  aliases: ['SPREADSHEET', 'TABLE'],
3987
4139
  isValid(value, settings, schema) {
3988
- return true;
4140
+ return isValidCsvString(value);
3989
4141
  },
3990
4142
  canBeValid(partialValue, settings, schema) {
3991
4143
  return true;
@@ -3997,8 +4149,7 @@ const CsvFormatDefinition = {
3997
4149
  {
3998
4150
  subvalueName: 'ROW',
3999
4151
  async mapValues(value, outputParameterName, settings, mapCallback) {
4000
- // TODO: [๐Ÿ‘จ๐Ÿพโ€๐Ÿคโ€๐Ÿ‘จ๐Ÿผ] DRY csv parsing
4001
- const csv = parse(value, { ...settings, ...MANDATORY_CSV_SETTINGS });
4152
+ const csv = csvParse(value, settings);
4002
4153
  if (csv.errors.length !== 0) {
4003
4154
  throw new CsvFormatError(spaceTrim((block) => `
4004
4155
  CSV parsing error
@@ -4028,8 +4179,7 @@ const CsvFormatDefinition = {
4028
4179
  {
4029
4180
  subvalueName: 'CELL',
4030
4181
  async mapValues(value, outputParameterName, settings, mapCallback) {
4031
- // TODO: [๐Ÿ‘จ๐Ÿพโ€๐Ÿคโ€๐Ÿ‘จ๐Ÿผ] DRY csv parsing
4032
- const csv = parse(value, { ...settings, ...MANDATORY_CSV_SETTINGS });
4182
+ const csv = csvParse(value, settings);
4033
4183
  if (csv.errors.length !== 0) {
4034
4184
  throw new CsvFormatError(spaceTrim((block) => `
4035
4185
  CSV parsing error
@@ -4139,6 +4289,30 @@ const TextFormatDefinition = {
4139
4289
  * TODO: [๐Ÿข] Allow to expect something inside each item of list and other formats
4140
4290
  */
4141
4291
 
4292
+ /**
4293
+ * Function to check if a string is valid XML
4294
+ *
4295
+ * @param value
4296
+ * @returns True if the string is a valid XML string, false otherwise
4297
+ *
4298
+ * @public exported from `@promptbook/utils`
4299
+ */
4300
+ function isValidXmlString(value) {
4301
+ try {
4302
+ const parser = new DOMParser();
4303
+ const parsedDocument = parser.parseFromString(value, 'application/xml');
4304
+ const parserError = parsedDocument.getElementsByTagName('parsererror');
4305
+ if (parserError.length > 0) {
4306
+ return false;
4307
+ }
4308
+ return true;
4309
+ }
4310
+ catch (error) {
4311
+ assertsError(error);
4312
+ return false;
4313
+ }
4314
+ }
4315
+
4142
4316
  /**
4143
4317
  * Definition for XML format
4144
4318
  *
@@ -4148,7 +4322,7 @@ const XmlFormatDefinition = {
4148
4322
  formatName: 'XML',
4149
4323
  mimeType: 'application/xml',
4150
4324
  isValid(value, settings, schema) {
4151
- return true;
4325
+ return isValidXmlString(value);
4152
4326
  },
4153
4327
  canBeValid(partialValue, settings, schema) {
4154
4328
  return true;
@@ -4721,9 +4895,7 @@ async function executeAttempts(options) {
4721
4895
  break scripts;
4722
4896
  }
4723
4897
  catch (error) {
4724
- if (!(error instanceof Error)) {
4725
- throw error;
4726
- }
4898
+ assertsError(error);
4727
4899
  if (error instanceof UnexpectedError) {
4728
4900
  throw error;
4729
4901
  }
@@ -4793,9 +4965,7 @@ async function executeAttempts(options) {
4793
4965
  break scripts;
4794
4966
  }
4795
4967
  catch (error) {
4796
- if (!(error instanceof Error)) {
4797
- throw error;
4798
- }
4968
+ assertsError(error);
4799
4969
  if (error instanceof UnexpectedError) {
4800
4970
  throw error;
4801
4971
  }
@@ -5038,13 +5208,79 @@ async function getExamplesForTask(task) {
5038
5208
  /**
5039
5209
  * @@@
5040
5210
  *
5211
+ * Here is the place where RAG (retrieval-augmented generation) happens
5212
+ *
5041
5213
  * @private internal utility of `createPipelineExecutor`
5042
5214
  */
5043
5215
  async function getKnowledgeForTask(options) {
5044
- const { preparedPipeline, task } = options;
5045
- return preparedPipeline.knowledgePieces.map(({ content }) => `- ${content}`).join('\n');
5216
+ const { tools, preparedPipeline, task } = options;
5217
+ const firstKnowlegePiece = preparedPipeline.knowledgePieces[0];
5218
+ const firstKnowlegeIndex = firstKnowlegePiece === null || firstKnowlegePiece === void 0 ? void 0 : firstKnowlegePiece.index[0];
5219
+ // <- TODO: Do not use just first knowledge piece and first index to determine embedding model, use also keyword search
5220
+ if (firstKnowlegePiece === undefined || firstKnowlegeIndex === undefined) {
5221
+ return 'No knowledge pieces found';
5222
+ }
5223
+ // TODO: [๐Ÿš] Make arrayable LLMs -> single LLM DRY
5224
+ const _llms = arrayableToArray(tools.llm);
5225
+ const llmTools = _llms.length === 1 ? _llms[0] : joinLlmExecutionTools(..._llms);
5226
+ const taskEmbeddingPrompt = {
5227
+ title: 'Knowledge Search',
5228
+ modelRequirements: {
5229
+ modelVariant: 'EMBEDDING',
5230
+ modelName: firstKnowlegeIndex.modelName,
5231
+ },
5232
+ content: task.content,
5233
+ parameters: {
5234
+ /* !!!!!!!! */
5235
+ },
5236
+ };
5237
+ const taskEmbeddingResult = await llmTools.callEmbeddingModel(taskEmbeddingPrompt);
5238
+ const knowledgePiecesWithRelevance = preparedPipeline.knowledgePieces.map((knowledgePiece) => {
5239
+ const { index } = knowledgePiece;
5240
+ const knowledgePieceIndex = index.find((i) => i.modelName === firstKnowlegeIndex.modelName);
5241
+ // <- TODO: Do not use just first knowledge piece and first index to determine embedding model
5242
+ if (knowledgePieceIndex === undefined) {
5243
+ return {
5244
+ content: knowledgePiece.content,
5245
+ relevance: 0,
5246
+ };
5247
+ }
5248
+ const relevance = computeCosineSimilarity(knowledgePieceIndex.position, taskEmbeddingResult.content);
5249
+ return {
5250
+ content: knowledgePiece.content,
5251
+ relevance,
5252
+ };
5253
+ });
5254
+ const knowledgePiecesSorted = knowledgePiecesWithRelevance.sort((a, b) => a.relevance - b.relevance);
5255
+ const knowledgePiecesLimited = knowledgePiecesSorted.slice(0, 5);
5256
+ console.log('!!! Embedding', {
5257
+ task,
5258
+ taskEmbeddingPrompt,
5259
+ taskEmbeddingResult,
5260
+ firstKnowlegePiece,
5261
+ firstKnowlegeIndex,
5262
+ knowledgePiecesWithRelevance,
5263
+ knowledgePiecesSorted,
5264
+ knowledgePiecesLimited,
5265
+ });
5266
+ return knowledgePiecesLimited.map(({ content }) => `- ${content}`).join('\n');
5046
5267
  // <- TODO: [๐Ÿง ] Some smart aggregation of knowledge pieces, single-line vs multi-line vs mixed
5047
5268
  }
5269
+ // TODO: !!!!!! Annotate + to new file
5270
+ function computeCosineSimilarity(embeddingVector1, embeddingVector2) {
5271
+ if (embeddingVector1.length !== embeddingVector2.length) {
5272
+ throw new TypeError('Embedding vectors must have the same length');
5273
+ }
5274
+ const dotProduct = embeddingVector1.reduce((sum, value, index) => sum + value * embeddingVector2[index], 0);
5275
+ const magnitude1 = Math.sqrt(embeddingVector1.reduce((sum, value) => sum + value * value, 0));
5276
+ const magnitude2 = Math.sqrt(embeddingVector2.reduce((sum, value) => sum + value * value, 0));
5277
+ return 1 - dotProduct / (magnitude1 * magnitude2);
5278
+ }
5279
+ /**
5280
+ * TODO: !!!! Verify if this is working
5281
+ * TODO: [โ™จ] Implement Better - use keyword search
5282
+ * TODO: [โ™จ] Examples of values
5283
+ */
5048
5284
 
5049
5285
  /**
5050
5286
  * @@@
@@ -5052,9 +5288,9 @@ async function getKnowledgeForTask(options) {
5052
5288
  * @private internal utility of `createPipelineExecutor`
5053
5289
  */
5054
5290
  async function getReservedParametersForTask(options) {
5055
- const { preparedPipeline, task, pipelineIdentification } = options;
5291
+ const { tools, preparedPipeline, task, pipelineIdentification } = options;
5056
5292
  const context = await getContextForTask(); // <- [๐Ÿ]
5057
- const knowledge = await getKnowledgeForTask({ preparedPipeline, task });
5293
+ const knowledge = await getKnowledgeForTask({ tools, preparedPipeline, task });
5058
5294
  const examples = await getExamplesForTask();
5059
5295
  const currentDate = new Date().toISOString(); // <- TODO: [๐Ÿง ][๐Ÿ’ฉ] Better
5060
5296
  const modelName = RESERVED_PARAMETER_MISSING_VALUE;
@@ -5116,6 +5352,7 @@ async function executeTask(options) {
5116
5352
  }
5117
5353
  const definedParameters = Object.freeze({
5118
5354
  ...(await getReservedParametersForTask({
5355
+ tools,
5119
5356
  preparedPipeline,
5120
5357
  task: currentTask,
5121
5358
  pipelineIdentification,
@@ -5416,9 +5653,7 @@ async function executePipeline(options) {
5416
5653
  await Promise.all(resolving);
5417
5654
  }
5418
5655
  catch (error /* <- Note: [3] */) {
5419
- if (!(error instanceof Error)) {
5420
- throw error;
5421
- }
5656
+ assertsError(error);
5422
5657
  // Note: No need to rethrow UnexpectedError
5423
5658
  // if (error instanceof UnexpectedError) {
5424
5659
  // Note: Count usage, [๐Ÿง ] Maybe put to separate function executionReportJsonToUsage + DRY [๐Ÿคนโ€โ™‚๏ธ]