@promptbook/node 0.92.0-9 → 0.92.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (206) hide show
  1. package/README.md +1 -5
  2. package/esm/index.es.js +801 -531
  3. package/esm/index.es.js.map +1 -1
  4. package/esm/typings/src/_packages/browser.index.d.ts +2 -0
  5. package/esm/typings/src/_packages/core.index.d.ts +26 -14
  6. package/esm/typings/src/_packages/types.index.d.ts +6 -2
  7. package/esm/typings/src/collection/PipelineCollection.d.ts +0 -2
  8. package/esm/typings/src/collection/SimplePipelineCollection.d.ts +1 -1
  9. package/esm/typings/src/commands/FOREACH/ForeachJson.d.ts +6 -6
  10. package/esm/typings/src/commands/FOREACH/foreachCommandParser.d.ts +0 -2
  11. package/esm/typings/src/commands/FORMFACTOR/formfactorCommandParser.d.ts +1 -1
  12. package/esm/typings/src/commands/_BOILERPLATE/boilerplateCommandParser.d.ts +1 -1
  13. package/esm/typings/src/commands/_common/types/CommandParser.d.ts +36 -28
  14. package/esm/typings/src/config.d.ts +41 -11
  15. package/esm/typings/src/constants.d.ts +43 -2
  16. package/esm/typings/src/conversion/parsePipeline.d.ts +2 -2
  17. package/esm/typings/src/errors/0-BoilerplateError.d.ts +2 -2
  18. package/esm/typings/src/errors/CollectionError.d.ts +1 -1
  19. package/esm/typings/src/executables/$provideExecutablesForNode.d.ts +1 -1
  20. package/esm/typings/src/executables/apps/locateLibreoffice.d.ts +2 -1
  21. package/esm/typings/src/executables/apps/locatePandoc.d.ts +2 -1
  22. package/esm/typings/src/executables/locateApp.d.ts +2 -2
  23. package/esm/typings/src/executables/platforms/locateAppOnLinux.d.ts +2 -1
  24. package/esm/typings/src/executables/platforms/locateAppOnMacOs.d.ts +2 -1
  25. package/esm/typings/src/executables/platforms/locateAppOnWindows.d.ts +2 -1
  26. package/esm/typings/src/execution/AbstractTaskResult.d.ts +1 -1
  27. package/esm/typings/src/execution/CommonToolsOptions.d.ts +3 -3
  28. package/esm/typings/src/execution/ExecutionTask.d.ts +19 -1
  29. package/esm/typings/src/execution/LlmExecutionToolsConstructor.d.ts +2 -1
  30. package/esm/typings/src/execution/PipelineExecutorResult.d.ts +4 -2
  31. package/esm/typings/src/execution/PromptbookFetch.d.ts +1 -1
  32. package/esm/typings/src/execution/ScriptExecutionTools.d.ts +1 -1
  33. package/esm/typings/src/execution/createPipelineExecutor/$OngoingTaskResult.d.ts +12 -9
  34. package/esm/typings/src/execution/createPipelineExecutor/10-executePipeline.d.ts +13 -10
  35. package/esm/typings/src/execution/createPipelineExecutor/20-executeTask.d.ts +12 -9
  36. package/esm/typings/src/execution/createPipelineExecutor/30-executeFormatSubvalues.d.ts +15 -3
  37. package/esm/typings/src/execution/createPipelineExecutor/40-executeAttempts.d.ts +21 -15
  38. package/esm/typings/src/execution/createPipelineExecutor/computeCosineSimilarity.d.ts +13 -0
  39. package/esm/typings/src/execution/createPipelineExecutor/filterJustOutputParameters.d.ts +7 -6
  40. package/esm/typings/src/execution/createPipelineExecutor/getContextForTask.d.ts +5 -1
  41. package/esm/typings/src/execution/createPipelineExecutor/getExamplesForTask.d.ts +1 -1
  42. package/esm/typings/src/execution/createPipelineExecutor/getKnowledgeForTask.d.ts +12 -9
  43. package/esm/typings/src/execution/createPipelineExecutor/getReservedParametersForTask.d.ts +18 -5
  44. package/esm/typings/src/execution/createPipelineExecutor/knowledgePiecesToString.d.ts +9 -0
  45. package/esm/typings/src/execution/execution-report/ExecutionReportJson.d.ts +1 -1
  46. package/esm/typings/src/execution/execution-report/ExecutionReportString.d.ts +1 -1
  47. package/esm/typings/src/execution/translation/automatic-translate/automatic-translators/LindatAutomaticTranslator.d.ts +4 -4
  48. package/esm/typings/src/execution/utils/checkExpectations.d.ts +3 -3
  49. package/esm/typings/src/execution/utils/uncertainNumber.d.ts +3 -2
  50. package/esm/typings/src/execution/utils/usageToWorktime.d.ts +1 -1
  51. package/esm/typings/src/formats/_common/{FormatDefinition.d.ts → FormatParser.d.ts} +8 -6
  52. package/esm/typings/src/formats/_common/FormatSubvalueParser.d.ts +66 -0
  53. package/esm/typings/src/formats/csv/CsvFormatParser.d.ts +17 -0
  54. package/esm/typings/src/formats/csv/CsvSettings.d.ts +2 -2
  55. package/esm/typings/src/formats/csv/utils/csvParse.d.ts +12 -0
  56. package/esm/typings/src/formats/csv/utils/isValidCsvString.d.ts +1 -1
  57. package/esm/typings/src/formats/index.d.ts +2 -2
  58. package/esm/typings/src/formats/json/JsonFormatParser.d.ts +19 -0
  59. package/esm/typings/src/formats/json/utils/isValidJsonString.d.ts +1 -1
  60. package/esm/typings/src/formats/json/utils/jsonParse.d.ts +0 -3
  61. package/esm/typings/src/formats/text/{TextFormatDefinition.d.ts → TextFormatParser.d.ts} +7 -7
  62. package/esm/typings/src/formats/xml/XmlFormatParser.d.ts +19 -0
  63. package/esm/typings/src/formats/xml/utils/isValidXmlString.d.ts +1 -1
  64. package/esm/typings/src/formfactors/_boilerplate/BoilerplateFormfactorDefinition.d.ts +3 -2
  65. package/esm/typings/src/formfactors/_common/AbstractFormfactorDefinition.d.ts +16 -7
  66. package/esm/typings/src/formfactors/_common/FormfactorDefinition.d.ts +3 -1
  67. package/esm/typings/src/formfactors/_common/string_formfactor_name.d.ts +2 -1
  68. package/esm/typings/src/formfactors/chatbot/ChatbotFormfactorDefinition.d.ts +2 -2
  69. package/esm/typings/src/formfactors/completion/CompletionFormfactorDefinition.d.ts +29 -0
  70. package/esm/typings/src/formfactors/generator/GeneratorFormfactorDefinition.d.ts +2 -1
  71. package/esm/typings/src/formfactors/generic/GenericFormfactorDefinition.d.ts +2 -2
  72. package/esm/typings/src/formfactors/index.d.ts +33 -8
  73. package/esm/typings/src/formfactors/matcher/MatcherFormfactorDefinition.d.ts +4 -2
  74. package/esm/typings/src/formfactors/sheets/SheetsFormfactorDefinition.d.ts +3 -2
  75. package/esm/typings/src/formfactors/translator/TranslatorFormfactorDefinition.d.ts +3 -2
  76. package/esm/typings/src/high-level-abstractions/index.d.ts +2 -2
  77. package/esm/typings/src/llm-providers/_common/register/$llmToolsMetadataRegister.d.ts +3 -3
  78. package/esm/typings/src/llm-providers/_common/register/$llmToolsRegister.d.ts +3 -3
  79. package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsConfigurationFromEnv.d.ts +4 -4
  80. package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground.d.ts +4 -3
  81. package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsFromEnv.d.ts +18 -5
  82. package/esm/typings/src/llm-providers/_common/register/LlmToolsConfiguration.d.ts +11 -4
  83. package/esm/typings/src/llm-providers/_common/register/LlmToolsMetadata.d.ts +21 -42
  84. package/esm/typings/src/llm-providers/_common/register/LlmToolsOptions.d.ts +9 -2
  85. package/esm/typings/src/llm-providers/_common/register/createLlmToolsFromConfiguration.d.ts +13 -4
  86. package/esm/typings/src/llm-providers/_common/utils/cache/CacheItem.d.ts +10 -5
  87. package/esm/typings/src/llm-providers/_common/utils/cache/CacheLlmToolsOptions.d.ts +11 -3
  88. package/esm/typings/src/llm-providers/_common/utils/cache/cacheLlmTools.d.ts +3 -3
  89. package/esm/typings/src/llm-providers/_common/utils/count-total-usage/limitTotalUsage.d.ts +5 -5
  90. package/esm/typings/src/llm-providers/anthropic-claude/AnthropicClaudeExecutionTools.d.ts +6 -0
  91. package/esm/typings/src/llm-providers/anthropic-claude/anthropic-claude-models.d.ts +1 -1
  92. package/esm/typings/src/llm-providers/azure-openai/AzureOpenAiExecutionTools.d.ts +1 -1
  93. package/esm/typings/src/llm-providers/azure-openai/AzureOpenAiExecutionToolsOptions.d.ts +4 -4
  94. package/esm/typings/src/llm-providers/deepseek/deepseek-models.d.ts +1 -1
  95. package/esm/typings/src/llm-providers/google/google-models.d.ts +1 -1
  96. package/esm/typings/src/llm-providers/openai/OpenAiAssistantExecutionTools.d.ts +1 -1
  97. package/esm/typings/src/llm-providers/openai/openai-models.d.ts +1 -1
  98. package/esm/typings/src/llm-providers/openai/register-configuration.d.ts +2 -2
  99. package/esm/typings/src/llm-providers/openai/register-constructor.d.ts +2 -2
  100. package/esm/typings/src/migrations/migratePipeline.d.ts +9 -0
  101. package/esm/typings/src/other/templates/getBookTemplates.d.ts +2 -2
  102. package/esm/typings/src/pipeline/PipelineInterface/PipelineInterface.d.ts +3 -3
  103. package/esm/typings/src/pipeline/PipelineInterface/constants.d.ts +1 -1
  104. package/esm/typings/src/pipeline/PipelineInterface/getPipelineInterface.d.ts +1 -1
  105. package/esm/typings/src/pipeline/PipelineInterface/isPipelineImplementingInterface.d.ts +5 -4
  106. package/esm/typings/src/pipeline/PipelineInterface/isPipelineInterfacesEqual.d.ts +1 -1
  107. package/esm/typings/src/pipeline/PipelineJson/CommonTaskJson.d.ts +10 -7
  108. package/esm/typings/src/pipeline/PipelineJson/ParameterJson.d.ts +1 -1
  109. package/esm/typings/src/pipeline/PipelineJson/PipelineJson.d.ts +3 -2
  110. package/esm/typings/src/pipeline/PipelineString.d.ts +3 -1
  111. package/esm/typings/src/pipeline/book-notation.d.ts +2 -2
  112. package/esm/typings/src/postprocessing/utils/extractBlock.d.ts +1 -1
  113. package/esm/typings/src/postprocessing/utils/extractJsonBlock.d.ts +2 -2
  114. package/esm/typings/src/prepare/prepareTasks.d.ts +8 -5
  115. package/esm/typings/src/remote-server/openapi.d.ts +1 -1
  116. package/esm/typings/src/remote-server/socket-types/listModels/PromptbookServer_ListModels_Response.d.ts +1 -1
  117. package/esm/typings/src/remote-server/startRemoteServer.d.ts +1 -1
  118. package/esm/typings/src/remote-server/types/RemoteServerOptions.d.ts +7 -6
  119. package/esm/typings/src/scrapers/_boilerplate/BoilerplateScraper.d.ts +3 -3
  120. package/esm/typings/src/scrapers/_boilerplate/createBoilerplateScraper.d.ts +1 -1
  121. package/esm/typings/src/scrapers/_boilerplate/register-metadata.d.ts +1 -1
  122. package/esm/typings/src/scrapers/_common/Converter.d.ts +3 -1
  123. package/esm/typings/src/scrapers/_common/Scraper.d.ts +4 -3
  124. package/esm/typings/src/scrapers/_common/ScraperIntermediateSource.d.ts +4 -2
  125. package/esm/typings/src/scrapers/_common/prepareKnowledgePieces.d.ts +2 -2
  126. package/esm/typings/src/scrapers/_common/register/$provideFilesystemForNode.d.ts +2 -1
  127. package/esm/typings/src/scrapers/_common/register/$provideScrapersForBrowser.d.ts +6 -3
  128. package/esm/typings/src/scrapers/_common/register/$provideScrapersForNode.d.ts +3 -5
  129. package/esm/typings/src/scrapers/_common/register/$scrapersMetadataRegister.d.ts +3 -3
  130. package/esm/typings/src/scrapers/_common/register/$scrapersRegister.d.ts +3 -2
  131. package/esm/typings/src/scrapers/_common/register/ScraperAndConverterMetadata.d.ts +8 -5
  132. package/esm/typings/src/scrapers/_common/register/ScraperConstructor.d.ts +2 -1
  133. package/esm/typings/src/scrapers/_common/utils/getScraperIntermediateSource.d.ts +6 -5
  134. package/esm/typings/src/scrapers/_common/utils/makeKnowledgeSourceHandler.d.ts +3 -1
  135. package/esm/typings/src/scrapers/document/createDocumentScraper.d.ts +1 -1
  136. package/esm/typings/src/scrapers/document-legacy/createLegacyDocumentScraper.d.ts +2 -1
  137. package/esm/typings/src/scrapers/markdown/createMarkdownScraper.d.ts +4 -1
  138. package/esm/typings/src/scrapers/markitdown/MarkitdownScraper.d.ts +1 -1
  139. package/esm/typings/src/scrapers/pdf/createPdfScraper.d.ts +2 -1
  140. package/esm/typings/src/scrapers/website/createWebsiteScraper.d.ts +3 -4
  141. package/esm/typings/src/scripting/javascript/JavascriptExecutionTools.d.ts +1 -1
  142. package/esm/typings/src/scripting/javascript/postprocessing-functions.d.ts +5 -1
  143. package/esm/typings/src/scripting/javascript/utils/preserve.d.ts +1 -1
  144. package/esm/typings/src/storage/_common/PromptbookStorage.d.ts +1 -1
  145. package/esm/typings/src/storage/file-cache-storage/FileCacheStorage.d.ts +12 -5
  146. package/esm/typings/src/storage/file-cache-storage/FileCacheStorageOptions.d.ts +4 -2
  147. package/esm/typings/src/storage/file-cache-storage/utils/nameToSubfolderPath.d.ts +2 -1
  148. package/esm/typings/src/storage/local-storage/getIndexedDbStorage.d.ts +11 -0
  149. package/esm/typings/src/storage/local-storage/utils/IndexedDbStorageOptions.d.ts +14 -0
  150. package/esm/typings/src/storage/local-storage/utils/makePromptbookStorageFromIndexedDb.d.ts +8 -0
  151. package/esm/typings/src/storage/local-storage/utils/makePromptbookStorageFromWebStorage.d.ts +2 -1
  152. package/esm/typings/src/types/IntermediateFilesStrategy.d.ts +2 -1
  153. package/esm/typings/src/types/ModelRequirements.d.ts +2 -2
  154. package/esm/typings/src/types/ModelVariant.d.ts +5 -5
  155. package/esm/typings/src/types/typeAliases.d.ts +22 -19
  156. package/esm/typings/src/utils/$Register.d.ts +8 -7
  157. package/esm/typings/src/utils/editable/edit-pipeline-string/addPipelineCommand.d.ts +2 -2
  158. package/esm/typings/src/utils/editable/edit-pipeline-string/deflatePipeline.d.ts +4 -1
  159. package/esm/typings/src/utils/editable/utils/isFlatPipeline.d.ts +2 -1
  160. package/esm/typings/src/utils/editable/utils/stringifyPipelineJson.d.ts +1 -1
  161. package/esm/typings/src/utils/environment/$getGlobalScope.d.ts +2 -1
  162. package/esm/typings/src/utils/expectation-counters/index.d.ts +1 -1
  163. package/esm/typings/src/utils/markdown/extractAllBlocksFromMarkdown.d.ts +2 -2
  164. package/esm/typings/src/utils/markdown/extractAllListItemsFromMarkdown.d.ts +1 -1
  165. package/esm/typings/src/utils/markdown/extractOneBlockFromMarkdown.d.ts +2 -2
  166. package/esm/typings/src/utils/normalization/nameToUriPart.d.ts +4 -4
  167. package/esm/typings/src/utils/normalization/nameToUriParts.d.ts +4 -4
  168. package/esm/typings/src/utils/normalization/normalize-to-kebab-case.d.ts +3 -3
  169. package/esm/typings/src/utils/normalization/normalizeTo_SCREAMING_CASE.d.ts +3 -3
  170. package/esm/typings/src/utils/normalization/normalizeTo_camelCase.d.ts +4 -4
  171. package/esm/typings/src/utils/normalization/normalizeTo_snake_case.d.ts +3 -3
  172. package/esm/typings/src/utils/normalization/removeDiacritics.d.ts +3 -3
  173. package/esm/typings/src/utils/normalization/searchKeywords.d.ts +4 -1
  174. package/esm/typings/src/utils/normalization/titleToName.d.ts +4 -4
  175. package/esm/typings/src/utils/organization/TODO_USE.d.ts +1 -1
  176. package/esm/typings/src/utils/organization/empty_object.d.ts +2 -2
  177. package/esm/typings/src/utils/organization/just.d.ts +1 -1
  178. package/esm/typings/src/utils/organization/just_empty_object.d.ts +4 -4
  179. package/esm/typings/src/utils/organization/keepUnused.d.ts +1 -1
  180. package/esm/typings/src/utils/parameters/mapAvailableToExpectedParameters.d.ts +7 -7
  181. package/esm/typings/src/utils/removeQuotes.d.ts +2 -2
  182. package/esm/typings/src/utils/serialization/clonePipeline.d.ts +4 -3
  183. package/esm/typings/src/utils/serialization/deepClone.d.ts +5 -1
  184. package/esm/typings/src/utils/trimCodeBlock.d.ts +1 -1
  185. package/esm/typings/src/utils/trimEndOfCodeBlock.d.ts +1 -1
  186. package/esm/typings/src/utils/unwrapResult.d.ts +2 -2
  187. package/esm/typings/src/utils/validators/javascriptName/isValidJavascriptName.d.ts +3 -3
  188. package/esm/typings/src/utils/validators/parameterName/validateParameterName.d.ts +5 -4
  189. package/esm/typings/src/utils/validators/semanticVersion/isValidPromptbookVersion.d.ts +1 -1
  190. package/esm/typings/src/utils/validators/semanticVersion/isValidSemanticVersion.d.ts +1 -1
  191. package/esm/typings/src/utils/validators/url/isHostnameOnPrivateNetwork.d.ts +1 -1
  192. package/esm/typings/src/utils/validators/url/isUrlOnPrivateNetwork.d.ts +1 -1
  193. package/esm/typings/src/utils/validators/url/isValidPipelineUrl.d.ts +1 -1
  194. package/esm/typings/src/utils/validators/url/isValidUrl.d.ts +1 -1
  195. package/esm/typings/src/version.d.ts +2 -1
  196. package/esm/typings/src/wizzard/wizzard.d.ts +1 -1
  197. package/package.json +14 -2
  198. package/umd/index.umd.js +804 -534
  199. package/umd/index.umd.js.map +1 -1
  200. package/esm/typings/src/formats/_common/FormatSubvalueDefinition.d.ts +0 -31
  201. package/esm/typings/src/formats/csv/CsvFormatDefinition.d.ts +0 -17
  202. package/esm/typings/src/formats/json/JsonFormatDefinition.d.ts +0 -19
  203. package/esm/typings/src/formats/xml/XmlFormatDefinition.d.ts +0 -19
  204. /package/esm/typings/src/llm-providers/{multiple → _multiple}/MultipleLlmExecutionTools.d.ts +0 -0
  205. /package/esm/typings/src/llm-providers/{multiple → _multiple}/joinLlmExecutionTools.d.ts +0 -0
  206. /package/esm/typings/src/llm-providers/{multiple → _multiple}/playground/playground.d.ts +0 -0
package/umd/index.umd.js CHANGED
@@ -1,8 +1,8 @@
1
1
  (function (global, factory) {
2
- typeof exports === 'object' && typeof module !== 'undefined' ? factory(exports, require('colors'), require('fs/promises'), require('path'), require('spacetrim'), require('jszip'), require('prettier'), require('prettier/parser-html'), require('rxjs'), require('crypto'), require('waitasecond'), require('papaparse'), require('crypto-js/enc-hex'), require('crypto-js/sha256'), require('crypto-js'), require('mime-types'), require('child_process'), require('dotenv')) :
3
- typeof define === 'function' && define.amd ? define(['exports', 'colors', 'fs/promises', 'path', 'spacetrim', 'jszip', 'prettier', 'prettier/parser-html', 'rxjs', 'crypto', 'waitasecond', 'papaparse', 'crypto-js/enc-hex', 'crypto-js/sha256', 'crypto-js', 'mime-types', 'child_process', 'dotenv'], factory) :
4
- (global = typeof globalThis !== 'undefined' ? globalThis : global || self, factory(global["promptbook-node"] = {}, global.colors, global.promises, global.path, global.spaceTrim, global.JSZip, global.prettier, global.parserHtml, global.rxjs, global.crypto, global.waitasecond, global.papaparse, global.hexEncoder, global.sha256, global.cryptoJs, global.mimeTypes, global.child_process, global.dotenv));
5
- })(this, (function (exports, colors, promises, path, spaceTrim, JSZip, prettier, parserHtml, rxjs, crypto, waitasecond, papaparse, hexEncoder, sha256, cryptoJs, mimeTypes, child_process, dotenv) { 'use strict';
2
+ typeof exports === 'object' && typeof module !== 'undefined' ? factory(exports, require('colors'), require('fs/promises'), require('path'), require('spacetrim'), require('jszip'), require('prettier'), require('prettier/parser-html'), require('crypto'), require('rxjs'), require('waitasecond'), require('papaparse'), require('crypto-js/enc-hex'), require('crypto-js/sha256'), require('crypto-js'), require('mime-types'), require('child_process'), require('dotenv')) :
3
+ typeof define === 'function' && define.amd ? define(['exports', 'colors', 'fs/promises', 'path', 'spacetrim', 'jszip', 'prettier', 'prettier/parser-html', 'crypto', 'rxjs', 'waitasecond', 'papaparse', 'crypto-js/enc-hex', 'crypto-js/sha256', 'crypto-js', 'mime-types', 'child_process', 'dotenv'], factory) :
4
+ (global = typeof globalThis !== 'undefined' ? globalThis : global || self, factory(global["promptbook-node"] = {}, global.colors, global.promises, global.path, global.spaceTrim, global.JSZip, global.prettier, global.parserHtml, global.crypto, global.rxjs, global.waitasecond, global.papaparse, global.hexEncoder, global.sha256, global.cryptoJs, global.mimeTypes, global.child_process, global.dotenv));
5
+ })(this, (function (exports, colors, promises, path, spaceTrim, JSZip, prettier, parserHtml, crypto, rxjs, waitasecond, papaparse, hexEncoder, sha256, cryptoJs, mimeTypes, child_process, dotenv) { 'use strict';
6
6
 
7
7
  function _interopDefaultLegacy (e) { return e && typeof e === 'object' && 'default' in e ? e : { 'default': e }; }
8
8
 
@@ -46,7 +46,7 @@
46
46
  * @generated
47
47
  * @see https://github.com/webgptorg/promptbook
48
48
  */
49
- const PROMPTBOOK_ENGINE_VERSION = '0.92.0-9';
49
+ const PROMPTBOOK_ENGINE_VERSION = '0.92.0';
50
50
  /**
51
51
  * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
52
52
  * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -56,7 +56,7 @@
56
56
  * Returns the same value that is passed as argument.
57
57
  * No side effects.
58
58
  *
59
- * Note: It can be usefull for:
59
+ * Note: It can be useful for:
60
60
  *
61
61
  * 1) Leveling indentation
62
62
  * 2) Putting always-true or always-false conditions without getting eslint errors
@@ -117,6 +117,21 @@
117
117
  * @public exported from `@promptbook/core`
118
118
  */
119
119
  const DEFAULT_MAX_FILE_SIZE = 100 * 1024 * 1024; // 100MB
120
+ /**
121
+ * Threshold value that determines when a dataset is considered "big"
122
+ * and may require special handling or optimizations
123
+ *
124
+ * For example, when error occurs in one item of the big dataset, it will not fail the whole pipeline
125
+ *
126
+ * @public exported from `@promptbook/core`
127
+ */
128
+ const BIG_DATASET_TRESHOLD = 50;
129
+ /**
130
+ * Placeholder text used to represent a placeholder value of failed operation
131
+ *
132
+ * @public exported from `@promptbook/core`
133
+ */
134
+ const FAILED_VALUE_PLACEHOLDER = '!?';
120
135
  // <- TODO: [🧠] Better system for generator warnings - not always "code" and "by `@promptbook/cli`"
121
136
  /**
122
137
  * The maximum number of iterations for a loops
@@ -211,7 +226,7 @@
211
226
  const DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME = `index`;
212
227
  // <- TODO: [🧜‍♂️]
213
228
  /**
214
- * @@@
229
+ * Default settings for parsing and generating CSV files in Promptbook.
215
230
  *
216
231
  * @public exported from `@promptbook/core`
217
232
  */
@@ -222,19 +237,19 @@
222
237
  skipEmptyLines: true,
223
238
  });
224
239
  /**
225
- * @@@
240
+ * Controls whether verbose logging is enabled by default throughout the application.
226
241
  *
227
242
  * @public exported from `@promptbook/core`
228
243
  */
229
244
  let DEFAULT_IS_VERBOSE = false;
230
245
  /**
231
- * @@@
246
+ * Controls whether auto-installation of dependencies is enabled by default.
232
247
  *
233
248
  * @public exported from `@promptbook/core`
234
249
  */
235
250
  const DEFAULT_IS_AUTO_INSTALLED = false;
236
251
  /**
237
- * @@@
252
+ * Indicates whether pipeline logic validation is enabled. When true, the pipeline logic is checked for consistency.
238
253
  *
239
254
  * @private within the repository
240
255
  */
@@ -302,7 +317,7 @@
302
317
  ${block(message)}
303
318
 
304
319
  Note: This error should not happen.
305
- It's probbably a bug in the pipeline collection
320
+ It's probably a bug in the pipeline collection
306
321
 
307
322
  Please report issue:
308
323
  ${block(getErrorReportUrl(new Error(message)).href)}
@@ -344,15 +359,12 @@
344
359
  }
345
360
  throw new Error(spaceTrim__default["default"]((block) => `
346
361
  ${block(error.message)}
347
-
362
+
348
363
  The JSON text:
349
364
  ${block(value)}
350
365
  `));
351
366
  }
352
367
  }
353
- /**
354
- * TODO: !!!! Use in Promptbook.studio
355
- */
356
368
 
357
369
  /**
358
370
  * Orders JSON object by keys
@@ -583,8 +595,12 @@
583
595
  */
584
596
 
585
597
  /**
586
- * @@@
598
+ * Creates a deep clone of the given object
599
+ *
600
+ * Note: This method only works for objects that are fully serializable to JSON and do not contain functions, Dates, or special types.
587
601
  *
602
+ * @param objectValue The object to clone.
603
+ * @returns A deep, writable clone of the input object.
588
604
  * @public exported from `@promptbook/utils`
589
605
  */
590
606
  function deepClone(objectValue) {
@@ -666,13 +682,13 @@
666
682
  */
667
683
  const REPLACING_NONCE = 'ptbkauk42kV2dzao34faw7FudQUHYPtW';
668
684
  /**
669
- * @@@
685
+ * Placeholder value indicating a parameter is missing its value.
670
686
  *
671
687
  * @private within the repository
672
688
  */
673
689
  const RESERVED_PARAMETER_MISSING_VALUE = 'MISSING-' + REPLACING_NONCE;
674
690
  /**
675
- * @@@
691
+ * Placeholder value indicating a parameter is restricted and cannot be used directly.
676
692
  *
677
693
  * @private within the repository
678
694
  */
@@ -733,7 +749,7 @@
733
749
  /**
734
750
  * Tests if given string is valid semantic version
735
751
  *
736
- * Note: There are two simmilar functions:
752
+ * Note: There are two similar functions:
737
753
  * - `isValidSemanticVersion` which tests any semantic version
738
754
  * - `isValidPromptbookVersion` *(this one)* which tests just Promptbook versions
739
755
  *
@@ -755,7 +771,7 @@
755
771
  *
756
772
  * @see https://www.npmjs.com/package/promptbook?activeTab=versions
757
773
  * Note: When you are using for example promptbook 2.0.0 and there already is promptbook 3.0.0 it don`t know about it.
758
- * Note: There are two simmilar functions:
774
+ * Note: There are two similar functions:
759
775
  * - `isValidSemanticVersion` which tests any semantic version
760
776
  * - `isValidPromptbookVersion` *(this one)* which tests just Promptbook versions
761
777
  *
@@ -776,7 +792,7 @@
776
792
  * Tests if given string is valid URL.
777
793
  *
778
794
  * Note: Dataurl are considered perfectly valid.
779
- * Note: There are two simmilar functions:
795
+ * Note: There are two similar functions:
780
796
  * - `isValidUrl` which tests any URL
781
797
  * - `isValidPipelineUrl` *(this one)* which tests just promptbook URL
782
798
  *
@@ -804,7 +820,7 @@
804
820
  /**
805
821
  * Tests if given string is valid pipeline URL URL.
806
822
  *
807
- * Note: There are two simmilar functions:
823
+ * Note: There are two similar functions:
808
824
  * - `isValidUrl` which tests any URL
809
825
  * - `isValidPipelineUrl` *(this one)* which tests just pipeline URL
810
826
  *
@@ -901,7 +917,7 @@
901
917
  ${block(pipelineIdentification)}
902
918
  `));
903
919
  }
904
- // TODO: [🧠] Maybe do here some propper JSON-schema / ZOD checking
920
+ // TODO: [🧠] Maybe do here some proper JSON-schema / ZOD checking
905
921
  if (!Array.isArray(pipeline.parameters)) {
906
922
  // TODO: [🧠] what is the correct error tp throw - maybe PromptbookSchemaError
907
923
  throw new ParseError(spaceTrim.spaceTrim((block) => `
@@ -912,7 +928,7 @@
912
928
  ${block(pipelineIdentification)}
913
929
  `));
914
930
  }
915
- // TODO: [🧠] Maybe do here some propper JSON-schema / ZOD checking
931
+ // TODO: [🧠] Maybe do here some proper JSON-schema / ZOD checking
916
932
  if (!Array.isArray(pipeline.tasks)) {
917
933
  // TODO: [🧠] what is the correct error tp throw - maybe PromptbookSchemaError
918
934
  throw new ParseError(spaceTrim.spaceTrim((block) => `
@@ -1223,7 +1239,7 @@
1223
1239
  * Function isValidJsonString will tell you if the string is valid JSON or not
1224
1240
  *
1225
1241
  * @param value The string to check
1226
- * @returns True if the string is a valid JSON string, false otherwise
1242
+ * @returns `true` if the string is a valid JSON string, false otherwise
1227
1243
  *
1228
1244
  * @public exported from `@promptbook/utils`
1229
1245
  */
@@ -1335,7 +1351,7 @@
1335
1351
  if (bookVersion !== `undefined`) {
1336
1352
  commands.push(`BOOK VERSION ${bookVersion}`);
1337
1353
  }
1338
- // TODO: [main] !!5 This increases size of the bundle and is probbably not necessary
1354
+ // TODO: [main] !!5 This increases size of the bundle and is probably not necessary
1339
1355
  pipelineString = prettifyMarkdown(pipelineString);
1340
1356
  for (const parameter of parameters.filter(({ isInput }) => isInput)) {
1341
1357
  commands.push(`INPUT PARAMETER ${taskParameterJsonToString(parameter)}`);
@@ -1545,7 +1561,7 @@
1545
1561
  /**
1546
1562
  * Constructs a pipeline collection from pipelines
1547
1563
  *
1548
- * @param pipelines @@@
1564
+ * @param pipelines Array of pipeline JSON objects to include in the collection
1549
1565
  *
1550
1566
  * Note: During the construction logic of all pipelines are validated
1551
1567
  * Note: It is not recommended to use this constructor directly, use `createCollectionFromJson` *(or other variant)* instead
@@ -1659,7 +1675,7 @@
1659
1675
  super(spaceTrim.spaceTrim((block) => `
1660
1676
  ${block(message)}
1661
1677
 
1662
- Note: You have probbably forgot to provide some tools for pipeline execution or preparation
1678
+ Note: You have probably forgot to provide some tools for pipeline execution or preparation
1663
1679
 
1664
1680
  `));
1665
1681
  this.name = 'MissingToolsError';
@@ -1675,15 +1691,18 @@
1675
1691
  * @public exported from `@promptbook/core`
1676
1692
  */
1677
1693
  function isPipelinePrepared(pipeline) {
1678
- // Note: Ignoring `pipeline.preparations` @@@
1679
- // Note: Ignoring `pipeline.knowledgePieces` @@@
1694
+ // Note: Ignoring `pipeline.preparations`
1695
+ // Note: Ignoring `pipeline.knowledgePieces`
1680
1696
  if (pipeline.title === undefined || pipeline.title === '' || pipeline.title === DEFAULT_BOOK_TITLE) {
1697
+ // console.log('Pipeline is not prepared because title is undefined or empty', pipeline);
1681
1698
  return false;
1682
1699
  }
1683
1700
  if (!pipeline.personas.every((persona) => persona.modelsRequirements !== undefined)) {
1701
+ // console.log('Pipeline is not prepared because personas are not prepared', pipeline.personas);
1684
1702
  return false;
1685
1703
  }
1686
1704
  if (!pipeline.knowledgeSources.every((knowledgeSource) => knowledgeSource.preparationIds !== undefined)) {
1705
+ //console.log('Pipeline is not prepared because knowledge sources are not prepared', pipeline.knowledgeSources);
1687
1706
  return false;
1688
1707
  }
1689
1708
  /*
@@ -1704,70 +1723,6 @@
1704
1723
  * - [♨] Are tasks prepared
1705
1724
  */
1706
1725
 
1707
- /**
1708
- * Generates random token
1709
- *
1710
- * Note: This function is cryptographically secure (it uses crypto.randomBytes internally)
1711
- *
1712
- * @private internal helper function
1713
- * @returns secure random token
1714
- */
1715
- function $randomToken(randomness) {
1716
- return crypto.randomBytes(randomness).toString('hex');
1717
- }
1718
- /**
1719
- * TODO: Maybe use nanoid instead https://github.com/ai/nanoid
1720
- */
1721
-
1722
- /**
1723
- * Recursively converts JSON strings to JSON objects
1724
-
1725
- * @public exported from `@promptbook/utils`
1726
- */
1727
- function jsonStringsToJsons(object) {
1728
- if (object === null) {
1729
- return object;
1730
- }
1731
- if (Array.isArray(object)) {
1732
- return object.map(jsonStringsToJsons);
1733
- }
1734
- if (typeof object !== 'object') {
1735
- return object;
1736
- }
1737
- const newObject = { ...object };
1738
- for (const [key, value] of Object.entries(object)) {
1739
- if (typeof value === 'string' && isValidJsonString(value)) {
1740
- newObject[key] = jsonParse(value);
1741
- }
1742
- else {
1743
- newObject[key] = jsonStringsToJsons(value);
1744
- }
1745
- }
1746
- return newObject;
1747
- }
1748
- /**
1749
- * TODO: Type the return type correctly
1750
- */
1751
-
1752
- /**
1753
- * This error indicates errors during the execution of the pipeline
1754
- *
1755
- * @public exported from `@promptbook/core`
1756
- */
1757
- class PipelineExecutionError extends Error {
1758
- constructor(message) {
1759
- // Added id parameter
1760
- super(message);
1761
- this.name = 'PipelineExecutionError';
1762
- // TODO: [🐙] DRY - Maybe $randomId
1763
- this.id = `error-${$randomToken(8 /* <- TODO: To global config + Use Base58 to avoid simmilar char conflicts */)}`;
1764
- Object.setPrototypeOf(this, PipelineExecutionError.prototype);
1765
- }
1766
- }
1767
- /**
1768
- * TODO: [🧠][🌂] Add id to all errors
1769
- */
1770
-
1771
1726
  /**
1772
1727
  * This error indicates problems parsing the format value
1773
1728
  *
@@ -1812,7 +1767,7 @@
1812
1767
  }
1813
1768
 
1814
1769
  /**
1815
- * This error indicates that the pipeline collection cannot be propperly loaded
1770
+ * This error indicates that the pipeline collection cannot be properly loaded
1816
1771
  *
1817
1772
  * @public exported from `@promptbook/core`
1818
1773
  */
@@ -1902,6 +1857,40 @@
1902
1857
  }
1903
1858
  }
1904
1859
 
1860
+ /**
1861
+ * Generates random token
1862
+ *
1863
+ * Note: This function is cryptographically secure (it uses crypto.randomBytes internally)
1864
+ *
1865
+ * @private internal helper function
1866
+ * @returns secure random token
1867
+ */
1868
+ function $randomToken(randomness) {
1869
+ return crypto.randomBytes(randomness).toString('hex');
1870
+ }
1871
+ /**
1872
+ * TODO: Maybe use nanoid instead https://github.com/ai/nanoid
1873
+ */
1874
+
1875
+ /**
1876
+ * This error indicates errors during the execution of the pipeline
1877
+ *
1878
+ * @public exported from `@promptbook/core`
1879
+ */
1880
+ class PipelineExecutionError extends Error {
1881
+ constructor(message) {
1882
+ // Added id parameter
1883
+ super(message);
1884
+ this.name = 'PipelineExecutionError';
1885
+ // TODO: [🐙] DRY - Maybe $randomId
1886
+ this.id = `error-${$randomToken(8 /* <- TODO: To global config + Use Base58 to avoid similar char conflicts */)}`;
1887
+ Object.setPrototypeOf(this, PipelineExecutionError.prototype);
1888
+ }
1889
+ }
1890
+ /**
1891
+ * TODO: [🧠][🌂] Add id to all errors
1892
+ */
1893
+
1905
1894
  /**
1906
1895
  * Error thrown when a fetch request fails
1907
1896
  *
@@ -1977,6 +1966,65 @@
1977
1966
  * Note: [💞] Ignore a discrepancy between file name and entity name
1978
1967
  */
1979
1968
 
1969
+ /**
1970
+ * Serializes an error into a [🚉] JSON-serializable object
1971
+ *
1972
+ * @public exported from `@promptbook/utils`
1973
+ */
1974
+ function serializeError(error) {
1975
+ const { name, message, stack } = error;
1976
+ const { id } = error;
1977
+ if (!Object.keys(ALL_ERRORS).includes(name)) {
1978
+ console.error(spaceTrim__default["default"]((block) => `
1979
+
1980
+ Cannot serialize error with name "${name}"
1981
+
1982
+ Authors of Promptbook probably forgot to add this error into the list of errors:
1983
+ https://github.com/webgptorg/promptbook/blob/main/src/errors/0-index.ts
1984
+
1985
+
1986
+ ${block(stack || message)}
1987
+
1988
+ `));
1989
+ }
1990
+ return {
1991
+ name: name,
1992
+ message,
1993
+ stack,
1994
+ id, // Include id in the serialized object
1995
+ };
1996
+ }
1997
+
1998
+ /**
1999
+ * Recursively converts JSON strings to JSON objects
2000
+
2001
+ * @public exported from `@promptbook/utils`
2002
+ */
2003
+ function jsonStringsToJsons(object) {
2004
+ if (object === null) {
2005
+ return object;
2006
+ }
2007
+ if (Array.isArray(object)) {
2008
+ return object.map(jsonStringsToJsons);
2009
+ }
2010
+ if (typeof object !== 'object') {
2011
+ return object;
2012
+ }
2013
+ const newObject = { ...object };
2014
+ for (const [key, value] of Object.entries(object)) {
2015
+ if (typeof value === 'string' && isValidJsonString(value)) {
2016
+ newObject[key] = jsonParse(value);
2017
+ }
2018
+ else {
2019
+ newObject[key] = jsonStringsToJsons(value);
2020
+ }
2021
+ }
2022
+ return newObject;
2023
+ }
2024
+ /**
2025
+ * TODO: Type the return type correctly
2026
+ */
2027
+
1980
2028
  /**
1981
2029
  * Deserializes the error object
1982
2030
  *
@@ -2051,8 +2099,9 @@
2051
2099
  */
2052
2100
  function createTask(options) {
2053
2101
  const { taskType, taskProcessCallback } = options;
2102
+ let { title } = options;
2054
2103
  // TODO: [🐙] DRY
2055
- const taskId = `${taskType.toLowerCase().substring(0, 4)}-${$randomToken(8 /* <- TODO: To global config + Use Base58 to avoid simmilar char conflicts */)}`;
2104
+ const taskId = `${taskType.toLowerCase().substring(0, 4)}-${$randomToken(8 /* <- TODO: To global config + Use Base58 to avoid similar char conflicts */)}`;
2056
2105
  let status = 'RUNNING';
2057
2106
  const createdAt = new Date();
2058
2107
  let updatedAt = createdAt;
@@ -2062,6 +2111,10 @@
2062
2111
  const partialResultSubject = new rxjs.Subject();
2063
2112
  // <- Note: Not using `BehaviorSubject` because on error we can't access the last value
2064
2113
  const finalResultPromise = /* not await */ taskProcessCallback((newOngoingResult) => {
2114
+ if (newOngoingResult.title) {
2115
+ title = newOngoingResult.title;
2116
+ }
2117
+ updatedAt = new Date();
2065
2118
  Object.assign(currentValue, newOngoingResult);
2066
2119
  // <- TODO: assign deep
2067
2120
  partialResultSubject.next(newOngoingResult);
@@ -2107,17 +2160,24 @@
2107
2160
  return {
2108
2161
  taskType,
2109
2162
  taskId,
2163
+ get promptbookVersion() {
2164
+ return PROMPTBOOK_ENGINE_VERSION;
2165
+ },
2166
+ get title() {
2167
+ return title;
2168
+ // <- Note: [1] These must be getters to allow changing the value in the future
2169
+ },
2110
2170
  get status() {
2111
2171
  return status;
2112
- // <- Note: [1] Theese must be getters to allow changing the value in the future
2172
+ // <- Note: [1] --||--
2113
2173
  },
2114
2174
  get createdAt() {
2115
2175
  return createdAt;
2116
- // <- Note: [1]
2176
+ // <- Note: [1] --||--
2117
2177
  },
2118
2178
  get updatedAt() {
2119
2179
  return updatedAt;
2120
- // <- Note: [1]
2180
+ // <- Note: [1] --||--
2121
2181
  },
2122
2182
  asPromise,
2123
2183
  asObservable() {
@@ -2125,15 +2185,15 @@
2125
2185
  },
2126
2186
  get errors() {
2127
2187
  return errors;
2128
- // <- Note: [1]
2188
+ // <- Note: [1] --||--
2129
2189
  },
2130
2190
  get warnings() {
2131
2191
  return warnings;
2132
- // <- Note: [1]
2192
+ // <- Note: [1] --||--
2133
2193
  },
2134
2194
  get currentValue() {
2135
2195
  return currentValue;
2136
- // <- Note: [1]
2196
+ // <- Note: [1] --||--
2137
2197
  },
2138
2198
  };
2139
2199
  }
@@ -2143,33 +2203,72 @@
2143
2203
  */
2144
2204
 
2145
2205
  /**
2146
- * Serializes an error into a [🚉] JSON-serializable object
2206
+ * Represents the uncertain value
2147
2207
  *
2148
- * @public exported from `@promptbook/utils`
2208
+ * @public exported from `@promptbook/core`
2209
+ */
2210
+ const ZERO_VALUE = $deepFreeze({ value: 0 });
2211
+ /**
2212
+ * Represents the uncertain value
2213
+ *
2214
+ * @public exported from `@promptbook/core`
2215
+ */
2216
+ const UNCERTAIN_ZERO_VALUE = $deepFreeze({ value: 0, isUncertain: true });
2217
+ /**
2218
+ * Represents the usage with no resources consumed
2219
+ *
2220
+ * @public exported from `@promptbook/core`
2221
+ */
2222
+ const ZERO_USAGE = $deepFreeze({
2223
+ price: ZERO_VALUE,
2224
+ input: {
2225
+ tokensCount: ZERO_VALUE,
2226
+ charactersCount: ZERO_VALUE,
2227
+ wordsCount: ZERO_VALUE,
2228
+ sentencesCount: ZERO_VALUE,
2229
+ linesCount: ZERO_VALUE,
2230
+ paragraphsCount: ZERO_VALUE,
2231
+ pagesCount: ZERO_VALUE,
2232
+ },
2233
+ output: {
2234
+ tokensCount: ZERO_VALUE,
2235
+ charactersCount: ZERO_VALUE,
2236
+ wordsCount: ZERO_VALUE,
2237
+ sentencesCount: ZERO_VALUE,
2238
+ linesCount: ZERO_VALUE,
2239
+ paragraphsCount: ZERO_VALUE,
2240
+ pagesCount: ZERO_VALUE,
2241
+ },
2242
+ });
2243
+ /**
2244
+ * Represents the usage with unknown resources consumed
2245
+ *
2246
+ * @public exported from `@promptbook/core`
2247
+ */
2248
+ const UNCERTAIN_USAGE = $deepFreeze({
2249
+ price: UNCERTAIN_ZERO_VALUE,
2250
+ input: {
2251
+ tokensCount: UNCERTAIN_ZERO_VALUE,
2252
+ charactersCount: UNCERTAIN_ZERO_VALUE,
2253
+ wordsCount: UNCERTAIN_ZERO_VALUE,
2254
+ sentencesCount: UNCERTAIN_ZERO_VALUE,
2255
+ linesCount: UNCERTAIN_ZERO_VALUE,
2256
+ paragraphsCount: UNCERTAIN_ZERO_VALUE,
2257
+ pagesCount: UNCERTAIN_ZERO_VALUE,
2258
+ },
2259
+ output: {
2260
+ tokensCount: UNCERTAIN_ZERO_VALUE,
2261
+ charactersCount: UNCERTAIN_ZERO_VALUE,
2262
+ wordsCount: UNCERTAIN_ZERO_VALUE,
2263
+ sentencesCount: UNCERTAIN_ZERO_VALUE,
2264
+ linesCount: UNCERTAIN_ZERO_VALUE,
2265
+ paragraphsCount: UNCERTAIN_ZERO_VALUE,
2266
+ pagesCount: UNCERTAIN_ZERO_VALUE,
2267
+ },
2268
+ });
2269
+ /**
2270
+ * Note: [💞] Ignore a discrepancy between file name and entity name
2149
2271
  */
2150
- function serializeError(error) {
2151
- const { name, message, stack } = error;
2152
- const { id } = error;
2153
- if (!Object.keys(ALL_ERRORS).includes(name)) {
2154
- console.error(spaceTrim__default["default"]((block) => `
2155
-
2156
- Cannot serialize error with name "${name}"
2157
-
2158
- Authors of Promptbook probably forgot to add this error into the list of errors:
2159
- https://github.com/webgptorg/promptbook/blob/main/src/errors/0-index.ts
2160
-
2161
-
2162
- ${block(stack || message)}
2163
-
2164
- `));
2165
- }
2166
- return {
2167
- name: name,
2168
- message,
2169
- stack,
2170
- id, // Include id in the serialized object
2171
- };
2172
- }
2173
2272
 
2174
2273
  /**
2175
2274
  * Format either small or big number
@@ -2252,74 +2351,6 @@
2252
2351
  }
2253
2352
  }
2254
2353
 
2255
- /**
2256
- * Represents the uncertain value
2257
- *
2258
- * @public exported from `@promptbook/core`
2259
- */
2260
- const ZERO_VALUE = $deepFreeze({ value: 0 });
2261
- /**
2262
- * Represents the uncertain value
2263
- *
2264
- * @public exported from `@promptbook/core`
2265
- */
2266
- const UNCERTAIN_ZERO_VALUE = $deepFreeze({ value: 0, isUncertain: true });
2267
- /**
2268
- * Represents the usage with no resources consumed
2269
- *
2270
- * @public exported from `@promptbook/core`
2271
- */
2272
- const ZERO_USAGE = $deepFreeze({
2273
- price: ZERO_VALUE,
2274
- input: {
2275
- tokensCount: ZERO_VALUE,
2276
- charactersCount: ZERO_VALUE,
2277
- wordsCount: ZERO_VALUE,
2278
- sentencesCount: ZERO_VALUE,
2279
- linesCount: ZERO_VALUE,
2280
- paragraphsCount: ZERO_VALUE,
2281
- pagesCount: ZERO_VALUE,
2282
- },
2283
- output: {
2284
- tokensCount: ZERO_VALUE,
2285
- charactersCount: ZERO_VALUE,
2286
- wordsCount: ZERO_VALUE,
2287
- sentencesCount: ZERO_VALUE,
2288
- linesCount: ZERO_VALUE,
2289
- paragraphsCount: ZERO_VALUE,
2290
- pagesCount: ZERO_VALUE,
2291
- },
2292
- });
2293
- /**
2294
- * Represents the usage with unknown resources consumed
2295
- *
2296
- * @public exported from `@promptbook/core`
2297
- */
2298
- $deepFreeze({
2299
- price: UNCERTAIN_ZERO_VALUE,
2300
- input: {
2301
- tokensCount: UNCERTAIN_ZERO_VALUE,
2302
- charactersCount: UNCERTAIN_ZERO_VALUE,
2303
- wordsCount: UNCERTAIN_ZERO_VALUE,
2304
- sentencesCount: UNCERTAIN_ZERO_VALUE,
2305
- linesCount: UNCERTAIN_ZERO_VALUE,
2306
- paragraphsCount: UNCERTAIN_ZERO_VALUE,
2307
- pagesCount: UNCERTAIN_ZERO_VALUE,
2308
- },
2309
- output: {
2310
- tokensCount: UNCERTAIN_ZERO_VALUE,
2311
- charactersCount: UNCERTAIN_ZERO_VALUE,
2312
- wordsCount: UNCERTAIN_ZERO_VALUE,
2313
- sentencesCount: UNCERTAIN_ZERO_VALUE,
2314
- linesCount: UNCERTAIN_ZERO_VALUE,
2315
- paragraphsCount: UNCERTAIN_ZERO_VALUE,
2316
- pagesCount: UNCERTAIN_ZERO_VALUE,
2317
- },
2318
- });
2319
- /**
2320
- * Note: [💞] Ignore a discrepancy between file name and entity name
2321
- */
2322
-
2323
2354
  /**
2324
2355
  * Function `addUsage` will add multiple usages into one
2325
2356
  *
@@ -2528,7 +2559,7 @@
2528
2559
  }
2529
2560
 
2530
2561
  /**
2531
- * @@@
2562
+ * Contains configuration options for parsing and generating CSV files, such as delimiters and quoting rules.
2532
2563
  *
2533
2564
  * @public exported from `@promptbook/core`
2534
2565
  */
@@ -2537,11 +2568,29 @@
2537
2568
  // encoding: 'utf-8',
2538
2569
  });
2539
2570
 
2571
+ /**
2572
+ * Converts a CSV string into an object
2573
+ *
2574
+ * Note: This is wrapper around `papaparse.parse()` with better autohealing
2575
+ *
2576
+ * @private - for now until `@promptbook/csv` is released
2577
+ */
2578
+ function csvParse(value /* <- TODO: string_csv */, settings, schema /* <- TODO: Make CSV Schemas */) {
2579
+ settings = { ...settings, ...MANDATORY_CSV_SETTINGS };
2580
+ // Note: Autoheal invalid '\n' characters
2581
+ if (settings.newline && !settings.newline.includes('\r') && value.includes('\r')) {
2582
+ console.warn('CSV string contains carriage return characters, but in the CSV settings the `newline` setting does not include them. Autohealing the CSV string.');
2583
+ value = value.replace(/\r\n/g, '\n').replace(/\r/g, '\n');
2584
+ }
2585
+ const csv = papaparse.parse(value, settings);
2586
+ return csv;
2587
+ }
2588
+
2540
2589
  /**
2541
2590
  * Function to check if a string is valid CSV
2542
2591
  *
2543
2592
  * @param value The string to check
2544
- * @returns True if the string is a valid CSV string, false otherwise
2593
+ * @returns `true` if the string is a valid CSV string, false otherwise
2545
2594
  *
2546
2595
  * @public exported from `@promptbook/utils`
2547
2596
  */
@@ -2565,7 +2614,7 @@
2565
2614
  * @public exported from `@promptbook/core`
2566
2615
  * <- TODO: [🏢] Export from package `@promptbook/csv`
2567
2616
  */
2568
- const CsvFormatDefinition = {
2617
+ const CsvFormatParser = {
2569
2618
  formatName: 'CSV',
2570
2619
  aliases: ['SPREADSHEET', 'TABLE'],
2571
2620
  isValid(value, settings, schema) {
@@ -2577,12 +2626,12 @@
2577
2626
  heal(value, settings, schema) {
2578
2627
  throw new Error('Not implemented');
2579
2628
  },
2580
- subvalueDefinitions: [
2629
+ subvalueParsers: [
2581
2630
  {
2582
2631
  subvalueName: 'ROW',
2583
- async mapValues(value, outputParameterName, settings, mapCallback) {
2584
- // TODO: [👨🏾‍🤝‍👨🏼] DRY csv parsing
2585
- const csv = papaparse.parse(value, { ...settings, ...MANDATORY_CSV_SETTINGS });
2632
+ async mapValues(options) {
2633
+ const { value, outputParameterName, settings, mapCallback, onProgress } = options;
2634
+ const csv = csvParse(value, settings);
2586
2635
  if (csv.errors.length !== 0) {
2587
2636
  throw new CsvFormatError(spaceTrim__default["default"]((block) => `
2588
2637
  CSV parsing error
@@ -2597,23 +2646,37 @@
2597
2646
  ${block(value)}
2598
2647
  `));
2599
2648
  }
2600
- const mappedData = await Promise.all(csv.data.map(async (row, index) => {
2649
+ const mappedData = [];
2650
+ const length = csv.data.length;
2651
+ for (let index = 0; index < length; index++) {
2652
+ const row = csv.data[index];
2601
2653
  if (row[outputParameterName]) {
2602
2654
  throw new CsvFormatError(`Can not overwrite existing column "${outputParameterName}" in CSV row`);
2603
2655
  }
2604
- return {
2656
+ const mappedRow = {
2605
2657
  ...row,
2606
- [outputParameterName]: await mapCallback(row, index),
2658
+ [outputParameterName]: await mapCallback(row, index, length),
2607
2659
  };
2608
- }));
2660
+ mappedData.push(mappedRow);
2661
+ if (onProgress) {
2662
+ // Note: Report the CSV with all rows mapped so far
2663
+ /*
2664
+ // TODO: [🛕] Report progress with all the rows including the pending ones
2665
+ const progressData = mappedData.map((row, i) =>
2666
+ i > index ? { ...row, [outputParameterName]: PENDING_VALUE_PLACEHOLDER } : row,
2667
+ );
2668
+ */
2669
+ await onProgress(papaparse.unparse(mappedData, { ...settings, ...MANDATORY_CSV_SETTINGS }));
2670
+ }
2671
+ }
2609
2672
  return papaparse.unparse(mappedData, { ...settings, ...MANDATORY_CSV_SETTINGS });
2610
2673
  },
2611
2674
  },
2612
2675
  {
2613
2676
  subvalueName: 'CELL',
2614
- async mapValues(value, outputParameterName, settings, mapCallback) {
2615
- // TODO: [👨🏾‍🤝‍👨🏼] DRY csv parsing
2616
- const csv = papaparse.parse(value, { ...settings, ...MANDATORY_CSV_SETTINGS });
2677
+ async mapValues(options) {
2678
+ const { value, settings, mapCallback, onProgress } = options;
2679
+ const csv = csvParse(value, settings);
2617
2680
  if (csv.errors.length !== 0) {
2618
2681
  throw new CsvFormatError(spaceTrim__default["default"]((block) => `
2619
2682
  CSV parsing error
@@ -2629,9 +2692,9 @@
2629
2692
  `));
2630
2693
  }
2631
2694
  const mappedData = await Promise.all(csv.data.map(async (row, rowIndex) => {
2632
- return /* not await */ Promise.all(Object.entries(row).map(async ([key, value], columnIndex) => {
2695
+ return /* not await */ Promise.all(Object.entries(row).map(async ([key, value], columnIndex, array) => {
2633
2696
  const index = rowIndex * Object.keys(row).length + columnIndex;
2634
- return /* not await */ mapCallback({ [key]: value }, index);
2697
+ return /* not await */ mapCallback({ [key]: value }, index, array.length);
2635
2698
  }));
2636
2699
  }));
2637
2700
  return papaparse.unparse(mappedData, { ...settings, ...MANDATORY_CSV_SETTINGS });
@@ -2640,10 +2703,10 @@
2640
2703
  ],
2641
2704
  };
2642
2705
  /**
2643
- * TODO: [🍓] In `CsvFormatDefinition` implement simple `isValid`
2644
- * TODO: [🍓] In `CsvFormatDefinition` implement partial `canBeValid`
2645
- * TODO: [🍓] In `CsvFormatDefinition` implement `heal
2646
- * TODO: [🍓] In `CsvFormatDefinition` implement `subvalueDefinitions`
2706
+ * TODO: [🍓] In `CsvFormatParser` implement simple `isValid`
2707
+ * TODO: [🍓] In `CsvFormatParser` implement partial `canBeValid`
2708
+ * TODO: [🍓] In `CsvFormatParser` implement `heal
2709
+ * TODO: [🍓] In `CsvFormatParser` implement `subvalueParsers`
2647
2710
  * TODO: [🏢] Allow to expect something inside CSV objects and other formats
2648
2711
  */
2649
2712
 
@@ -2652,7 +2715,7 @@
2652
2715
  *
2653
2716
  * @private still in development [🏢]
2654
2717
  */
2655
- const JsonFormatDefinition = {
2718
+ const JsonFormatParser = {
2656
2719
  formatName: 'JSON',
2657
2720
  mimeType: 'application/json',
2658
2721
  isValid(value, settings, schema) {
@@ -2664,28 +2727,28 @@
2664
2727
  heal(value, settings, schema) {
2665
2728
  throw new Error('Not implemented');
2666
2729
  },
2667
- subvalueDefinitions: [],
2730
+ subvalueParsers: [],
2668
2731
  };
2669
2732
  /**
2670
- * TODO: [🧠] Maybe propper instance of object
2733
+ * TODO: [🧠] Maybe proper instance of object
2671
2734
  * TODO: [0] Make string_serialized_json
2672
2735
  * TODO: [1] Make type for JSON Settings and Schema
2673
2736
  * TODO: [🧠] What to use for validating JSONs - JSON Schema, ZoD, typescript types/interfaces,...?
2674
- * TODO: [🍓] In `JsonFormatDefinition` implement simple `isValid`
2675
- * TODO: [🍓] In `JsonFormatDefinition` implement partial `canBeValid`
2676
- * TODO: [🍓] In `JsonFormatDefinition` implement `heal
2677
- * TODO: [🍓] In `JsonFormatDefinition` implement `subvalueDefinitions`
2737
+ * TODO: [🍓] In `JsonFormatParser` implement simple `isValid`
2738
+ * TODO: [🍓] In `JsonFormatParser` implement partial `canBeValid`
2739
+ * TODO: [🍓] In `JsonFormatParser` implement `heal
2740
+ * TODO: [🍓] In `JsonFormatParser` implement `subvalueParsers`
2678
2741
  * TODO: [🏢] Allow to expect something inside JSON objects and other formats
2679
2742
  */
2680
2743
 
2681
2744
  /**
2682
2745
  * Definition for any text - this will be always valid
2683
2746
  *
2684
- * Note: This is not useful for validation, but for splitting and mapping with `subvalueDefinitions`
2747
+ * Note: This is not useful for validation, but for splitting and mapping with `subvalueParsers`
2685
2748
  *
2686
2749
  * @public exported from `@promptbook/core`
2687
2750
  */
2688
- const TextFormatDefinition = {
2751
+ const TextFormatParser = {
2689
2752
  formatName: 'TEXT',
2690
2753
  isValid(value) {
2691
2754
  return typeof value === 'string';
@@ -2694,19 +2757,20 @@
2694
2757
  return typeof partialValue === 'string';
2695
2758
  },
2696
2759
  heal() {
2697
- throw new UnexpectedError('It does not make sense to call `TextFormatDefinition.heal`');
2760
+ throw new UnexpectedError('It does not make sense to call `TextFormatParser.heal`');
2698
2761
  },
2699
- subvalueDefinitions: [
2762
+ subvalueParsers: [
2700
2763
  {
2701
2764
  subvalueName: 'LINE',
2702
- async mapValues(value, outputParameterName, settings, mapCallback) {
2765
+ async mapValues(options) {
2766
+ const { value, mapCallback, onProgress } = options;
2703
2767
  const lines = value.split('\n');
2704
- const mappedLines = await Promise.all(lines.map((lineContent, lineNumber) =>
2768
+ const mappedLines = await Promise.all(lines.map((lineContent, lineNumber, array) =>
2705
2769
  // TODO: [🧠] Maybe option to skip empty line
2706
2770
  /* not await */ mapCallback({
2707
2771
  lineContent,
2708
2772
  // TODO: [🧠] Maybe also put here `lineNumber`
2709
- }, lineNumber)));
2773
+ }, lineNumber, array.length)));
2710
2774
  return mappedLines.join('\n');
2711
2775
  },
2712
2776
  },
@@ -2716,10 +2780,10 @@
2716
2780
  /**
2717
2781
  * TODO: [1] Make type for XML Text and Schema
2718
2782
  * TODO: [🧠][🤠] Here should be all words, characters, lines, paragraphs, pages available as subvalues
2719
- * TODO: [🍓] In `TextFormatDefinition` implement simple `isValid`
2720
- * TODO: [🍓] In `TextFormatDefinition` implement partial `canBeValid`
2721
- * TODO: [🍓] In `TextFormatDefinition` implement `heal
2722
- * TODO: [🍓] In `TextFormatDefinition` implement `subvalueDefinitions`
2783
+ * TODO: [🍓] In `TextFormatParser` implement simple `isValid`
2784
+ * TODO: [🍓] In `TextFormatParser` implement partial `canBeValid`
2785
+ * TODO: [🍓] In `TextFormatParser` implement `heal
2786
+ * TODO: [🍓] In `TextFormatParser` implement `subvalueParsers`
2723
2787
  * TODO: [🏢] Allow to expect something inside each item of list and other formats
2724
2788
  */
2725
2789
 
@@ -2727,7 +2791,7 @@
2727
2791
  * Function to check if a string is valid XML
2728
2792
  *
2729
2793
  * @param value
2730
- * @returns True if the string is a valid XML string, false otherwise
2794
+ * @returns `true` if the string is a valid XML string, false otherwise
2731
2795
  *
2732
2796
  * @public exported from `@promptbook/utils`
2733
2797
  */
@@ -2752,7 +2816,7 @@
2752
2816
  *
2753
2817
  * @private still in development [🏢]
2754
2818
  */
2755
- const XmlFormatDefinition = {
2819
+ const XmlFormatParser = {
2756
2820
  formatName: 'XML',
2757
2821
  mimeType: 'application/xml',
2758
2822
  isValid(value, settings, schema) {
@@ -2764,17 +2828,17 @@
2764
2828
  heal(value, settings, schema) {
2765
2829
  throw new Error('Not implemented');
2766
2830
  },
2767
- subvalueDefinitions: [],
2831
+ subvalueParsers: [],
2768
2832
  };
2769
2833
  /**
2770
- * TODO: [🧠] Maybe propper instance of object
2834
+ * TODO: [🧠] Maybe proper instance of object
2771
2835
  * TODO: [0] Make string_serialized_xml
2772
2836
  * TODO: [1] Make type for XML Settings and Schema
2773
2837
  * TODO: [🧠] What to use for validating XMLs - XSD,...
2774
- * TODO: [🍓] In `XmlFormatDefinition` implement simple `isValid`
2775
- * TODO: [🍓] In `XmlFormatDefinition` implement partial `canBeValid`
2776
- * TODO: [🍓] In `XmlFormatDefinition` implement `heal
2777
- * TODO: [🍓] In `XmlFormatDefinition` implement `subvalueDefinitions`
2838
+ * TODO: [🍓] In `XmlFormatParser` implement simple `isValid`
2839
+ * TODO: [🍓] In `XmlFormatParser` implement partial `canBeValid`
2840
+ * TODO: [🍓] In `XmlFormatParser` implement `heal
2841
+ * TODO: [🍓] In `XmlFormatParser` implement `subvalueParsers`
2778
2842
  * TODO: [🏢] Allow to expect something inside XML and other formats
2779
2843
  */
2780
2844
 
@@ -2783,24 +2847,19 @@
2783
2847
  *
2784
2848
  * @private internal index of `...` <- TODO [🏢]
2785
2849
  */
2786
- const FORMAT_DEFINITIONS = [
2787
- JsonFormatDefinition,
2788
- XmlFormatDefinition,
2789
- TextFormatDefinition,
2790
- CsvFormatDefinition,
2791
- ];
2850
+ const FORMAT_DEFINITIONS = [JsonFormatParser, XmlFormatParser, TextFormatParser, CsvFormatParser];
2792
2851
  /**
2793
2852
  * Note: [💞] Ignore a discrepancy between file name and entity name
2794
2853
  */
2795
2854
 
2796
2855
  /**
2797
- * Maps available parameters to expected parameters
2856
+ * Maps available parameters to expected parameters for a pipeline task.
2798
2857
  *
2799
2858
  * The strategy is:
2800
- * 1) @@@
2801
- * 2) @@@
2859
+ * 1) First, match parameters by name where both available and expected.
2860
+ * 2) Then, if there are unmatched expected and available parameters, map them by order.
2802
2861
  *
2803
- * @throws {PipelineExecutionError} @@@
2862
+ * @throws {PipelineExecutionError} If the number of unmatched expected and available parameters does not match, or mapping is ambiguous.
2804
2863
  * @private within the repository used in `createPipelineExecutor`
2805
2864
  */
2806
2865
  function mapAvailableToExpectedParameters(options) {
@@ -2823,7 +2882,7 @@
2823
2882
  else if (!availableParametersNames.has(parameterName) && expectedParameterNames.has(parameterName)) ;
2824
2883
  }
2825
2884
  if (expectedParameterNames.size === 0) {
2826
- // Note: [👨‍👨‍👧] Now we can freeze `mappedParameters` to prevent @@@
2885
+ // Note: [👨‍👨‍👧] Now we can freeze `mappedParameters` to prevent accidental modifications after mapping
2827
2886
  Object.freeze(mappedParameters);
2828
2887
  return mappedParameters;
2829
2888
  }
@@ -2854,7 +2913,7 @@
2854
2913
  for (let i = 0; i < expectedParameterNames.size; i++) {
2855
2914
  mappedParameters[expectedParameterNamesArray[i]] = availableParameters[availableParametersNamesArray[i]];
2856
2915
  }
2857
- // Note: [👨‍👨‍👧] Now we can freeze `mappedParameters` to prevent @@@
2916
+ // Note: [👨‍👨‍👧] Now we can freeze `mappedParameters` to prevent accidental modifications after mapping
2858
2917
  Object.freeze(mappedParameters);
2859
2918
  return mappedParameters;
2860
2919
  }
@@ -2876,29 +2935,40 @@
2876
2935
  return 'Multiple LLM Providers';
2877
2936
  }
2878
2937
  get description() {
2879
- return this.llmExecutionTools.map(({ title }, index) => `${index + 1}) \`${title}\``).join('\n');
2938
+ const innerModelsTitlesAndDescriptions = this.llmExecutionTools
2939
+ .map(({ title, description }, index) => {
2940
+ const headLine = `${index + 1}) \`${title}\``;
2941
+ if (description === undefined) {
2942
+ return headLine;
2943
+ }
2944
+ return spaceTrim__default["default"]((block) => `
2945
+ ${headLine}
2946
+
2947
+ ${ /* <- Note: Indenting the description: */block(description)}
2948
+ `);
2949
+ })
2950
+ .join('\n\n');
2951
+ return spaceTrim__default["default"]((block) => `
2952
+ Multiple LLM Providers:
2953
+
2954
+ ${block(innerModelsTitlesAndDescriptions)}
2955
+ `);
2880
2956
  }
2881
2957
  /**
2882
2958
  * Check the configuration of all execution tools
2883
2959
  */
2884
2960
  async checkConfiguration() {
2885
- // TODO: Maybe do it in parallel
2886
- for (const llmExecutionTools of this.llmExecutionTools) {
2887
- await llmExecutionTools.checkConfiguration();
2888
- }
2961
+ // Note: Run checks in parallel
2962
+ await Promise.all(this.llmExecutionTools.map((tools) => tools.checkConfiguration()));
2889
2963
  }
2890
2964
  /**
2891
2965
  * List all available models that can be used
2892
2966
  * This lists is a combination of all available models from all execution tools
2893
2967
  */
2894
2968
  async listModels() {
2895
- const availableModels = [];
2896
- for (const llmExecutionTools of this.llmExecutionTools) {
2897
- // TODO: [🪂] Obtain models in parallel
2898
- const models = await llmExecutionTools.listModels();
2899
- availableModels.push(...models);
2900
- }
2901
- return availableModels;
2969
+ // Obtain all models in parallel and flatten
2970
+ const modelArrays = await Promise.all(this.llmExecutionTools.map((tools) => tools.listModels()));
2971
+ return modelArrays.flat();
2902
2972
  }
2903
2973
  /**
2904
2974
  * Calls the best available chat model
@@ -3054,8 +3124,8 @@
3054
3124
  /**
3055
3125
  * Extracts all code blocks from markdown.
3056
3126
  *
3057
- * Note: There are multiple simmilar function:
3058
- * - `extractBlock` just extracts the content of the code block which is also used as build-in function for postprocessing
3127
+ * Note: There are multiple similar functions:
3128
+ * - `extractBlock` just extracts the content of the code block which is also used as built-in function for postprocessing
3059
3129
  * - `extractJsonBlock` extracts exactly one valid JSON code block
3060
3130
  * - `extractOneBlockFromMarkdown` extracts exactly one code block with language of the code block
3061
3131
  * - `extractAllBlocksFromMarkdown` extracts all code blocks with language of the code block
@@ -3105,7 +3175,7 @@
3105
3175
  if (currentCodeBlock.content !== '') {
3106
3176
  currentCodeBlock.content += '\n';
3107
3177
  }
3108
- currentCodeBlock.content += line.split('\\`\\`\\`').join('```') /* <- TODO: Maybe make propper unescape */;
3178
+ currentCodeBlock.content += line.split('\\`\\`\\`').join('```') /* <- TODO: Maybe make proper unescape */;
3109
3179
  }
3110
3180
  }
3111
3181
  if (currentCodeBlock !== null) {
@@ -3125,7 +3195,7 @@
3125
3195
  * - When there are multiple JSON code blocks the function throws a `ParseError`
3126
3196
  *
3127
3197
  * Note: It is not important if marked as ```json BUT if it is VALID JSON
3128
- * Note: There are multiple simmilar function:
3198
+ * Note: There are multiple similar function:
3129
3199
  * - `extractBlock` just extracts the content of the code block which is also used as build-in function for postprocessing
3130
3200
  * - `extractJsonBlock` extracts exactly one valid JSON code block
3131
3201
  * - `extractOneBlockFromMarkdown` extracts exactly one code block with language of the code block
@@ -3150,7 +3220,7 @@
3150
3220
  }
3151
3221
  /**
3152
3222
  * TODO: Add some auto-healing logic + extract YAML, JSON5, TOML, etc.
3153
- * TODO: [🏢] Make this logic part of `JsonFormatDefinition` or `isValidJsonString`
3223
+ * TODO: [🏢] Make this logic part of `JsonFormatParser` or `isValidJsonString`
3154
3224
  */
3155
3225
 
3156
3226
  /**
@@ -3176,7 +3246,7 @@
3176
3246
  * Just says that the variable is not used but should be kept
3177
3247
  * No side effects.
3178
3248
  *
3179
- * Note: It can be usefull for:
3249
+ * Note: It can be useful for:
3180
3250
  *
3181
3251
  * 1) Suppressing eager optimization of unused imports
3182
3252
  * 2) Suppressing eslint errors of unused variables in the tests
@@ -3603,10 +3673,10 @@
3603
3673
  */
3604
3674
 
3605
3675
  /**
3606
- * @@@
3676
+ * Removes diacritic marks (accents) from characters in a string.
3607
3677
  *
3608
- * @param input @@@
3609
- * @returns @@@
3678
+ * @param input The string containing diacritics to be normalized.
3679
+ * @returns The string with diacritics removed or normalized.
3610
3680
  * @public exported from `@promptbook/utils`
3611
3681
  */
3612
3682
  function removeDiacritics(input) {
@@ -3649,14 +3719,14 @@
3649
3719
  PAGES: countPages,
3650
3720
  };
3651
3721
  /**
3652
- * TODO: [🧠][🤠] This should be probbably as part of `TextFormatDefinition`
3722
+ * TODO: [🧠][🤠] This should be probably as part of `TextFormatParser`
3653
3723
  * Note: [💞] Ignore a discrepancy between file name and entity name
3654
3724
  */
3655
3725
 
3656
3726
  /**
3657
3727
  * Function checkExpectations will check if the expectations on given value are met
3658
3728
  *
3659
- * Note: There are two simmilar functions:
3729
+ * Note: There are two similar functions:
3660
3730
  * - `checkExpectations` which throws an error if the expectations are not met
3661
3731
  * - `isPassingExpectations` which returns a boolean
3662
3732
  *
@@ -3677,13 +3747,17 @@
3677
3747
  }
3678
3748
  /**
3679
3749
  * TODO: [💝] Unite object for expecting amount and format
3680
- * TODO: [🧠][🤠] This should be part of `TextFormatDefinition`
3750
+ * TODO: [🧠][🤠] This should be part of `TextFormatParser`
3681
3751
  * Note: [💝] and [🤠] are interconnected together
3682
3752
  */
3683
3753
 
3684
3754
  /**
3685
- * @@@
3755
+ * Executes a pipeline task with multiple attempts, including joker and retry logic. Handles different task types
3756
+ * (prompt, script, dialog, etc.), applies postprocessing, checks expectations, and updates the execution report.
3757
+ * Throws errors if execution fails after all attempts.
3686
3758
  *
3759
+ * @param options - The options for execution, including task, parameters, pipeline, and configuration.
3760
+ * @returns The result string of the executed task.
3687
3761
  * @private internal utility of `createPipelineExecutor`
3688
3762
  */
3689
3763
  async function executeAttempts(options) {
@@ -3905,7 +3979,7 @@
3905
3979
  if (task.format) {
3906
3980
  if (task.format === 'JSON') {
3907
3981
  if (!isValidJsonString($ongoingTaskResult.$resultString || '')) {
3908
- // TODO: [🏢] Do more universally via `FormatDefinition`
3982
+ // TODO: [🏢] Do more universally via `FormatParser`
3909
3983
  try {
3910
3984
  $ongoingTaskResult.$resultString = extractJsonBlock($ongoingTaskResult.$resultString || '');
3911
3985
  }
@@ -4007,12 +4081,16 @@
4007
4081
  */
4008
4082
 
4009
4083
  /**
4010
- * @@@
4084
+ * Executes a pipeline task that requires mapping or iterating over subvalues of a parameter (such as rows in a CSV).
4085
+ * Handles format and subformat resolution, error handling, and progress reporting.
4086
+ *
4087
+ * @param options - Options for execution, including task details and progress callback.
4088
+ * @returns The result of the subvalue mapping or execution attempts.
4011
4089
  *
4012
4090
  * @private internal utility of `createPipelineExecutor`
4013
4091
  */
4014
4092
  async function executeFormatSubvalues(options) {
4015
- const { task, jokerParameterNames, parameters, priority, csvSettings, pipelineIdentification } = options;
4093
+ const { task, jokerParameterNames, parameters, priority, csvSettings, onProgress, pipelineIdentification } = options;
4016
4094
  if (task.foreach === undefined) {
4017
4095
  return /* not await */ executeAttempts(options);
4018
4096
  }
@@ -4043,16 +4121,16 @@
4043
4121
  ${block(pipelineIdentification)}
4044
4122
  `));
4045
4123
  }
4046
- const subvalueDefinition = formatDefinition.subvalueDefinitions.find((subvalueDefinition) => [subvalueDefinition.subvalueName, ...(subvalueDefinition.aliases || [])].includes(task.foreach.subformatName));
4047
- if (subvalueDefinition === undefined) {
4124
+ const subvalueParser = formatDefinition.subvalueParsers.find((subvalueParser) => [subvalueParser.subvalueName, ...(subvalueParser.aliases || [])].includes(task.foreach.subformatName));
4125
+ if (subvalueParser === undefined) {
4048
4126
  throw new UnexpectedError(
4049
4127
  // <- TODO: [🧠][🧐] Should be formats fixed per promptbook version or behave as plugins (=> change UnexpectedError)
4050
4128
  spaceTrim__default["default"]((block) => `
4051
4129
  Unsupported subformat name "${task.foreach.subformatName}" for format "${task.foreach.formatName}"
4052
4130
 
4053
4131
  Available subformat names for format "${formatDefinition.formatName}":
4054
- ${block(formatDefinition.subvalueDefinitions
4055
- .map((subvalueDefinition) => subvalueDefinition.subvalueName)
4132
+ ${block(formatDefinition.subvalueParsers
4133
+ .map((subvalueParser) => subvalueParser.subvalueName)
4056
4134
  .map((subvalueName) => `- ${subvalueName}`)
4057
4135
  .join('\n'))}
4058
4136
 
@@ -4064,55 +4142,85 @@
4064
4142
  let formatSettings;
4065
4143
  if (formatDefinition.formatName === 'CSV') {
4066
4144
  formatSettings = csvSettings;
4067
- // <- TODO: [🤹‍♂️] More universal, make simmilar pattern for other formats for example \n vs \r\n in text
4068
- }
4069
- const resultString = await subvalueDefinition.mapValues(parameterValue, task.foreach.outputSubparameterName, formatSettings, async (subparameters, index) => {
4070
- let mappedParameters;
4071
- // TODO: [🤹‍♂️][🪂] Limit to N concurrent executions
4072
- // TODO: When done [🐚] Report progress also for each subvalue here
4073
- try {
4074
- mappedParameters = mapAvailableToExpectedParameters({
4075
- expectedParameters: Object.fromEntries(task.foreach.inputSubparameterNames.map((subparameterName) => [subparameterName, null])),
4076
- availableParameters: subparameters,
4077
- });
4078
- }
4079
- catch (error) {
4080
- if (!(error instanceof PipelineExecutionError)) {
4081
- throw error;
4145
+ // <- TODO: [🤹‍♂️] More universal, make similar pattern for other formats for example \n vs \r\n in text
4146
+ }
4147
+ const resultString = await subvalueParser.mapValues({
4148
+ value: parameterValue,
4149
+ outputParameterName: task.foreach.outputSubparameterName,
4150
+ settings: formatSettings,
4151
+ onProgress(partialResultString) {
4152
+ return onProgress(Object.freeze({
4153
+ [task.resultingParameterName]: partialResultString,
4154
+ }));
4155
+ },
4156
+ async mapCallback(subparameters, index, length) {
4157
+ let mappedParameters;
4158
+ try {
4159
+ mappedParameters = mapAvailableToExpectedParameters({
4160
+ expectedParameters: Object.fromEntries(task.foreach.inputSubparameterNames.map((subparameterName) => [subparameterName, null])),
4161
+ availableParameters: subparameters,
4162
+ });
4082
4163
  }
4083
- throw new PipelineExecutionError(spaceTrim__default["default"]((block) => `
4084
- ${error.message}
4164
+ catch (error) {
4165
+ if (!(error instanceof PipelineExecutionError)) {
4166
+ throw error;
4167
+ }
4168
+ const highLevelError = new PipelineExecutionError(spaceTrim__default["default"]((block) => `
4169
+ ${error.message}
4085
4170
 
4086
- This is error in FOREACH command
4087
- You have probbably passed wrong data to pipeline or wrong data was generated which are processed by FOREACH command
4171
+ This is error in FOREACH command when mapping ${formatDefinition.formatName} ${subvalueParser.subvalueName} data (${index + 1}/${length})
4172
+ You have probably passed wrong data to pipeline or wrong data was generated which are processed by FOREACH command
4088
4173
 
4089
- ${block(pipelineIdentification)}
4090
- Subparameter index: ${index}
4091
- `));
4092
- }
4093
- const allSubparameters = {
4094
- ...parameters,
4095
- ...mappedParameters,
4096
- };
4097
- // Note: [👨‍👨‍👧] Now we can freeze `subparameters` because we are sure that all and only used parameters are defined and are not going to be changed
4098
- Object.freeze(allSubparameters);
4099
- const subresultString = await executeAttempts({
4100
- ...options,
4101
- priority: priority + index,
4102
- parameters: allSubparameters,
4103
- pipelineIdentification: spaceTrim__default["default"]((block) => `
4104
- ${block(pipelineIdentification)}
4105
- Subparameter index: ${index}
4106
- `),
4107
- });
4108
- return subresultString;
4174
+ ${block(pipelineIdentification)}
4175
+ `));
4176
+ if (length > BIG_DATASET_TRESHOLD) {
4177
+ console.error(highLevelError);
4178
+ return FAILED_VALUE_PLACEHOLDER;
4179
+ }
4180
+ throw highLevelError;
4181
+ }
4182
+ const allSubparameters = {
4183
+ ...parameters,
4184
+ ...mappedParameters,
4185
+ };
4186
+ Object.freeze(allSubparameters);
4187
+ try {
4188
+ const subresultString = await executeAttempts({
4189
+ ...options,
4190
+ priority: priority + index,
4191
+ parameters: allSubparameters,
4192
+ pipelineIdentification: spaceTrim__default["default"]((block) => `
4193
+ ${block(pipelineIdentification)}
4194
+ Subparameter index: ${index}
4195
+ `),
4196
+ });
4197
+ return subresultString;
4198
+ }
4199
+ catch (error) {
4200
+ if (length > BIG_DATASET_TRESHOLD) {
4201
+ console.error(spaceTrim__default["default"]((block) => `
4202
+ ${error.message}
4203
+
4204
+ This is error in FOREACH command when processing ${formatDefinition.formatName} ${subvalueParser.subvalueName} data (${index + 1}/${length})
4205
+
4206
+ ${block(pipelineIdentification)}
4207
+ `));
4208
+ return FAILED_VALUE_PLACEHOLDER;
4209
+ }
4210
+ throw error;
4211
+ }
4212
+ },
4109
4213
  });
4110
4214
  return resultString;
4111
4215
  }
4112
4216
 
4113
4217
  /**
4114
- * @@@
4218
+ * Returns the context for a given task, typically used to provide additional information or variables
4219
+ * required for the execution of the task within a pipeline. The context is returned as a string value
4220
+ * that may include markdown formatting.
4115
4221
  *
4222
+ * @param task - The task for which the context is being generated. This should be a deeply immutable TaskJson object.
4223
+ * @returns The context as a string, formatted as markdown and parameter value.
4116
4224
  * @private internal utility of `createPipelineExecutor`
4117
4225
  */
4118
4226
  async function getContextForTask(task) {
@@ -4120,7 +4228,7 @@
4120
4228
  }
4121
4229
 
4122
4230
  /**
4123
- * @@@
4231
+ * Retrieves example values or templates for a given task, used to guide or validate pipeline execution.
4124
4232
  *
4125
4233
  * @private internal utility of `createPipelineExecutor`
4126
4234
  */
@@ -4129,91 +4237,128 @@
4129
4237
  }
4130
4238
 
4131
4239
  /**
4132
- * @@@
4240
+ * Computes the cosine similarity between two embedding vectors
4241
+ *
4242
+ * Note: This is helping function for RAG (retrieval-augmented generation)
4243
+ *
4244
+ * @param embeddingVector1
4245
+ * @param embeddingVector2
4246
+ * @returns Cosine similarity between the two vectors
4247
+ *
4248
+ * @public exported from `@promptbook/core`
4249
+ */
4250
+ function computeCosineSimilarity(embeddingVector1, embeddingVector2) {
4251
+ if (embeddingVector1.length !== embeddingVector2.length) {
4252
+ throw new TypeError('Embedding vectors must have the same length');
4253
+ }
4254
+ const dotProduct = embeddingVector1.reduce((sum, value, index) => sum + value * embeddingVector2[index], 0);
4255
+ const magnitude1 = Math.sqrt(embeddingVector1.reduce((sum, value) => sum + value * value, 0));
4256
+ const magnitude2 = Math.sqrt(embeddingVector2.reduce((sum, value) => sum + value * value, 0));
4257
+ return 1 - dotProduct / (magnitude1 * magnitude2);
4258
+ }
4259
+
4260
+ /**
4261
+ *
4262
+ * @param knowledgePieces
4263
+ * @returns
4133
4264
  *
4134
- * Here is the place where RAG (retrieval-augmented generation) happens
4265
+ * @private internal utility of `createPipelineExecutor`
4266
+ */
4267
+ function knowledgePiecesToString(knowledgePieces) {
4268
+ return knowledgePieces
4269
+ .map((knowledgePiece) => {
4270
+ const { content } = knowledgePiece;
4271
+ return `- ${content}`;
4272
+ })
4273
+ .join('\n');
4274
+ // <- TODO: [🧠] Some smarter aggregation of knowledge pieces, single-line vs multi-line vs mixed
4275
+ }
4276
+
4277
+ /**
4278
+ * Retrieves the most relevant knowledge pieces for a given task using embedding-based similarity search.
4279
+ * This is where retrieval-augmented generation (RAG) is performed to enhance the task with external knowledge.
4135
4280
  *
4136
4281
  * @private internal utility of `createPipelineExecutor`
4137
4282
  */
4138
4283
  async function getKnowledgeForTask(options) {
4139
- const { tools, preparedPipeline, task } = options;
4284
+ const { tools, preparedPipeline, task, parameters } = options;
4140
4285
  const firstKnowlegePiece = preparedPipeline.knowledgePieces[0];
4141
4286
  const firstKnowlegeIndex = firstKnowlegePiece === null || firstKnowlegePiece === void 0 ? void 0 : firstKnowlegePiece.index[0];
4142
4287
  // <- TODO: Do not use just first knowledge piece and first index to determine embedding model, use also keyword search
4143
4288
  if (firstKnowlegePiece === undefined || firstKnowlegeIndex === undefined) {
4144
- return 'No knowledge pieces found';
4289
+ return ''; // <- Note: Np knowledge present, return empty string
4145
4290
  }
4146
- // TODO: [🚐] Make arrayable LLMs -> single LLM DRY
4147
- const _llms = arrayableToArray(tools.llm);
4148
- const llmTools = _llms.length === 1 ? _llms[0] : joinLlmExecutionTools(..._llms);
4149
- const taskEmbeddingPrompt = {
4150
- title: 'Knowledge Search',
4151
- modelRequirements: {
4152
- modelVariant: 'EMBEDDING',
4153
- modelName: firstKnowlegeIndex.modelName,
4154
- },
4155
- content: task.content,
4156
- parameters: {
4157
- /* !!!!!!!! */
4158
- },
4159
- };
4160
- const taskEmbeddingResult = await llmTools.callEmbeddingModel(taskEmbeddingPrompt);
4161
- const knowledgePiecesWithRelevance = preparedPipeline.knowledgePieces.map((knowledgePiece) => {
4162
- const { index } = knowledgePiece;
4163
- const knowledgePieceIndex = index.find((i) => i.modelName === firstKnowlegeIndex.modelName);
4164
- // <- TODO: Do not use just first knowledge piece and first index to determine embedding model
4165
- if (knowledgePieceIndex === undefined) {
4291
+ try {
4292
+ // TODO: [🚐] Make arrayable LLMs -> single LLM DRY
4293
+ const _llms = arrayableToArray(tools.llm);
4294
+ const llmTools = _llms.length === 1 ? _llms[0] : joinLlmExecutionTools(..._llms);
4295
+ const taskEmbeddingPrompt = {
4296
+ title: 'Knowledge Search',
4297
+ modelRequirements: {
4298
+ modelVariant: 'EMBEDDING',
4299
+ modelName: firstKnowlegeIndex.modelName,
4300
+ },
4301
+ content: task.content,
4302
+ parameters,
4303
+ };
4304
+ const taskEmbeddingResult = await llmTools.callEmbeddingModel(taskEmbeddingPrompt);
4305
+ const knowledgePiecesWithRelevance = preparedPipeline.knowledgePieces.map((knowledgePiece) => {
4306
+ const { index } = knowledgePiece;
4307
+ const knowledgePieceIndex = index.find((i) => i.modelName === firstKnowlegeIndex.modelName);
4308
+ // <- TODO: Do not use just first knowledge piece and first index to determine embedding model
4309
+ if (knowledgePieceIndex === undefined) {
4310
+ return {
4311
+ content: knowledgePiece.content,
4312
+ relevance: 0,
4313
+ };
4314
+ }
4315
+ const relevance = computeCosineSimilarity(knowledgePieceIndex.position, taskEmbeddingResult.content);
4166
4316
  return {
4167
4317
  content: knowledgePiece.content,
4168
- relevance: 0,
4318
+ relevance,
4169
4319
  };
4170
- }
4171
- const relevance = computeCosineSimilarity(knowledgePieceIndex.position, taskEmbeddingResult.content);
4172
- return {
4173
- content: knowledgePiece.content,
4174
- relevance,
4175
- };
4176
- });
4177
- const knowledgePiecesSorted = knowledgePiecesWithRelevance.sort((a, b) => a.relevance - b.relevance);
4178
- const knowledgePiecesLimited = knowledgePiecesSorted.slice(0, 5);
4179
- console.log('!!! Embedding', {
4180
- task,
4181
- taskEmbeddingPrompt,
4182
- taskEmbeddingResult,
4183
- firstKnowlegePiece,
4184
- firstKnowlegeIndex,
4185
- knowledgePiecesWithRelevance,
4186
- knowledgePiecesSorted,
4187
- knowledgePiecesLimited,
4188
- });
4189
- return knowledgePiecesLimited.map(({ content }) => `- ${content}`).join('\n');
4190
- // <- TODO: [🧠] Some smart aggregation of knowledge pieces, single-line vs multi-line vs mixed
4191
- }
4192
- // TODO: !!!!!! Annotate + to new file
4193
- function computeCosineSimilarity(embeddingVector1, embeddingVector2) {
4194
- if (embeddingVector1.length !== embeddingVector2.length) {
4195
- throw new TypeError('Embedding vectors must have the same length');
4320
+ });
4321
+ const knowledgePiecesSorted = knowledgePiecesWithRelevance.sort((a, b) => a.relevance - b.relevance);
4322
+ const knowledgePiecesLimited = knowledgePiecesSorted.slice(0, 5);
4323
+ /*
4324
+ console.log('`getKnowledgeForTask` Embedding', {
4325
+ task,
4326
+ taskEmbeddingPrompt,
4327
+ taskEmbeddingResult,
4328
+ firstKnowlegePiece,
4329
+ firstKnowlegeIndex,
4330
+ knowledgePiecesWithRelevance,
4331
+ knowledgePiecesSorted,
4332
+ knowledgePiecesLimited,
4333
+ });
4334
+ */
4335
+ return knowledgePiecesToString(knowledgePiecesLimited);
4336
+ }
4337
+ catch (error) {
4338
+ assertsError(error);
4339
+ console.error('Error in `getKnowledgeForTask`', error);
4340
+ // Note: If the LLM fails, just return all knowledge pieces
4341
+ return knowledgePiecesToString(preparedPipeline.knowledgePieces);
4196
4342
  }
4197
- const dotProduct = embeddingVector1.reduce((sum, value, index) => sum + value * embeddingVector2[index], 0);
4198
- const magnitude1 = Math.sqrt(embeddingVector1.reduce((sum, value) => sum + value * value, 0));
4199
- const magnitude2 = Math.sqrt(embeddingVector2.reduce((sum, value) => sum + value * value, 0));
4200
- return 1 - dotProduct / (magnitude1 * magnitude2);
4201
4343
  }
4202
4344
  /**
4203
- * TODO: !!!! Verify if this is working
4204
4345
  * TODO: [♨] Implement Better - use keyword search
4205
4346
  * TODO: [♨] Examples of values
4206
4347
  */
4207
4348
 
4208
4349
  /**
4209
- * @@@
4350
+ * Retrieves all reserved parameters for a given pipeline task, including context, knowledge, examples, and metadata.
4351
+ * Ensures all reserved parameters are defined and throws if any are missing.
4352
+ *
4353
+ * @param options - Options including tools, pipeline, task, and context.
4354
+ * @returns An object containing all reserved parameters for the task.
4210
4355
  *
4211
4356
  * @private internal utility of `createPipelineExecutor`
4212
4357
  */
4213
4358
  async function getReservedParametersForTask(options) {
4214
- const { tools, preparedPipeline, task, pipelineIdentification } = options;
4359
+ const { tools, preparedPipeline, task, parameters, pipelineIdentification, isVerbose } = options;
4215
4360
  const context = await getContextForTask(); // <- [🏍]
4216
- const knowledge = await getKnowledgeForTask({ tools, preparedPipeline, task });
4361
+ const knowledge = await getKnowledgeForTask({ tools, preparedPipeline, task, parameters });
4217
4362
  const examples = await getExamplesForTask();
4218
4363
  const currentDate = new Date().toISOString(); // <- TODO: [🧠][💩] Better
4219
4364
  const modelName = RESERVED_PARAMETER_MISSING_VALUE;
@@ -4225,6 +4370,9 @@
4225
4370
  currentDate,
4226
4371
  modelName,
4227
4372
  };
4373
+ if (isVerbose) {
4374
+ console.info('Reserved parameters for task:', { options, reservedParameters });
4375
+ }
4228
4376
  // Note: Doublecheck that ALL reserved parameters are defined:
4229
4377
  for (const parameterName of RESERVED_PARAMETER_NAMES) {
4230
4378
  if (reservedParameters[parameterName] === undefined) {
@@ -4239,23 +4387,21 @@
4239
4387
  }
4240
4388
 
4241
4389
  /**
4242
- * @@@
4390
+ * Executes a single task within a pipeline, handling parameter validation, error checking, and progress reporting.
4391
+ *
4392
+ * @param options - Options for execution, including the task, pipeline, parameters, and callbacks.
4393
+ * @returns The output parameters produced by the task.
4243
4394
  *
4244
4395
  * @private internal utility of `createPipelineExecutor`
4245
4396
  */
4246
4397
  async function executeTask(options) {
4247
4398
  const { currentTask, preparedPipeline, parametersToPass, tools, onProgress, $executionReport, pipelineIdentification, maxExecutionAttempts, maxParallelCount, csvSettings, isVerbose, rootDirname, cacheDirname, intermediateFilesStrategy, isAutoInstalled, isNotPreparedWarningSupressed, } = options;
4248
4399
  const priority = preparedPipeline.tasks.length - preparedPipeline.tasks.indexOf(currentTask);
4249
- await onProgress({
4250
- outputParameters: {
4251
- [currentTask.resultingParameterName]: '', // <- TODO: [🧠] What is the best value here?
4252
- },
4253
- });
4254
4400
  // Note: Check consistency of used and dependent parameters which was also done in `validatePipeline`, but it’s good to doublecheck
4255
4401
  const usedParameterNames = extractParameterNamesFromTask(currentTask);
4256
4402
  const dependentParameterNames = new Set(currentTask.dependentParameterNames);
4257
4403
  // TODO: [👩🏾‍🤝‍👩🏻] Use here `mapAvailableToExpectedParameters`
4258
- if (union(difference(usedParameterNames, dependentParameterNames), difference(dependentParameterNames, usedParameterNames)).size !== 0) {
4404
+ if (difference(union(difference(usedParameterNames, dependentParameterNames), difference(dependentParameterNames, usedParameterNames)), new Set(RESERVED_PARAMETER_NAMES)).size !== 0) {
4259
4405
  throw new UnexpectedError(spaceTrim.spaceTrim((block) => `
4260
4406
  Dependent parameters are not consistent with used parameters:
4261
4407
 
@@ -4273,13 +4419,16 @@
4273
4419
 
4274
4420
  `));
4275
4421
  }
4422
+ const reservedParameters = await getReservedParametersForTask({
4423
+ tools,
4424
+ preparedPipeline,
4425
+ task: currentTask,
4426
+ pipelineIdentification,
4427
+ parameters: parametersToPass,
4428
+ isVerbose,
4429
+ });
4276
4430
  const definedParameters = Object.freeze({
4277
- ...(await getReservedParametersForTask({
4278
- tools,
4279
- preparedPipeline,
4280
- task: currentTask,
4281
- pipelineIdentification,
4282
- })),
4431
+ ...reservedParameters,
4283
4432
  ...parametersToPass,
4284
4433
  });
4285
4434
  const definedParameterNames = new Set(Object.keys(definedParameters));
@@ -4324,6 +4473,7 @@
4324
4473
  preparedPipeline,
4325
4474
  tools,
4326
4475
  $executionReport,
4476
+ onProgress,
4327
4477
  pipelineIdentification,
4328
4478
  maxExecutionAttempts,
4329
4479
  maxParallelCount,
@@ -4351,7 +4501,8 @@
4351
4501
  */
4352
4502
 
4353
4503
  /**
4354
- * @@@
4504
+ * Filters and returns only the output parameters from the provided pipeline execution options.
4505
+ * Adds warnings for any expected output parameters that are missing.
4355
4506
  *
4356
4507
  * @private internal utility of `createPipelineExecutor`
4357
4508
  */
@@ -4376,9 +4527,12 @@
4376
4527
  }
4377
4528
 
4378
4529
  /**
4379
- * @@@
4530
+ * Executes an entire pipeline, resolving tasks in dependency order, handling errors, and reporting progress.
4531
+ *
4532
+ * Note: This is not a `PipelineExecutor` (which is bound to a single pipeline), but a utility function used by `createPipelineExecutor` to create a `PipelineExecutor`.
4380
4533
  *
4381
- * Note: This is not a `PipelineExecutor` (which is binded with one exact pipeline), but a utility function of `createPipelineExecutor` which creates `PipelineExecutor`
4534
+ * @param options - Options for execution, including input parameters, pipeline, and callbacks.
4535
+ * @returns The result of the pipeline execution, including output parameters, errors, and usage statistics.
4382
4536
  *
4383
4537
  * @private internal utility of `createPipelineExecutor`
4384
4538
  */
@@ -4701,10 +4855,27 @@
4701
4855
  cacheDirname,
4702
4856
  intermediateFilesStrategy,
4703
4857
  isAutoInstalled,
4858
+ }).catch((error) => {
4859
+ assertsError(error);
4860
+ return exportJson({
4861
+ name: 'pipelineExecutorResult',
4862
+ message: `Unuccessful PipelineExecutorResult, last catch`,
4863
+ order: [],
4864
+ value: {
4865
+ isSuccessful: false,
4866
+ errors: [serializeError(error)],
4867
+ warnings: [],
4868
+ usage: UNCERTAIN_USAGE,
4869
+ executionReport: null,
4870
+ outputParameters: {},
4871
+ preparedPipeline,
4872
+ },
4873
+ });
4704
4874
  });
4705
4875
  };
4706
4876
  const pipelineExecutor = (inputParameters) => createTask({
4707
4877
  taskType: 'EXECUTION',
4878
+ title: pipeline.title,
4708
4879
  taskProcessCallback(updateOngoingResult) {
4709
4880
  return pipelineExecutorWithCallback(inputParameters, async (newOngoingResult) => {
4710
4881
  updateOngoingResult(newOngoingResult);
@@ -4756,12 +4927,14 @@
4756
4927
  const spending = new rxjs.Subject();
4757
4928
  const proxyTools = {
4758
4929
  get title() {
4759
- // TODO: [🧠] Maybe put here some suffix
4760
- return llmTools.title;
4930
+ return `${llmTools.title} (+usage)`;
4931
+ // <- TODO: [🧈] Maybe standartize the suffix when wrapping `LlmExecutionTools` up
4932
+ // <- TODO: [🧈][🧠] Does it make sence to suffix "(+usage)"?
4761
4933
  },
4762
4934
  get description() {
4763
- // TODO: [🧠] Maybe put here some suffix
4764
- return llmTools.description;
4935
+ return `${llmTools.description} (+usage)`;
4936
+ // <- TODO: [🧈] Maybe standartize the suffix when wrapping `LlmExecutionTools` up
4937
+ // <- TODO: [🧈][🧠] Does it make sence to suffix "(+usage)"?
4765
4938
  },
4766
4939
  checkConfiguration() {
4767
4940
  return /* not await */ llmTools.checkConfiguration();
@@ -4884,7 +5057,8 @@
4884
5057
  */
4885
5058
 
4886
5059
  /**
4887
- * @@@
5060
+ * Safely retrieves the global scope object (window in browser, global in Node.js)
5061
+ * regardless of the JavaScript environment in which the code is running
4888
5062
  *
4889
5063
  * Note: `$` is used to indicate that this function is not a pure function - it access global scope
4890
5064
  *
@@ -4895,10 +5069,10 @@
4895
5069
  }
4896
5070
 
4897
5071
  /**
4898
- * @@@
5072
+ * Normalizes a text string to SCREAMING_CASE (all uppercase with underscores).
4899
5073
  *
4900
- * @param text @@@
4901
- * @returns @@@
5074
+ * @param text The text string to be converted to SCREAMING_CASE format.
5075
+ * @returns The normalized text in SCREAMING_CASE format.
4902
5076
  * @example 'HELLO_WORLD'
4903
5077
  * @example 'I_LOVE_PROMPTBOOK'
4904
5078
  * @public exported from `@promptbook/utils`
@@ -4950,10 +5124,10 @@
4950
5124
  */
4951
5125
 
4952
5126
  /**
4953
- * @@@
5127
+ * Normalizes a text string to snake_case format.
4954
5128
  *
4955
- * @param text @@@
4956
- * @returns @@@
5129
+ * @param text The text string to be converted to snake_case format.
5130
+ * @returns The normalized text in snake_case format.
4957
5131
  * @example 'hello_world'
4958
5132
  * @example 'i_love_promptbook'
4959
5133
  * @public exported from `@promptbook/utils`
@@ -4963,11 +5137,11 @@
4963
5137
  }
4964
5138
 
4965
5139
  /**
4966
- * Register is @@@
5140
+ * Global registry for storing and managing registered entities of a given type.
4967
5141
  *
4968
5142
  * Note: `$` is used to indicate that this function is not a pure function - it accesses and adds variables in global scope.
4969
5143
  *
4970
- * @private internal utility, exported are only signleton instances of this class
5144
+ * @private internal utility, exported are only singleton instances of this class
4971
5145
  */
4972
5146
  class $Register {
4973
5147
  constructor(registerName) {
@@ -5011,10 +5185,10 @@
5011
5185
  }
5012
5186
 
5013
5187
  /**
5014
- * @@@
5188
+ * Global registry for storing metadata about all available scrapers and converters.
5015
5189
  *
5016
- * Note: `$` is used to indicate that this interacts with the global scope
5017
- * @singleton Only one instance of each register is created per build, but thare can be more @@@
5190
+ * Note: `$` is used to indicate that this interacts with the global scope.
5191
+ * @singleton Only one instance of each register is created per build, but there can be more in different contexts (e.g., tests).
5018
5192
  * @public exported from `@promptbook/core`
5019
5193
  */
5020
5194
  const $scrapersMetadataRegister = new $Register('scrapers_metadata');
@@ -5023,10 +5197,11 @@
5023
5197
  */
5024
5198
 
5025
5199
  /**
5026
- * @@@
5200
+ * Registry for all available scrapers in the system.
5201
+ * Central point for registering and accessing different types of content scrapers.
5027
5202
  *
5028
5203
  * Note: `$` is used to indicate that this interacts with the global scope
5029
- * @singleton Only one instance of each register is created per build, but thare can be more @@@
5204
+ * @singleton Only one instance of each register is created per build, but there can be more than one in different build modules
5030
5205
  * @public exported from `@promptbook/core`
5031
5206
  */
5032
5207
  const $scrapersRegister = new $Register('scraper_constructors');
@@ -5122,10 +5297,10 @@
5122
5297
  */
5123
5298
 
5124
5299
  /**
5125
- * @@@
5300
+ * Converts a given text to kebab-case format.
5126
5301
  *
5127
- * @param text @@@
5128
- * @returns @@@
5302
+ * @param text The text to be converted.
5303
+ * @returns The kebab-case formatted string.
5129
5304
  * @example 'hello-world'
5130
5305
  * @example 'i-love-promptbook'
5131
5306
  * @public exported from `@promptbook/utils`
@@ -5194,7 +5369,8 @@
5194
5369
  */
5195
5370
 
5196
5371
  /**
5197
- * @@@
5372
+ * Converts a name to a properly formatted subfolder path for cache storage.
5373
+ * Handles normalization and path formatting to create consistent cache directory structures.
5198
5374
  *
5199
5375
  * @private for `FileCacheStorage`
5200
5376
  */
@@ -5274,11 +5450,11 @@
5274
5450
  }
5275
5451
 
5276
5452
  /**
5277
- * @@@
5453
+ * Converts a title string into a normalized name.
5278
5454
  *
5279
- * @param value @@@
5280
- * @returns @@@
5281
- * @example @@@
5455
+ * @param value The title string to be converted to a name.
5456
+ * @returns A normalized name derived from the input title.
5457
+ * @example 'Hello World!' -> 'hello-world'
5282
5458
  * @public exported from `@promptbook/utils`
5283
5459
  */
5284
5460
  function titleToName(value) {
@@ -5329,7 +5505,9 @@
5329
5505
  */
5330
5506
 
5331
5507
  /**
5332
- * @@@
5508
+ * Factory function that creates a handler for processing knowledge sources.
5509
+ * Provides standardized processing of different types of knowledge sources
5510
+ * across various scraper implementations.
5333
5511
  *
5334
5512
  * @public exported from `@promptbook/core`
5335
5513
  */
@@ -5468,7 +5646,7 @@
5468
5646
  }
5469
5647
 
5470
5648
  /**
5471
- * Prepares the knowle
5649
+ * Prepares the knowledge pieces
5472
5650
  *
5473
5651
  * @see https://github.com/webgptorg/promptbook/discussions/41
5474
5652
  * @public exported from `@promptbook/core`
@@ -5564,15 +5742,18 @@
5564
5742
  * TODO: [🧊] In future one preparation can take data from previous preparation and save tokens and time
5565
5743
  * Put `knowledgePieces` into `PrepareKnowledgeOptions`
5566
5744
  * TODO: [🪂] More than max things can run in parallel by acident [1,[2a,2b,_],[3a,3b,_]]
5567
- * TODO: [🧠][❎] Do here propper M:N mapping
5745
+ * TODO: [🧠][❎] Do here proper M:N mapping
5568
5746
  * [x] One source can make multiple pieces
5569
5747
  * [ ] One piece can have multiple sources
5570
5748
  */
5571
5749
 
5572
5750
  /**
5573
- * @@@
5751
+ * Prepares tasks by adding knowledge to the prompt and ensuring all necessary parameters are included.
5574
5752
  *
5575
- * @public exported from `@promptbook/core`
5753
+ * @param tasks Sequence of tasks that are chained together to form a pipeline
5754
+ * @returns A promise that resolves to the prepared tasks.
5755
+ *
5756
+ * @private internal utility of `preparePipeline`
5576
5757
  */
5577
5758
  async function prepareTasks(pipeline, tools, options) {
5578
5759
  const { maxParallelCount = DEFAULT_MAX_PARALLEL_COUNT } = options;
@@ -5610,7 +5791,7 @@
5610
5791
  return { tasksPrepared };
5611
5792
  }
5612
5793
  /**
5613
- * TODO: [😂] Adding knowledge should be convert to async high-level abstractions, simmilar thing with expectations to sync high-level abstractions
5794
+ * TODO: [😂] Adding knowledge should be convert to async high-level abstractions, similar thing with expectations to sync high-level abstractions
5614
5795
  * TODO: [🧠] Add context to each task (if missing)
5615
5796
  * TODO: [🧠] What is better name `prepareTask` or `prepareTaskAndParameters`
5616
5797
  * TODO: [♨][main] !!3 Prepare index the examples and maybe tasks
@@ -5746,7 +5927,7 @@
5746
5927
  order: ORDER_OF_PIPELINE_JSON,
5747
5928
  value: {
5748
5929
  ...pipeline,
5749
- // <- TODO: Probbably deeply clone the pipeline because `$exportJson` freezes the subobjects
5930
+ // <- TODO: Probably deeply clone the pipeline because `$exportJson` freezes the subobjects
5750
5931
  title,
5751
5932
  knowledgeSources: knowledgeSourcesPrepared,
5752
5933
  knowledgePieces: knowledgePiecesPrepared,
@@ -6022,7 +6203,7 @@
6022
6203
  throw new ParseError(`Task section and example section must end with return statement -> {parameterName}`);
6023
6204
  };
6024
6205
  if ($taskJson.content === undefined) {
6025
- throw new UnexpectedError(`Content is missing in the taskJson - probbably commands are applied in wrong order`);
6206
+ throw new UnexpectedError(`Content is missing in the taskJson - probably commands are applied in wrong order`);
6026
6207
  }
6027
6208
  if (command.taskType === 'EXAMPLE') {
6028
6209
  expectResultingParameterName();
@@ -6090,7 +6271,7 @@
6090
6271
  /**
6091
6272
  * Parses the boilerplate command
6092
6273
  *
6093
- * Note: @@@ This command is used as boilerplate for new commands - it should NOT be used in any `.book` file
6274
+ * Note: @@ This command is used as boilerplate for new commands - it should NOT be used in any `.book` file
6094
6275
  *
6095
6276
  * @see `documentationUrl` for more details
6096
6277
  * @private within the commands folder
@@ -6478,11 +6659,11 @@
6478
6659
  };
6479
6660
 
6480
6661
  /**
6481
- * @@@
6662
+ * Normalizes a given text to camelCase format.
6482
6663
  *
6483
- * @param text @@@
6484
- * @param _isFirstLetterCapital @@@
6485
- * @returns @@@
6664
+ * @param text The text to be normalized.
6665
+ * @param _isFirstLetterCapital Whether the first letter should be capitalized.
6666
+ * @returns The camelCase formatted string.
6486
6667
  * @example 'helloWorld'
6487
6668
  * @example 'iLovePromptbook'
6488
6669
  * @public exported from `@promptbook/utils`
@@ -6532,9 +6713,9 @@
6532
6713
  /**
6533
6714
  * Removes quotes from a string
6534
6715
  *
6535
- * Tip: This is very usefull for post-processing of the result of the LLM model
6716
+ * Tip: This is very useful for post-processing of the result of the LLM model
6536
6717
  * Note: This function removes only the same quotes from the beginning and the end of the string
6537
- * Note: There are two simmilar functions:
6718
+ * Note: There are two similar functions:
6538
6719
  * - `removeQuotes` which removes only bounding quotes
6539
6720
  * - `unwrapResult` which removes whole introduce sentence
6540
6721
  *
@@ -6546,18 +6727,19 @@
6546
6727
  if (text.startsWith('"') && text.endsWith('"')) {
6547
6728
  return text.slice(1, -1);
6548
6729
  }
6549
- if (text.startsWith('\'') && text.endsWith('\'')) {
6730
+ if (text.startsWith("'") && text.endsWith("'")) {
6550
6731
  return text.slice(1, -1);
6551
6732
  }
6552
6733
  return text;
6553
6734
  }
6554
6735
 
6555
6736
  /**
6556
- * Function `validateParameterName` will @@@
6737
+ * Function `validateParameterName` will normalize and validate a parameter name for use in pipelines.
6738
+ * It removes diacritics, emojis, and quotes, normalizes to camelCase, and checks for reserved names and invalid characters.
6557
6739
  *
6558
- * @param parameterName @@@
6559
- * @returns @@@
6560
- * @throws {ParseError} @@@
6740
+ * @param parameterName The parameter name to validate and normalize.
6741
+ * @returns The validated and normalized parameter name.
6742
+ * @throws {ParseError} If the parameter name is empty, reserved, or contains invalid characters.
6561
6743
  * @private within the repository
6562
6744
  */
6563
6745
  function validateParameterName(parameterName) {
@@ -6627,8 +6809,6 @@
6627
6809
  /**
6628
6810
  * Parses the foreach command
6629
6811
  *
6630
- * Note: @@@ This command is used as foreach for new commands - it should NOT be used in any `.book` file
6631
- *
6632
6812
  * @see `documentationUrl` for more details
6633
6813
  * @public exported from `@promptbook/editable`
6634
6814
  */
@@ -6685,14 +6865,14 @@
6685
6865
  `));
6686
6866
  // <- TODO: [🏢] List all supported format names
6687
6867
  }
6688
- const subvalueDefinition = formatDefinition.subvalueDefinitions.find((subvalueDefinition) => [subvalueDefinition.subvalueName, ...(subvalueDefinition.aliases || [])].includes(subformatName));
6689
- if (subvalueDefinition === undefined) {
6868
+ const subvalueParser = formatDefinition.subvalueParsers.find((subvalueParser) => [subvalueParser.subvalueName, ...(subvalueParser.aliases || [])].includes(subformatName));
6869
+ if (subvalueParser === undefined) {
6690
6870
  throw new ParseError(spaceTrim__default["default"]((block) => `
6691
6871
  Unsupported subformat name "${subformatName}" for format "${formatName}"
6692
6872
 
6693
6873
  Available subformat names for format "${formatDefinition.formatName}":
6694
- ${block(formatDefinition.subvalueDefinitions
6695
- .map((subvalueDefinition) => subvalueDefinition.subvalueName)
6874
+ ${block(formatDefinition.subvalueParsers
6875
+ .map((subvalueParser) => subvalueParser.subvalueName)
6696
6876
  .map((subvalueName) => `- ${subvalueName}`)
6697
6877
  .join('\n'))}
6698
6878
  `));
@@ -6869,14 +7049,14 @@
6869
7049
  };
6870
7050
 
6871
7051
  /**
6872
- * @@@
7052
+ * Chatbot form factor definition for conversational interfaces that interact with users in a chat-like manner.
6873
7053
  *
6874
7054
  * @public exported from `@promptbook/core`
6875
7055
  */
6876
7056
  const ChatbotFormfactorDefinition = {
6877
7057
  name: 'CHATBOT',
6878
7058
  aliasNames: ['CHAT'],
6879
- description: `@@@`,
7059
+ description: `A chatbot form factor for conversational user interfaces.`,
6880
7060
  documentationUrl: `https://github.com/webgptorg/promptbook/discussions/174`,
6881
7061
  pipelineInterface: {
6882
7062
  inputParameters: [
@@ -6903,7 +7083,45 @@
6903
7083
  };
6904
7084
 
6905
7085
  /**
6906
- * Generator is form of app that @@@
7086
+ * Completion is formfactor that emulates completion models
7087
+ *
7088
+ * @public exported from `@promptbook/core`
7089
+ */
7090
+ const CompletionFormfactorDefinition = {
7091
+ name: 'COMPLETION',
7092
+ description: `Completion is formfactor that emulates completion models`,
7093
+ documentationUrl: `https://github.com/webgptorg/promptbook/discussions/@@`,
7094
+ // <- TODO: https://github.com/webgptorg/promptbook/discussions/new?category=concepts
7095
+ // "🔠 Completion Formfactor"
7096
+ pipelineInterface: {
7097
+ inputParameters: [
7098
+ {
7099
+ name: 'inputText',
7100
+ description: `Input text to be completed`,
7101
+ isInput: true,
7102
+ isOutput: false,
7103
+ },
7104
+ {
7105
+ name: 'instructions',
7106
+ description: `Additional instructions for the model, for example the required length, empty by default`,
7107
+ isInput: true,
7108
+ isOutput: false,
7109
+ },
7110
+ ],
7111
+ outputParameters: [
7112
+ {
7113
+ name: 'followingText',
7114
+ description: `Text that follows the input text`,
7115
+ isInput: false,
7116
+ isOutput: true,
7117
+ },
7118
+ ],
7119
+ },
7120
+ };
7121
+
7122
+ /**
7123
+ * Generator form factor represents an application that generates content or data based on user input or predefined rules.
7124
+ * This form factor is used for apps that produce outputs, such as text, images, or other media, based on provided input.
6907
7125
  *
6908
7126
  * @public exported from `@promptbook/core`
6909
7127
  */
@@ -6932,7 +7150,7 @@
6932
7150
  };
6933
7151
 
6934
7152
  /**
6935
- * @@@
7153
+ * Pipeline interface which is equivalent to `any`
6936
7154
  *
6937
7155
  * @see https://github.com/webgptorg/promptbook/discussions/171
6938
7156
  *
@@ -6947,13 +7165,13 @@
6947
7165
  */
6948
7166
 
6949
7167
  /**
6950
- * @@@
7168
+ * A generic pipeline
6951
7169
  *
6952
7170
  * @public exported from `@promptbook/core`
6953
7171
  */
6954
7172
  const GenericFormfactorDefinition = {
6955
7173
  name: 'GENERIC',
6956
- description: `@@@`,
7174
+ description: `A generic pipeline`,
6957
7175
  documentationUrl: `https://github.com/webgptorg/promptbook/discussions/173`,
6958
7176
  pipelineInterface: GENERIC_PIPELINE_INTERFACE,
6959
7177
  };
@@ -6988,17 +7206,20 @@
6988
7206
  };
6989
7207
 
6990
7208
  /**
6991
- * Matcher is form of app that @@@
7209
+ * Matcher is form of app that evaluates (spreadsheet) content against defined criteria or patterns,
7210
+ * determining if it matches or meets specific requirements. Used for classification,
7211
+ * validation, filtering, and quality assessment of inputs.
6992
7212
  *
6993
7213
  * @public exported from `@promptbook/core`
6994
7214
  */
6995
7215
  const MatcherFormfactorDefinition = {
6996
7216
  name: 'EXPERIMENTAL_MATCHER',
6997
- description: `@@@`,
7217
+ description: `An evaluation system that determines whether content meets specific criteria or patterns.
7218
+ Used for content validation, quality assessment, and intelligent filtering tasks. Currently in experimental phase.`,
6998
7219
  documentationUrl: `https://github.com/webgptorg/promptbook/discussions/177`,
6999
7220
  pipelineInterface: {
7000
7221
  inputParameters: [
7001
- /* @@@ */
7222
+ /* Input parameters for content to be matched and criteria to match against */
7002
7223
  {
7003
7224
  name: 'nonce',
7004
7225
  description: 'Just to prevent EXPERIMENTAL_MATCHER to be set as implicit formfactor',
@@ -7007,20 +7228,21 @@
7007
7228
  },
7008
7229
  ],
7009
7230
  outputParameters: [
7010
- /* @@@ */
7231
+ /* Output parameters containing match results, confidence scores, and relevant metadata */
7011
7232
  ],
7012
7233
  },
7013
7234
  };
7014
7235
 
7015
7236
  /**
7016
- * Sheets is form of app that @@@
7237
+ * Sheets is form of app that processes tabular data in CSV format, allowing transformation
7238
+ * and analysis of structured data through AI-powered operations
7017
7239
  *
7018
7240
  * @public exported from `@promptbook/core`
7019
7241
  */
7020
7242
  const SheetsFormfactorDefinition = {
7021
7243
  name: 'SHEETS',
7022
7244
  aliasNames: ['SHEETS', 'SHEET'],
7023
- description: `@@@`,
7245
+ description: `A formfactor for processing spreadsheet-like data in CSV format, enabling AI transformations on tabular data`,
7024
7246
  documentationUrl: `https://github.com/webgptorg/promptbook/discussions/176`,
7025
7247
  pipelineInterface: {
7026
7248
  inputParameters: [
@@ -7043,13 +7265,16 @@
7043
7265
  };
7044
7266
 
7045
7267
  /**
7046
- * Translator is form of app that @@@
7268
+ * Translator is form of app that transforms input text from one form to another,
7269
+ * such as language translation, style conversion, tone modification, or other text transformations.
7047
7270
  *
7048
7271
  * @public exported from `@promptbook/core`
7049
7272
  */
7050
7273
  const TranslatorFormfactorDefinition = {
7051
7274
  name: 'TRANSLATOR',
7052
- description: `@@@`,
7275
+ description: `A text transformation system that converts input content into different forms,
7276
+ including language translations, paraphrasing, style conversions, and tone adjustments.
7277
+ This form factor takes one input and produces one transformed output.`,
7053
7278
  documentationUrl: `https://github.com/webgptorg/promptbook/discussions/175`,
7054
7279
  pipelineInterface: {
7055
7280
  inputParameters: [
@@ -7086,6 +7311,8 @@
7086
7311
  MatcherFormfactorDefinition,
7087
7312
  GeneratorFormfactorDefinition,
7088
7313
  ImageGeneratorFormfactorDefinition,
7314
+ CompletionFormfactorDefinition,
7315
+ // <- [🛬] When making new formfactor, copy the _boilerplate and link it here
7089
7316
  ];
7090
7317
  /**
7091
7318
  * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -7094,7 +7321,7 @@
7094
7321
  /**
7095
7322
  * Parses the formfactor command
7096
7323
  *
7097
- * Note: @@@ This command is used as formfactor for new commands - it should NOT be used in any `.book` file
7324
+ * Note: This command is used as a formfactor for new commands and defines the app type format - it should NOT be used in any `.book` file
7098
7325
  *
7099
7326
  * @see `documentationUrl` for more details
7100
7327
  * @public exported from `@promptbook/editable`
@@ -7116,7 +7343,7 @@
7116
7343
  /**
7117
7344
  * Description of the FORMFACTOR command
7118
7345
  */
7119
- description: `@@`,
7346
+ description: `Specifies the application type and interface requirements that this promptbook should conform to`,
7120
7347
  /**
7121
7348
  * Link to documentation
7122
7349
  */
@@ -7259,8 +7486,7 @@
7259
7486
  };
7260
7487
 
7261
7488
  /**
7262
- * @@@
7263
- *
7489
+ * @see {@link ModelVariant}
7264
7490
  * @public exported from `@promptbook/core`
7265
7491
  */
7266
7492
  const MODEL_VARIANTS = ['COMPLETION', 'CHAT', 'EMBEDDING' /* <- TODO [🏳] */ /* <- [🤖] */];
@@ -7692,10 +7918,10 @@
7692
7918
  }
7693
7919
 
7694
7920
  /**
7695
- * @@@
7921
+ * Checks if the given value is a valid JavaScript identifier name.
7696
7922
  *
7697
- * @param javascriptName @@@
7698
- * @returns @@@
7923
+ * @param javascriptName The value to check for JavaScript identifier validity.
7924
+ * @returns `true` if the value is a valid JavaScript name, false otherwise.
7699
7925
  * @public exported from `@promptbook/utils`
7700
7926
  */
7701
7927
  function isValidJavascriptName(javascriptName) {
@@ -8156,7 +8382,7 @@
8156
8382
  // Arg1 Arg2 Arg3 | FOO
8157
8383
  {
8158
8384
  const commandNameRaw = items.slice(-1).join('_');
8159
- const args = items.slice(0, -1); // <- Note: This is probbably not correct
8385
+ const args = items.slice(0, -1); // <- Note: This is probably not correct
8160
8386
  const rawArgs = raw
8161
8387
  .substring(0, raw.length - commandNameRaw.length)
8162
8388
  .trim();
@@ -8175,7 +8401,10 @@
8175
8401
  `));
8176
8402
  }
8177
8403
  /**
8178
- * @@@
8404
+ * Generates a markdown-formatted message listing all supported commands
8405
+ * with their descriptions and documentation links
8406
+ *
8407
+ * @returns A formatted markdown string containing all available commands and their details
8179
8408
  */
8180
8409
  function getSupportedCommandsMessage() {
8181
8410
  return COMMANDS.flatMap(({ name, aliasNames, description, documentationUrl }) =>
@@ -8186,7 +8415,10 @@
8186
8415
  ]).join('\n');
8187
8416
  }
8188
8417
  /**
8189
- * @@@
8418
+ * Attempts to parse a command variant using the provided input parameters
8419
+ *
8420
+ * @param input Object containing command parsing information including raw command text and normalized values
8421
+ * @returns A parsed Command object if successful, or null if the command cannot be parsed
8190
8422
  */
8191
8423
  function parseCommandVariant(input) {
8192
8424
  const { commandNameRaw, usagePlace, normalized, args, raw, rawArgs } = input;
@@ -8233,7 +8465,7 @@
8233
8465
  }
8234
8466
 
8235
8467
  /**
8236
- * @@@
8468
+ * Extracts the interface (input and output parameters) from a pipeline.
8237
8469
  *
8238
8470
  * @deprecated https://github.com/webgptorg/promptbook/pull/186
8239
8471
  * @see https://github.com/webgptorg/promptbook/discussions/171
@@ -8266,7 +8498,7 @@
8266
8498
  }
8267
8499
 
8268
8500
  /**
8269
- * @@@
8501
+ * Checks if two pipeline interfaces are structurally identical.
8270
8502
  *
8271
8503
  * @deprecated https://github.com/webgptorg/promptbook/pull/186
8272
8504
  * @see https://github.com/webgptorg/promptbook/discussions/171
@@ -8298,10 +8530,11 @@
8298
8530
  }
8299
8531
 
8300
8532
  /**
8301
- * @@@
8533
+ * Checks if a given pipeline satisfies the requirements of a specified pipeline interface.
8302
8534
  *
8303
8535
  * @deprecated https://github.com/webgptorg/promptbook/pull/186
8304
8536
  * @see https://github.com/webgptorg/promptbook/discussions/171
8537
+ * @returns `true` if the pipeline implements the interface, `false` otherwise.
8305
8538
  *
8306
8539
  * @public exported from `@promptbook/core`
8307
8540
  */
@@ -8487,7 +8720,8 @@
8487
8720
  }
8488
8721
 
8489
8722
  /**
8490
- * @@@
8723
+ * Utility to determine if a pipeline string is in flat format.
8724
+ * A flat pipeline is a simple text without proper structure (headers, blocks, etc).
8491
8725
  *
8492
8726
  * @public exported from `@promptbook/editable`
8493
8727
  */
@@ -8508,7 +8742,10 @@
8508
8742
  }
8509
8743
 
8510
8744
  /**
8511
- * @@@
8745
+ * Converts a pipeline structure to its string representation.
8746
+ *
8747
+ * Transforms a flat, simple pipeline into a properly formatted pipeline string
8748
+ * with sections for title, prompt, and return statement.
8512
8749
  *
8513
8750
  * @public exported from `@promptbook/editable`
8514
8751
  */
@@ -8565,7 +8802,7 @@
8565
8802
  * Note: It can not work with html syntax and comments
8566
8803
  *
8567
8804
  * @param markdown any valid markdown
8568
- * @returns @@@
8805
+ * @returns An array of strings, each representing an individual list item found in the markdown
8569
8806
  * @public exported from `@promptbook/markdown-utils`
8570
8807
  */
8571
8808
  function extractAllListItemsFromMarkdown(markdown) {
@@ -8590,8 +8827,8 @@
8590
8827
  *
8591
8828
  * - When there are multiple or no code blocks the function throws a `ParseError`
8592
8829
  *
8593
- * Note: There are multiple simmilar function:
8594
- * - `extractBlock` just extracts the content of the code block which is also used as build-in function for postprocessing
8830
+ * Note: There are multiple similar functions:
8831
+ * - `extractBlock` just extracts the content of the code block which is also used as built-in function for postprocessing
8595
8832
  * - `extractJsonBlock` extracts exactly one valid JSON code block
8596
8833
  * - `extractOneBlockFromMarkdown` extracts exactly one code block with language of the code block
8597
8834
  * - `extractAllBlocksFromMarkdown` extracts all code blocks with language of the code block
@@ -8746,7 +8983,7 @@
8746
8983
  * Compile pipeline from string (markdown) format to JSON format synchronously
8747
8984
  *
8748
8985
  * Note: There are 3 similar functions:
8749
- * - `compilePipeline` **(preferred)** - which propperly compiles the promptbook and use embedding for external knowledge
8986
+ * - `compilePipeline` **(preferred)** - which properly compiles the promptbook and uses embedding for external knowledge
8750
8987
  * - `parsePipeline` - use only if you need to compile promptbook synchronously and it contains NO external knowledge
8751
8988
  * - `preparePipeline` - just one step in the compilation process
8752
8989
  *
@@ -9183,7 +9420,7 @@
9183
9420
  * TODO: Use spaceTrim more effectively
9184
9421
  * TODO: [🧠] Parameter flags - isInput, isOutput, isInternal
9185
9422
  * TODO: [🥞] Not optimal parsing because `splitMarkdownIntoSections` is executed twice with same string, once through `flattenMarkdown` and second directly here
9186
- * TODO: [♈] Probbably move expectations from tasks to parameters
9423
+ * TODO: [♈] Probably move expectations from tasks to parameters
9187
9424
  * TODO: [🛠] Actions, instruments (and maybe knowledge) => Functions and tools
9188
9425
  * TODO: [🍙] Make some standard order of json properties
9189
9426
  */
@@ -9400,7 +9637,8 @@
9400
9637
  */
9401
9638
 
9402
9639
  /**
9403
- * @@@
9640
+ * Attempts to locate the specified application on a Linux system using the 'which' command.
9641
+ * Returns the path to the executable if found, or null otherwise.
9404
9642
  *
9405
9643
  * @private within the repository
9406
9644
  */
@@ -9420,7 +9658,8 @@
9420
9658
  */
9421
9659
 
9422
9660
  /**
9423
- * @@@
9661
+ * Provides filesystem access (for example for Node.js-based scrapers)
9662
+ * Creates a standardized filesystem interface that scrapers can use for file operations.
9424
9663
  *
9425
9664
  * @public exported from `@promptbook/node`
9426
9665
  */
@@ -9466,7 +9705,8 @@
9466
9705
  // eslint-disable-next-line @typescript-eslint/no-var-requires
9467
9706
  const userhome = require('userhome');
9468
9707
  /**
9469
- * @@@
9708
+ * Attempts to locate the specified application on a macOS system by checking standard application paths and using mdfind.
9709
+ * Returns the path to the executable if found, or null otherwise.
9470
9710
  *
9471
9711
  * @private within the repository
9472
9712
  */
@@ -9498,7 +9738,8 @@
9498
9738
  */
9499
9739
 
9500
9740
  /**
9501
- * @@@
9741
+ * Attempts to locate the specified application on a Windows system by searching common installation directories.
9742
+ * Returns the path to the executable if found, or null otherwise.
9502
9743
  *
9503
9744
  * @private within the repository
9504
9745
  */
@@ -9569,7 +9810,8 @@
9569
9810
  */
9570
9811
 
9571
9812
  /**
9572
- * @@@
9813
+ * Locates the LibreOffice executable on the current system by searching platform-specific paths.
9814
+ * Returns the path to the executable if found, or null otherwise.
9573
9815
  *
9574
9816
  * @private within the repository
9575
9817
  */
@@ -9587,7 +9829,8 @@
9587
9829
  */
9588
9830
 
9589
9831
  /**
9590
- * @@@
9832
+ * Locates the Pandoc executable on the current system by searching platform-specific paths.
9833
+ * Returns the path to the executable if found, or null otherwise.
9591
9834
  *
9592
9835
  * @private within the repository
9593
9836
  */
@@ -9605,7 +9848,7 @@
9605
9848
  */
9606
9849
 
9607
9850
  /**
9608
- * @@@
9851
+ * Provides paths to required executables (i.e. as Pandoc and LibreOffice) for Node.js environments.
9609
9852
  *
9610
9853
  * @public exported from `@promptbook/node`
9611
9854
  */
@@ -9625,10 +9868,10 @@
9625
9868
  */
9626
9869
 
9627
9870
  /**
9628
- * @@@
9871
+ * Register for LLM tools metadata.
9629
9872
  *
9630
9873
  * Note: `$` is used to indicate that this interacts with the global scope
9631
- * @singleton Only one instance of each register is created per build, but thare can be more @@@
9874
+ * @singleton Only one instance of each register is created per build, but there can be more instances across different builds or environments.
9632
9875
  * @public exported from `@promptbook/core`
9633
9876
  */
9634
9877
  const $llmToolsMetadataRegister = new $Register('llm_tools_metadata');
@@ -9656,10 +9899,10 @@
9656
9899
  */
9657
9900
 
9658
9901
  /**
9659
- * @@@
9902
+ * Register for LLM tools.
9660
9903
  *
9661
9904
  * Note: `$` is used to indicate that this interacts with the global scope
9662
- * @singleton Only one instance of each register is created per build, but thare can be more @@@
9905
+ * @singleton Only one instance of each register is created per build, but there can be more instances across different builds or environments.
9663
9906
  * @public exported from `@promptbook/core`
9664
9907
  */
9665
9908
  const $llmToolsRegister = new $Register('llm_execution_tools_constructors');
@@ -9852,9 +10095,8 @@
9852
10095
  */
9853
10096
 
9854
10097
  /**
9855
- * @@@
10098
+ * Provides LLM tools configuration by reading environment variables.
9856
10099
  *
9857
- * @@@ .env
9858
10100
  * Note: `$` is used to indicate that this function is not a pure function - it uses filesystem to access `.env` file
9859
10101
  *
9860
10102
  * It looks for environment variables:
@@ -9862,7 +10104,8 @@
9862
10104
  * - `process.env.ANTHROPIC_CLAUDE_API_KEY`
9863
10105
  * - ...
9864
10106
  *
9865
- * @returns @@@
10107
+ * @see Environment variables documentation or .env file for required variables.
10108
+ * @returns A promise that resolves to the LLM tools configuration, or null if configuration is incomplete or missing.
9866
10109
  * @public exported from `@promptbook/node`
9867
10110
  */
9868
10111
  async function $provideLlmToolsConfigurationFromEnv() {
@@ -9884,11 +10127,16 @@
9884
10127
  */
9885
10128
 
9886
10129
  /**
9887
- * @@@
10130
+ * Creates LLM execution tools from provided configuration objects
10131
+ *
10132
+ * Instantiates and configures LLM tool instances for each configuration entry,
10133
+ * combining them into a unified interface via MultipleLlmExecutionTools.
9888
10134
  *
9889
10135
  * Note: This function is not cached, every call creates new instance of `MultipleLlmExecutionTools`
9890
10136
  *
9891
- * @returns @@@
10137
+ * @param configuration Array of LLM tool configurations to instantiate
10138
+ * @param options Additional options for configuring the LLM tools
10139
+ * @returns A unified interface combining all successfully instantiated LLM tools
9892
10140
  * @public exported from `@promptbook/core`
9893
10141
  */
9894
10142
  function createLlmToolsFromConfiguration(configuration, options = {}) {
@@ -9927,7 +10175,11 @@
9927
10175
  /**
9928
10176
  * TODO: [🎌] Together with `createLlmToolsFromConfiguration` + 'EXECUTION_TOOLS_CLASSES' gets to `@promptbook/core` ALL model providers, make this more efficient
9929
10177
  * TODO: [🧠][🎌] Dynamically install required providers
9930
- * TODO: @@@ write discussion about this - wizzard
10178
+ * TODO: We should implement an interactive configuration wizard that would:
10179
+ * 1. Detect which LLM providers are available in the environment
10180
+ * 2. Guide users through required configuration settings for each provider
10181
+ * 3. Allow testing connections before completing setup
10182
+ * 4. Generate appropriate configuration code for application integration
9931
10183
  * TODO: [🧠][🍛] Which name is better `createLlmToolsFromConfig` or `createLlmToolsFromConfiguration`?
9932
10184
  * TODO: [🧠] Is there some meaningfull way how to test this util
9933
10185
  * TODO: This should be maybe not under `_common` but under `utils`
@@ -9935,11 +10187,14 @@
9935
10187
  */
9936
10188
 
9937
10189
  /**
9938
- * @@@
10190
+ * Automatically configures LLM tools from environment variables in Node.js
10191
+ *
10192
+ * This utility function detects available LLM providers based on environment variables
10193
+ * and creates properly configured LLM execution tools for each detected provider.
9939
10194
  *
9940
10195
  * Note: This function is not cached, every call creates new instance of `MultipleLlmExecutionTools`
9941
10196
  *
9942
- * @@@ .env
10197
+ * Supports environment variables from .env files when dotenv is configured
9943
10198
  * Note: `$` is used to indicate that this function is not a pure function - it uses filesystem to access `.env` file
9944
10199
  *
9945
10200
  * It looks for environment variables:
@@ -9947,7 +10202,8 @@
9947
10202
  * - `process.env.ANTHROPIC_CLAUDE_API_KEY`
9948
10203
  * - ...
9949
10204
  *
9950
- * @returns @@@
10205
+ * @param options Configuration options for the LLM tools
10206
+ * @returns A unified interface containing all detected and configured LLM tools
9951
10207
  * @public exported from `@promptbook/node`
9952
10208
  */
9953
10209
  async function $provideLlmToolsFromEnv(options = {}) {
@@ -9973,7 +10229,16 @@
9973
10229
  return createLlmToolsFromConfiguration(configuration, options);
9974
10230
  }
9975
10231
  /**
9976
- * TODO: @@@ write `$provideLlmToolsFromEnv` vs `$provideLlmToolsConfigurationFromEnv` vs `createLlmToolsFromConfiguration`
10232
+ * TODO: The architecture for LLM tools configuration consists of three key functions:
10233
+ * 1. `$provideLlmToolsFromEnv` - High-level function that detects available providers from env vars and returns ready-to-use LLM tools
10234
+ * 2. `$provideLlmToolsConfigurationFromEnv` - Middle layer that extracts configuration objects from environment variables
10235
+ * 3. `createLlmToolsFromConfiguration` - Low-level function that instantiates LLM tools from explicit configuration
10236
+ *
10237
+ * This layered approach allows flexibility in how tools are configured:
10238
+ * - Use $provideLlmToolsFromEnv for automatic detection and setup in Node.js environments
10239
+ * - Use $provideLlmToolsConfigurationFromEnv to extract config objects for modification before instantiation
10240
+ * - Use createLlmToolsFromConfiguration for explicit control over tool configurations
10241
+ *
9977
10242
  * TODO: [🧠][🍛] Which name is better `$provideLlmToolsFromEnv` or `$provideLlmToolsFromEnvironment`?
9978
10243
  * TODO: [🧠] Is there some meaningfull way how to test this util
9979
10244
  * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
@@ -9983,11 +10248,9 @@
9983
10248
  */
9984
10249
 
9985
10250
  /**
9986
- * @@@
9987
- *
9988
- * 1) @@@
9989
- * 2) @@@
9990
- *
10251
+ * Provides a collection of scrapers optimized for Node.js environment.
10252
+ * 1) `provideScrapersForNode` use as default
10253
+ * 2) `provideScrapersForBrowser` use in limited browser environment *
9991
10254
  * @public exported from `@promptbook/node`
9992
10255
  */
9993
10256
  async function $provideScrapersForNode(tools, options) {
@@ -10094,11 +10357,11 @@
10094
10357
  }
10095
10358
 
10096
10359
  /**
10097
- * @@@
10360
+ * Converts a name string into a URI-compatible format.
10098
10361
  *
10099
- * @param name @@@
10100
- * @returns @@@
10101
- * @example @@@
10362
+ * @param name The string to be converted to a URI-compatible format.
10363
+ * @returns A URI-compatible string derived from the input name.
10364
+ * @example 'Hello World' -> 'hello-world'
10102
10365
  * @public exported from `@promptbook/utils`
10103
10366
  */
10104
10367
  function nameToUriPart(name) {
@@ -10112,11 +10375,11 @@
10112
10375
  }
10113
10376
 
10114
10377
  /**
10115
- * @@@
10378
+ * Converts a given name into URI-compatible parts.
10116
10379
  *
10117
- * @param name @@@
10118
- * @returns @@@
10119
- * @example @@@
10380
+ * @param name The name to be converted into URI parts.
10381
+ * @returns An array of URI-compatible parts derived from the name.
10382
+ * @example 'Example Name' -> ['example', 'name']
10120
10383
  * @public exported from `@promptbook/utils`
10121
10384
  */
10122
10385
  function nameToUriParts(name) {
@@ -10183,7 +10446,7 @@
10183
10446
  /**
10184
10447
  * Function trimCodeBlock will trim starting and ending code block from the string if it is present.
10185
10448
  *
10186
- * Note: This is usefull for post-processing of the result of the chat LLM model
10449
+ * Note: This is useful for post-processing of the result of the chat LLM model
10187
10450
  * when the model wraps the result in the (markdown) code block.
10188
10451
  *
10189
10452
  * @public exported from `@promptbook/utils`
@@ -10202,7 +10465,7 @@
10202
10465
  /**
10203
10466
  * Function trimEndOfCodeBlock will remove ending code block from the string if it is present.
10204
10467
  *
10205
- * Note: This is usefull for post-processing of the result of the completion LLM model
10468
+ * Note: This is useful for post-processing of the result of the completion LLM model
10206
10469
  * if you want to start code block in the prompt but you don't want to end it in the result.
10207
10470
  *
10208
10471
  * @public exported from `@promptbook/utils`
@@ -10217,9 +10480,9 @@
10217
10480
  /**
10218
10481
  * Removes quotes and optional introduce text from a string
10219
10482
  *
10220
- * Tip: This is very usefull for post-processing of the result of the LLM model
10483
+ * Tip: This is very useful for post-processing of the result of the LLM model
10221
10484
  * Note: This function trims the text and removes whole introduce sentence if it is present
10222
- * Note: There are two simmilar functions:
10485
+ * Note: There are two similar functions:
10223
10486
  * - `removeQuotes` which removes only bounding quotes
10224
10487
  * - `unwrapResult` which removes whole introduce sentence
10225
10488
  *
@@ -10290,7 +10553,7 @@
10290
10553
  *
10291
10554
  * - When there are multiple or no code blocks the function throws a `ParseError`
10292
10555
  *
10293
- * Note: There are multiple simmilar function:
10556
+ * Note: There are multiple similar function:
10294
10557
  * - `extractBlock` just extracts the content of the code block which is also used as build-in function for postprocessing
10295
10558
  * - `extractJsonBlock` extracts exactly one valid JSON code block
10296
10559
  * - `extractOneBlockFromMarkdown` extracts exactly one code block with language of the code block
@@ -10327,7 +10590,7 @@
10327
10590
  })();
10328
10591
  }
10329
10592
  /**
10330
- * TODO: Probbably remove in favour of `keepImported`
10593
+ * TODO: Probably remove in favour of `keepImported`
10331
10594
  * TODO: [1] This maybe does memory leak
10332
10595
  */
10333
10596
 
@@ -10531,7 +10794,7 @@
10531
10794
  */
10532
10795
 
10533
10796
  /**
10534
- * Placeholder for better implementation of JavascriptExecutionTools - some propper sandboxing
10797
+ * Placeholder for better implementation of JavascriptExecutionTools - some proper sandboxing
10535
10798
  *
10536
10799
  * @alias JavascriptExecutionTools
10537
10800
  * @public exported from `@promptbook/javascript`
@@ -10916,14 +11179,17 @@
10916
11179
  return pipelineJsonStringified;
10917
11180
  }
10918
11181
  /**
10919
- * TODO: [🐝] Not Working propperly @see https://promptbook.studio/examples/mixed-knowledge.book
11182
+ * TODO: [🐝] Not Working properly @see https://promptbook.studio/examples/mixed-knowledge.book
10920
11183
  * TODO: [🧠][0] Maybe rename to `stringifyPipelineJson`, `stringifyIndexedJson`,...
10921
11184
  * TODO: [🧠] Maybe more elegant solution than replacing via regex
10922
11185
  * TODO: [🍙] Make some standard order of json properties
10923
11186
  */
10924
11187
 
10925
11188
  /**
10926
- * @@@
11189
+ * A storage implementation that caches data in files organized in a directory structure.
11190
+ * Provides methods for retrieving, storing, and managing cached data on the filesystem.
11191
+ *
11192
+ * This class implements the PromptbookStorage interface for filesystem-based caching.
10927
11193
  *
10928
11194
  * @public exported from `@promptbook/node`
10929
11195
  */
@@ -10936,7 +11202,8 @@
10936
11202
  }
10937
11203
  }
10938
11204
  /**
10939
- * @@@
11205
+ * Converts a storage key to a filesystem path where the data should be stored.
11206
+ * Creates a consistent, deterministic file path based on the key string.
10940
11207
  */
10941
11208
  getFilenameForKey(key) {
10942
11209
  // TODO: [👬] DRY
@@ -10948,7 +11215,8 @@
10948
11215
  ...nameToSubfolderPath(hash /* <- TODO: [🎎] Maybe add some SHA256 prefix */), `${name.substring(0, MAX_FILENAME_LENGTH)}.json`);
10949
11216
  }
10950
11217
  /**
10951
- * @@@ Returns the current value associated with the given key, or null if the given key does not exist in the list associated with the object.
11218
+ * Returns the current value associated with the given key, or null if the given key does not exist.
11219
+ * Retrieves the cached data from the file system storage.
10952
11220
  */
10953
11221
  async getItem(key) {
10954
11222
  const filename = this.getFilenameForKey(key);
@@ -10961,7 +11229,8 @@
10961
11229
  return value;
10962
11230
  }
10963
11231
  /**
10964
- * @@@ Sets the value of the pair identified by key to value, creating a new key/value pair if none existed for key previously.
11232
+ * Sets the value of the pair identified by key to value, creating a new key/value pair if none existed for key previously.
11233
+ * Persists data to the file system, creating necessary directory structure if it doesn't exist.
10965
11234
  */
10966
11235
  async setItem(key, value) {
10967
11236
  const filename = this.getFilenameForKey(key);
@@ -10973,7 +11242,8 @@
10973
11242
  await promises.writeFile(filename, fileContent, 'utf-8');
10974
11243
  }
10975
11244
  /**
10976
- * @@@ Removes the key/value pair with the given key from the list associated with the object, if a key/value pair with the given key exists.
11245
+ * Removes the key/value pair with the given key from the storage, if a key/value pair with the given key exists.
11246
+ * Deletes the corresponding file from the filesystem.
10977
11247
  */
10978
11248
  async removeItem(key) {
10979
11249
  const filename = this.getFilenameForKey(key);