@promptbook/node 0.92.0-9 → 0.92.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -5
- package/esm/index.es.js +801 -531
- package/esm/index.es.js.map +1 -1
- package/esm/typings/src/_packages/browser.index.d.ts +2 -0
- package/esm/typings/src/_packages/core.index.d.ts +26 -14
- package/esm/typings/src/_packages/types.index.d.ts +6 -2
- package/esm/typings/src/collection/PipelineCollection.d.ts +0 -2
- package/esm/typings/src/collection/SimplePipelineCollection.d.ts +1 -1
- package/esm/typings/src/commands/FOREACH/ForeachJson.d.ts +6 -6
- package/esm/typings/src/commands/FOREACH/foreachCommandParser.d.ts +0 -2
- package/esm/typings/src/commands/FORMFACTOR/formfactorCommandParser.d.ts +1 -1
- package/esm/typings/src/commands/_BOILERPLATE/boilerplateCommandParser.d.ts +1 -1
- package/esm/typings/src/commands/_common/types/CommandParser.d.ts +36 -28
- package/esm/typings/src/config.d.ts +41 -11
- package/esm/typings/src/constants.d.ts +43 -2
- package/esm/typings/src/conversion/parsePipeline.d.ts +2 -2
- package/esm/typings/src/errors/0-BoilerplateError.d.ts +2 -2
- package/esm/typings/src/errors/CollectionError.d.ts +1 -1
- package/esm/typings/src/executables/$provideExecutablesForNode.d.ts +1 -1
- package/esm/typings/src/executables/apps/locateLibreoffice.d.ts +2 -1
- package/esm/typings/src/executables/apps/locatePandoc.d.ts +2 -1
- package/esm/typings/src/executables/locateApp.d.ts +2 -2
- package/esm/typings/src/executables/platforms/locateAppOnLinux.d.ts +2 -1
- package/esm/typings/src/executables/platforms/locateAppOnMacOs.d.ts +2 -1
- package/esm/typings/src/executables/platforms/locateAppOnWindows.d.ts +2 -1
- package/esm/typings/src/execution/AbstractTaskResult.d.ts +1 -1
- package/esm/typings/src/execution/CommonToolsOptions.d.ts +3 -3
- package/esm/typings/src/execution/ExecutionTask.d.ts +19 -1
- package/esm/typings/src/execution/LlmExecutionToolsConstructor.d.ts +2 -1
- package/esm/typings/src/execution/PipelineExecutorResult.d.ts +4 -2
- package/esm/typings/src/execution/PromptbookFetch.d.ts +1 -1
- package/esm/typings/src/execution/ScriptExecutionTools.d.ts +1 -1
- package/esm/typings/src/execution/createPipelineExecutor/$OngoingTaskResult.d.ts +12 -9
- package/esm/typings/src/execution/createPipelineExecutor/10-executePipeline.d.ts +13 -10
- package/esm/typings/src/execution/createPipelineExecutor/20-executeTask.d.ts +12 -9
- package/esm/typings/src/execution/createPipelineExecutor/30-executeFormatSubvalues.d.ts +15 -3
- package/esm/typings/src/execution/createPipelineExecutor/40-executeAttempts.d.ts +21 -15
- package/esm/typings/src/execution/createPipelineExecutor/computeCosineSimilarity.d.ts +13 -0
- package/esm/typings/src/execution/createPipelineExecutor/filterJustOutputParameters.d.ts +7 -6
- package/esm/typings/src/execution/createPipelineExecutor/getContextForTask.d.ts +5 -1
- package/esm/typings/src/execution/createPipelineExecutor/getExamplesForTask.d.ts +1 -1
- package/esm/typings/src/execution/createPipelineExecutor/getKnowledgeForTask.d.ts +12 -9
- package/esm/typings/src/execution/createPipelineExecutor/getReservedParametersForTask.d.ts +18 -5
- package/esm/typings/src/execution/createPipelineExecutor/knowledgePiecesToString.d.ts +9 -0
- package/esm/typings/src/execution/execution-report/ExecutionReportJson.d.ts +1 -1
- package/esm/typings/src/execution/execution-report/ExecutionReportString.d.ts +1 -1
- package/esm/typings/src/execution/translation/automatic-translate/automatic-translators/LindatAutomaticTranslator.d.ts +4 -4
- package/esm/typings/src/execution/utils/checkExpectations.d.ts +3 -3
- package/esm/typings/src/execution/utils/uncertainNumber.d.ts +3 -2
- package/esm/typings/src/execution/utils/usageToWorktime.d.ts +1 -1
- package/esm/typings/src/formats/_common/{FormatDefinition.d.ts → FormatParser.d.ts} +8 -6
- package/esm/typings/src/formats/_common/FormatSubvalueParser.d.ts +66 -0
- package/esm/typings/src/formats/csv/CsvFormatParser.d.ts +17 -0
- package/esm/typings/src/formats/csv/CsvSettings.d.ts +2 -2
- package/esm/typings/src/formats/csv/utils/csvParse.d.ts +12 -0
- package/esm/typings/src/formats/csv/utils/isValidCsvString.d.ts +1 -1
- package/esm/typings/src/formats/index.d.ts +2 -2
- package/esm/typings/src/formats/json/JsonFormatParser.d.ts +19 -0
- package/esm/typings/src/formats/json/utils/isValidJsonString.d.ts +1 -1
- package/esm/typings/src/formats/json/utils/jsonParse.d.ts +0 -3
- package/esm/typings/src/formats/text/{TextFormatDefinition.d.ts → TextFormatParser.d.ts} +7 -7
- package/esm/typings/src/formats/xml/XmlFormatParser.d.ts +19 -0
- package/esm/typings/src/formats/xml/utils/isValidXmlString.d.ts +1 -1
- package/esm/typings/src/formfactors/_boilerplate/BoilerplateFormfactorDefinition.d.ts +3 -2
- package/esm/typings/src/formfactors/_common/AbstractFormfactorDefinition.d.ts +16 -7
- package/esm/typings/src/formfactors/_common/FormfactorDefinition.d.ts +3 -1
- package/esm/typings/src/formfactors/_common/string_formfactor_name.d.ts +2 -1
- package/esm/typings/src/formfactors/chatbot/ChatbotFormfactorDefinition.d.ts +2 -2
- package/esm/typings/src/formfactors/completion/CompletionFormfactorDefinition.d.ts +29 -0
- package/esm/typings/src/formfactors/generator/GeneratorFormfactorDefinition.d.ts +2 -1
- package/esm/typings/src/formfactors/generic/GenericFormfactorDefinition.d.ts +2 -2
- package/esm/typings/src/formfactors/index.d.ts +33 -8
- package/esm/typings/src/formfactors/matcher/MatcherFormfactorDefinition.d.ts +4 -2
- package/esm/typings/src/formfactors/sheets/SheetsFormfactorDefinition.d.ts +3 -2
- package/esm/typings/src/formfactors/translator/TranslatorFormfactorDefinition.d.ts +3 -2
- package/esm/typings/src/high-level-abstractions/index.d.ts +2 -2
- package/esm/typings/src/llm-providers/_common/register/$llmToolsMetadataRegister.d.ts +3 -3
- package/esm/typings/src/llm-providers/_common/register/$llmToolsRegister.d.ts +3 -3
- package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsConfigurationFromEnv.d.ts +4 -4
- package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground.d.ts +4 -3
- package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsFromEnv.d.ts +18 -5
- package/esm/typings/src/llm-providers/_common/register/LlmToolsConfiguration.d.ts +11 -4
- package/esm/typings/src/llm-providers/_common/register/LlmToolsMetadata.d.ts +21 -42
- package/esm/typings/src/llm-providers/_common/register/LlmToolsOptions.d.ts +9 -2
- package/esm/typings/src/llm-providers/_common/register/createLlmToolsFromConfiguration.d.ts +13 -4
- package/esm/typings/src/llm-providers/_common/utils/cache/CacheItem.d.ts +10 -5
- package/esm/typings/src/llm-providers/_common/utils/cache/CacheLlmToolsOptions.d.ts +11 -3
- package/esm/typings/src/llm-providers/_common/utils/cache/cacheLlmTools.d.ts +3 -3
- package/esm/typings/src/llm-providers/_common/utils/count-total-usage/limitTotalUsage.d.ts +5 -5
- package/esm/typings/src/llm-providers/anthropic-claude/AnthropicClaudeExecutionTools.d.ts +6 -0
- package/esm/typings/src/llm-providers/anthropic-claude/anthropic-claude-models.d.ts +1 -1
- package/esm/typings/src/llm-providers/azure-openai/AzureOpenAiExecutionTools.d.ts +1 -1
- package/esm/typings/src/llm-providers/azure-openai/AzureOpenAiExecutionToolsOptions.d.ts +4 -4
- package/esm/typings/src/llm-providers/deepseek/deepseek-models.d.ts +1 -1
- package/esm/typings/src/llm-providers/google/google-models.d.ts +1 -1
- package/esm/typings/src/llm-providers/openai/OpenAiAssistantExecutionTools.d.ts +1 -1
- package/esm/typings/src/llm-providers/openai/openai-models.d.ts +1 -1
- package/esm/typings/src/llm-providers/openai/register-configuration.d.ts +2 -2
- package/esm/typings/src/llm-providers/openai/register-constructor.d.ts +2 -2
- package/esm/typings/src/migrations/migratePipeline.d.ts +9 -0
- package/esm/typings/src/other/templates/getBookTemplates.d.ts +2 -2
- package/esm/typings/src/pipeline/PipelineInterface/PipelineInterface.d.ts +3 -3
- package/esm/typings/src/pipeline/PipelineInterface/constants.d.ts +1 -1
- package/esm/typings/src/pipeline/PipelineInterface/getPipelineInterface.d.ts +1 -1
- package/esm/typings/src/pipeline/PipelineInterface/isPipelineImplementingInterface.d.ts +5 -4
- package/esm/typings/src/pipeline/PipelineInterface/isPipelineInterfacesEqual.d.ts +1 -1
- package/esm/typings/src/pipeline/PipelineJson/CommonTaskJson.d.ts +10 -7
- package/esm/typings/src/pipeline/PipelineJson/ParameterJson.d.ts +1 -1
- package/esm/typings/src/pipeline/PipelineJson/PipelineJson.d.ts +3 -2
- package/esm/typings/src/pipeline/PipelineString.d.ts +3 -1
- package/esm/typings/src/pipeline/book-notation.d.ts +2 -2
- package/esm/typings/src/postprocessing/utils/extractBlock.d.ts +1 -1
- package/esm/typings/src/postprocessing/utils/extractJsonBlock.d.ts +2 -2
- package/esm/typings/src/prepare/prepareTasks.d.ts +8 -5
- package/esm/typings/src/remote-server/openapi.d.ts +1 -1
- package/esm/typings/src/remote-server/socket-types/listModels/PromptbookServer_ListModels_Response.d.ts +1 -1
- package/esm/typings/src/remote-server/startRemoteServer.d.ts +1 -1
- package/esm/typings/src/remote-server/types/RemoteServerOptions.d.ts +7 -6
- package/esm/typings/src/scrapers/_boilerplate/BoilerplateScraper.d.ts +3 -3
- package/esm/typings/src/scrapers/_boilerplate/createBoilerplateScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/_boilerplate/register-metadata.d.ts +1 -1
- package/esm/typings/src/scrapers/_common/Converter.d.ts +3 -1
- package/esm/typings/src/scrapers/_common/Scraper.d.ts +4 -3
- package/esm/typings/src/scrapers/_common/ScraperIntermediateSource.d.ts +4 -2
- package/esm/typings/src/scrapers/_common/prepareKnowledgePieces.d.ts +2 -2
- package/esm/typings/src/scrapers/_common/register/$provideFilesystemForNode.d.ts +2 -1
- package/esm/typings/src/scrapers/_common/register/$provideScrapersForBrowser.d.ts +6 -3
- package/esm/typings/src/scrapers/_common/register/$provideScrapersForNode.d.ts +3 -5
- package/esm/typings/src/scrapers/_common/register/$scrapersMetadataRegister.d.ts +3 -3
- package/esm/typings/src/scrapers/_common/register/$scrapersRegister.d.ts +3 -2
- package/esm/typings/src/scrapers/_common/register/ScraperAndConverterMetadata.d.ts +8 -5
- package/esm/typings/src/scrapers/_common/register/ScraperConstructor.d.ts +2 -1
- package/esm/typings/src/scrapers/_common/utils/getScraperIntermediateSource.d.ts +6 -5
- package/esm/typings/src/scrapers/_common/utils/makeKnowledgeSourceHandler.d.ts +3 -1
- package/esm/typings/src/scrapers/document/createDocumentScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/document-legacy/createLegacyDocumentScraper.d.ts +2 -1
- package/esm/typings/src/scrapers/markdown/createMarkdownScraper.d.ts +4 -1
- package/esm/typings/src/scrapers/markitdown/MarkitdownScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/pdf/createPdfScraper.d.ts +2 -1
- package/esm/typings/src/scrapers/website/createWebsiteScraper.d.ts +3 -4
- package/esm/typings/src/scripting/javascript/JavascriptExecutionTools.d.ts +1 -1
- package/esm/typings/src/scripting/javascript/postprocessing-functions.d.ts +5 -1
- package/esm/typings/src/scripting/javascript/utils/preserve.d.ts +1 -1
- package/esm/typings/src/storage/_common/PromptbookStorage.d.ts +1 -1
- package/esm/typings/src/storage/file-cache-storage/FileCacheStorage.d.ts +12 -5
- package/esm/typings/src/storage/file-cache-storage/FileCacheStorageOptions.d.ts +4 -2
- package/esm/typings/src/storage/file-cache-storage/utils/nameToSubfolderPath.d.ts +2 -1
- package/esm/typings/src/storage/local-storage/getIndexedDbStorage.d.ts +11 -0
- package/esm/typings/src/storage/local-storage/utils/IndexedDbStorageOptions.d.ts +14 -0
- package/esm/typings/src/storage/local-storage/utils/makePromptbookStorageFromIndexedDb.d.ts +8 -0
- package/esm/typings/src/storage/local-storage/utils/makePromptbookStorageFromWebStorage.d.ts +2 -1
- package/esm/typings/src/types/IntermediateFilesStrategy.d.ts +2 -1
- package/esm/typings/src/types/ModelRequirements.d.ts +2 -2
- package/esm/typings/src/types/ModelVariant.d.ts +5 -5
- package/esm/typings/src/types/typeAliases.d.ts +22 -19
- package/esm/typings/src/utils/$Register.d.ts +8 -7
- package/esm/typings/src/utils/editable/edit-pipeline-string/addPipelineCommand.d.ts +2 -2
- package/esm/typings/src/utils/editable/edit-pipeline-string/deflatePipeline.d.ts +4 -1
- package/esm/typings/src/utils/editable/utils/isFlatPipeline.d.ts +2 -1
- package/esm/typings/src/utils/editable/utils/stringifyPipelineJson.d.ts +1 -1
- package/esm/typings/src/utils/environment/$getGlobalScope.d.ts +2 -1
- package/esm/typings/src/utils/expectation-counters/index.d.ts +1 -1
- package/esm/typings/src/utils/markdown/extractAllBlocksFromMarkdown.d.ts +2 -2
- package/esm/typings/src/utils/markdown/extractAllListItemsFromMarkdown.d.ts +1 -1
- package/esm/typings/src/utils/markdown/extractOneBlockFromMarkdown.d.ts +2 -2
- package/esm/typings/src/utils/normalization/nameToUriPart.d.ts +4 -4
- package/esm/typings/src/utils/normalization/nameToUriParts.d.ts +4 -4
- package/esm/typings/src/utils/normalization/normalize-to-kebab-case.d.ts +3 -3
- package/esm/typings/src/utils/normalization/normalizeTo_SCREAMING_CASE.d.ts +3 -3
- package/esm/typings/src/utils/normalization/normalizeTo_camelCase.d.ts +4 -4
- package/esm/typings/src/utils/normalization/normalizeTo_snake_case.d.ts +3 -3
- package/esm/typings/src/utils/normalization/removeDiacritics.d.ts +3 -3
- package/esm/typings/src/utils/normalization/searchKeywords.d.ts +4 -1
- package/esm/typings/src/utils/normalization/titleToName.d.ts +4 -4
- package/esm/typings/src/utils/organization/TODO_USE.d.ts +1 -1
- package/esm/typings/src/utils/organization/empty_object.d.ts +2 -2
- package/esm/typings/src/utils/organization/just.d.ts +1 -1
- package/esm/typings/src/utils/organization/just_empty_object.d.ts +4 -4
- package/esm/typings/src/utils/organization/keepUnused.d.ts +1 -1
- package/esm/typings/src/utils/parameters/mapAvailableToExpectedParameters.d.ts +7 -7
- package/esm/typings/src/utils/removeQuotes.d.ts +2 -2
- package/esm/typings/src/utils/serialization/clonePipeline.d.ts +4 -3
- package/esm/typings/src/utils/serialization/deepClone.d.ts +5 -1
- package/esm/typings/src/utils/trimCodeBlock.d.ts +1 -1
- package/esm/typings/src/utils/trimEndOfCodeBlock.d.ts +1 -1
- package/esm/typings/src/utils/unwrapResult.d.ts +2 -2
- package/esm/typings/src/utils/validators/javascriptName/isValidJavascriptName.d.ts +3 -3
- package/esm/typings/src/utils/validators/parameterName/validateParameterName.d.ts +5 -4
- package/esm/typings/src/utils/validators/semanticVersion/isValidPromptbookVersion.d.ts +1 -1
- package/esm/typings/src/utils/validators/semanticVersion/isValidSemanticVersion.d.ts +1 -1
- package/esm/typings/src/utils/validators/url/isHostnameOnPrivateNetwork.d.ts +1 -1
- package/esm/typings/src/utils/validators/url/isUrlOnPrivateNetwork.d.ts +1 -1
- package/esm/typings/src/utils/validators/url/isValidPipelineUrl.d.ts +1 -1
- package/esm/typings/src/utils/validators/url/isValidUrl.d.ts +1 -1
- package/esm/typings/src/version.d.ts +2 -1
- package/esm/typings/src/wizzard/wizzard.d.ts +1 -1
- package/package.json +14 -2
- package/umd/index.umd.js +804 -534
- package/umd/index.umd.js.map +1 -1
- package/esm/typings/src/formats/_common/FormatSubvalueDefinition.d.ts +0 -31
- package/esm/typings/src/formats/csv/CsvFormatDefinition.d.ts +0 -17
- package/esm/typings/src/formats/json/JsonFormatDefinition.d.ts +0 -19
- package/esm/typings/src/formats/xml/XmlFormatDefinition.d.ts +0 -19
- /package/esm/typings/src/llm-providers/{multiple → _multiple}/MultipleLlmExecutionTools.d.ts +0 -0
- /package/esm/typings/src/llm-providers/{multiple → _multiple}/joinLlmExecutionTools.d.ts +0 -0
- /package/esm/typings/src/llm-providers/{multiple → _multiple}/playground/playground.d.ts +0 -0
package/umd/index.umd.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
(function (global, factory) {
|
|
2
|
-
typeof exports === 'object' && typeof module !== 'undefined' ? factory(exports, require('colors'), require('fs/promises'), require('path'), require('spacetrim'), require('jszip'), require('prettier'), require('prettier/parser-html'), require('
|
|
3
|
-
typeof define === 'function' && define.amd ? define(['exports', 'colors', 'fs/promises', 'path', 'spacetrim', 'jszip', 'prettier', 'prettier/parser-html', '
|
|
4
|
-
(global = typeof globalThis !== 'undefined' ? globalThis : global || self, factory(global["promptbook-node"] = {}, global.colors, global.promises, global.path, global.spaceTrim, global.JSZip, global.prettier, global.parserHtml, global.
|
|
5
|
-
})(this, (function (exports, colors, promises, path, spaceTrim, JSZip, prettier, parserHtml,
|
|
2
|
+
typeof exports === 'object' && typeof module !== 'undefined' ? factory(exports, require('colors'), require('fs/promises'), require('path'), require('spacetrim'), require('jszip'), require('prettier'), require('prettier/parser-html'), require('crypto'), require('rxjs'), require('waitasecond'), require('papaparse'), require('crypto-js/enc-hex'), require('crypto-js/sha256'), require('crypto-js'), require('mime-types'), require('child_process'), require('dotenv')) :
|
|
3
|
+
typeof define === 'function' && define.amd ? define(['exports', 'colors', 'fs/promises', 'path', 'spacetrim', 'jszip', 'prettier', 'prettier/parser-html', 'crypto', 'rxjs', 'waitasecond', 'papaparse', 'crypto-js/enc-hex', 'crypto-js/sha256', 'crypto-js', 'mime-types', 'child_process', 'dotenv'], factory) :
|
|
4
|
+
(global = typeof globalThis !== 'undefined' ? globalThis : global || self, factory(global["promptbook-node"] = {}, global.colors, global.promises, global.path, global.spaceTrim, global.JSZip, global.prettier, global.parserHtml, global.crypto, global.rxjs, global.waitasecond, global.papaparse, global.hexEncoder, global.sha256, global.cryptoJs, global.mimeTypes, global.child_process, global.dotenv));
|
|
5
|
+
})(this, (function (exports, colors, promises, path, spaceTrim, JSZip, prettier, parserHtml, crypto, rxjs, waitasecond, papaparse, hexEncoder, sha256, cryptoJs, mimeTypes, child_process, dotenv) { 'use strict';
|
|
6
6
|
|
|
7
7
|
function _interopDefaultLegacy (e) { return e && typeof e === 'object' && 'default' in e ? e : { 'default': e }; }
|
|
8
8
|
|
|
@@ -46,7 +46,7 @@
|
|
|
46
46
|
* @generated
|
|
47
47
|
* @see https://github.com/webgptorg/promptbook
|
|
48
48
|
*/
|
|
49
|
-
const PROMPTBOOK_ENGINE_VERSION = '0.92.0
|
|
49
|
+
const PROMPTBOOK_ENGINE_VERSION = '0.92.0';
|
|
50
50
|
/**
|
|
51
51
|
* TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
|
|
52
52
|
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
@@ -56,7 +56,7 @@
|
|
|
56
56
|
* Returns the same value that is passed as argument.
|
|
57
57
|
* No side effects.
|
|
58
58
|
*
|
|
59
|
-
* Note: It can be
|
|
59
|
+
* Note: It can be useful for:
|
|
60
60
|
*
|
|
61
61
|
* 1) Leveling indentation
|
|
62
62
|
* 2) Putting always-true or always-false conditions without getting eslint errors
|
|
@@ -117,6 +117,21 @@
|
|
|
117
117
|
* @public exported from `@promptbook/core`
|
|
118
118
|
*/
|
|
119
119
|
const DEFAULT_MAX_FILE_SIZE = 100 * 1024 * 1024; // 100MB
|
|
120
|
+
/**
|
|
121
|
+
* Threshold value that determines when a dataset is considered "big"
|
|
122
|
+
* and may require special handling or optimizations
|
|
123
|
+
*
|
|
124
|
+
* For example, when error occurs in one item of the big dataset, it will not fail the whole pipeline
|
|
125
|
+
*
|
|
126
|
+
* @public exported from `@promptbook/core`
|
|
127
|
+
*/
|
|
128
|
+
const BIG_DATASET_TRESHOLD = 50;
|
|
129
|
+
/**
|
|
130
|
+
* Placeholder text used to represent a placeholder value of failed operation
|
|
131
|
+
*
|
|
132
|
+
* @public exported from `@promptbook/core`
|
|
133
|
+
*/
|
|
134
|
+
const FAILED_VALUE_PLACEHOLDER = '!?';
|
|
120
135
|
// <- TODO: [🧠] Better system for generator warnings - not always "code" and "by `@promptbook/cli`"
|
|
121
136
|
/**
|
|
122
137
|
* The maximum number of iterations for a loops
|
|
@@ -211,7 +226,7 @@
|
|
|
211
226
|
const DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME = `index`;
|
|
212
227
|
// <- TODO: [🧜♂️]
|
|
213
228
|
/**
|
|
214
|
-
*
|
|
229
|
+
* Default settings for parsing and generating CSV files in Promptbook.
|
|
215
230
|
*
|
|
216
231
|
* @public exported from `@promptbook/core`
|
|
217
232
|
*/
|
|
@@ -222,19 +237,19 @@
|
|
|
222
237
|
skipEmptyLines: true,
|
|
223
238
|
});
|
|
224
239
|
/**
|
|
225
|
-
*
|
|
240
|
+
* Controls whether verbose logging is enabled by default throughout the application.
|
|
226
241
|
*
|
|
227
242
|
* @public exported from `@promptbook/core`
|
|
228
243
|
*/
|
|
229
244
|
let DEFAULT_IS_VERBOSE = false;
|
|
230
245
|
/**
|
|
231
|
-
*
|
|
246
|
+
* Controls whether auto-installation of dependencies is enabled by default.
|
|
232
247
|
*
|
|
233
248
|
* @public exported from `@promptbook/core`
|
|
234
249
|
*/
|
|
235
250
|
const DEFAULT_IS_AUTO_INSTALLED = false;
|
|
236
251
|
/**
|
|
237
|
-
*
|
|
252
|
+
* Indicates whether pipeline logic validation is enabled. When true, the pipeline logic is checked for consistency.
|
|
238
253
|
*
|
|
239
254
|
* @private within the repository
|
|
240
255
|
*/
|
|
@@ -302,7 +317,7 @@
|
|
|
302
317
|
${block(message)}
|
|
303
318
|
|
|
304
319
|
Note: This error should not happen.
|
|
305
|
-
It's
|
|
320
|
+
It's probably a bug in the pipeline collection
|
|
306
321
|
|
|
307
322
|
Please report issue:
|
|
308
323
|
${block(getErrorReportUrl(new Error(message)).href)}
|
|
@@ -344,15 +359,12 @@
|
|
|
344
359
|
}
|
|
345
360
|
throw new Error(spaceTrim__default["default"]((block) => `
|
|
346
361
|
${block(error.message)}
|
|
347
|
-
|
|
362
|
+
|
|
348
363
|
The JSON text:
|
|
349
364
|
${block(value)}
|
|
350
365
|
`));
|
|
351
366
|
}
|
|
352
367
|
}
|
|
353
|
-
/**
|
|
354
|
-
* TODO: !!!! Use in Promptbook.studio
|
|
355
|
-
*/
|
|
356
368
|
|
|
357
369
|
/**
|
|
358
370
|
* Orders JSON object by keys
|
|
@@ -583,8 +595,12 @@
|
|
|
583
595
|
*/
|
|
584
596
|
|
|
585
597
|
/**
|
|
586
|
-
*
|
|
598
|
+
* Creates a deep clone of the given object
|
|
599
|
+
*
|
|
600
|
+
* Note: This method only works for objects that are fully serializable to JSON and do not contain functions, Dates, or special types.
|
|
587
601
|
*
|
|
602
|
+
* @param objectValue The object to clone.
|
|
603
|
+
* @returns A deep, writable clone of the input object.
|
|
588
604
|
* @public exported from `@promptbook/utils`
|
|
589
605
|
*/
|
|
590
606
|
function deepClone(objectValue) {
|
|
@@ -666,13 +682,13 @@
|
|
|
666
682
|
*/
|
|
667
683
|
const REPLACING_NONCE = 'ptbkauk42kV2dzao34faw7FudQUHYPtW';
|
|
668
684
|
/**
|
|
669
|
-
*
|
|
685
|
+
* Placeholder value indicating a parameter is missing its value.
|
|
670
686
|
*
|
|
671
687
|
* @private within the repository
|
|
672
688
|
*/
|
|
673
689
|
const RESERVED_PARAMETER_MISSING_VALUE = 'MISSING-' + REPLACING_NONCE;
|
|
674
690
|
/**
|
|
675
|
-
*
|
|
691
|
+
* Placeholder value indicating a parameter is restricted and cannot be used directly.
|
|
676
692
|
*
|
|
677
693
|
* @private within the repository
|
|
678
694
|
*/
|
|
@@ -733,7 +749,7 @@
|
|
|
733
749
|
/**
|
|
734
750
|
* Tests if given string is valid semantic version
|
|
735
751
|
*
|
|
736
|
-
* Note: There are two
|
|
752
|
+
* Note: There are two similar functions:
|
|
737
753
|
* - `isValidSemanticVersion` which tests any semantic version
|
|
738
754
|
* - `isValidPromptbookVersion` *(this one)* which tests just Promptbook versions
|
|
739
755
|
*
|
|
@@ -755,7 +771,7 @@
|
|
|
755
771
|
*
|
|
756
772
|
* @see https://www.npmjs.com/package/promptbook?activeTab=versions
|
|
757
773
|
* Note: When you are using for example promptbook 2.0.0 and there already is promptbook 3.0.0 it don`t know about it.
|
|
758
|
-
* Note: There are two
|
|
774
|
+
* Note: There are two similar functions:
|
|
759
775
|
* - `isValidSemanticVersion` which tests any semantic version
|
|
760
776
|
* - `isValidPromptbookVersion` *(this one)* which tests just Promptbook versions
|
|
761
777
|
*
|
|
@@ -776,7 +792,7 @@
|
|
|
776
792
|
* Tests if given string is valid URL.
|
|
777
793
|
*
|
|
778
794
|
* Note: Dataurl are considered perfectly valid.
|
|
779
|
-
* Note: There are two
|
|
795
|
+
* Note: There are two similar functions:
|
|
780
796
|
* - `isValidUrl` which tests any URL
|
|
781
797
|
* - `isValidPipelineUrl` *(this one)* which tests just promptbook URL
|
|
782
798
|
*
|
|
@@ -804,7 +820,7 @@
|
|
|
804
820
|
/**
|
|
805
821
|
* Tests if given string is valid pipeline URL URL.
|
|
806
822
|
*
|
|
807
|
-
* Note: There are two
|
|
823
|
+
* Note: There are two similar functions:
|
|
808
824
|
* - `isValidUrl` which tests any URL
|
|
809
825
|
* - `isValidPipelineUrl` *(this one)* which tests just pipeline URL
|
|
810
826
|
*
|
|
@@ -901,7 +917,7 @@
|
|
|
901
917
|
${block(pipelineIdentification)}
|
|
902
918
|
`));
|
|
903
919
|
}
|
|
904
|
-
// TODO: [🧠] Maybe do here some
|
|
920
|
+
// TODO: [🧠] Maybe do here some proper JSON-schema / ZOD checking
|
|
905
921
|
if (!Array.isArray(pipeline.parameters)) {
|
|
906
922
|
// TODO: [🧠] what is the correct error tp throw - maybe PromptbookSchemaError
|
|
907
923
|
throw new ParseError(spaceTrim.spaceTrim((block) => `
|
|
@@ -912,7 +928,7 @@
|
|
|
912
928
|
${block(pipelineIdentification)}
|
|
913
929
|
`));
|
|
914
930
|
}
|
|
915
|
-
// TODO: [🧠] Maybe do here some
|
|
931
|
+
// TODO: [🧠] Maybe do here some proper JSON-schema / ZOD checking
|
|
916
932
|
if (!Array.isArray(pipeline.tasks)) {
|
|
917
933
|
// TODO: [🧠] what is the correct error tp throw - maybe PromptbookSchemaError
|
|
918
934
|
throw new ParseError(spaceTrim.spaceTrim((block) => `
|
|
@@ -1223,7 +1239,7 @@
|
|
|
1223
1239
|
* Function isValidJsonString will tell you if the string is valid JSON or not
|
|
1224
1240
|
*
|
|
1225
1241
|
* @param value The string to check
|
|
1226
|
-
* @returns
|
|
1242
|
+
* @returns `true` if the string is a valid JSON string, false otherwise
|
|
1227
1243
|
*
|
|
1228
1244
|
* @public exported from `@promptbook/utils`
|
|
1229
1245
|
*/
|
|
@@ -1335,7 +1351,7 @@
|
|
|
1335
1351
|
if (bookVersion !== `undefined`) {
|
|
1336
1352
|
commands.push(`BOOK VERSION ${bookVersion}`);
|
|
1337
1353
|
}
|
|
1338
|
-
// TODO: [main] !!5 This increases size of the bundle and is
|
|
1354
|
+
// TODO: [main] !!5 This increases size of the bundle and is probably not necessary
|
|
1339
1355
|
pipelineString = prettifyMarkdown(pipelineString);
|
|
1340
1356
|
for (const parameter of parameters.filter(({ isInput }) => isInput)) {
|
|
1341
1357
|
commands.push(`INPUT PARAMETER ${taskParameterJsonToString(parameter)}`);
|
|
@@ -1545,7 +1561,7 @@
|
|
|
1545
1561
|
/**
|
|
1546
1562
|
* Constructs a pipeline collection from pipelines
|
|
1547
1563
|
*
|
|
1548
|
-
* @param pipelines
|
|
1564
|
+
* @param pipelines Array of pipeline JSON objects to include in the collection
|
|
1549
1565
|
*
|
|
1550
1566
|
* Note: During the construction logic of all pipelines are validated
|
|
1551
1567
|
* Note: It is not recommended to use this constructor directly, use `createCollectionFromJson` *(or other variant)* instead
|
|
@@ -1659,7 +1675,7 @@
|
|
|
1659
1675
|
super(spaceTrim.spaceTrim((block) => `
|
|
1660
1676
|
${block(message)}
|
|
1661
1677
|
|
|
1662
|
-
Note: You have
|
|
1678
|
+
Note: You have probably forgot to provide some tools for pipeline execution or preparation
|
|
1663
1679
|
|
|
1664
1680
|
`));
|
|
1665
1681
|
this.name = 'MissingToolsError';
|
|
@@ -1675,15 +1691,18 @@
|
|
|
1675
1691
|
* @public exported from `@promptbook/core`
|
|
1676
1692
|
*/
|
|
1677
1693
|
function isPipelinePrepared(pipeline) {
|
|
1678
|
-
// Note: Ignoring `pipeline.preparations`
|
|
1679
|
-
// Note: Ignoring `pipeline.knowledgePieces`
|
|
1694
|
+
// Note: Ignoring `pipeline.preparations`
|
|
1695
|
+
// Note: Ignoring `pipeline.knowledgePieces`
|
|
1680
1696
|
if (pipeline.title === undefined || pipeline.title === '' || pipeline.title === DEFAULT_BOOK_TITLE) {
|
|
1697
|
+
// console.log('Pipeline is not prepared because title is undefined or empty', pipeline);
|
|
1681
1698
|
return false;
|
|
1682
1699
|
}
|
|
1683
1700
|
if (!pipeline.personas.every((persona) => persona.modelsRequirements !== undefined)) {
|
|
1701
|
+
// console.log('Pipeline is not prepared because personas are not prepared', pipeline.personas);
|
|
1684
1702
|
return false;
|
|
1685
1703
|
}
|
|
1686
1704
|
if (!pipeline.knowledgeSources.every((knowledgeSource) => knowledgeSource.preparationIds !== undefined)) {
|
|
1705
|
+
//console.log('Pipeline is not prepared because knowledge sources are not prepared', pipeline.knowledgeSources);
|
|
1687
1706
|
return false;
|
|
1688
1707
|
}
|
|
1689
1708
|
/*
|
|
@@ -1704,70 +1723,6 @@
|
|
|
1704
1723
|
* - [♨] Are tasks prepared
|
|
1705
1724
|
*/
|
|
1706
1725
|
|
|
1707
|
-
/**
|
|
1708
|
-
* Generates random token
|
|
1709
|
-
*
|
|
1710
|
-
* Note: This function is cryptographically secure (it uses crypto.randomBytes internally)
|
|
1711
|
-
*
|
|
1712
|
-
* @private internal helper function
|
|
1713
|
-
* @returns secure random token
|
|
1714
|
-
*/
|
|
1715
|
-
function $randomToken(randomness) {
|
|
1716
|
-
return crypto.randomBytes(randomness).toString('hex');
|
|
1717
|
-
}
|
|
1718
|
-
/**
|
|
1719
|
-
* TODO: Maybe use nanoid instead https://github.com/ai/nanoid
|
|
1720
|
-
*/
|
|
1721
|
-
|
|
1722
|
-
/**
|
|
1723
|
-
* Recursively converts JSON strings to JSON objects
|
|
1724
|
-
|
|
1725
|
-
* @public exported from `@promptbook/utils`
|
|
1726
|
-
*/
|
|
1727
|
-
function jsonStringsToJsons(object) {
|
|
1728
|
-
if (object === null) {
|
|
1729
|
-
return object;
|
|
1730
|
-
}
|
|
1731
|
-
if (Array.isArray(object)) {
|
|
1732
|
-
return object.map(jsonStringsToJsons);
|
|
1733
|
-
}
|
|
1734
|
-
if (typeof object !== 'object') {
|
|
1735
|
-
return object;
|
|
1736
|
-
}
|
|
1737
|
-
const newObject = { ...object };
|
|
1738
|
-
for (const [key, value] of Object.entries(object)) {
|
|
1739
|
-
if (typeof value === 'string' && isValidJsonString(value)) {
|
|
1740
|
-
newObject[key] = jsonParse(value);
|
|
1741
|
-
}
|
|
1742
|
-
else {
|
|
1743
|
-
newObject[key] = jsonStringsToJsons(value);
|
|
1744
|
-
}
|
|
1745
|
-
}
|
|
1746
|
-
return newObject;
|
|
1747
|
-
}
|
|
1748
|
-
/**
|
|
1749
|
-
* TODO: Type the return type correctly
|
|
1750
|
-
*/
|
|
1751
|
-
|
|
1752
|
-
/**
|
|
1753
|
-
* This error indicates errors during the execution of the pipeline
|
|
1754
|
-
*
|
|
1755
|
-
* @public exported from `@promptbook/core`
|
|
1756
|
-
*/
|
|
1757
|
-
class PipelineExecutionError extends Error {
|
|
1758
|
-
constructor(message) {
|
|
1759
|
-
// Added id parameter
|
|
1760
|
-
super(message);
|
|
1761
|
-
this.name = 'PipelineExecutionError';
|
|
1762
|
-
// TODO: [🐙] DRY - Maybe $randomId
|
|
1763
|
-
this.id = `error-${$randomToken(8 /* <- TODO: To global config + Use Base58 to avoid simmilar char conflicts */)}`;
|
|
1764
|
-
Object.setPrototypeOf(this, PipelineExecutionError.prototype);
|
|
1765
|
-
}
|
|
1766
|
-
}
|
|
1767
|
-
/**
|
|
1768
|
-
* TODO: [🧠][🌂] Add id to all errors
|
|
1769
|
-
*/
|
|
1770
|
-
|
|
1771
1726
|
/**
|
|
1772
1727
|
* This error indicates problems parsing the format value
|
|
1773
1728
|
*
|
|
@@ -1812,7 +1767,7 @@
|
|
|
1812
1767
|
}
|
|
1813
1768
|
|
|
1814
1769
|
/**
|
|
1815
|
-
* This error indicates that the pipeline collection cannot be
|
|
1770
|
+
* This error indicates that the pipeline collection cannot be properly loaded
|
|
1816
1771
|
*
|
|
1817
1772
|
* @public exported from `@promptbook/core`
|
|
1818
1773
|
*/
|
|
@@ -1902,6 +1857,40 @@
|
|
|
1902
1857
|
}
|
|
1903
1858
|
}
|
|
1904
1859
|
|
|
1860
|
+
/**
|
|
1861
|
+
* Generates random token
|
|
1862
|
+
*
|
|
1863
|
+
* Note: This function is cryptographically secure (it uses crypto.randomBytes internally)
|
|
1864
|
+
*
|
|
1865
|
+
* @private internal helper function
|
|
1866
|
+
* @returns secure random token
|
|
1867
|
+
*/
|
|
1868
|
+
function $randomToken(randomness) {
|
|
1869
|
+
return crypto.randomBytes(randomness).toString('hex');
|
|
1870
|
+
}
|
|
1871
|
+
/**
|
|
1872
|
+
* TODO: Maybe use nanoid instead https://github.com/ai/nanoid
|
|
1873
|
+
*/
|
|
1874
|
+
|
|
1875
|
+
/**
|
|
1876
|
+
* This error indicates errors during the execution of the pipeline
|
|
1877
|
+
*
|
|
1878
|
+
* @public exported from `@promptbook/core`
|
|
1879
|
+
*/
|
|
1880
|
+
class PipelineExecutionError extends Error {
|
|
1881
|
+
constructor(message) {
|
|
1882
|
+
// Added id parameter
|
|
1883
|
+
super(message);
|
|
1884
|
+
this.name = 'PipelineExecutionError';
|
|
1885
|
+
// TODO: [🐙] DRY - Maybe $randomId
|
|
1886
|
+
this.id = `error-${$randomToken(8 /* <- TODO: To global config + Use Base58 to avoid similar char conflicts */)}`;
|
|
1887
|
+
Object.setPrototypeOf(this, PipelineExecutionError.prototype);
|
|
1888
|
+
}
|
|
1889
|
+
}
|
|
1890
|
+
/**
|
|
1891
|
+
* TODO: [🧠][🌂] Add id to all errors
|
|
1892
|
+
*/
|
|
1893
|
+
|
|
1905
1894
|
/**
|
|
1906
1895
|
* Error thrown when a fetch request fails
|
|
1907
1896
|
*
|
|
@@ -1977,6 +1966,65 @@
|
|
|
1977
1966
|
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
1978
1967
|
*/
|
|
1979
1968
|
|
|
1969
|
+
/**
|
|
1970
|
+
* Serializes an error into a [🚉] JSON-serializable object
|
|
1971
|
+
*
|
|
1972
|
+
* @public exported from `@promptbook/utils`
|
|
1973
|
+
*/
|
|
1974
|
+
function serializeError(error) {
|
|
1975
|
+
const { name, message, stack } = error;
|
|
1976
|
+
const { id } = error;
|
|
1977
|
+
if (!Object.keys(ALL_ERRORS).includes(name)) {
|
|
1978
|
+
console.error(spaceTrim__default["default"]((block) => `
|
|
1979
|
+
|
|
1980
|
+
Cannot serialize error with name "${name}"
|
|
1981
|
+
|
|
1982
|
+
Authors of Promptbook probably forgot to add this error into the list of errors:
|
|
1983
|
+
https://github.com/webgptorg/promptbook/blob/main/src/errors/0-index.ts
|
|
1984
|
+
|
|
1985
|
+
|
|
1986
|
+
${block(stack || message)}
|
|
1987
|
+
|
|
1988
|
+
`));
|
|
1989
|
+
}
|
|
1990
|
+
return {
|
|
1991
|
+
name: name,
|
|
1992
|
+
message,
|
|
1993
|
+
stack,
|
|
1994
|
+
id, // Include id in the serialized object
|
|
1995
|
+
};
|
|
1996
|
+
}
|
|
1997
|
+
|
|
1998
|
+
/**
|
|
1999
|
+
* Recursively converts JSON strings to JSON objects
|
|
2000
|
+
|
|
2001
|
+
* @public exported from `@promptbook/utils`
|
|
2002
|
+
*/
|
|
2003
|
+
function jsonStringsToJsons(object) {
|
|
2004
|
+
if (object === null) {
|
|
2005
|
+
return object;
|
|
2006
|
+
}
|
|
2007
|
+
if (Array.isArray(object)) {
|
|
2008
|
+
return object.map(jsonStringsToJsons);
|
|
2009
|
+
}
|
|
2010
|
+
if (typeof object !== 'object') {
|
|
2011
|
+
return object;
|
|
2012
|
+
}
|
|
2013
|
+
const newObject = { ...object };
|
|
2014
|
+
for (const [key, value] of Object.entries(object)) {
|
|
2015
|
+
if (typeof value === 'string' && isValidJsonString(value)) {
|
|
2016
|
+
newObject[key] = jsonParse(value);
|
|
2017
|
+
}
|
|
2018
|
+
else {
|
|
2019
|
+
newObject[key] = jsonStringsToJsons(value);
|
|
2020
|
+
}
|
|
2021
|
+
}
|
|
2022
|
+
return newObject;
|
|
2023
|
+
}
|
|
2024
|
+
/**
|
|
2025
|
+
* TODO: Type the return type correctly
|
|
2026
|
+
*/
|
|
2027
|
+
|
|
1980
2028
|
/**
|
|
1981
2029
|
* Deserializes the error object
|
|
1982
2030
|
*
|
|
@@ -2051,8 +2099,9 @@
|
|
|
2051
2099
|
*/
|
|
2052
2100
|
function createTask(options) {
|
|
2053
2101
|
const { taskType, taskProcessCallback } = options;
|
|
2102
|
+
let { title } = options;
|
|
2054
2103
|
// TODO: [🐙] DRY
|
|
2055
|
-
const taskId = `${taskType.toLowerCase().substring(0, 4)}-${$randomToken(8 /* <- TODO: To global config + Use Base58 to avoid
|
|
2104
|
+
const taskId = `${taskType.toLowerCase().substring(0, 4)}-${$randomToken(8 /* <- TODO: To global config + Use Base58 to avoid similar char conflicts */)}`;
|
|
2056
2105
|
let status = 'RUNNING';
|
|
2057
2106
|
const createdAt = new Date();
|
|
2058
2107
|
let updatedAt = createdAt;
|
|
@@ -2062,6 +2111,10 @@
|
|
|
2062
2111
|
const partialResultSubject = new rxjs.Subject();
|
|
2063
2112
|
// <- Note: Not using `BehaviorSubject` because on error we can't access the last value
|
|
2064
2113
|
const finalResultPromise = /* not await */ taskProcessCallback((newOngoingResult) => {
|
|
2114
|
+
if (newOngoingResult.title) {
|
|
2115
|
+
title = newOngoingResult.title;
|
|
2116
|
+
}
|
|
2117
|
+
updatedAt = new Date();
|
|
2065
2118
|
Object.assign(currentValue, newOngoingResult);
|
|
2066
2119
|
// <- TODO: assign deep
|
|
2067
2120
|
partialResultSubject.next(newOngoingResult);
|
|
@@ -2107,17 +2160,24 @@
|
|
|
2107
2160
|
return {
|
|
2108
2161
|
taskType,
|
|
2109
2162
|
taskId,
|
|
2163
|
+
get promptbookVersion() {
|
|
2164
|
+
return PROMPTBOOK_ENGINE_VERSION;
|
|
2165
|
+
},
|
|
2166
|
+
get title() {
|
|
2167
|
+
return title;
|
|
2168
|
+
// <- Note: [1] These must be getters to allow changing the value in the future
|
|
2169
|
+
},
|
|
2110
2170
|
get status() {
|
|
2111
2171
|
return status;
|
|
2112
|
-
// <- Note: [1]
|
|
2172
|
+
// <- Note: [1] --||--
|
|
2113
2173
|
},
|
|
2114
2174
|
get createdAt() {
|
|
2115
2175
|
return createdAt;
|
|
2116
|
-
// <- Note: [1]
|
|
2176
|
+
// <- Note: [1] --||--
|
|
2117
2177
|
},
|
|
2118
2178
|
get updatedAt() {
|
|
2119
2179
|
return updatedAt;
|
|
2120
|
-
// <- Note: [1]
|
|
2180
|
+
// <- Note: [1] --||--
|
|
2121
2181
|
},
|
|
2122
2182
|
asPromise,
|
|
2123
2183
|
asObservable() {
|
|
@@ -2125,15 +2185,15 @@
|
|
|
2125
2185
|
},
|
|
2126
2186
|
get errors() {
|
|
2127
2187
|
return errors;
|
|
2128
|
-
// <- Note: [1]
|
|
2188
|
+
// <- Note: [1] --||--
|
|
2129
2189
|
},
|
|
2130
2190
|
get warnings() {
|
|
2131
2191
|
return warnings;
|
|
2132
|
-
// <- Note: [1]
|
|
2192
|
+
// <- Note: [1] --||--
|
|
2133
2193
|
},
|
|
2134
2194
|
get currentValue() {
|
|
2135
2195
|
return currentValue;
|
|
2136
|
-
// <- Note: [1]
|
|
2196
|
+
// <- Note: [1] --||--
|
|
2137
2197
|
},
|
|
2138
2198
|
};
|
|
2139
2199
|
}
|
|
@@ -2143,33 +2203,72 @@
|
|
|
2143
2203
|
*/
|
|
2144
2204
|
|
|
2145
2205
|
/**
|
|
2146
|
-
*
|
|
2206
|
+
* Represents the uncertain value
|
|
2147
2207
|
*
|
|
2148
|
-
* @public exported from `@promptbook/
|
|
2208
|
+
* @public exported from `@promptbook/core`
|
|
2209
|
+
*/
|
|
2210
|
+
const ZERO_VALUE = $deepFreeze({ value: 0 });
|
|
2211
|
+
/**
|
|
2212
|
+
* Represents the uncertain value
|
|
2213
|
+
*
|
|
2214
|
+
* @public exported from `@promptbook/core`
|
|
2215
|
+
*/
|
|
2216
|
+
const UNCERTAIN_ZERO_VALUE = $deepFreeze({ value: 0, isUncertain: true });
|
|
2217
|
+
/**
|
|
2218
|
+
* Represents the usage with no resources consumed
|
|
2219
|
+
*
|
|
2220
|
+
* @public exported from `@promptbook/core`
|
|
2221
|
+
*/
|
|
2222
|
+
const ZERO_USAGE = $deepFreeze({
|
|
2223
|
+
price: ZERO_VALUE,
|
|
2224
|
+
input: {
|
|
2225
|
+
tokensCount: ZERO_VALUE,
|
|
2226
|
+
charactersCount: ZERO_VALUE,
|
|
2227
|
+
wordsCount: ZERO_VALUE,
|
|
2228
|
+
sentencesCount: ZERO_VALUE,
|
|
2229
|
+
linesCount: ZERO_VALUE,
|
|
2230
|
+
paragraphsCount: ZERO_VALUE,
|
|
2231
|
+
pagesCount: ZERO_VALUE,
|
|
2232
|
+
},
|
|
2233
|
+
output: {
|
|
2234
|
+
tokensCount: ZERO_VALUE,
|
|
2235
|
+
charactersCount: ZERO_VALUE,
|
|
2236
|
+
wordsCount: ZERO_VALUE,
|
|
2237
|
+
sentencesCount: ZERO_VALUE,
|
|
2238
|
+
linesCount: ZERO_VALUE,
|
|
2239
|
+
paragraphsCount: ZERO_VALUE,
|
|
2240
|
+
pagesCount: ZERO_VALUE,
|
|
2241
|
+
},
|
|
2242
|
+
});
|
|
2243
|
+
/**
|
|
2244
|
+
* Represents the usage with unknown resources consumed
|
|
2245
|
+
*
|
|
2246
|
+
* @public exported from `@promptbook/core`
|
|
2247
|
+
*/
|
|
2248
|
+
const UNCERTAIN_USAGE = $deepFreeze({
|
|
2249
|
+
price: UNCERTAIN_ZERO_VALUE,
|
|
2250
|
+
input: {
|
|
2251
|
+
tokensCount: UNCERTAIN_ZERO_VALUE,
|
|
2252
|
+
charactersCount: UNCERTAIN_ZERO_VALUE,
|
|
2253
|
+
wordsCount: UNCERTAIN_ZERO_VALUE,
|
|
2254
|
+
sentencesCount: UNCERTAIN_ZERO_VALUE,
|
|
2255
|
+
linesCount: UNCERTAIN_ZERO_VALUE,
|
|
2256
|
+
paragraphsCount: UNCERTAIN_ZERO_VALUE,
|
|
2257
|
+
pagesCount: UNCERTAIN_ZERO_VALUE,
|
|
2258
|
+
},
|
|
2259
|
+
output: {
|
|
2260
|
+
tokensCount: UNCERTAIN_ZERO_VALUE,
|
|
2261
|
+
charactersCount: UNCERTAIN_ZERO_VALUE,
|
|
2262
|
+
wordsCount: UNCERTAIN_ZERO_VALUE,
|
|
2263
|
+
sentencesCount: UNCERTAIN_ZERO_VALUE,
|
|
2264
|
+
linesCount: UNCERTAIN_ZERO_VALUE,
|
|
2265
|
+
paragraphsCount: UNCERTAIN_ZERO_VALUE,
|
|
2266
|
+
pagesCount: UNCERTAIN_ZERO_VALUE,
|
|
2267
|
+
},
|
|
2268
|
+
});
|
|
2269
|
+
/**
|
|
2270
|
+
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
2149
2271
|
*/
|
|
2150
|
-
function serializeError(error) {
|
|
2151
|
-
const { name, message, stack } = error;
|
|
2152
|
-
const { id } = error;
|
|
2153
|
-
if (!Object.keys(ALL_ERRORS).includes(name)) {
|
|
2154
|
-
console.error(spaceTrim__default["default"]((block) => `
|
|
2155
|
-
|
|
2156
|
-
Cannot serialize error with name "${name}"
|
|
2157
|
-
|
|
2158
|
-
Authors of Promptbook probably forgot to add this error into the list of errors:
|
|
2159
|
-
https://github.com/webgptorg/promptbook/blob/main/src/errors/0-index.ts
|
|
2160
|
-
|
|
2161
|
-
|
|
2162
|
-
${block(stack || message)}
|
|
2163
|
-
|
|
2164
|
-
`));
|
|
2165
|
-
}
|
|
2166
|
-
return {
|
|
2167
|
-
name: name,
|
|
2168
|
-
message,
|
|
2169
|
-
stack,
|
|
2170
|
-
id, // Include id in the serialized object
|
|
2171
|
-
};
|
|
2172
|
-
}
|
|
2173
2272
|
|
|
2174
2273
|
/**
|
|
2175
2274
|
* Format either small or big number
|
|
@@ -2252,74 +2351,6 @@
|
|
|
2252
2351
|
}
|
|
2253
2352
|
}
|
|
2254
2353
|
|
|
2255
|
-
/**
|
|
2256
|
-
* Represents the uncertain value
|
|
2257
|
-
*
|
|
2258
|
-
* @public exported from `@promptbook/core`
|
|
2259
|
-
*/
|
|
2260
|
-
const ZERO_VALUE = $deepFreeze({ value: 0 });
|
|
2261
|
-
/**
|
|
2262
|
-
* Represents the uncertain value
|
|
2263
|
-
*
|
|
2264
|
-
* @public exported from `@promptbook/core`
|
|
2265
|
-
*/
|
|
2266
|
-
const UNCERTAIN_ZERO_VALUE = $deepFreeze({ value: 0, isUncertain: true });
|
|
2267
|
-
/**
|
|
2268
|
-
* Represents the usage with no resources consumed
|
|
2269
|
-
*
|
|
2270
|
-
* @public exported from `@promptbook/core`
|
|
2271
|
-
*/
|
|
2272
|
-
const ZERO_USAGE = $deepFreeze({
|
|
2273
|
-
price: ZERO_VALUE,
|
|
2274
|
-
input: {
|
|
2275
|
-
tokensCount: ZERO_VALUE,
|
|
2276
|
-
charactersCount: ZERO_VALUE,
|
|
2277
|
-
wordsCount: ZERO_VALUE,
|
|
2278
|
-
sentencesCount: ZERO_VALUE,
|
|
2279
|
-
linesCount: ZERO_VALUE,
|
|
2280
|
-
paragraphsCount: ZERO_VALUE,
|
|
2281
|
-
pagesCount: ZERO_VALUE,
|
|
2282
|
-
},
|
|
2283
|
-
output: {
|
|
2284
|
-
tokensCount: ZERO_VALUE,
|
|
2285
|
-
charactersCount: ZERO_VALUE,
|
|
2286
|
-
wordsCount: ZERO_VALUE,
|
|
2287
|
-
sentencesCount: ZERO_VALUE,
|
|
2288
|
-
linesCount: ZERO_VALUE,
|
|
2289
|
-
paragraphsCount: ZERO_VALUE,
|
|
2290
|
-
pagesCount: ZERO_VALUE,
|
|
2291
|
-
},
|
|
2292
|
-
});
|
|
2293
|
-
/**
|
|
2294
|
-
* Represents the usage with unknown resources consumed
|
|
2295
|
-
*
|
|
2296
|
-
* @public exported from `@promptbook/core`
|
|
2297
|
-
*/
|
|
2298
|
-
$deepFreeze({
|
|
2299
|
-
price: UNCERTAIN_ZERO_VALUE,
|
|
2300
|
-
input: {
|
|
2301
|
-
tokensCount: UNCERTAIN_ZERO_VALUE,
|
|
2302
|
-
charactersCount: UNCERTAIN_ZERO_VALUE,
|
|
2303
|
-
wordsCount: UNCERTAIN_ZERO_VALUE,
|
|
2304
|
-
sentencesCount: UNCERTAIN_ZERO_VALUE,
|
|
2305
|
-
linesCount: UNCERTAIN_ZERO_VALUE,
|
|
2306
|
-
paragraphsCount: UNCERTAIN_ZERO_VALUE,
|
|
2307
|
-
pagesCount: UNCERTAIN_ZERO_VALUE,
|
|
2308
|
-
},
|
|
2309
|
-
output: {
|
|
2310
|
-
tokensCount: UNCERTAIN_ZERO_VALUE,
|
|
2311
|
-
charactersCount: UNCERTAIN_ZERO_VALUE,
|
|
2312
|
-
wordsCount: UNCERTAIN_ZERO_VALUE,
|
|
2313
|
-
sentencesCount: UNCERTAIN_ZERO_VALUE,
|
|
2314
|
-
linesCount: UNCERTAIN_ZERO_VALUE,
|
|
2315
|
-
paragraphsCount: UNCERTAIN_ZERO_VALUE,
|
|
2316
|
-
pagesCount: UNCERTAIN_ZERO_VALUE,
|
|
2317
|
-
},
|
|
2318
|
-
});
|
|
2319
|
-
/**
|
|
2320
|
-
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
2321
|
-
*/
|
|
2322
|
-
|
|
2323
2354
|
/**
|
|
2324
2355
|
* Function `addUsage` will add multiple usages into one
|
|
2325
2356
|
*
|
|
@@ -2528,7 +2559,7 @@
|
|
|
2528
2559
|
}
|
|
2529
2560
|
|
|
2530
2561
|
/**
|
|
2531
|
-
*
|
|
2562
|
+
* Contains configuration options for parsing and generating CSV files, such as delimiters and quoting rules.
|
|
2532
2563
|
*
|
|
2533
2564
|
* @public exported from `@promptbook/core`
|
|
2534
2565
|
*/
|
|
@@ -2537,11 +2568,29 @@
|
|
|
2537
2568
|
// encoding: 'utf-8',
|
|
2538
2569
|
});
|
|
2539
2570
|
|
|
2571
|
+
/**
|
|
2572
|
+
* Converts a CSV string into an object
|
|
2573
|
+
*
|
|
2574
|
+
* Note: This is wrapper around `papaparse.parse()` with better autohealing
|
|
2575
|
+
*
|
|
2576
|
+
* @private - for now until `@promptbook/csv` is released
|
|
2577
|
+
*/
|
|
2578
|
+
function csvParse(value /* <- TODO: string_csv */, settings, schema /* <- TODO: Make CSV Schemas */) {
|
|
2579
|
+
settings = { ...settings, ...MANDATORY_CSV_SETTINGS };
|
|
2580
|
+
// Note: Autoheal invalid '\n' characters
|
|
2581
|
+
if (settings.newline && !settings.newline.includes('\r') && value.includes('\r')) {
|
|
2582
|
+
console.warn('CSV string contains carriage return characters, but in the CSV settings the `newline` setting does not include them. Autohealing the CSV string.');
|
|
2583
|
+
value = value.replace(/\r\n/g, '\n').replace(/\r/g, '\n');
|
|
2584
|
+
}
|
|
2585
|
+
const csv = papaparse.parse(value, settings);
|
|
2586
|
+
return csv;
|
|
2587
|
+
}
|
|
2588
|
+
|
|
2540
2589
|
/**
|
|
2541
2590
|
* Function to check if a string is valid CSV
|
|
2542
2591
|
*
|
|
2543
2592
|
* @param value The string to check
|
|
2544
|
-
* @returns
|
|
2593
|
+
* @returns `true` if the string is a valid CSV string, false otherwise
|
|
2545
2594
|
*
|
|
2546
2595
|
* @public exported from `@promptbook/utils`
|
|
2547
2596
|
*/
|
|
@@ -2565,7 +2614,7 @@
|
|
|
2565
2614
|
* @public exported from `@promptbook/core`
|
|
2566
2615
|
* <- TODO: [🏢] Export from package `@promptbook/csv`
|
|
2567
2616
|
*/
|
|
2568
|
-
const
|
|
2617
|
+
const CsvFormatParser = {
|
|
2569
2618
|
formatName: 'CSV',
|
|
2570
2619
|
aliases: ['SPREADSHEET', 'TABLE'],
|
|
2571
2620
|
isValid(value, settings, schema) {
|
|
@@ -2577,12 +2626,12 @@
|
|
|
2577
2626
|
heal(value, settings, schema) {
|
|
2578
2627
|
throw new Error('Not implemented');
|
|
2579
2628
|
},
|
|
2580
|
-
|
|
2629
|
+
subvalueParsers: [
|
|
2581
2630
|
{
|
|
2582
2631
|
subvalueName: 'ROW',
|
|
2583
|
-
async mapValues(
|
|
2584
|
-
|
|
2585
|
-
const csv =
|
|
2632
|
+
async mapValues(options) {
|
|
2633
|
+
const { value, outputParameterName, settings, mapCallback, onProgress } = options;
|
|
2634
|
+
const csv = csvParse(value, settings);
|
|
2586
2635
|
if (csv.errors.length !== 0) {
|
|
2587
2636
|
throw new CsvFormatError(spaceTrim__default["default"]((block) => `
|
|
2588
2637
|
CSV parsing error
|
|
@@ -2597,23 +2646,37 @@
|
|
|
2597
2646
|
${block(value)}
|
|
2598
2647
|
`));
|
|
2599
2648
|
}
|
|
2600
|
-
const mappedData =
|
|
2649
|
+
const mappedData = [];
|
|
2650
|
+
const length = csv.data.length;
|
|
2651
|
+
for (let index = 0; index < length; index++) {
|
|
2652
|
+
const row = csv.data[index];
|
|
2601
2653
|
if (row[outputParameterName]) {
|
|
2602
2654
|
throw new CsvFormatError(`Can not overwrite existing column "${outputParameterName}" in CSV row`);
|
|
2603
2655
|
}
|
|
2604
|
-
|
|
2656
|
+
const mappedRow = {
|
|
2605
2657
|
...row,
|
|
2606
|
-
[outputParameterName]: await mapCallback(row, index),
|
|
2658
|
+
[outputParameterName]: await mapCallback(row, index, length),
|
|
2607
2659
|
};
|
|
2608
|
-
|
|
2660
|
+
mappedData.push(mappedRow);
|
|
2661
|
+
if (onProgress) {
|
|
2662
|
+
// Note: Report the CSV with all rows mapped so far
|
|
2663
|
+
/*
|
|
2664
|
+
// TODO: [🛕] Report progress with all the rows including the pending ones
|
|
2665
|
+
const progressData = mappedData.map((row, i) =>
|
|
2666
|
+
i > index ? { ...row, [outputParameterName]: PENDING_VALUE_PLACEHOLDER } : row,
|
|
2667
|
+
);
|
|
2668
|
+
*/
|
|
2669
|
+
await onProgress(papaparse.unparse(mappedData, { ...settings, ...MANDATORY_CSV_SETTINGS }));
|
|
2670
|
+
}
|
|
2671
|
+
}
|
|
2609
2672
|
return papaparse.unparse(mappedData, { ...settings, ...MANDATORY_CSV_SETTINGS });
|
|
2610
2673
|
},
|
|
2611
2674
|
},
|
|
2612
2675
|
{
|
|
2613
2676
|
subvalueName: 'CELL',
|
|
2614
|
-
async mapValues(
|
|
2615
|
-
|
|
2616
|
-
const csv =
|
|
2677
|
+
async mapValues(options) {
|
|
2678
|
+
const { value, settings, mapCallback, onProgress } = options;
|
|
2679
|
+
const csv = csvParse(value, settings);
|
|
2617
2680
|
if (csv.errors.length !== 0) {
|
|
2618
2681
|
throw new CsvFormatError(spaceTrim__default["default"]((block) => `
|
|
2619
2682
|
CSV parsing error
|
|
@@ -2629,9 +2692,9 @@
|
|
|
2629
2692
|
`));
|
|
2630
2693
|
}
|
|
2631
2694
|
const mappedData = await Promise.all(csv.data.map(async (row, rowIndex) => {
|
|
2632
|
-
return /* not await */ Promise.all(Object.entries(row).map(async ([key, value], columnIndex) => {
|
|
2695
|
+
return /* not await */ Promise.all(Object.entries(row).map(async ([key, value], columnIndex, array) => {
|
|
2633
2696
|
const index = rowIndex * Object.keys(row).length + columnIndex;
|
|
2634
|
-
return /* not await */ mapCallback({ [key]: value }, index);
|
|
2697
|
+
return /* not await */ mapCallback({ [key]: value }, index, array.length);
|
|
2635
2698
|
}));
|
|
2636
2699
|
}));
|
|
2637
2700
|
return papaparse.unparse(mappedData, { ...settings, ...MANDATORY_CSV_SETTINGS });
|
|
@@ -2640,10 +2703,10 @@
|
|
|
2640
2703
|
],
|
|
2641
2704
|
};
|
|
2642
2705
|
/**
|
|
2643
|
-
* TODO: [🍓] In `
|
|
2644
|
-
* TODO: [🍓] In `
|
|
2645
|
-
* TODO: [🍓] In `
|
|
2646
|
-
* TODO: [🍓] In `
|
|
2706
|
+
* TODO: [🍓] In `CsvFormatParser` implement simple `isValid`
|
|
2707
|
+
* TODO: [🍓] In `CsvFormatParser` implement partial `canBeValid`
|
|
2708
|
+
* TODO: [🍓] In `CsvFormatParser` implement `heal
|
|
2709
|
+
* TODO: [🍓] In `CsvFormatParser` implement `subvalueParsers`
|
|
2647
2710
|
* TODO: [🏢] Allow to expect something inside CSV objects and other formats
|
|
2648
2711
|
*/
|
|
2649
2712
|
|
|
@@ -2652,7 +2715,7 @@
|
|
|
2652
2715
|
*
|
|
2653
2716
|
* @private still in development [🏢]
|
|
2654
2717
|
*/
|
|
2655
|
-
const
|
|
2718
|
+
const JsonFormatParser = {
|
|
2656
2719
|
formatName: 'JSON',
|
|
2657
2720
|
mimeType: 'application/json',
|
|
2658
2721
|
isValid(value, settings, schema) {
|
|
@@ -2664,28 +2727,28 @@
|
|
|
2664
2727
|
heal(value, settings, schema) {
|
|
2665
2728
|
throw new Error('Not implemented');
|
|
2666
2729
|
},
|
|
2667
|
-
|
|
2730
|
+
subvalueParsers: [],
|
|
2668
2731
|
};
|
|
2669
2732
|
/**
|
|
2670
|
-
* TODO: [🧠] Maybe
|
|
2733
|
+
* TODO: [🧠] Maybe proper instance of object
|
|
2671
2734
|
* TODO: [0] Make string_serialized_json
|
|
2672
2735
|
* TODO: [1] Make type for JSON Settings and Schema
|
|
2673
2736
|
* TODO: [🧠] What to use for validating JSONs - JSON Schema, ZoD, typescript types/interfaces,...?
|
|
2674
|
-
* TODO: [🍓] In `
|
|
2675
|
-
* TODO: [🍓] In `
|
|
2676
|
-
* TODO: [🍓] In `
|
|
2677
|
-
* TODO: [🍓] In `
|
|
2737
|
+
* TODO: [🍓] In `JsonFormatParser` implement simple `isValid`
|
|
2738
|
+
* TODO: [🍓] In `JsonFormatParser` implement partial `canBeValid`
|
|
2739
|
+
* TODO: [🍓] In `JsonFormatParser` implement `heal
|
|
2740
|
+
* TODO: [🍓] In `JsonFormatParser` implement `subvalueParsers`
|
|
2678
2741
|
* TODO: [🏢] Allow to expect something inside JSON objects and other formats
|
|
2679
2742
|
*/
|
|
2680
2743
|
|
|
2681
2744
|
/**
|
|
2682
2745
|
* Definition for any text - this will be always valid
|
|
2683
2746
|
*
|
|
2684
|
-
* Note: This is not useful for validation, but for splitting and mapping with `
|
|
2747
|
+
* Note: This is not useful for validation, but for splitting and mapping with `subvalueParsers`
|
|
2685
2748
|
*
|
|
2686
2749
|
* @public exported from `@promptbook/core`
|
|
2687
2750
|
*/
|
|
2688
|
-
const
|
|
2751
|
+
const TextFormatParser = {
|
|
2689
2752
|
formatName: 'TEXT',
|
|
2690
2753
|
isValid(value) {
|
|
2691
2754
|
return typeof value === 'string';
|
|
@@ -2694,19 +2757,20 @@
|
|
|
2694
2757
|
return typeof partialValue === 'string';
|
|
2695
2758
|
},
|
|
2696
2759
|
heal() {
|
|
2697
|
-
throw new UnexpectedError('It does not make sense to call `
|
|
2760
|
+
throw new UnexpectedError('It does not make sense to call `TextFormatParser.heal`');
|
|
2698
2761
|
},
|
|
2699
|
-
|
|
2762
|
+
subvalueParsers: [
|
|
2700
2763
|
{
|
|
2701
2764
|
subvalueName: 'LINE',
|
|
2702
|
-
async mapValues(
|
|
2765
|
+
async mapValues(options) {
|
|
2766
|
+
const { value, mapCallback, onProgress } = options;
|
|
2703
2767
|
const lines = value.split('\n');
|
|
2704
|
-
const mappedLines = await Promise.all(lines.map((lineContent, lineNumber) =>
|
|
2768
|
+
const mappedLines = await Promise.all(lines.map((lineContent, lineNumber, array) =>
|
|
2705
2769
|
// TODO: [🧠] Maybe option to skip empty line
|
|
2706
2770
|
/* not await */ mapCallback({
|
|
2707
2771
|
lineContent,
|
|
2708
2772
|
// TODO: [🧠] Maybe also put here `lineNumber`
|
|
2709
|
-
}, lineNumber)));
|
|
2773
|
+
}, lineNumber, array.length)));
|
|
2710
2774
|
return mappedLines.join('\n');
|
|
2711
2775
|
},
|
|
2712
2776
|
},
|
|
@@ -2716,10 +2780,10 @@
|
|
|
2716
2780
|
/**
|
|
2717
2781
|
* TODO: [1] Make type for XML Text and Schema
|
|
2718
2782
|
* TODO: [🧠][🤠] Here should be all words, characters, lines, paragraphs, pages available as subvalues
|
|
2719
|
-
* TODO: [🍓] In `
|
|
2720
|
-
* TODO: [🍓] In `
|
|
2721
|
-
* TODO: [🍓] In `
|
|
2722
|
-
* TODO: [🍓] In `
|
|
2783
|
+
* TODO: [🍓] In `TextFormatParser` implement simple `isValid`
|
|
2784
|
+
* TODO: [🍓] In `TextFormatParser` implement partial `canBeValid`
|
|
2785
|
+
* TODO: [🍓] In `TextFormatParser` implement `heal
|
|
2786
|
+
* TODO: [🍓] In `TextFormatParser` implement `subvalueParsers`
|
|
2723
2787
|
* TODO: [🏢] Allow to expect something inside each item of list and other formats
|
|
2724
2788
|
*/
|
|
2725
2789
|
|
|
@@ -2727,7 +2791,7 @@
|
|
|
2727
2791
|
* Function to check if a string is valid XML
|
|
2728
2792
|
*
|
|
2729
2793
|
* @param value
|
|
2730
|
-
* @returns
|
|
2794
|
+
* @returns `true` if the string is a valid XML string, false otherwise
|
|
2731
2795
|
*
|
|
2732
2796
|
* @public exported from `@promptbook/utils`
|
|
2733
2797
|
*/
|
|
@@ -2752,7 +2816,7 @@
|
|
|
2752
2816
|
*
|
|
2753
2817
|
* @private still in development [🏢]
|
|
2754
2818
|
*/
|
|
2755
|
-
const
|
|
2819
|
+
const XmlFormatParser = {
|
|
2756
2820
|
formatName: 'XML',
|
|
2757
2821
|
mimeType: 'application/xml',
|
|
2758
2822
|
isValid(value, settings, schema) {
|
|
@@ -2764,17 +2828,17 @@
|
|
|
2764
2828
|
heal(value, settings, schema) {
|
|
2765
2829
|
throw new Error('Not implemented');
|
|
2766
2830
|
},
|
|
2767
|
-
|
|
2831
|
+
subvalueParsers: [],
|
|
2768
2832
|
};
|
|
2769
2833
|
/**
|
|
2770
|
-
* TODO: [🧠] Maybe
|
|
2834
|
+
* TODO: [🧠] Maybe proper instance of object
|
|
2771
2835
|
* TODO: [0] Make string_serialized_xml
|
|
2772
2836
|
* TODO: [1] Make type for XML Settings and Schema
|
|
2773
2837
|
* TODO: [🧠] What to use for validating XMLs - XSD,...
|
|
2774
|
-
* TODO: [🍓] In `
|
|
2775
|
-
* TODO: [🍓] In `
|
|
2776
|
-
* TODO: [🍓] In `
|
|
2777
|
-
* TODO: [🍓] In `
|
|
2838
|
+
* TODO: [🍓] In `XmlFormatParser` implement simple `isValid`
|
|
2839
|
+
* TODO: [🍓] In `XmlFormatParser` implement partial `canBeValid`
|
|
2840
|
+
* TODO: [🍓] In `XmlFormatParser` implement `heal
|
|
2841
|
+
* TODO: [🍓] In `XmlFormatParser` implement `subvalueParsers`
|
|
2778
2842
|
* TODO: [🏢] Allow to expect something inside XML and other formats
|
|
2779
2843
|
*/
|
|
2780
2844
|
|
|
@@ -2783,24 +2847,19 @@
|
|
|
2783
2847
|
*
|
|
2784
2848
|
* @private internal index of `...` <- TODO [🏢]
|
|
2785
2849
|
*/
|
|
2786
|
-
const FORMAT_DEFINITIONS = [
|
|
2787
|
-
JsonFormatDefinition,
|
|
2788
|
-
XmlFormatDefinition,
|
|
2789
|
-
TextFormatDefinition,
|
|
2790
|
-
CsvFormatDefinition,
|
|
2791
|
-
];
|
|
2850
|
+
const FORMAT_DEFINITIONS = [JsonFormatParser, XmlFormatParser, TextFormatParser, CsvFormatParser];
|
|
2792
2851
|
/**
|
|
2793
2852
|
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
2794
2853
|
*/
|
|
2795
2854
|
|
|
2796
2855
|
/**
|
|
2797
|
-
* Maps available parameters to expected parameters
|
|
2856
|
+
* Maps available parameters to expected parameters for a pipeline task.
|
|
2798
2857
|
*
|
|
2799
2858
|
* The strategy is:
|
|
2800
|
-
* 1)
|
|
2801
|
-
* 2)
|
|
2859
|
+
* 1) First, match parameters by name where both available and expected.
|
|
2860
|
+
* 2) Then, if there are unmatched expected and available parameters, map them by order.
|
|
2802
2861
|
*
|
|
2803
|
-
* @throws {PipelineExecutionError}
|
|
2862
|
+
* @throws {PipelineExecutionError} If the number of unmatched expected and available parameters does not match, or mapping is ambiguous.
|
|
2804
2863
|
* @private within the repository used in `createPipelineExecutor`
|
|
2805
2864
|
*/
|
|
2806
2865
|
function mapAvailableToExpectedParameters(options) {
|
|
@@ -2823,7 +2882,7 @@
|
|
|
2823
2882
|
else if (!availableParametersNames.has(parameterName) && expectedParameterNames.has(parameterName)) ;
|
|
2824
2883
|
}
|
|
2825
2884
|
if (expectedParameterNames.size === 0) {
|
|
2826
|
-
// Note: [👨👨👧] Now we can freeze `mappedParameters` to prevent
|
|
2885
|
+
// Note: [👨👨👧] Now we can freeze `mappedParameters` to prevent accidental modifications after mapping
|
|
2827
2886
|
Object.freeze(mappedParameters);
|
|
2828
2887
|
return mappedParameters;
|
|
2829
2888
|
}
|
|
@@ -2854,7 +2913,7 @@
|
|
|
2854
2913
|
for (let i = 0; i < expectedParameterNames.size; i++) {
|
|
2855
2914
|
mappedParameters[expectedParameterNamesArray[i]] = availableParameters[availableParametersNamesArray[i]];
|
|
2856
2915
|
}
|
|
2857
|
-
// Note: [👨👨👧] Now we can freeze `mappedParameters` to prevent
|
|
2916
|
+
// Note: [👨👨👧] Now we can freeze `mappedParameters` to prevent accidental modifications after mapping
|
|
2858
2917
|
Object.freeze(mappedParameters);
|
|
2859
2918
|
return mappedParameters;
|
|
2860
2919
|
}
|
|
@@ -2876,29 +2935,40 @@
|
|
|
2876
2935
|
return 'Multiple LLM Providers';
|
|
2877
2936
|
}
|
|
2878
2937
|
get description() {
|
|
2879
|
-
|
|
2938
|
+
const innerModelsTitlesAndDescriptions = this.llmExecutionTools
|
|
2939
|
+
.map(({ title, description }, index) => {
|
|
2940
|
+
const headLine = `${index + 1}) \`${title}\``;
|
|
2941
|
+
if (description === undefined) {
|
|
2942
|
+
return headLine;
|
|
2943
|
+
}
|
|
2944
|
+
return spaceTrim__default["default"]((block) => `
|
|
2945
|
+
${headLine}
|
|
2946
|
+
|
|
2947
|
+
${ /* <- Note: Indenting the description: */block(description)}
|
|
2948
|
+
`);
|
|
2949
|
+
})
|
|
2950
|
+
.join('\n\n');
|
|
2951
|
+
return spaceTrim__default["default"]((block) => `
|
|
2952
|
+
Multiple LLM Providers:
|
|
2953
|
+
|
|
2954
|
+
${block(innerModelsTitlesAndDescriptions)}
|
|
2955
|
+
`);
|
|
2880
2956
|
}
|
|
2881
2957
|
/**
|
|
2882
2958
|
* Check the configuration of all execution tools
|
|
2883
2959
|
*/
|
|
2884
2960
|
async checkConfiguration() {
|
|
2885
|
-
//
|
|
2886
|
-
|
|
2887
|
-
await llmExecutionTools.checkConfiguration();
|
|
2888
|
-
}
|
|
2961
|
+
// Note: Run checks in parallel
|
|
2962
|
+
await Promise.all(this.llmExecutionTools.map((tools) => tools.checkConfiguration()));
|
|
2889
2963
|
}
|
|
2890
2964
|
/**
|
|
2891
2965
|
* List all available models that can be used
|
|
2892
2966
|
* This lists is a combination of all available models from all execution tools
|
|
2893
2967
|
*/
|
|
2894
2968
|
async listModels() {
|
|
2895
|
-
|
|
2896
|
-
|
|
2897
|
-
|
|
2898
|
-
const models = await llmExecutionTools.listModels();
|
|
2899
|
-
availableModels.push(...models);
|
|
2900
|
-
}
|
|
2901
|
-
return availableModels;
|
|
2969
|
+
// Obtain all models in parallel and flatten
|
|
2970
|
+
const modelArrays = await Promise.all(this.llmExecutionTools.map((tools) => tools.listModels()));
|
|
2971
|
+
return modelArrays.flat();
|
|
2902
2972
|
}
|
|
2903
2973
|
/**
|
|
2904
2974
|
* Calls the best available chat model
|
|
@@ -3054,8 +3124,8 @@
|
|
|
3054
3124
|
/**
|
|
3055
3125
|
* Extracts all code blocks from markdown.
|
|
3056
3126
|
*
|
|
3057
|
-
* Note: There are multiple
|
|
3058
|
-
* - `extractBlock` just extracts the content of the code block which is also used as
|
|
3127
|
+
* Note: There are multiple similar functions:
|
|
3128
|
+
* - `extractBlock` just extracts the content of the code block which is also used as built-in function for postprocessing
|
|
3059
3129
|
* - `extractJsonBlock` extracts exactly one valid JSON code block
|
|
3060
3130
|
* - `extractOneBlockFromMarkdown` extracts exactly one code block with language of the code block
|
|
3061
3131
|
* - `extractAllBlocksFromMarkdown` extracts all code blocks with language of the code block
|
|
@@ -3105,7 +3175,7 @@
|
|
|
3105
3175
|
if (currentCodeBlock.content !== '') {
|
|
3106
3176
|
currentCodeBlock.content += '\n';
|
|
3107
3177
|
}
|
|
3108
|
-
currentCodeBlock.content += line.split('\\`\\`\\`').join('```') /* <- TODO: Maybe make
|
|
3178
|
+
currentCodeBlock.content += line.split('\\`\\`\\`').join('```') /* <- TODO: Maybe make proper unescape */;
|
|
3109
3179
|
}
|
|
3110
3180
|
}
|
|
3111
3181
|
if (currentCodeBlock !== null) {
|
|
@@ -3125,7 +3195,7 @@
|
|
|
3125
3195
|
* - When there are multiple JSON code blocks the function throws a `ParseError`
|
|
3126
3196
|
*
|
|
3127
3197
|
* Note: It is not important if marked as ```json BUT if it is VALID JSON
|
|
3128
|
-
* Note: There are multiple
|
|
3198
|
+
* Note: There are multiple similar function:
|
|
3129
3199
|
* - `extractBlock` just extracts the content of the code block which is also used as build-in function for postprocessing
|
|
3130
3200
|
* - `extractJsonBlock` extracts exactly one valid JSON code block
|
|
3131
3201
|
* - `extractOneBlockFromMarkdown` extracts exactly one code block with language of the code block
|
|
@@ -3150,7 +3220,7 @@
|
|
|
3150
3220
|
}
|
|
3151
3221
|
/**
|
|
3152
3222
|
* TODO: Add some auto-healing logic + extract YAML, JSON5, TOML, etc.
|
|
3153
|
-
* TODO: [🏢] Make this logic part of `
|
|
3223
|
+
* TODO: [🏢] Make this logic part of `JsonFormatParser` or `isValidJsonString`
|
|
3154
3224
|
*/
|
|
3155
3225
|
|
|
3156
3226
|
/**
|
|
@@ -3176,7 +3246,7 @@
|
|
|
3176
3246
|
* Just says that the variable is not used but should be kept
|
|
3177
3247
|
* No side effects.
|
|
3178
3248
|
*
|
|
3179
|
-
* Note: It can be
|
|
3249
|
+
* Note: It can be useful for:
|
|
3180
3250
|
*
|
|
3181
3251
|
* 1) Suppressing eager optimization of unused imports
|
|
3182
3252
|
* 2) Suppressing eslint errors of unused variables in the tests
|
|
@@ -3603,10 +3673,10 @@
|
|
|
3603
3673
|
*/
|
|
3604
3674
|
|
|
3605
3675
|
/**
|
|
3606
|
-
*
|
|
3676
|
+
* Removes diacritic marks (accents) from characters in a string.
|
|
3607
3677
|
*
|
|
3608
|
-
* @param input
|
|
3609
|
-
* @returns
|
|
3678
|
+
* @param input The string containing diacritics to be normalized.
|
|
3679
|
+
* @returns The string with diacritics removed or normalized.
|
|
3610
3680
|
* @public exported from `@promptbook/utils`
|
|
3611
3681
|
*/
|
|
3612
3682
|
function removeDiacritics(input) {
|
|
@@ -3649,14 +3719,14 @@
|
|
|
3649
3719
|
PAGES: countPages,
|
|
3650
3720
|
};
|
|
3651
3721
|
/**
|
|
3652
|
-
* TODO: [🧠][🤠] This should be
|
|
3722
|
+
* TODO: [🧠][🤠] This should be probably as part of `TextFormatParser`
|
|
3653
3723
|
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
3654
3724
|
*/
|
|
3655
3725
|
|
|
3656
3726
|
/**
|
|
3657
3727
|
* Function checkExpectations will check if the expectations on given value are met
|
|
3658
3728
|
*
|
|
3659
|
-
* Note: There are two
|
|
3729
|
+
* Note: There are two similar functions:
|
|
3660
3730
|
* - `checkExpectations` which throws an error if the expectations are not met
|
|
3661
3731
|
* - `isPassingExpectations` which returns a boolean
|
|
3662
3732
|
*
|
|
@@ -3677,13 +3747,17 @@
|
|
|
3677
3747
|
}
|
|
3678
3748
|
/**
|
|
3679
3749
|
* TODO: [💝] Unite object for expecting amount and format
|
|
3680
|
-
* TODO: [🧠][🤠] This should be part of `
|
|
3750
|
+
* TODO: [🧠][🤠] This should be part of `TextFormatParser`
|
|
3681
3751
|
* Note: [💝] and [🤠] are interconnected together
|
|
3682
3752
|
*/
|
|
3683
3753
|
|
|
3684
3754
|
/**
|
|
3685
|
-
*
|
|
3755
|
+
* Executes a pipeline task with multiple attempts, including joker and retry logic. Handles different task types
|
|
3756
|
+
* (prompt, script, dialog, etc.), applies postprocessing, checks expectations, and updates the execution report.
|
|
3757
|
+
* Throws errors if execution fails after all attempts.
|
|
3686
3758
|
*
|
|
3759
|
+
* @param options - The options for execution, including task, parameters, pipeline, and configuration.
|
|
3760
|
+
* @returns The result string of the executed task.
|
|
3687
3761
|
* @private internal utility of `createPipelineExecutor`
|
|
3688
3762
|
*/
|
|
3689
3763
|
async function executeAttempts(options) {
|
|
@@ -3905,7 +3979,7 @@
|
|
|
3905
3979
|
if (task.format) {
|
|
3906
3980
|
if (task.format === 'JSON') {
|
|
3907
3981
|
if (!isValidJsonString($ongoingTaskResult.$resultString || '')) {
|
|
3908
|
-
// TODO: [🏢] Do more universally via `
|
|
3982
|
+
// TODO: [🏢] Do more universally via `FormatParser`
|
|
3909
3983
|
try {
|
|
3910
3984
|
$ongoingTaskResult.$resultString = extractJsonBlock($ongoingTaskResult.$resultString || '');
|
|
3911
3985
|
}
|
|
@@ -4007,12 +4081,16 @@
|
|
|
4007
4081
|
*/
|
|
4008
4082
|
|
|
4009
4083
|
/**
|
|
4010
|
-
*
|
|
4084
|
+
* Executes a pipeline task that requires mapping or iterating over subvalues of a parameter (such as rows in a CSV).
|
|
4085
|
+
* Handles format and subformat resolution, error handling, and progress reporting.
|
|
4086
|
+
*
|
|
4087
|
+
* @param options - Options for execution, including task details and progress callback.
|
|
4088
|
+
* @returns The result of the subvalue mapping or execution attempts.
|
|
4011
4089
|
*
|
|
4012
4090
|
* @private internal utility of `createPipelineExecutor`
|
|
4013
4091
|
*/
|
|
4014
4092
|
async function executeFormatSubvalues(options) {
|
|
4015
|
-
const { task, jokerParameterNames, parameters, priority, csvSettings, pipelineIdentification } = options;
|
|
4093
|
+
const { task, jokerParameterNames, parameters, priority, csvSettings, onProgress, pipelineIdentification } = options;
|
|
4016
4094
|
if (task.foreach === undefined) {
|
|
4017
4095
|
return /* not await */ executeAttempts(options);
|
|
4018
4096
|
}
|
|
@@ -4043,16 +4121,16 @@
|
|
|
4043
4121
|
${block(pipelineIdentification)}
|
|
4044
4122
|
`));
|
|
4045
4123
|
}
|
|
4046
|
-
const
|
|
4047
|
-
if (
|
|
4124
|
+
const subvalueParser = formatDefinition.subvalueParsers.find((subvalueParser) => [subvalueParser.subvalueName, ...(subvalueParser.aliases || [])].includes(task.foreach.subformatName));
|
|
4125
|
+
if (subvalueParser === undefined) {
|
|
4048
4126
|
throw new UnexpectedError(
|
|
4049
4127
|
// <- TODO: [🧠][🧐] Should be formats fixed per promptbook version or behave as plugins (=> change UnexpectedError)
|
|
4050
4128
|
spaceTrim__default["default"]((block) => `
|
|
4051
4129
|
Unsupported subformat name "${task.foreach.subformatName}" for format "${task.foreach.formatName}"
|
|
4052
4130
|
|
|
4053
4131
|
Available subformat names for format "${formatDefinition.formatName}":
|
|
4054
|
-
${block(formatDefinition.
|
|
4055
|
-
.map((
|
|
4132
|
+
${block(formatDefinition.subvalueParsers
|
|
4133
|
+
.map((subvalueParser) => subvalueParser.subvalueName)
|
|
4056
4134
|
.map((subvalueName) => `- ${subvalueName}`)
|
|
4057
4135
|
.join('\n'))}
|
|
4058
4136
|
|
|
@@ -4064,55 +4142,85 @@
|
|
|
4064
4142
|
let formatSettings;
|
|
4065
4143
|
if (formatDefinition.formatName === 'CSV') {
|
|
4066
4144
|
formatSettings = csvSettings;
|
|
4067
|
-
// <- TODO: [🤹♂️] More universal, make
|
|
4068
|
-
}
|
|
4069
|
-
const resultString = await
|
|
4070
|
-
|
|
4071
|
-
|
|
4072
|
-
|
|
4073
|
-
|
|
4074
|
-
|
|
4075
|
-
|
|
4076
|
-
|
|
4077
|
-
|
|
4078
|
-
|
|
4079
|
-
|
|
4080
|
-
|
|
4081
|
-
|
|
4145
|
+
// <- TODO: [🤹♂️] More universal, make similar pattern for other formats for example \n vs \r\n in text
|
|
4146
|
+
}
|
|
4147
|
+
const resultString = await subvalueParser.mapValues({
|
|
4148
|
+
value: parameterValue,
|
|
4149
|
+
outputParameterName: task.foreach.outputSubparameterName,
|
|
4150
|
+
settings: formatSettings,
|
|
4151
|
+
onProgress(partialResultString) {
|
|
4152
|
+
return onProgress(Object.freeze({
|
|
4153
|
+
[task.resultingParameterName]: partialResultString,
|
|
4154
|
+
}));
|
|
4155
|
+
},
|
|
4156
|
+
async mapCallback(subparameters, index, length) {
|
|
4157
|
+
let mappedParameters;
|
|
4158
|
+
try {
|
|
4159
|
+
mappedParameters = mapAvailableToExpectedParameters({
|
|
4160
|
+
expectedParameters: Object.fromEntries(task.foreach.inputSubparameterNames.map((subparameterName) => [subparameterName, null])),
|
|
4161
|
+
availableParameters: subparameters,
|
|
4162
|
+
});
|
|
4082
4163
|
}
|
|
4083
|
-
|
|
4084
|
-
|
|
4164
|
+
catch (error) {
|
|
4165
|
+
if (!(error instanceof PipelineExecutionError)) {
|
|
4166
|
+
throw error;
|
|
4167
|
+
}
|
|
4168
|
+
const highLevelError = new PipelineExecutionError(spaceTrim__default["default"]((block) => `
|
|
4169
|
+
${error.message}
|
|
4085
4170
|
|
|
4086
|
-
|
|
4087
|
-
|
|
4171
|
+
This is error in FOREACH command when mapping ${formatDefinition.formatName} ${subvalueParser.subvalueName} data (${index + 1}/${length})
|
|
4172
|
+
You have probably passed wrong data to pipeline or wrong data was generated which are processed by FOREACH command
|
|
4088
4173
|
|
|
4089
|
-
|
|
4090
|
-
|
|
4091
|
-
|
|
4092
|
-
|
|
4093
|
-
|
|
4094
|
-
|
|
4095
|
-
|
|
4096
|
-
|
|
4097
|
-
|
|
4098
|
-
|
|
4099
|
-
|
|
4100
|
-
|
|
4101
|
-
|
|
4102
|
-
|
|
4103
|
-
|
|
4104
|
-
|
|
4105
|
-
|
|
4106
|
-
|
|
4107
|
-
|
|
4108
|
-
|
|
4174
|
+
${block(pipelineIdentification)}
|
|
4175
|
+
`));
|
|
4176
|
+
if (length > BIG_DATASET_TRESHOLD) {
|
|
4177
|
+
console.error(highLevelError);
|
|
4178
|
+
return FAILED_VALUE_PLACEHOLDER;
|
|
4179
|
+
}
|
|
4180
|
+
throw highLevelError;
|
|
4181
|
+
}
|
|
4182
|
+
const allSubparameters = {
|
|
4183
|
+
...parameters,
|
|
4184
|
+
...mappedParameters,
|
|
4185
|
+
};
|
|
4186
|
+
Object.freeze(allSubparameters);
|
|
4187
|
+
try {
|
|
4188
|
+
const subresultString = await executeAttempts({
|
|
4189
|
+
...options,
|
|
4190
|
+
priority: priority + index,
|
|
4191
|
+
parameters: allSubparameters,
|
|
4192
|
+
pipelineIdentification: spaceTrim__default["default"]((block) => `
|
|
4193
|
+
${block(pipelineIdentification)}
|
|
4194
|
+
Subparameter index: ${index}
|
|
4195
|
+
`),
|
|
4196
|
+
});
|
|
4197
|
+
return subresultString;
|
|
4198
|
+
}
|
|
4199
|
+
catch (error) {
|
|
4200
|
+
if (length > BIG_DATASET_TRESHOLD) {
|
|
4201
|
+
console.error(spaceTrim__default["default"]((block) => `
|
|
4202
|
+
${error.message}
|
|
4203
|
+
|
|
4204
|
+
This is error in FOREACH command when processing ${formatDefinition.formatName} ${subvalueParser.subvalueName} data (${index + 1}/${length})
|
|
4205
|
+
|
|
4206
|
+
${block(pipelineIdentification)}
|
|
4207
|
+
`));
|
|
4208
|
+
return FAILED_VALUE_PLACEHOLDER;
|
|
4209
|
+
}
|
|
4210
|
+
throw error;
|
|
4211
|
+
}
|
|
4212
|
+
},
|
|
4109
4213
|
});
|
|
4110
4214
|
return resultString;
|
|
4111
4215
|
}
|
|
4112
4216
|
|
|
4113
4217
|
/**
|
|
4114
|
-
*
|
|
4218
|
+
* Returns the context for a given task, typically used to provide additional information or variables
|
|
4219
|
+
* required for the execution of the task within a pipeline. The context is returned as a string value
|
|
4220
|
+
* that may include markdown formatting.
|
|
4115
4221
|
*
|
|
4222
|
+
* @param task - The task for which the context is being generated. This should be a deeply immutable TaskJson object.
|
|
4223
|
+
* @returns The context as a string, formatted as markdown and parameter value.
|
|
4116
4224
|
* @private internal utility of `createPipelineExecutor`
|
|
4117
4225
|
*/
|
|
4118
4226
|
async function getContextForTask(task) {
|
|
@@ -4120,7 +4228,7 @@
|
|
|
4120
4228
|
}
|
|
4121
4229
|
|
|
4122
4230
|
/**
|
|
4123
|
-
*
|
|
4231
|
+
* Retrieves example values or templates for a given task, used to guide or validate pipeline execution.
|
|
4124
4232
|
*
|
|
4125
4233
|
* @private internal utility of `createPipelineExecutor`
|
|
4126
4234
|
*/
|
|
@@ -4129,91 +4237,128 @@
|
|
|
4129
4237
|
}
|
|
4130
4238
|
|
|
4131
4239
|
/**
|
|
4132
|
-
*
|
|
4240
|
+
* Computes the cosine similarity between two embedding vectors
|
|
4241
|
+
*
|
|
4242
|
+
* Note: This is helping function for RAG (retrieval-augmented generation)
|
|
4243
|
+
*
|
|
4244
|
+
* @param embeddingVector1
|
|
4245
|
+
* @param embeddingVector2
|
|
4246
|
+
* @returns Cosine similarity between the two vectors
|
|
4247
|
+
*
|
|
4248
|
+
* @public exported from `@promptbook/core`
|
|
4249
|
+
*/
|
|
4250
|
+
function computeCosineSimilarity(embeddingVector1, embeddingVector2) {
|
|
4251
|
+
if (embeddingVector1.length !== embeddingVector2.length) {
|
|
4252
|
+
throw new TypeError('Embedding vectors must have the same length');
|
|
4253
|
+
}
|
|
4254
|
+
const dotProduct = embeddingVector1.reduce((sum, value, index) => sum + value * embeddingVector2[index], 0);
|
|
4255
|
+
const magnitude1 = Math.sqrt(embeddingVector1.reduce((sum, value) => sum + value * value, 0));
|
|
4256
|
+
const magnitude2 = Math.sqrt(embeddingVector2.reduce((sum, value) => sum + value * value, 0));
|
|
4257
|
+
return 1 - dotProduct / (magnitude1 * magnitude2);
|
|
4258
|
+
}
|
|
4259
|
+
|
|
4260
|
+
/**
|
|
4261
|
+
*
|
|
4262
|
+
* @param knowledgePieces
|
|
4263
|
+
* @returns
|
|
4133
4264
|
*
|
|
4134
|
-
*
|
|
4265
|
+
* @private internal utility of `createPipelineExecutor`
|
|
4266
|
+
*/
|
|
4267
|
+
function knowledgePiecesToString(knowledgePieces) {
|
|
4268
|
+
return knowledgePieces
|
|
4269
|
+
.map((knowledgePiece) => {
|
|
4270
|
+
const { content } = knowledgePiece;
|
|
4271
|
+
return `- ${content}`;
|
|
4272
|
+
})
|
|
4273
|
+
.join('\n');
|
|
4274
|
+
// <- TODO: [🧠] Some smarter aggregation of knowledge pieces, single-line vs multi-line vs mixed
|
|
4275
|
+
}
|
|
4276
|
+
|
|
4277
|
+
/**
|
|
4278
|
+
* Retrieves the most relevant knowledge pieces for a given task using embedding-based similarity search.
|
|
4279
|
+
* This is where retrieval-augmented generation (RAG) is performed to enhance the task with external knowledge.
|
|
4135
4280
|
*
|
|
4136
4281
|
* @private internal utility of `createPipelineExecutor`
|
|
4137
4282
|
*/
|
|
4138
4283
|
async function getKnowledgeForTask(options) {
|
|
4139
|
-
const { tools, preparedPipeline, task } = options;
|
|
4284
|
+
const { tools, preparedPipeline, task, parameters } = options;
|
|
4140
4285
|
const firstKnowlegePiece = preparedPipeline.knowledgePieces[0];
|
|
4141
4286
|
const firstKnowlegeIndex = firstKnowlegePiece === null || firstKnowlegePiece === void 0 ? void 0 : firstKnowlegePiece.index[0];
|
|
4142
4287
|
// <- TODO: Do not use just first knowledge piece and first index to determine embedding model, use also keyword search
|
|
4143
4288
|
if (firstKnowlegePiece === undefined || firstKnowlegeIndex === undefined) {
|
|
4144
|
-
return '
|
|
4289
|
+
return ''; // <- Note: Np knowledge present, return empty string
|
|
4145
4290
|
}
|
|
4146
|
-
|
|
4147
|
-
|
|
4148
|
-
|
|
4149
|
-
|
|
4150
|
-
|
|
4151
|
-
|
|
4152
|
-
|
|
4153
|
-
|
|
4154
|
-
|
|
4155
|
-
|
|
4156
|
-
|
|
4157
|
-
|
|
4158
|
-
}
|
|
4159
|
-
|
|
4160
|
-
|
|
4161
|
-
|
|
4162
|
-
|
|
4163
|
-
|
|
4164
|
-
|
|
4165
|
-
|
|
4291
|
+
try {
|
|
4292
|
+
// TODO: [🚐] Make arrayable LLMs -> single LLM DRY
|
|
4293
|
+
const _llms = arrayableToArray(tools.llm);
|
|
4294
|
+
const llmTools = _llms.length === 1 ? _llms[0] : joinLlmExecutionTools(..._llms);
|
|
4295
|
+
const taskEmbeddingPrompt = {
|
|
4296
|
+
title: 'Knowledge Search',
|
|
4297
|
+
modelRequirements: {
|
|
4298
|
+
modelVariant: 'EMBEDDING',
|
|
4299
|
+
modelName: firstKnowlegeIndex.modelName,
|
|
4300
|
+
},
|
|
4301
|
+
content: task.content,
|
|
4302
|
+
parameters,
|
|
4303
|
+
};
|
|
4304
|
+
const taskEmbeddingResult = await llmTools.callEmbeddingModel(taskEmbeddingPrompt);
|
|
4305
|
+
const knowledgePiecesWithRelevance = preparedPipeline.knowledgePieces.map((knowledgePiece) => {
|
|
4306
|
+
const { index } = knowledgePiece;
|
|
4307
|
+
const knowledgePieceIndex = index.find((i) => i.modelName === firstKnowlegeIndex.modelName);
|
|
4308
|
+
// <- TODO: Do not use just first knowledge piece and first index to determine embedding model
|
|
4309
|
+
if (knowledgePieceIndex === undefined) {
|
|
4310
|
+
return {
|
|
4311
|
+
content: knowledgePiece.content,
|
|
4312
|
+
relevance: 0,
|
|
4313
|
+
};
|
|
4314
|
+
}
|
|
4315
|
+
const relevance = computeCosineSimilarity(knowledgePieceIndex.position, taskEmbeddingResult.content);
|
|
4166
4316
|
return {
|
|
4167
4317
|
content: knowledgePiece.content,
|
|
4168
|
-
relevance
|
|
4318
|
+
relevance,
|
|
4169
4319
|
};
|
|
4170
|
-
}
|
|
4171
|
-
const
|
|
4172
|
-
|
|
4173
|
-
|
|
4174
|
-
|
|
4175
|
-
|
|
4176
|
-
|
|
4177
|
-
|
|
4178
|
-
|
|
4179
|
-
|
|
4180
|
-
|
|
4181
|
-
|
|
4182
|
-
|
|
4183
|
-
|
|
4184
|
-
|
|
4185
|
-
|
|
4186
|
-
|
|
4187
|
-
|
|
4188
|
-
|
|
4189
|
-
|
|
4190
|
-
|
|
4191
|
-
|
|
4192
|
-
// TODO: !!!!!! Annotate + to new file
|
|
4193
|
-
function computeCosineSimilarity(embeddingVector1, embeddingVector2) {
|
|
4194
|
-
if (embeddingVector1.length !== embeddingVector2.length) {
|
|
4195
|
-
throw new TypeError('Embedding vectors must have the same length');
|
|
4320
|
+
});
|
|
4321
|
+
const knowledgePiecesSorted = knowledgePiecesWithRelevance.sort((a, b) => a.relevance - b.relevance);
|
|
4322
|
+
const knowledgePiecesLimited = knowledgePiecesSorted.slice(0, 5);
|
|
4323
|
+
/*
|
|
4324
|
+
console.log('`getKnowledgeForTask` Embedding', {
|
|
4325
|
+
task,
|
|
4326
|
+
taskEmbeddingPrompt,
|
|
4327
|
+
taskEmbeddingResult,
|
|
4328
|
+
firstKnowlegePiece,
|
|
4329
|
+
firstKnowlegeIndex,
|
|
4330
|
+
knowledgePiecesWithRelevance,
|
|
4331
|
+
knowledgePiecesSorted,
|
|
4332
|
+
knowledgePiecesLimited,
|
|
4333
|
+
});
|
|
4334
|
+
*/
|
|
4335
|
+
return knowledgePiecesToString(knowledgePiecesLimited);
|
|
4336
|
+
}
|
|
4337
|
+
catch (error) {
|
|
4338
|
+
assertsError(error);
|
|
4339
|
+
console.error('Error in `getKnowledgeForTask`', error);
|
|
4340
|
+
// Note: If the LLM fails, just return all knowledge pieces
|
|
4341
|
+
return knowledgePiecesToString(preparedPipeline.knowledgePieces);
|
|
4196
4342
|
}
|
|
4197
|
-
const dotProduct = embeddingVector1.reduce((sum, value, index) => sum + value * embeddingVector2[index], 0);
|
|
4198
|
-
const magnitude1 = Math.sqrt(embeddingVector1.reduce((sum, value) => sum + value * value, 0));
|
|
4199
|
-
const magnitude2 = Math.sqrt(embeddingVector2.reduce((sum, value) => sum + value * value, 0));
|
|
4200
|
-
return 1 - dotProduct / (magnitude1 * magnitude2);
|
|
4201
4343
|
}
|
|
4202
4344
|
/**
|
|
4203
|
-
* TODO: !!!! Verify if this is working
|
|
4204
4345
|
* TODO: [♨] Implement Better - use keyword search
|
|
4205
4346
|
* TODO: [♨] Examples of values
|
|
4206
4347
|
*/
|
|
4207
4348
|
|
|
4208
4349
|
/**
|
|
4209
|
-
*
|
|
4350
|
+
* Retrieves all reserved parameters for a given pipeline task, including context, knowledge, examples, and metadata.
|
|
4351
|
+
* Ensures all reserved parameters are defined and throws if any are missing.
|
|
4352
|
+
*
|
|
4353
|
+
* @param options - Options including tools, pipeline, task, and context.
|
|
4354
|
+
* @returns An object containing all reserved parameters for the task.
|
|
4210
4355
|
*
|
|
4211
4356
|
* @private internal utility of `createPipelineExecutor`
|
|
4212
4357
|
*/
|
|
4213
4358
|
async function getReservedParametersForTask(options) {
|
|
4214
|
-
const { tools, preparedPipeline, task, pipelineIdentification } = options;
|
|
4359
|
+
const { tools, preparedPipeline, task, parameters, pipelineIdentification, isVerbose } = options;
|
|
4215
4360
|
const context = await getContextForTask(); // <- [🏍]
|
|
4216
|
-
const knowledge = await getKnowledgeForTask({ tools, preparedPipeline, task });
|
|
4361
|
+
const knowledge = await getKnowledgeForTask({ tools, preparedPipeline, task, parameters });
|
|
4217
4362
|
const examples = await getExamplesForTask();
|
|
4218
4363
|
const currentDate = new Date().toISOString(); // <- TODO: [🧠][💩] Better
|
|
4219
4364
|
const modelName = RESERVED_PARAMETER_MISSING_VALUE;
|
|
@@ -4225,6 +4370,9 @@
|
|
|
4225
4370
|
currentDate,
|
|
4226
4371
|
modelName,
|
|
4227
4372
|
};
|
|
4373
|
+
if (isVerbose) {
|
|
4374
|
+
console.info('Reserved parameters for task:', { options, reservedParameters });
|
|
4375
|
+
}
|
|
4228
4376
|
// Note: Doublecheck that ALL reserved parameters are defined:
|
|
4229
4377
|
for (const parameterName of RESERVED_PARAMETER_NAMES) {
|
|
4230
4378
|
if (reservedParameters[parameterName] === undefined) {
|
|
@@ -4239,23 +4387,21 @@
|
|
|
4239
4387
|
}
|
|
4240
4388
|
|
|
4241
4389
|
/**
|
|
4242
|
-
*
|
|
4390
|
+
* Executes a single task within a pipeline, handling parameter validation, error checking, and progress reporting.
|
|
4391
|
+
*
|
|
4392
|
+
* @param options - Options for execution, including the task, pipeline, parameters, and callbacks.
|
|
4393
|
+
* @returns The output parameters produced by the task.
|
|
4243
4394
|
*
|
|
4244
4395
|
* @private internal utility of `createPipelineExecutor`
|
|
4245
4396
|
*/
|
|
4246
4397
|
async function executeTask(options) {
|
|
4247
4398
|
const { currentTask, preparedPipeline, parametersToPass, tools, onProgress, $executionReport, pipelineIdentification, maxExecutionAttempts, maxParallelCount, csvSettings, isVerbose, rootDirname, cacheDirname, intermediateFilesStrategy, isAutoInstalled, isNotPreparedWarningSupressed, } = options;
|
|
4248
4399
|
const priority = preparedPipeline.tasks.length - preparedPipeline.tasks.indexOf(currentTask);
|
|
4249
|
-
await onProgress({
|
|
4250
|
-
outputParameters: {
|
|
4251
|
-
[currentTask.resultingParameterName]: '', // <- TODO: [🧠] What is the best value here?
|
|
4252
|
-
},
|
|
4253
|
-
});
|
|
4254
4400
|
// Note: Check consistency of used and dependent parameters which was also done in `validatePipeline`, but it’s good to doublecheck
|
|
4255
4401
|
const usedParameterNames = extractParameterNamesFromTask(currentTask);
|
|
4256
4402
|
const dependentParameterNames = new Set(currentTask.dependentParameterNames);
|
|
4257
4403
|
// TODO: [👩🏾🤝👩🏻] Use here `mapAvailableToExpectedParameters`
|
|
4258
|
-
if (union(difference(usedParameterNames, dependentParameterNames), difference(dependentParameterNames, usedParameterNames)).size !== 0) {
|
|
4404
|
+
if (difference(union(difference(usedParameterNames, dependentParameterNames), difference(dependentParameterNames, usedParameterNames)), new Set(RESERVED_PARAMETER_NAMES)).size !== 0) {
|
|
4259
4405
|
throw new UnexpectedError(spaceTrim.spaceTrim((block) => `
|
|
4260
4406
|
Dependent parameters are not consistent with used parameters:
|
|
4261
4407
|
|
|
@@ -4273,13 +4419,16 @@
|
|
|
4273
4419
|
|
|
4274
4420
|
`));
|
|
4275
4421
|
}
|
|
4422
|
+
const reservedParameters = await getReservedParametersForTask({
|
|
4423
|
+
tools,
|
|
4424
|
+
preparedPipeline,
|
|
4425
|
+
task: currentTask,
|
|
4426
|
+
pipelineIdentification,
|
|
4427
|
+
parameters: parametersToPass,
|
|
4428
|
+
isVerbose,
|
|
4429
|
+
});
|
|
4276
4430
|
const definedParameters = Object.freeze({
|
|
4277
|
-
...
|
|
4278
|
-
tools,
|
|
4279
|
-
preparedPipeline,
|
|
4280
|
-
task: currentTask,
|
|
4281
|
-
pipelineIdentification,
|
|
4282
|
-
})),
|
|
4431
|
+
...reservedParameters,
|
|
4283
4432
|
...parametersToPass,
|
|
4284
4433
|
});
|
|
4285
4434
|
const definedParameterNames = new Set(Object.keys(definedParameters));
|
|
@@ -4324,6 +4473,7 @@
|
|
|
4324
4473
|
preparedPipeline,
|
|
4325
4474
|
tools,
|
|
4326
4475
|
$executionReport,
|
|
4476
|
+
onProgress,
|
|
4327
4477
|
pipelineIdentification,
|
|
4328
4478
|
maxExecutionAttempts,
|
|
4329
4479
|
maxParallelCount,
|
|
@@ -4351,7 +4501,8 @@
|
|
|
4351
4501
|
*/
|
|
4352
4502
|
|
|
4353
4503
|
/**
|
|
4354
|
-
*
|
|
4504
|
+
* Filters and returns only the output parameters from the provided pipeline execution options.
|
|
4505
|
+
* Adds warnings for any expected output parameters that are missing.
|
|
4355
4506
|
*
|
|
4356
4507
|
* @private internal utility of `createPipelineExecutor`
|
|
4357
4508
|
*/
|
|
@@ -4376,9 +4527,12 @@
|
|
|
4376
4527
|
}
|
|
4377
4528
|
|
|
4378
4529
|
/**
|
|
4379
|
-
*
|
|
4530
|
+
* Executes an entire pipeline, resolving tasks in dependency order, handling errors, and reporting progress.
|
|
4531
|
+
*
|
|
4532
|
+
* Note: This is not a `PipelineExecutor` (which is bound to a single pipeline), but a utility function used by `createPipelineExecutor` to create a `PipelineExecutor`.
|
|
4380
4533
|
*
|
|
4381
|
-
*
|
|
4534
|
+
* @param options - Options for execution, including input parameters, pipeline, and callbacks.
|
|
4535
|
+
* @returns The result of the pipeline execution, including output parameters, errors, and usage statistics.
|
|
4382
4536
|
*
|
|
4383
4537
|
* @private internal utility of `createPipelineExecutor`
|
|
4384
4538
|
*/
|
|
@@ -4701,10 +4855,27 @@
|
|
|
4701
4855
|
cacheDirname,
|
|
4702
4856
|
intermediateFilesStrategy,
|
|
4703
4857
|
isAutoInstalled,
|
|
4858
|
+
}).catch((error) => {
|
|
4859
|
+
assertsError(error);
|
|
4860
|
+
return exportJson({
|
|
4861
|
+
name: 'pipelineExecutorResult',
|
|
4862
|
+
message: `Unuccessful PipelineExecutorResult, last catch`,
|
|
4863
|
+
order: [],
|
|
4864
|
+
value: {
|
|
4865
|
+
isSuccessful: false,
|
|
4866
|
+
errors: [serializeError(error)],
|
|
4867
|
+
warnings: [],
|
|
4868
|
+
usage: UNCERTAIN_USAGE,
|
|
4869
|
+
executionReport: null,
|
|
4870
|
+
outputParameters: {},
|
|
4871
|
+
preparedPipeline,
|
|
4872
|
+
},
|
|
4873
|
+
});
|
|
4704
4874
|
});
|
|
4705
4875
|
};
|
|
4706
4876
|
const pipelineExecutor = (inputParameters) => createTask({
|
|
4707
4877
|
taskType: 'EXECUTION',
|
|
4878
|
+
title: pipeline.title,
|
|
4708
4879
|
taskProcessCallback(updateOngoingResult) {
|
|
4709
4880
|
return pipelineExecutorWithCallback(inputParameters, async (newOngoingResult) => {
|
|
4710
4881
|
updateOngoingResult(newOngoingResult);
|
|
@@ -4756,12 +4927,14 @@
|
|
|
4756
4927
|
const spending = new rxjs.Subject();
|
|
4757
4928
|
const proxyTools = {
|
|
4758
4929
|
get title() {
|
|
4759
|
-
|
|
4760
|
-
|
|
4930
|
+
return `${llmTools.title} (+usage)`;
|
|
4931
|
+
// <- TODO: [🧈] Maybe standartize the suffix when wrapping `LlmExecutionTools` up
|
|
4932
|
+
// <- TODO: [🧈][🧠] Does it make sence to suffix "(+usage)"?
|
|
4761
4933
|
},
|
|
4762
4934
|
get description() {
|
|
4763
|
-
|
|
4764
|
-
|
|
4935
|
+
return `${llmTools.description} (+usage)`;
|
|
4936
|
+
// <- TODO: [🧈] Maybe standartize the suffix when wrapping `LlmExecutionTools` up
|
|
4937
|
+
// <- TODO: [🧈][🧠] Does it make sence to suffix "(+usage)"?
|
|
4765
4938
|
},
|
|
4766
4939
|
checkConfiguration() {
|
|
4767
4940
|
return /* not await */ llmTools.checkConfiguration();
|
|
@@ -4884,7 +5057,8 @@
|
|
|
4884
5057
|
*/
|
|
4885
5058
|
|
|
4886
5059
|
/**
|
|
4887
|
-
*
|
|
5060
|
+
* Safely retrieves the global scope object (window in browser, global in Node.js)
|
|
5061
|
+
* regardless of the JavaScript environment in which the code is running
|
|
4888
5062
|
*
|
|
4889
5063
|
* Note: `$` is used to indicate that this function is not a pure function - it access global scope
|
|
4890
5064
|
*
|
|
@@ -4895,10 +5069,10 @@
|
|
|
4895
5069
|
}
|
|
4896
5070
|
|
|
4897
5071
|
/**
|
|
4898
|
-
*
|
|
5072
|
+
* Normalizes a text string to SCREAMING_CASE (all uppercase with underscores).
|
|
4899
5073
|
*
|
|
4900
|
-
* @param text
|
|
4901
|
-
* @returns
|
|
5074
|
+
* @param text The text string to be converted to SCREAMING_CASE format.
|
|
5075
|
+
* @returns The normalized text in SCREAMING_CASE format.
|
|
4902
5076
|
* @example 'HELLO_WORLD'
|
|
4903
5077
|
* @example 'I_LOVE_PROMPTBOOK'
|
|
4904
5078
|
* @public exported from `@promptbook/utils`
|
|
@@ -4950,10 +5124,10 @@
|
|
|
4950
5124
|
*/
|
|
4951
5125
|
|
|
4952
5126
|
/**
|
|
4953
|
-
*
|
|
5127
|
+
* Normalizes a text string to snake_case format.
|
|
4954
5128
|
*
|
|
4955
|
-
* @param text
|
|
4956
|
-
* @returns
|
|
5129
|
+
* @param text The text string to be converted to snake_case format.
|
|
5130
|
+
* @returns The normalized text in snake_case format.
|
|
4957
5131
|
* @example 'hello_world'
|
|
4958
5132
|
* @example 'i_love_promptbook'
|
|
4959
5133
|
* @public exported from `@promptbook/utils`
|
|
@@ -4963,11 +5137,11 @@
|
|
|
4963
5137
|
}
|
|
4964
5138
|
|
|
4965
5139
|
/**
|
|
4966
|
-
*
|
|
5140
|
+
* Global registry for storing and managing registered entities of a given type.
|
|
4967
5141
|
*
|
|
4968
5142
|
* Note: `$` is used to indicate that this function is not a pure function - it accesses and adds variables in global scope.
|
|
4969
5143
|
*
|
|
4970
|
-
* @private internal utility, exported are only
|
|
5144
|
+
* @private internal utility, exported are only singleton instances of this class
|
|
4971
5145
|
*/
|
|
4972
5146
|
class $Register {
|
|
4973
5147
|
constructor(registerName) {
|
|
@@ -5011,10 +5185,10 @@
|
|
|
5011
5185
|
}
|
|
5012
5186
|
|
|
5013
5187
|
/**
|
|
5014
|
-
*
|
|
5188
|
+
* Global registry for storing metadata about all available scrapers and converters.
|
|
5015
5189
|
*
|
|
5016
|
-
* Note: `$` is used to indicate that this interacts with the global scope
|
|
5017
|
-
* @singleton Only one instance of each register is created per build, but
|
|
5190
|
+
* Note: `$` is used to indicate that this interacts with the global scope.
|
|
5191
|
+
* @singleton Only one instance of each register is created per build, but there can be more in different contexts (e.g., tests).
|
|
5018
5192
|
* @public exported from `@promptbook/core`
|
|
5019
5193
|
*/
|
|
5020
5194
|
const $scrapersMetadataRegister = new $Register('scrapers_metadata');
|
|
@@ -5023,10 +5197,11 @@
|
|
|
5023
5197
|
*/
|
|
5024
5198
|
|
|
5025
5199
|
/**
|
|
5026
|
-
*
|
|
5200
|
+
* Registry for all available scrapers in the system.
|
|
5201
|
+
* Central point for registering and accessing different types of content scrapers.
|
|
5027
5202
|
*
|
|
5028
5203
|
* Note: `$` is used to indicate that this interacts with the global scope
|
|
5029
|
-
* @singleton Only one instance of each register is created per build, but
|
|
5204
|
+
* @singleton Only one instance of each register is created per build, but there can be more than one in different build modules
|
|
5030
5205
|
* @public exported from `@promptbook/core`
|
|
5031
5206
|
*/
|
|
5032
5207
|
const $scrapersRegister = new $Register('scraper_constructors');
|
|
@@ -5122,10 +5297,10 @@
|
|
|
5122
5297
|
*/
|
|
5123
5298
|
|
|
5124
5299
|
/**
|
|
5125
|
-
*
|
|
5300
|
+
* Converts a given text to kebab-case format.
|
|
5126
5301
|
*
|
|
5127
|
-
* @param text
|
|
5128
|
-
* @returns
|
|
5302
|
+
* @param text The text to be converted.
|
|
5303
|
+
* @returns The kebab-case formatted string.
|
|
5129
5304
|
* @example 'hello-world'
|
|
5130
5305
|
* @example 'i-love-promptbook'
|
|
5131
5306
|
* @public exported from `@promptbook/utils`
|
|
@@ -5194,7 +5369,8 @@
|
|
|
5194
5369
|
*/
|
|
5195
5370
|
|
|
5196
5371
|
/**
|
|
5197
|
-
*
|
|
5372
|
+
* Converts a name to a properly formatted subfolder path for cache storage.
|
|
5373
|
+
* Handles normalization and path formatting to create consistent cache directory structures.
|
|
5198
5374
|
*
|
|
5199
5375
|
* @private for `FileCacheStorage`
|
|
5200
5376
|
*/
|
|
@@ -5274,11 +5450,11 @@
|
|
|
5274
5450
|
}
|
|
5275
5451
|
|
|
5276
5452
|
/**
|
|
5277
|
-
*
|
|
5453
|
+
* Converts a title string into a normalized name.
|
|
5278
5454
|
*
|
|
5279
|
-
* @param value
|
|
5280
|
-
* @returns
|
|
5281
|
-
* @example
|
|
5455
|
+
* @param value The title string to be converted to a name.
|
|
5456
|
+
* @returns A normalized name derived from the input title.
|
|
5457
|
+
* @example 'Hello World!' -> 'hello-world'
|
|
5282
5458
|
* @public exported from `@promptbook/utils`
|
|
5283
5459
|
*/
|
|
5284
5460
|
function titleToName(value) {
|
|
@@ -5329,7 +5505,9 @@
|
|
|
5329
5505
|
*/
|
|
5330
5506
|
|
|
5331
5507
|
/**
|
|
5332
|
-
*
|
|
5508
|
+
* Factory function that creates a handler for processing knowledge sources.
|
|
5509
|
+
* Provides standardized processing of different types of knowledge sources
|
|
5510
|
+
* across various scraper implementations.
|
|
5333
5511
|
*
|
|
5334
5512
|
* @public exported from `@promptbook/core`
|
|
5335
5513
|
*/
|
|
@@ -5468,7 +5646,7 @@
|
|
|
5468
5646
|
}
|
|
5469
5647
|
|
|
5470
5648
|
/**
|
|
5471
|
-
* Prepares the
|
|
5649
|
+
* Prepares the knowledge pieces
|
|
5472
5650
|
*
|
|
5473
5651
|
* @see https://github.com/webgptorg/promptbook/discussions/41
|
|
5474
5652
|
* @public exported from `@promptbook/core`
|
|
@@ -5564,15 +5742,18 @@
|
|
|
5564
5742
|
* TODO: [🧊] In future one preparation can take data from previous preparation and save tokens and time
|
|
5565
5743
|
* Put `knowledgePieces` into `PrepareKnowledgeOptions`
|
|
5566
5744
|
* TODO: [🪂] More than max things can run in parallel by acident [1,[2a,2b,_],[3a,3b,_]]
|
|
5567
|
-
* TODO: [🧠][❎] Do here
|
|
5745
|
+
* TODO: [🧠][❎] Do here proper M:N mapping
|
|
5568
5746
|
* [x] One source can make multiple pieces
|
|
5569
5747
|
* [ ] One piece can have multiple sources
|
|
5570
5748
|
*/
|
|
5571
5749
|
|
|
5572
5750
|
/**
|
|
5573
|
-
*
|
|
5751
|
+
* Prepares tasks by adding knowledge to the prompt and ensuring all necessary parameters are included.
|
|
5574
5752
|
*
|
|
5575
|
-
* @
|
|
5753
|
+
* @param tasks Sequence of tasks that are chained together to form a pipeline
|
|
5754
|
+
* @returns A promise that resolves to the prepared tasks.
|
|
5755
|
+
*
|
|
5756
|
+
* @private internal utility of `preparePipeline`
|
|
5576
5757
|
*/
|
|
5577
5758
|
async function prepareTasks(pipeline, tools, options) {
|
|
5578
5759
|
const { maxParallelCount = DEFAULT_MAX_PARALLEL_COUNT } = options;
|
|
@@ -5610,7 +5791,7 @@
|
|
|
5610
5791
|
return { tasksPrepared };
|
|
5611
5792
|
}
|
|
5612
5793
|
/**
|
|
5613
|
-
* TODO: [😂] Adding knowledge should be convert to async high-level abstractions,
|
|
5794
|
+
* TODO: [😂] Adding knowledge should be convert to async high-level abstractions, similar thing with expectations to sync high-level abstractions
|
|
5614
5795
|
* TODO: [🧠] Add context to each task (if missing)
|
|
5615
5796
|
* TODO: [🧠] What is better name `prepareTask` or `prepareTaskAndParameters`
|
|
5616
5797
|
* TODO: [♨][main] !!3 Prepare index the examples and maybe tasks
|
|
@@ -5746,7 +5927,7 @@
|
|
|
5746
5927
|
order: ORDER_OF_PIPELINE_JSON,
|
|
5747
5928
|
value: {
|
|
5748
5929
|
...pipeline,
|
|
5749
|
-
// <- TODO:
|
|
5930
|
+
// <- TODO: Probably deeply clone the pipeline because `$exportJson` freezes the subobjects
|
|
5750
5931
|
title,
|
|
5751
5932
|
knowledgeSources: knowledgeSourcesPrepared,
|
|
5752
5933
|
knowledgePieces: knowledgePiecesPrepared,
|
|
@@ -6022,7 +6203,7 @@
|
|
|
6022
6203
|
throw new ParseError(`Task section and example section must end with return statement -> {parameterName}`);
|
|
6023
6204
|
};
|
|
6024
6205
|
if ($taskJson.content === undefined) {
|
|
6025
|
-
throw new UnexpectedError(`Content is missing in the taskJson -
|
|
6206
|
+
throw new UnexpectedError(`Content is missing in the taskJson - probably commands are applied in wrong order`);
|
|
6026
6207
|
}
|
|
6027
6208
|
if (command.taskType === 'EXAMPLE') {
|
|
6028
6209
|
expectResultingParameterName();
|
|
@@ -6090,7 +6271,7 @@
|
|
|
6090
6271
|
/**
|
|
6091
6272
|
* Parses the boilerplate command
|
|
6092
6273
|
*
|
|
6093
|
-
* Note:
|
|
6274
|
+
* Note: @@ This command is used as boilerplate for new commands - it should NOT be used in any `.book` file
|
|
6094
6275
|
*
|
|
6095
6276
|
* @see `documentationUrl` for more details
|
|
6096
6277
|
* @private within the commands folder
|
|
@@ -6478,11 +6659,11 @@
|
|
|
6478
6659
|
};
|
|
6479
6660
|
|
|
6480
6661
|
/**
|
|
6481
|
-
*
|
|
6662
|
+
* Normalizes a given text to camelCase format.
|
|
6482
6663
|
*
|
|
6483
|
-
* @param text
|
|
6484
|
-
* @param _isFirstLetterCapital
|
|
6485
|
-
* @returns
|
|
6664
|
+
* @param text The text to be normalized.
|
|
6665
|
+
* @param _isFirstLetterCapital Whether the first letter should be capitalized.
|
|
6666
|
+
* @returns The camelCase formatted string.
|
|
6486
6667
|
* @example 'helloWorld'
|
|
6487
6668
|
* @example 'iLovePromptbook'
|
|
6488
6669
|
* @public exported from `@promptbook/utils`
|
|
@@ -6532,9 +6713,9 @@
|
|
|
6532
6713
|
/**
|
|
6533
6714
|
* Removes quotes from a string
|
|
6534
6715
|
*
|
|
6535
|
-
* Tip: This is very
|
|
6716
|
+
* Tip: This is very useful for post-processing of the result of the LLM model
|
|
6536
6717
|
* Note: This function removes only the same quotes from the beginning and the end of the string
|
|
6537
|
-
* Note: There are two
|
|
6718
|
+
* Note: There are two similar functions:
|
|
6538
6719
|
* - `removeQuotes` which removes only bounding quotes
|
|
6539
6720
|
* - `unwrapResult` which removes whole introduce sentence
|
|
6540
6721
|
*
|
|
@@ -6546,18 +6727,19 @@
|
|
|
6546
6727
|
if (text.startsWith('"') && text.endsWith('"')) {
|
|
6547
6728
|
return text.slice(1, -1);
|
|
6548
6729
|
}
|
|
6549
|
-
if (text.startsWith('
|
|
6730
|
+
if (text.startsWith("'") && text.endsWith("'")) {
|
|
6550
6731
|
return text.slice(1, -1);
|
|
6551
6732
|
}
|
|
6552
6733
|
return text;
|
|
6553
6734
|
}
|
|
6554
6735
|
|
|
6555
6736
|
/**
|
|
6556
|
-
* Function `validateParameterName` will
|
|
6737
|
+
* Function `validateParameterName` will normalize and validate a parameter name for use in pipelines.
|
|
6738
|
+
* It removes diacritics, emojis, and quotes, normalizes to camelCase, and checks for reserved names and invalid characters.
|
|
6557
6739
|
*
|
|
6558
|
-
* @param parameterName
|
|
6559
|
-
* @returns
|
|
6560
|
-
* @throws {ParseError}
|
|
6740
|
+
* @param parameterName The parameter name to validate and normalize.
|
|
6741
|
+
* @returns The validated and normalized parameter name.
|
|
6742
|
+
* @throws {ParseError} If the parameter name is empty, reserved, or contains invalid characters.
|
|
6561
6743
|
* @private within the repository
|
|
6562
6744
|
*/
|
|
6563
6745
|
function validateParameterName(parameterName) {
|
|
@@ -6627,8 +6809,6 @@
|
|
|
6627
6809
|
/**
|
|
6628
6810
|
* Parses the foreach command
|
|
6629
6811
|
*
|
|
6630
|
-
* Note: @@@ This command is used as foreach for new commands - it should NOT be used in any `.book` file
|
|
6631
|
-
*
|
|
6632
6812
|
* @see `documentationUrl` for more details
|
|
6633
6813
|
* @public exported from `@promptbook/editable`
|
|
6634
6814
|
*/
|
|
@@ -6685,14 +6865,14 @@
|
|
|
6685
6865
|
`));
|
|
6686
6866
|
// <- TODO: [🏢] List all supported format names
|
|
6687
6867
|
}
|
|
6688
|
-
const
|
|
6689
|
-
if (
|
|
6868
|
+
const subvalueParser = formatDefinition.subvalueParsers.find((subvalueParser) => [subvalueParser.subvalueName, ...(subvalueParser.aliases || [])].includes(subformatName));
|
|
6869
|
+
if (subvalueParser === undefined) {
|
|
6690
6870
|
throw new ParseError(spaceTrim__default["default"]((block) => `
|
|
6691
6871
|
Unsupported subformat name "${subformatName}" for format "${formatName}"
|
|
6692
6872
|
|
|
6693
6873
|
Available subformat names for format "${formatDefinition.formatName}":
|
|
6694
|
-
${block(formatDefinition.
|
|
6695
|
-
.map((
|
|
6874
|
+
${block(formatDefinition.subvalueParsers
|
|
6875
|
+
.map((subvalueParser) => subvalueParser.subvalueName)
|
|
6696
6876
|
.map((subvalueName) => `- ${subvalueName}`)
|
|
6697
6877
|
.join('\n'))}
|
|
6698
6878
|
`));
|
|
@@ -6869,14 +7049,14 @@
|
|
|
6869
7049
|
};
|
|
6870
7050
|
|
|
6871
7051
|
/**
|
|
6872
|
-
*
|
|
7052
|
+
* Chatbot form factor definition for conversational interfaces that interact with users in a chat-like manner.
|
|
6873
7053
|
*
|
|
6874
7054
|
* @public exported from `@promptbook/core`
|
|
6875
7055
|
*/
|
|
6876
7056
|
const ChatbotFormfactorDefinition = {
|
|
6877
7057
|
name: 'CHATBOT',
|
|
6878
7058
|
aliasNames: ['CHAT'],
|
|
6879
|
-
description:
|
|
7059
|
+
description: `A chatbot form factor for conversational user interfaces.`,
|
|
6880
7060
|
documentationUrl: `https://github.com/webgptorg/promptbook/discussions/174`,
|
|
6881
7061
|
pipelineInterface: {
|
|
6882
7062
|
inputParameters: [
|
|
@@ -6903,7 +7083,45 @@
|
|
|
6903
7083
|
};
|
|
6904
7084
|
|
|
6905
7085
|
/**
|
|
6906
|
-
*
|
|
7086
|
+
* Completion is formfactor that emulates completion models
|
|
7087
|
+
*
|
|
7088
|
+
* @public exported from `@promptbook/core`
|
|
7089
|
+
*/
|
|
7090
|
+
const CompletionFormfactorDefinition = {
|
|
7091
|
+
name: 'COMPLETION',
|
|
7092
|
+
description: `Completion is formfactor that emulates completion models`,
|
|
7093
|
+
documentationUrl: `https://github.com/webgptorg/promptbook/discussions/@@`,
|
|
7094
|
+
// <- TODO: https://github.com/webgptorg/promptbook/discussions/new?category=concepts
|
|
7095
|
+
// "🔠 Completion Formfactor"
|
|
7096
|
+
pipelineInterface: {
|
|
7097
|
+
inputParameters: [
|
|
7098
|
+
{
|
|
7099
|
+
name: 'inputText',
|
|
7100
|
+
description: `Input text to be completed`,
|
|
7101
|
+
isInput: true,
|
|
7102
|
+
isOutput: false,
|
|
7103
|
+
},
|
|
7104
|
+
{
|
|
7105
|
+
name: 'instructions',
|
|
7106
|
+
description: `Additional instructions for the model, for example the required length, empty by default`,
|
|
7107
|
+
isInput: true,
|
|
7108
|
+
isOutput: false,
|
|
7109
|
+
},
|
|
7110
|
+
],
|
|
7111
|
+
outputParameters: [
|
|
7112
|
+
{
|
|
7113
|
+
name: 'followingText',
|
|
7114
|
+
description: `Text that follows the input text`,
|
|
7115
|
+
isInput: false,
|
|
7116
|
+
isOutput: true,
|
|
7117
|
+
},
|
|
7118
|
+
],
|
|
7119
|
+
},
|
|
7120
|
+
};
|
|
7121
|
+
|
|
7122
|
+
/**
|
|
7123
|
+
* Generator form factor represents an application that generates content or data based on user input or predefined rules.
|
|
7124
|
+
* This form factor is used for apps that produce outputs, such as text, images, or other media, based on provided input.
|
|
6907
7125
|
*
|
|
6908
7126
|
* @public exported from `@promptbook/core`
|
|
6909
7127
|
*/
|
|
@@ -6932,7 +7150,7 @@
|
|
|
6932
7150
|
};
|
|
6933
7151
|
|
|
6934
7152
|
/**
|
|
6935
|
-
*
|
|
7153
|
+
* Pipeline interface which is equivalent to `any`
|
|
6936
7154
|
*
|
|
6937
7155
|
* @see https://github.com/webgptorg/promptbook/discussions/171
|
|
6938
7156
|
*
|
|
@@ -6947,13 +7165,13 @@
|
|
|
6947
7165
|
*/
|
|
6948
7166
|
|
|
6949
7167
|
/**
|
|
6950
|
-
*
|
|
7168
|
+
* A generic pipeline
|
|
6951
7169
|
*
|
|
6952
7170
|
* @public exported from `@promptbook/core`
|
|
6953
7171
|
*/
|
|
6954
7172
|
const GenericFormfactorDefinition = {
|
|
6955
7173
|
name: 'GENERIC',
|
|
6956
|
-
description:
|
|
7174
|
+
description: `A generic pipeline`,
|
|
6957
7175
|
documentationUrl: `https://github.com/webgptorg/promptbook/discussions/173`,
|
|
6958
7176
|
pipelineInterface: GENERIC_PIPELINE_INTERFACE,
|
|
6959
7177
|
};
|
|
@@ -6988,17 +7206,20 @@
|
|
|
6988
7206
|
};
|
|
6989
7207
|
|
|
6990
7208
|
/**
|
|
6991
|
-
* Matcher is form of app that
|
|
7209
|
+
* Matcher is form of app that evaluates (spreadsheet) content against defined criteria or patterns,
|
|
7210
|
+
* determining if it matches or meets specific requirements. Used for classification,
|
|
7211
|
+
* validation, filtering, and quality assessment of inputs.
|
|
6992
7212
|
*
|
|
6993
7213
|
* @public exported from `@promptbook/core`
|
|
6994
7214
|
*/
|
|
6995
7215
|
const MatcherFormfactorDefinition = {
|
|
6996
7216
|
name: 'EXPERIMENTAL_MATCHER',
|
|
6997
|
-
description:
|
|
7217
|
+
description: `An evaluation system that determines whether content meets specific criteria or patterns.
|
|
7218
|
+
Used for content validation, quality assessment, and intelligent filtering tasks. Currently in experimental phase.`,
|
|
6998
7219
|
documentationUrl: `https://github.com/webgptorg/promptbook/discussions/177`,
|
|
6999
7220
|
pipelineInterface: {
|
|
7000
7221
|
inputParameters: [
|
|
7001
|
-
/*
|
|
7222
|
+
/* Input parameters for content to be matched and criteria to match against */
|
|
7002
7223
|
{
|
|
7003
7224
|
name: 'nonce',
|
|
7004
7225
|
description: 'Just to prevent EXPERIMENTAL_MATCHER to be set as implicit formfactor',
|
|
@@ -7007,20 +7228,21 @@
|
|
|
7007
7228
|
},
|
|
7008
7229
|
],
|
|
7009
7230
|
outputParameters: [
|
|
7010
|
-
/*
|
|
7231
|
+
/* Output parameters containing match results, confidence scores, and relevant metadata */
|
|
7011
7232
|
],
|
|
7012
7233
|
},
|
|
7013
7234
|
};
|
|
7014
7235
|
|
|
7015
7236
|
/**
|
|
7016
|
-
* Sheets is form of app that
|
|
7237
|
+
* Sheets is form of app that processes tabular data in CSV format, allowing transformation
|
|
7238
|
+
* and analysis of structured data through AI-powered operations
|
|
7017
7239
|
*
|
|
7018
7240
|
* @public exported from `@promptbook/core`
|
|
7019
7241
|
*/
|
|
7020
7242
|
const SheetsFormfactorDefinition = {
|
|
7021
7243
|
name: 'SHEETS',
|
|
7022
7244
|
aliasNames: ['SHEETS', 'SHEET'],
|
|
7023
|
-
description:
|
|
7245
|
+
description: `A formfactor for processing spreadsheet-like data in CSV format, enabling AI transformations on tabular data`,
|
|
7024
7246
|
documentationUrl: `https://github.com/webgptorg/promptbook/discussions/176`,
|
|
7025
7247
|
pipelineInterface: {
|
|
7026
7248
|
inputParameters: [
|
|
@@ -7043,13 +7265,16 @@
|
|
|
7043
7265
|
};
|
|
7044
7266
|
|
|
7045
7267
|
/**
|
|
7046
|
-
* Translator is form of app that
|
|
7268
|
+
* Translator is form of app that transforms input text from one form to another,
|
|
7269
|
+
* such as language translation, style conversion, tone modification, or other text transformations.
|
|
7047
7270
|
*
|
|
7048
7271
|
* @public exported from `@promptbook/core`
|
|
7049
7272
|
*/
|
|
7050
7273
|
const TranslatorFormfactorDefinition = {
|
|
7051
7274
|
name: 'TRANSLATOR',
|
|
7052
|
-
description:
|
|
7275
|
+
description: `A text transformation system that converts input content into different forms,
|
|
7276
|
+
including language translations, paraphrasing, style conversions, and tone adjustments.
|
|
7277
|
+
This form factor takes one input and produces one transformed output.`,
|
|
7053
7278
|
documentationUrl: `https://github.com/webgptorg/promptbook/discussions/175`,
|
|
7054
7279
|
pipelineInterface: {
|
|
7055
7280
|
inputParameters: [
|
|
@@ -7086,6 +7311,8 @@
|
|
|
7086
7311
|
MatcherFormfactorDefinition,
|
|
7087
7312
|
GeneratorFormfactorDefinition,
|
|
7088
7313
|
ImageGeneratorFormfactorDefinition,
|
|
7314
|
+
CompletionFormfactorDefinition,
|
|
7315
|
+
// <- [🛬] When making new formfactor, copy the _boilerplate and link it here
|
|
7089
7316
|
];
|
|
7090
7317
|
/**
|
|
7091
7318
|
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
@@ -7094,7 +7321,7 @@
|
|
|
7094
7321
|
/**
|
|
7095
7322
|
* Parses the formfactor command
|
|
7096
7323
|
*
|
|
7097
|
-
* Note:
|
|
7324
|
+
* Note: This command is used as a formfactor for new commands and defines the app type format - it should NOT be used in any `.book` file
|
|
7098
7325
|
*
|
|
7099
7326
|
* @see `documentationUrl` for more details
|
|
7100
7327
|
* @public exported from `@promptbook/editable`
|
|
@@ -7116,7 +7343,7 @@
|
|
|
7116
7343
|
/**
|
|
7117
7344
|
* Description of the FORMFACTOR command
|
|
7118
7345
|
*/
|
|
7119
|
-
description:
|
|
7346
|
+
description: `Specifies the application type and interface requirements that this promptbook should conform to`,
|
|
7120
7347
|
/**
|
|
7121
7348
|
* Link to documentation
|
|
7122
7349
|
*/
|
|
@@ -7259,8 +7486,7 @@
|
|
|
7259
7486
|
};
|
|
7260
7487
|
|
|
7261
7488
|
/**
|
|
7262
|
-
*
|
|
7263
|
-
*
|
|
7489
|
+
* @see {@link ModelVariant}
|
|
7264
7490
|
* @public exported from `@promptbook/core`
|
|
7265
7491
|
*/
|
|
7266
7492
|
const MODEL_VARIANTS = ['COMPLETION', 'CHAT', 'EMBEDDING' /* <- TODO [🏳] */ /* <- [🤖] */];
|
|
@@ -7692,10 +7918,10 @@
|
|
|
7692
7918
|
}
|
|
7693
7919
|
|
|
7694
7920
|
/**
|
|
7695
|
-
*
|
|
7921
|
+
* Checks if the given value is a valid JavaScript identifier name.
|
|
7696
7922
|
*
|
|
7697
|
-
* @param javascriptName
|
|
7698
|
-
* @returns
|
|
7923
|
+
* @param javascriptName The value to check for JavaScript identifier validity.
|
|
7924
|
+
* @returns `true` if the value is a valid JavaScript name, false otherwise.
|
|
7699
7925
|
* @public exported from `@promptbook/utils`
|
|
7700
7926
|
*/
|
|
7701
7927
|
function isValidJavascriptName(javascriptName) {
|
|
@@ -8156,7 +8382,7 @@
|
|
|
8156
8382
|
// Arg1 Arg2 Arg3 | FOO
|
|
8157
8383
|
{
|
|
8158
8384
|
const commandNameRaw = items.slice(-1).join('_');
|
|
8159
|
-
const args = items.slice(0, -1); // <- Note: This is
|
|
8385
|
+
const args = items.slice(0, -1); // <- Note: This is probably not correct
|
|
8160
8386
|
const rawArgs = raw
|
|
8161
8387
|
.substring(0, raw.length - commandNameRaw.length)
|
|
8162
8388
|
.trim();
|
|
@@ -8175,7 +8401,10 @@
|
|
|
8175
8401
|
`));
|
|
8176
8402
|
}
|
|
8177
8403
|
/**
|
|
8178
|
-
*
|
|
8404
|
+
* Generates a markdown-formatted message listing all supported commands
|
|
8405
|
+
* with their descriptions and documentation links
|
|
8406
|
+
*
|
|
8407
|
+
* @returns A formatted markdown string containing all available commands and their details
|
|
8179
8408
|
*/
|
|
8180
8409
|
function getSupportedCommandsMessage() {
|
|
8181
8410
|
return COMMANDS.flatMap(({ name, aliasNames, description, documentationUrl }) =>
|
|
@@ -8186,7 +8415,10 @@
|
|
|
8186
8415
|
]).join('\n');
|
|
8187
8416
|
}
|
|
8188
8417
|
/**
|
|
8189
|
-
*
|
|
8418
|
+
* Attempts to parse a command variant using the provided input parameters
|
|
8419
|
+
*
|
|
8420
|
+
* @param input Object containing command parsing information including raw command text and normalized values
|
|
8421
|
+
* @returns A parsed Command object if successful, or null if the command cannot be parsed
|
|
8190
8422
|
*/
|
|
8191
8423
|
function parseCommandVariant(input) {
|
|
8192
8424
|
const { commandNameRaw, usagePlace, normalized, args, raw, rawArgs } = input;
|
|
@@ -8233,7 +8465,7 @@
|
|
|
8233
8465
|
}
|
|
8234
8466
|
|
|
8235
8467
|
/**
|
|
8236
|
-
*
|
|
8468
|
+
* Extracts the interface (input and output parameters) from a pipeline.
|
|
8237
8469
|
*
|
|
8238
8470
|
* @deprecated https://github.com/webgptorg/promptbook/pull/186
|
|
8239
8471
|
* @see https://github.com/webgptorg/promptbook/discussions/171
|
|
@@ -8266,7 +8498,7 @@
|
|
|
8266
8498
|
}
|
|
8267
8499
|
|
|
8268
8500
|
/**
|
|
8269
|
-
*
|
|
8501
|
+
* Checks if two pipeline interfaces are structurally identical.
|
|
8270
8502
|
*
|
|
8271
8503
|
* @deprecated https://github.com/webgptorg/promptbook/pull/186
|
|
8272
8504
|
* @see https://github.com/webgptorg/promptbook/discussions/171
|
|
@@ -8298,10 +8530,11 @@
|
|
|
8298
8530
|
}
|
|
8299
8531
|
|
|
8300
8532
|
/**
|
|
8301
|
-
*
|
|
8533
|
+
* Checks if a given pipeline satisfies the requirements of a specified pipeline interface.
|
|
8302
8534
|
*
|
|
8303
8535
|
* @deprecated https://github.com/webgptorg/promptbook/pull/186
|
|
8304
8536
|
* @see https://github.com/webgptorg/promptbook/discussions/171
|
|
8537
|
+
* @returns `true` if the pipeline implements the interface, `false` otherwise.
|
|
8305
8538
|
*
|
|
8306
8539
|
* @public exported from `@promptbook/core`
|
|
8307
8540
|
*/
|
|
@@ -8487,7 +8720,8 @@
|
|
|
8487
8720
|
}
|
|
8488
8721
|
|
|
8489
8722
|
/**
|
|
8490
|
-
*
|
|
8723
|
+
* Utility to determine if a pipeline string is in flat format.
|
|
8724
|
+
* A flat pipeline is a simple text without proper structure (headers, blocks, etc).
|
|
8491
8725
|
*
|
|
8492
8726
|
* @public exported from `@promptbook/editable`
|
|
8493
8727
|
*/
|
|
@@ -8508,7 +8742,10 @@
|
|
|
8508
8742
|
}
|
|
8509
8743
|
|
|
8510
8744
|
/**
|
|
8511
|
-
*
|
|
8745
|
+
* Converts a pipeline structure to its string representation.
|
|
8746
|
+
*
|
|
8747
|
+
* Transforms a flat, simple pipeline into a properly formatted pipeline string
|
|
8748
|
+
* with sections for title, prompt, and return statement.
|
|
8512
8749
|
*
|
|
8513
8750
|
* @public exported from `@promptbook/editable`
|
|
8514
8751
|
*/
|
|
@@ -8565,7 +8802,7 @@
|
|
|
8565
8802
|
* Note: It can not work with html syntax and comments
|
|
8566
8803
|
*
|
|
8567
8804
|
* @param markdown any valid markdown
|
|
8568
|
-
* @returns
|
|
8805
|
+
* @returns An array of strings, each representing an individual list item found in the markdown
|
|
8569
8806
|
* @public exported from `@promptbook/markdown-utils`
|
|
8570
8807
|
*/
|
|
8571
8808
|
function extractAllListItemsFromMarkdown(markdown) {
|
|
@@ -8590,8 +8827,8 @@
|
|
|
8590
8827
|
*
|
|
8591
8828
|
* - When there are multiple or no code blocks the function throws a `ParseError`
|
|
8592
8829
|
*
|
|
8593
|
-
* Note: There are multiple
|
|
8594
|
-
* - `extractBlock` just extracts the content of the code block which is also used as
|
|
8830
|
+
* Note: There are multiple similar functions:
|
|
8831
|
+
* - `extractBlock` just extracts the content of the code block which is also used as built-in function for postprocessing
|
|
8595
8832
|
* - `extractJsonBlock` extracts exactly one valid JSON code block
|
|
8596
8833
|
* - `extractOneBlockFromMarkdown` extracts exactly one code block with language of the code block
|
|
8597
8834
|
* - `extractAllBlocksFromMarkdown` extracts all code blocks with language of the code block
|
|
@@ -8746,7 +8983,7 @@
|
|
|
8746
8983
|
* Compile pipeline from string (markdown) format to JSON format synchronously
|
|
8747
8984
|
*
|
|
8748
8985
|
* Note: There are 3 similar functions:
|
|
8749
|
-
* - `compilePipeline` **(preferred)** - which
|
|
8986
|
+
* - `compilePipeline` **(preferred)** - which properly compiles the promptbook and uses embedding for external knowledge
|
|
8750
8987
|
* - `parsePipeline` - use only if you need to compile promptbook synchronously and it contains NO external knowledge
|
|
8751
8988
|
* - `preparePipeline` - just one step in the compilation process
|
|
8752
8989
|
*
|
|
@@ -9183,7 +9420,7 @@
|
|
|
9183
9420
|
* TODO: Use spaceTrim more effectively
|
|
9184
9421
|
* TODO: [🧠] Parameter flags - isInput, isOutput, isInternal
|
|
9185
9422
|
* TODO: [🥞] Not optimal parsing because `splitMarkdownIntoSections` is executed twice with same string, once through `flattenMarkdown` and second directly here
|
|
9186
|
-
* TODO: [♈]
|
|
9423
|
+
* TODO: [♈] Probably move expectations from tasks to parameters
|
|
9187
9424
|
* TODO: [🛠] Actions, instruments (and maybe knowledge) => Functions and tools
|
|
9188
9425
|
* TODO: [🍙] Make some standard order of json properties
|
|
9189
9426
|
*/
|
|
@@ -9400,7 +9637,8 @@
|
|
|
9400
9637
|
*/
|
|
9401
9638
|
|
|
9402
9639
|
/**
|
|
9403
|
-
*
|
|
9640
|
+
* Attempts to locate the specified application on a Linux system using the 'which' command.
|
|
9641
|
+
* Returns the path to the executable if found, or null otherwise.
|
|
9404
9642
|
*
|
|
9405
9643
|
* @private within the repository
|
|
9406
9644
|
*/
|
|
@@ -9420,7 +9658,8 @@
|
|
|
9420
9658
|
*/
|
|
9421
9659
|
|
|
9422
9660
|
/**
|
|
9423
|
-
*
|
|
9661
|
+
* Provides filesystem access (for example for Node.js-based scrapers)
|
|
9662
|
+
* Creates a standardized filesystem interface that scrapers can use for file operations.
|
|
9424
9663
|
*
|
|
9425
9664
|
* @public exported from `@promptbook/node`
|
|
9426
9665
|
*/
|
|
@@ -9466,7 +9705,8 @@
|
|
|
9466
9705
|
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
|
9467
9706
|
const userhome = require('userhome');
|
|
9468
9707
|
/**
|
|
9469
|
-
*
|
|
9708
|
+
* Attempts to locate the specified application on a macOS system by checking standard application paths and using mdfind.
|
|
9709
|
+
* Returns the path to the executable if found, or null otherwise.
|
|
9470
9710
|
*
|
|
9471
9711
|
* @private within the repository
|
|
9472
9712
|
*/
|
|
@@ -9498,7 +9738,8 @@
|
|
|
9498
9738
|
*/
|
|
9499
9739
|
|
|
9500
9740
|
/**
|
|
9501
|
-
*
|
|
9741
|
+
* Attempts to locate the specified application on a Windows system by searching common installation directories.
|
|
9742
|
+
* Returns the path to the executable if found, or null otherwise.
|
|
9502
9743
|
*
|
|
9503
9744
|
* @private within the repository
|
|
9504
9745
|
*/
|
|
@@ -9569,7 +9810,8 @@
|
|
|
9569
9810
|
*/
|
|
9570
9811
|
|
|
9571
9812
|
/**
|
|
9572
|
-
*
|
|
9813
|
+
* Locates the LibreOffice executable on the current system by searching platform-specific paths.
|
|
9814
|
+
* Returns the path to the executable if found, or null otherwise.
|
|
9573
9815
|
*
|
|
9574
9816
|
* @private within the repository
|
|
9575
9817
|
*/
|
|
@@ -9587,7 +9829,8 @@
|
|
|
9587
9829
|
*/
|
|
9588
9830
|
|
|
9589
9831
|
/**
|
|
9590
|
-
*
|
|
9832
|
+
* Locates the Pandoc executable on the current system by searching platform-specific paths.
|
|
9833
|
+
* Returns the path to the executable if found, or null otherwise.
|
|
9591
9834
|
*
|
|
9592
9835
|
* @private within the repository
|
|
9593
9836
|
*/
|
|
@@ -9605,7 +9848,7 @@
|
|
|
9605
9848
|
*/
|
|
9606
9849
|
|
|
9607
9850
|
/**
|
|
9608
|
-
*
|
|
9851
|
+
* Provides paths to required executables (i.e. as Pandoc and LibreOffice) for Node.js environments.
|
|
9609
9852
|
*
|
|
9610
9853
|
* @public exported from `@promptbook/node`
|
|
9611
9854
|
*/
|
|
@@ -9625,10 +9868,10 @@
|
|
|
9625
9868
|
*/
|
|
9626
9869
|
|
|
9627
9870
|
/**
|
|
9628
|
-
*
|
|
9871
|
+
* Register for LLM tools metadata.
|
|
9629
9872
|
*
|
|
9630
9873
|
* Note: `$` is used to indicate that this interacts with the global scope
|
|
9631
|
-
* @singleton Only one instance of each register is created per build, but
|
|
9874
|
+
* @singleton Only one instance of each register is created per build, but there can be more instances across different builds or environments.
|
|
9632
9875
|
* @public exported from `@promptbook/core`
|
|
9633
9876
|
*/
|
|
9634
9877
|
const $llmToolsMetadataRegister = new $Register('llm_tools_metadata');
|
|
@@ -9656,10 +9899,10 @@
|
|
|
9656
9899
|
*/
|
|
9657
9900
|
|
|
9658
9901
|
/**
|
|
9659
|
-
*
|
|
9902
|
+
* Register for LLM tools.
|
|
9660
9903
|
*
|
|
9661
9904
|
* Note: `$` is used to indicate that this interacts with the global scope
|
|
9662
|
-
* @singleton Only one instance of each register is created per build, but
|
|
9905
|
+
* @singleton Only one instance of each register is created per build, but there can be more instances across different builds or environments.
|
|
9663
9906
|
* @public exported from `@promptbook/core`
|
|
9664
9907
|
*/
|
|
9665
9908
|
const $llmToolsRegister = new $Register('llm_execution_tools_constructors');
|
|
@@ -9852,9 +10095,8 @@
|
|
|
9852
10095
|
*/
|
|
9853
10096
|
|
|
9854
10097
|
/**
|
|
9855
|
-
*
|
|
10098
|
+
* Provides LLM tools configuration by reading environment variables.
|
|
9856
10099
|
*
|
|
9857
|
-
* @@@ .env
|
|
9858
10100
|
* Note: `$` is used to indicate that this function is not a pure function - it uses filesystem to access `.env` file
|
|
9859
10101
|
*
|
|
9860
10102
|
* It looks for environment variables:
|
|
@@ -9862,7 +10104,8 @@
|
|
|
9862
10104
|
* - `process.env.ANTHROPIC_CLAUDE_API_KEY`
|
|
9863
10105
|
* - ...
|
|
9864
10106
|
*
|
|
9865
|
-
* @
|
|
10107
|
+
* @see Environment variables documentation or .env file for required variables.
|
|
10108
|
+
* @returns A promise that resolves to the LLM tools configuration, or null if configuration is incomplete or missing.
|
|
9866
10109
|
* @public exported from `@promptbook/node`
|
|
9867
10110
|
*/
|
|
9868
10111
|
async function $provideLlmToolsConfigurationFromEnv() {
|
|
@@ -9884,11 +10127,16 @@
|
|
|
9884
10127
|
*/
|
|
9885
10128
|
|
|
9886
10129
|
/**
|
|
9887
|
-
*
|
|
10130
|
+
* Creates LLM execution tools from provided configuration objects
|
|
10131
|
+
*
|
|
10132
|
+
* Instantiates and configures LLM tool instances for each configuration entry,
|
|
10133
|
+
* combining them into a unified interface via MultipleLlmExecutionTools.
|
|
9888
10134
|
*
|
|
9889
10135
|
* Note: This function is not cached, every call creates new instance of `MultipleLlmExecutionTools`
|
|
9890
10136
|
*
|
|
9891
|
-
* @
|
|
10137
|
+
* @param configuration Array of LLM tool configurations to instantiate
|
|
10138
|
+
* @param options Additional options for configuring the LLM tools
|
|
10139
|
+
* @returns A unified interface combining all successfully instantiated LLM tools
|
|
9892
10140
|
* @public exported from `@promptbook/core`
|
|
9893
10141
|
*/
|
|
9894
10142
|
function createLlmToolsFromConfiguration(configuration, options = {}) {
|
|
@@ -9927,7 +10175,11 @@
|
|
|
9927
10175
|
/**
|
|
9928
10176
|
* TODO: [🎌] Together with `createLlmToolsFromConfiguration` + 'EXECUTION_TOOLS_CLASSES' gets to `@promptbook/core` ALL model providers, make this more efficient
|
|
9929
10177
|
* TODO: [🧠][🎌] Dynamically install required providers
|
|
9930
|
-
* TODO:
|
|
10178
|
+
* TODO: We should implement an interactive configuration wizard that would:
|
|
10179
|
+
* 1. Detect which LLM providers are available in the environment
|
|
10180
|
+
* 2. Guide users through required configuration settings for each provider
|
|
10181
|
+
* 3. Allow testing connections before completing setup
|
|
10182
|
+
* 4. Generate appropriate configuration code for application integration
|
|
9931
10183
|
* TODO: [🧠][🍛] Which name is better `createLlmToolsFromConfig` or `createLlmToolsFromConfiguration`?
|
|
9932
10184
|
* TODO: [🧠] Is there some meaningfull way how to test this util
|
|
9933
10185
|
* TODO: This should be maybe not under `_common` but under `utils`
|
|
@@ -9935,11 +10187,14 @@
|
|
|
9935
10187
|
*/
|
|
9936
10188
|
|
|
9937
10189
|
/**
|
|
9938
|
-
*
|
|
10190
|
+
* Automatically configures LLM tools from environment variables in Node.js
|
|
10191
|
+
*
|
|
10192
|
+
* This utility function detects available LLM providers based on environment variables
|
|
10193
|
+
* and creates properly configured LLM execution tools for each detected provider.
|
|
9939
10194
|
*
|
|
9940
10195
|
* Note: This function is not cached, every call creates new instance of `MultipleLlmExecutionTools`
|
|
9941
10196
|
*
|
|
9942
|
-
*
|
|
10197
|
+
* Supports environment variables from .env files when dotenv is configured
|
|
9943
10198
|
* Note: `$` is used to indicate that this function is not a pure function - it uses filesystem to access `.env` file
|
|
9944
10199
|
*
|
|
9945
10200
|
* It looks for environment variables:
|
|
@@ -9947,7 +10202,8 @@
|
|
|
9947
10202
|
* - `process.env.ANTHROPIC_CLAUDE_API_KEY`
|
|
9948
10203
|
* - ...
|
|
9949
10204
|
*
|
|
9950
|
-
* @
|
|
10205
|
+
* @param options Configuration options for the LLM tools
|
|
10206
|
+
* @returns A unified interface containing all detected and configured LLM tools
|
|
9951
10207
|
* @public exported from `@promptbook/node`
|
|
9952
10208
|
*/
|
|
9953
10209
|
async function $provideLlmToolsFromEnv(options = {}) {
|
|
@@ -9973,7 +10229,16 @@
|
|
|
9973
10229
|
return createLlmToolsFromConfiguration(configuration, options);
|
|
9974
10230
|
}
|
|
9975
10231
|
/**
|
|
9976
|
-
* TODO:
|
|
10232
|
+
* TODO: The architecture for LLM tools configuration consists of three key functions:
|
|
10233
|
+
* 1. `$provideLlmToolsFromEnv` - High-level function that detects available providers from env vars and returns ready-to-use LLM tools
|
|
10234
|
+
* 2. `$provideLlmToolsConfigurationFromEnv` - Middle layer that extracts configuration objects from environment variables
|
|
10235
|
+
* 3. `createLlmToolsFromConfiguration` - Low-level function that instantiates LLM tools from explicit configuration
|
|
10236
|
+
*
|
|
10237
|
+
* This layered approach allows flexibility in how tools are configured:
|
|
10238
|
+
* - Use $provideLlmToolsFromEnv for automatic detection and setup in Node.js environments
|
|
10239
|
+
* - Use $provideLlmToolsConfigurationFromEnv to extract config objects for modification before instantiation
|
|
10240
|
+
* - Use createLlmToolsFromConfiguration for explicit control over tool configurations
|
|
10241
|
+
*
|
|
9977
10242
|
* TODO: [🧠][🍛] Which name is better `$provideLlmToolsFromEnv` or `$provideLlmToolsFromEnvironment`?
|
|
9978
10243
|
* TODO: [🧠] Is there some meaningfull way how to test this util
|
|
9979
10244
|
* Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
|
|
@@ -9983,11 +10248,9 @@
|
|
|
9983
10248
|
*/
|
|
9984
10249
|
|
|
9985
10250
|
/**
|
|
9986
|
-
*
|
|
9987
|
-
*
|
|
9988
|
-
*
|
|
9989
|
-
* 2) @@@
|
|
9990
|
-
*
|
|
10251
|
+
* Provides a collection of scrapers optimized for Node.js environment.
|
|
10252
|
+
* 1) `provideScrapersForNode` use as default
|
|
10253
|
+
* 2) `provideScrapersForBrowser` use in limited browser environment *
|
|
9991
10254
|
* @public exported from `@promptbook/node`
|
|
9992
10255
|
*/
|
|
9993
10256
|
async function $provideScrapersForNode(tools, options) {
|
|
@@ -10094,11 +10357,11 @@
|
|
|
10094
10357
|
}
|
|
10095
10358
|
|
|
10096
10359
|
/**
|
|
10097
|
-
*
|
|
10360
|
+
* Converts a name string into a URI-compatible format.
|
|
10098
10361
|
*
|
|
10099
|
-
* @param name
|
|
10100
|
-
* @returns
|
|
10101
|
-
* @example
|
|
10362
|
+
* @param name The string to be converted to a URI-compatible format.
|
|
10363
|
+
* @returns A URI-compatible string derived from the input name.
|
|
10364
|
+
* @example 'Hello World' -> 'hello-world'
|
|
10102
10365
|
* @public exported from `@promptbook/utils`
|
|
10103
10366
|
*/
|
|
10104
10367
|
function nameToUriPart(name) {
|
|
@@ -10112,11 +10375,11 @@
|
|
|
10112
10375
|
}
|
|
10113
10376
|
|
|
10114
10377
|
/**
|
|
10115
|
-
*
|
|
10378
|
+
* Converts a given name into URI-compatible parts.
|
|
10116
10379
|
*
|
|
10117
|
-
* @param name
|
|
10118
|
-
* @returns
|
|
10119
|
-
* @example
|
|
10380
|
+
* @param name The name to be converted into URI parts.
|
|
10381
|
+
* @returns An array of URI-compatible parts derived from the name.
|
|
10382
|
+
* @example 'Example Name' -> ['example', 'name']
|
|
10120
10383
|
* @public exported from `@promptbook/utils`
|
|
10121
10384
|
*/
|
|
10122
10385
|
function nameToUriParts(name) {
|
|
@@ -10183,7 +10446,7 @@
|
|
|
10183
10446
|
/**
|
|
10184
10447
|
* Function trimCodeBlock will trim starting and ending code block from the string if it is present.
|
|
10185
10448
|
*
|
|
10186
|
-
* Note: This is
|
|
10449
|
+
* Note: This is useful for post-processing of the result of the chat LLM model
|
|
10187
10450
|
* when the model wraps the result in the (markdown) code block.
|
|
10188
10451
|
*
|
|
10189
10452
|
* @public exported from `@promptbook/utils`
|
|
@@ -10202,7 +10465,7 @@
|
|
|
10202
10465
|
/**
|
|
10203
10466
|
* Function trimEndOfCodeBlock will remove ending code block from the string if it is present.
|
|
10204
10467
|
*
|
|
10205
|
-
* Note: This is
|
|
10468
|
+
* Note: This is useful for post-processing of the result of the completion LLM model
|
|
10206
10469
|
* if you want to start code block in the prompt but you don't want to end it in the result.
|
|
10207
10470
|
*
|
|
10208
10471
|
* @public exported from `@promptbook/utils`
|
|
@@ -10217,9 +10480,9 @@
|
|
|
10217
10480
|
/**
|
|
10218
10481
|
* Removes quotes and optional introduce text from a string
|
|
10219
10482
|
*
|
|
10220
|
-
* Tip: This is very
|
|
10483
|
+
* Tip: This is very useful for post-processing of the result of the LLM model
|
|
10221
10484
|
* Note: This function trims the text and removes whole introduce sentence if it is present
|
|
10222
|
-
* Note: There are two
|
|
10485
|
+
* Note: There are two similar functions:
|
|
10223
10486
|
* - `removeQuotes` which removes only bounding quotes
|
|
10224
10487
|
* - `unwrapResult` which removes whole introduce sentence
|
|
10225
10488
|
*
|
|
@@ -10290,7 +10553,7 @@
|
|
|
10290
10553
|
*
|
|
10291
10554
|
* - When there are multiple or no code blocks the function throws a `ParseError`
|
|
10292
10555
|
*
|
|
10293
|
-
* Note: There are multiple
|
|
10556
|
+
* Note: There are multiple similar function:
|
|
10294
10557
|
* - `extractBlock` just extracts the content of the code block which is also used as build-in function for postprocessing
|
|
10295
10558
|
* - `extractJsonBlock` extracts exactly one valid JSON code block
|
|
10296
10559
|
* - `extractOneBlockFromMarkdown` extracts exactly one code block with language of the code block
|
|
@@ -10327,7 +10590,7 @@
|
|
|
10327
10590
|
})();
|
|
10328
10591
|
}
|
|
10329
10592
|
/**
|
|
10330
|
-
* TODO:
|
|
10593
|
+
* TODO: Probably remove in favour of `keepImported`
|
|
10331
10594
|
* TODO: [1] This maybe does memory leak
|
|
10332
10595
|
*/
|
|
10333
10596
|
|
|
@@ -10531,7 +10794,7 @@
|
|
|
10531
10794
|
*/
|
|
10532
10795
|
|
|
10533
10796
|
/**
|
|
10534
|
-
* Placeholder for better implementation of JavascriptExecutionTools - some
|
|
10797
|
+
* Placeholder for better implementation of JavascriptExecutionTools - some proper sandboxing
|
|
10535
10798
|
*
|
|
10536
10799
|
* @alias JavascriptExecutionTools
|
|
10537
10800
|
* @public exported from `@promptbook/javascript`
|
|
@@ -10916,14 +11179,17 @@
|
|
|
10916
11179
|
return pipelineJsonStringified;
|
|
10917
11180
|
}
|
|
10918
11181
|
/**
|
|
10919
|
-
* TODO: [🐝] Not Working
|
|
11182
|
+
* TODO: [🐝] Not Working properly @see https://promptbook.studio/examples/mixed-knowledge.book
|
|
10920
11183
|
* TODO: [🧠][0] Maybe rename to `stringifyPipelineJson`, `stringifyIndexedJson`,...
|
|
10921
11184
|
* TODO: [🧠] Maybe more elegant solution than replacing via regex
|
|
10922
11185
|
* TODO: [🍙] Make some standard order of json properties
|
|
10923
11186
|
*/
|
|
10924
11187
|
|
|
10925
11188
|
/**
|
|
10926
|
-
*
|
|
11189
|
+
* A storage implementation that caches data in files organized in a directory structure.
|
|
11190
|
+
* Provides methods for retrieving, storing, and managing cached data on the filesystem.
|
|
11191
|
+
*
|
|
11192
|
+
* This class implements the PromptbookStorage interface for filesystem-based caching.
|
|
10927
11193
|
*
|
|
10928
11194
|
* @public exported from `@promptbook/node`
|
|
10929
11195
|
*/
|
|
@@ -10936,7 +11202,8 @@
|
|
|
10936
11202
|
}
|
|
10937
11203
|
}
|
|
10938
11204
|
/**
|
|
10939
|
-
*
|
|
11205
|
+
* Converts a storage key to a filesystem path where the data should be stored.
|
|
11206
|
+
* Creates a consistent, deterministic file path based on the key string.
|
|
10940
11207
|
*/
|
|
10941
11208
|
getFilenameForKey(key) {
|
|
10942
11209
|
// TODO: [👬] DRY
|
|
@@ -10948,7 +11215,8 @@
|
|
|
10948
11215
|
...nameToSubfolderPath(hash /* <- TODO: [🎎] Maybe add some SHA256 prefix */), `${name.substring(0, MAX_FILENAME_LENGTH)}.json`);
|
|
10949
11216
|
}
|
|
10950
11217
|
/**
|
|
10951
|
-
*
|
|
11218
|
+
* Returns the current value associated with the given key, or null if the given key does not exist.
|
|
11219
|
+
* Retrieves the cached data from the file system storage.
|
|
10952
11220
|
*/
|
|
10953
11221
|
async getItem(key) {
|
|
10954
11222
|
const filename = this.getFilenameForKey(key);
|
|
@@ -10961,7 +11229,8 @@
|
|
|
10961
11229
|
return value;
|
|
10962
11230
|
}
|
|
10963
11231
|
/**
|
|
10964
|
-
*
|
|
11232
|
+
* Sets the value of the pair identified by key to value, creating a new key/value pair if none existed for key previously.
|
|
11233
|
+
* Persists data to the file system, creating necessary directory structure if it doesn't exist.
|
|
10965
11234
|
*/
|
|
10966
11235
|
async setItem(key, value) {
|
|
10967
11236
|
const filename = this.getFilenameForKey(key);
|
|
@@ -10973,7 +11242,8 @@
|
|
|
10973
11242
|
await promises.writeFile(filename, fileContent, 'utf-8');
|
|
10974
11243
|
}
|
|
10975
11244
|
/**
|
|
10976
|
-
*
|
|
11245
|
+
* Removes the key/value pair with the given key from the storage, if a key/value pair with the given key exists.
|
|
11246
|
+
* Deletes the corresponding file from the filesystem.
|
|
10977
11247
|
*/
|
|
10978
11248
|
async removeItem(key) {
|
|
10979
11249
|
const filename = this.getFilenameForKey(key);
|