@promptbook/node 0.92.0-8 → 0.92.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -5
- package/esm/index.es.js +801 -531
- package/esm/index.es.js.map +1 -1
- package/esm/typings/src/_packages/browser.index.d.ts +2 -0
- package/esm/typings/src/_packages/core.index.d.ts +26 -14
- package/esm/typings/src/_packages/types.index.d.ts +6 -2
- package/esm/typings/src/collection/PipelineCollection.d.ts +0 -2
- package/esm/typings/src/collection/SimplePipelineCollection.d.ts +1 -1
- package/esm/typings/src/commands/FOREACH/ForeachJson.d.ts +6 -6
- package/esm/typings/src/commands/FOREACH/foreachCommandParser.d.ts +0 -2
- package/esm/typings/src/commands/FORMFACTOR/formfactorCommandParser.d.ts +1 -1
- package/esm/typings/src/commands/_BOILERPLATE/boilerplateCommandParser.d.ts +1 -1
- package/esm/typings/src/commands/_common/types/CommandParser.d.ts +36 -28
- package/esm/typings/src/config.d.ts +41 -11
- package/esm/typings/src/constants.d.ts +43 -2
- package/esm/typings/src/conversion/parsePipeline.d.ts +2 -2
- package/esm/typings/src/errors/0-BoilerplateError.d.ts +2 -2
- package/esm/typings/src/errors/CollectionError.d.ts +1 -1
- package/esm/typings/src/executables/$provideExecutablesForNode.d.ts +1 -1
- package/esm/typings/src/executables/apps/locateLibreoffice.d.ts +2 -1
- package/esm/typings/src/executables/apps/locatePandoc.d.ts +2 -1
- package/esm/typings/src/executables/locateApp.d.ts +2 -2
- package/esm/typings/src/executables/platforms/locateAppOnLinux.d.ts +2 -1
- package/esm/typings/src/executables/platforms/locateAppOnMacOs.d.ts +2 -1
- package/esm/typings/src/executables/platforms/locateAppOnWindows.d.ts +2 -1
- package/esm/typings/src/execution/AbstractTaskResult.d.ts +1 -1
- package/esm/typings/src/execution/CommonToolsOptions.d.ts +5 -1
- package/esm/typings/src/execution/ExecutionTask.d.ts +19 -1
- package/esm/typings/src/execution/LlmExecutionToolsConstructor.d.ts +2 -1
- package/esm/typings/src/execution/PipelineExecutorResult.d.ts +4 -2
- package/esm/typings/src/execution/PromptbookFetch.d.ts +1 -1
- package/esm/typings/src/execution/ScriptExecutionTools.d.ts +1 -1
- package/esm/typings/src/execution/createPipelineExecutor/$OngoingTaskResult.d.ts +12 -9
- package/esm/typings/src/execution/createPipelineExecutor/10-executePipeline.d.ts +13 -10
- package/esm/typings/src/execution/createPipelineExecutor/20-executeTask.d.ts +12 -9
- package/esm/typings/src/execution/createPipelineExecutor/30-executeFormatSubvalues.d.ts +15 -3
- package/esm/typings/src/execution/createPipelineExecutor/40-executeAttempts.d.ts +21 -15
- package/esm/typings/src/execution/createPipelineExecutor/computeCosineSimilarity.d.ts +13 -0
- package/esm/typings/src/execution/createPipelineExecutor/filterJustOutputParameters.d.ts +7 -6
- package/esm/typings/src/execution/createPipelineExecutor/getContextForTask.d.ts +5 -1
- package/esm/typings/src/execution/createPipelineExecutor/getExamplesForTask.d.ts +1 -1
- package/esm/typings/src/execution/createPipelineExecutor/getKnowledgeForTask.d.ts +12 -9
- package/esm/typings/src/execution/createPipelineExecutor/getReservedParametersForTask.d.ts +18 -5
- package/esm/typings/src/execution/createPipelineExecutor/knowledgePiecesToString.d.ts +9 -0
- package/esm/typings/src/execution/execution-report/ExecutionReportJson.d.ts +1 -1
- package/esm/typings/src/execution/execution-report/ExecutionReportString.d.ts +1 -1
- package/esm/typings/src/execution/translation/automatic-translate/automatic-translators/LindatAutomaticTranslator.d.ts +4 -4
- package/esm/typings/src/execution/utils/checkExpectations.d.ts +3 -3
- package/esm/typings/src/execution/utils/uncertainNumber.d.ts +3 -2
- package/esm/typings/src/execution/utils/usageToWorktime.d.ts +1 -1
- package/esm/typings/src/formats/_common/{FormatDefinition.d.ts → FormatParser.d.ts} +8 -6
- package/esm/typings/src/formats/_common/FormatSubvalueParser.d.ts +66 -0
- package/esm/typings/src/formats/csv/CsvFormatParser.d.ts +17 -0
- package/esm/typings/src/formats/csv/CsvSettings.d.ts +2 -2
- package/esm/typings/src/formats/csv/utils/csvParse.d.ts +12 -0
- package/esm/typings/src/formats/csv/utils/isValidCsvString.d.ts +1 -1
- package/esm/typings/src/formats/index.d.ts +2 -2
- package/esm/typings/src/formats/json/JsonFormatParser.d.ts +19 -0
- package/esm/typings/src/formats/json/utils/isValidJsonString.d.ts +1 -1
- package/esm/typings/src/formats/json/utils/jsonParse.d.ts +0 -3
- package/esm/typings/src/formats/text/{TextFormatDefinition.d.ts → TextFormatParser.d.ts} +7 -7
- package/esm/typings/src/formats/xml/XmlFormatParser.d.ts +19 -0
- package/esm/typings/src/formats/xml/utils/isValidXmlString.d.ts +1 -1
- package/esm/typings/src/formfactors/_boilerplate/BoilerplateFormfactorDefinition.d.ts +3 -2
- package/esm/typings/src/formfactors/_common/AbstractFormfactorDefinition.d.ts +16 -7
- package/esm/typings/src/formfactors/_common/FormfactorDefinition.d.ts +3 -1
- package/esm/typings/src/formfactors/_common/string_formfactor_name.d.ts +2 -1
- package/esm/typings/src/formfactors/chatbot/ChatbotFormfactorDefinition.d.ts +2 -2
- package/esm/typings/src/formfactors/completion/CompletionFormfactorDefinition.d.ts +29 -0
- package/esm/typings/src/formfactors/generator/GeneratorFormfactorDefinition.d.ts +2 -1
- package/esm/typings/src/formfactors/generic/GenericFormfactorDefinition.d.ts +2 -2
- package/esm/typings/src/formfactors/index.d.ts +33 -8
- package/esm/typings/src/formfactors/matcher/MatcherFormfactorDefinition.d.ts +4 -2
- package/esm/typings/src/formfactors/sheets/SheetsFormfactorDefinition.d.ts +3 -2
- package/esm/typings/src/formfactors/translator/TranslatorFormfactorDefinition.d.ts +3 -2
- package/esm/typings/src/high-level-abstractions/index.d.ts +2 -2
- package/esm/typings/src/llm-providers/_common/register/$llmToolsMetadataRegister.d.ts +3 -3
- package/esm/typings/src/llm-providers/_common/register/$llmToolsRegister.d.ts +3 -3
- package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsConfigurationFromEnv.d.ts +4 -4
- package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground.d.ts +4 -3
- package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsFromEnv.d.ts +18 -5
- package/esm/typings/src/llm-providers/_common/register/LlmToolsConfiguration.d.ts +11 -4
- package/esm/typings/src/llm-providers/_common/register/LlmToolsMetadata.d.ts +21 -42
- package/esm/typings/src/llm-providers/_common/register/LlmToolsOptions.d.ts +9 -2
- package/esm/typings/src/llm-providers/_common/register/createLlmToolsFromConfiguration.d.ts +13 -4
- package/esm/typings/src/llm-providers/_common/utils/cache/CacheItem.d.ts +10 -5
- package/esm/typings/src/llm-providers/_common/utils/cache/CacheLlmToolsOptions.d.ts +11 -3
- package/esm/typings/src/llm-providers/_common/utils/cache/cacheLlmTools.d.ts +3 -3
- package/esm/typings/src/llm-providers/_common/utils/count-total-usage/limitTotalUsage.d.ts +5 -5
- package/esm/typings/src/llm-providers/anthropic-claude/AnthropicClaudeExecutionTools.d.ts +6 -0
- package/esm/typings/src/llm-providers/anthropic-claude/anthropic-claude-models.d.ts +1 -1
- package/esm/typings/src/llm-providers/azure-openai/AzureOpenAiExecutionTools.d.ts +5 -1
- package/esm/typings/src/llm-providers/azure-openai/AzureOpenAiExecutionToolsOptions.d.ts +4 -4
- package/esm/typings/src/llm-providers/deepseek/deepseek-models.d.ts +1 -1
- package/esm/typings/src/llm-providers/google/google-models.d.ts +1 -1
- package/esm/typings/src/llm-providers/openai/OpenAiAssistantExecutionTools.d.ts +1 -1
- package/esm/typings/src/llm-providers/openai/OpenAiExecutionTools.d.ts +4 -0
- package/esm/typings/src/llm-providers/openai/openai-models.d.ts +1 -1
- package/esm/typings/src/llm-providers/openai/register-configuration.d.ts +2 -2
- package/esm/typings/src/llm-providers/openai/register-constructor.d.ts +2 -2
- package/esm/typings/src/migrations/migratePipeline.d.ts +9 -0
- package/esm/typings/src/other/templates/getBookTemplates.d.ts +2 -2
- package/esm/typings/src/pipeline/PipelineInterface/PipelineInterface.d.ts +3 -3
- package/esm/typings/src/pipeline/PipelineInterface/constants.d.ts +1 -1
- package/esm/typings/src/pipeline/PipelineInterface/getPipelineInterface.d.ts +1 -1
- package/esm/typings/src/pipeline/PipelineInterface/isPipelineImplementingInterface.d.ts +5 -4
- package/esm/typings/src/pipeline/PipelineInterface/isPipelineInterfacesEqual.d.ts +1 -1
- package/esm/typings/src/pipeline/PipelineJson/CommonTaskJson.d.ts +10 -7
- package/esm/typings/src/pipeline/PipelineJson/ParameterJson.d.ts +1 -1
- package/esm/typings/src/pipeline/PipelineJson/PipelineJson.d.ts +3 -2
- package/esm/typings/src/pipeline/PipelineString.d.ts +3 -1
- package/esm/typings/src/pipeline/book-notation.d.ts +2 -2
- package/esm/typings/src/postprocessing/utils/extractBlock.d.ts +1 -1
- package/esm/typings/src/postprocessing/utils/extractJsonBlock.d.ts +2 -2
- package/esm/typings/src/prepare/prepareTasks.d.ts +8 -5
- package/esm/typings/src/remote-server/openapi.d.ts +1 -1
- package/esm/typings/src/remote-server/socket-types/listModels/PromptbookServer_ListModels_Response.d.ts +1 -1
- package/esm/typings/src/remote-server/startRemoteServer.d.ts +1 -1
- package/esm/typings/src/remote-server/types/RemoteServerOptions.d.ts +7 -6
- package/esm/typings/src/scrapers/_boilerplate/BoilerplateScraper.d.ts +3 -3
- package/esm/typings/src/scrapers/_boilerplate/createBoilerplateScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/_boilerplate/register-metadata.d.ts +1 -1
- package/esm/typings/src/scrapers/_common/Converter.d.ts +3 -1
- package/esm/typings/src/scrapers/_common/Scraper.d.ts +4 -3
- package/esm/typings/src/scrapers/_common/ScraperIntermediateSource.d.ts +4 -2
- package/esm/typings/src/scrapers/_common/prepareKnowledgePieces.d.ts +2 -2
- package/esm/typings/src/scrapers/_common/register/$provideFilesystemForNode.d.ts +2 -1
- package/esm/typings/src/scrapers/_common/register/$provideScrapersForBrowser.d.ts +6 -3
- package/esm/typings/src/scrapers/_common/register/$provideScrapersForNode.d.ts +3 -5
- package/esm/typings/src/scrapers/_common/register/$scrapersMetadataRegister.d.ts +3 -3
- package/esm/typings/src/scrapers/_common/register/$scrapersRegister.d.ts +3 -2
- package/esm/typings/src/scrapers/_common/register/ScraperAndConverterMetadata.d.ts +8 -5
- package/esm/typings/src/scrapers/_common/register/ScraperConstructor.d.ts +2 -1
- package/esm/typings/src/scrapers/_common/utils/getScraperIntermediateSource.d.ts +6 -5
- package/esm/typings/src/scrapers/_common/utils/makeKnowledgeSourceHandler.d.ts +3 -1
- package/esm/typings/src/scrapers/document/createDocumentScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/document-legacy/createLegacyDocumentScraper.d.ts +2 -1
- package/esm/typings/src/scrapers/markdown/createMarkdownScraper.d.ts +4 -1
- package/esm/typings/src/scrapers/markitdown/MarkitdownScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/pdf/createPdfScraper.d.ts +2 -1
- package/esm/typings/src/scrapers/website/createWebsiteScraper.d.ts +3 -4
- package/esm/typings/src/scripting/javascript/JavascriptExecutionTools.d.ts +1 -1
- package/esm/typings/src/scripting/javascript/postprocessing-functions.d.ts +5 -1
- package/esm/typings/src/scripting/javascript/utils/preserve.d.ts +1 -1
- package/esm/typings/src/storage/_common/PromptbookStorage.d.ts +1 -1
- package/esm/typings/src/storage/file-cache-storage/FileCacheStorage.d.ts +12 -5
- package/esm/typings/src/storage/file-cache-storage/FileCacheStorageOptions.d.ts +4 -2
- package/esm/typings/src/storage/file-cache-storage/utils/nameToSubfolderPath.d.ts +2 -1
- package/esm/typings/src/storage/local-storage/getIndexedDbStorage.d.ts +11 -0
- package/esm/typings/src/storage/local-storage/utils/IndexedDbStorageOptions.d.ts +14 -0
- package/esm/typings/src/storage/local-storage/utils/makePromptbookStorageFromIndexedDb.d.ts +8 -0
- package/esm/typings/src/storage/local-storage/utils/makePromptbookStorageFromWebStorage.d.ts +2 -1
- package/esm/typings/src/types/IntermediateFilesStrategy.d.ts +2 -1
- package/esm/typings/src/types/ModelRequirements.d.ts +2 -2
- package/esm/typings/src/types/ModelVariant.d.ts +5 -5
- package/esm/typings/src/types/typeAliases.d.ts +22 -19
- package/esm/typings/src/utils/$Register.d.ts +8 -7
- package/esm/typings/src/utils/editable/edit-pipeline-string/addPipelineCommand.d.ts +2 -2
- package/esm/typings/src/utils/editable/edit-pipeline-string/deflatePipeline.d.ts +4 -1
- package/esm/typings/src/utils/editable/utils/isFlatPipeline.d.ts +2 -1
- package/esm/typings/src/utils/editable/utils/stringifyPipelineJson.d.ts +1 -1
- package/esm/typings/src/utils/environment/$getGlobalScope.d.ts +2 -1
- package/esm/typings/src/utils/expectation-counters/index.d.ts +1 -1
- package/esm/typings/src/utils/markdown/extractAllBlocksFromMarkdown.d.ts +2 -2
- package/esm/typings/src/utils/markdown/extractAllListItemsFromMarkdown.d.ts +1 -1
- package/esm/typings/src/utils/markdown/extractOneBlockFromMarkdown.d.ts +2 -2
- package/esm/typings/src/utils/normalization/nameToUriPart.d.ts +4 -4
- package/esm/typings/src/utils/normalization/nameToUriParts.d.ts +4 -4
- package/esm/typings/src/utils/normalization/normalize-to-kebab-case.d.ts +3 -3
- package/esm/typings/src/utils/normalization/normalizeTo_SCREAMING_CASE.d.ts +3 -3
- package/esm/typings/src/utils/normalization/normalizeTo_camelCase.d.ts +4 -4
- package/esm/typings/src/utils/normalization/normalizeTo_snake_case.d.ts +3 -3
- package/esm/typings/src/utils/normalization/removeDiacritics.d.ts +3 -3
- package/esm/typings/src/utils/normalization/searchKeywords.d.ts +4 -1
- package/esm/typings/src/utils/normalization/titleToName.d.ts +4 -4
- package/esm/typings/src/utils/organization/TODO_USE.d.ts +1 -1
- package/esm/typings/src/utils/organization/empty_object.d.ts +2 -2
- package/esm/typings/src/utils/organization/just.d.ts +1 -1
- package/esm/typings/src/utils/organization/just_empty_object.d.ts +4 -4
- package/esm/typings/src/utils/organization/keepUnused.d.ts +1 -1
- package/esm/typings/src/utils/parameters/mapAvailableToExpectedParameters.d.ts +7 -7
- package/esm/typings/src/utils/removeQuotes.d.ts +2 -2
- package/esm/typings/src/utils/serialization/clonePipeline.d.ts +4 -3
- package/esm/typings/src/utils/serialization/deepClone.d.ts +5 -1
- package/esm/typings/src/utils/trimCodeBlock.d.ts +1 -1
- package/esm/typings/src/utils/trimEndOfCodeBlock.d.ts +1 -1
- package/esm/typings/src/utils/unwrapResult.d.ts +2 -2
- package/esm/typings/src/utils/validators/javascriptName/isValidJavascriptName.d.ts +3 -3
- package/esm/typings/src/utils/validators/parameterName/validateParameterName.d.ts +5 -4
- package/esm/typings/src/utils/validators/semanticVersion/isValidPromptbookVersion.d.ts +1 -1
- package/esm/typings/src/utils/validators/semanticVersion/isValidSemanticVersion.d.ts +1 -1
- package/esm/typings/src/utils/validators/url/isHostnameOnPrivateNetwork.d.ts +1 -1
- package/esm/typings/src/utils/validators/url/isUrlOnPrivateNetwork.d.ts +1 -1
- package/esm/typings/src/utils/validators/url/isValidPipelineUrl.d.ts +1 -1
- package/esm/typings/src/utils/validators/url/isValidUrl.d.ts +1 -1
- package/esm/typings/src/version.d.ts +2 -1
- package/esm/typings/src/wizzard/wizzard.d.ts +1 -1
- package/package.json +14 -2
- package/umd/index.umd.js +804 -534
- package/umd/index.umd.js.map +1 -1
- package/esm/typings/src/formats/_common/FormatSubvalueDefinition.d.ts +0 -31
- package/esm/typings/src/formats/csv/CsvFormatDefinition.d.ts +0 -17
- package/esm/typings/src/formats/json/JsonFormatDefinition.d.ts +0 -19
- package/esm/typings/src/formats/xml/XmlFormatDefinition.d.ts +0 -19
- /package/esm/typings/src/llm-providers/{multiple → _multiple}/MultipleLlmExecutionTools.d.ts +0 -0
- /package/esm/typings/src/llm-providers/{multiple → _multiple}/joinLlmExecutionTools.d.ts +0 -0
- /package/esm/typings/src/llm-providers/{multiple → _multiple}/playground/playground.d.ts +0 -0
package/esm/index.es.js
CHANGED
|
@@ -5,8 +5,8 @@ import spaceTrim, { spaceTrim as spaceTrim$1 } from 'spacetrim';
|
|
|
5
5
|
import JSZip from 'jszip';
|
|
6
6
|
import { format } from 'prettier';
|
|
7
7
|
import parserHtml from 'prettier/parser-html';
|
|
8
|
-
import { Subject } from 'rxjs';
|
|
9
8
|
import { randomBytes } from 'crypto';
|
|
9
|
+
import { Subject } from 'rxjs';
|
|
10
10
|
import { forTime } from 'waitasecond';
|
|
11
11
|
import { parse, unparse } from 'papaparse';
|
|
12
12
|
import hexEncoder from 'crypto-js/enc-hex';
|
|
@@ -30,7 +30,7 @@ const BOOK_LANGUAGE_VERSION = '1.0.0';
|
|
|
30
30
|
* @generated
|
|
31
31
|
* @see https://github.com/webgptorg/promptbook
|
|
32
32
|
*/
|
|
33
|
-
const PROMPTBOOK_ENGINE_VERSION = '0.92.0
|
|
33
|
+
const PROMPTBOOK_ENGINE_VERSION = '0.92.0';
|
|
34
34
|
/**
|
|
35
35
|
* TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
|
|
36
36
|
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
@@ -40,7 +40,7 @@ const PROMPTBOOK_ENGINE_VERSION = '0.92.0-8';
|
|
|
40
40
|
* Returns the same value that is passed as argument.
|
|
41
41
|
* No side effects.
|
|
42
42
|
*
|
|
43
|
-
* Note: It can be
|
|
43
|
+
* Note: It can be useful for:
|
|
44
44
|
*
|
|
45
45
|
* 1) Leveling indentation
|
|
46
46
|
* 2) Putting always-true or always-false conditions without getting eslint errors
|
|
@@ -101,6 +101,21 @@ const DEFAULT_BOOK_OUTPUT_PARAMETER_NAME = 'result';
|
|
|
101
101
|
* @public exported from `@promptbook/core`
|
|
102
102
|
*/
|
|
103
103
|
const DEFAULT_MAX_FILE_SIZE = 100 * 1024 * 1024; // 100MB
|
|
104
|
+
/**
|
|
105
|
+
* Threshold value that determines when a dataset is considered "big"
|
|
106
|
+
* and may require special handling or optimizations
|
|
107
|
+
*
|
|
108
|
+
* For example, when error occurs in one item of the big dataset, it will not fail the whole pipeline
|
|
109
|
+
*
|
|
110
|
+
* @public exported from `@promptbook/core`
|
|
111
|
+
*/
|
|
112
|
+
const BIG_DATASET_TRESHOLD = 50;
|
|
113
|
+
/**
|
|
114
|
+
* Placeholder text used to represent a placeholder value of failed operation
|
|
115
|
+
*
|
|
116
|
+
* @public exported from `@promptbook/core`
|
|
117
|
+
*/
|
|
118
|
+
const FAILED_VALUE_PLACEHOLDER = '!?';
|
|
104
119
|
// <- TODO: [🧠] Better system for generator warnings - not always "code" and "by `@promptbook/cli`"
|
|
105
120
|
/**
|
|
106
121
|
* The maximum number of iterations for a loops
|
|
@@ -195,7 +210,7 @@ ex-port const WIZZARD_APP_ID: string_app_id = 'wizzard';
|
|
|
195
210
|
const DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME = `index`;
|
|
196
211
|
// <- TODO: [🧜♂️]
|
|
197
212
|
/**
|
|
198
|
-
*
|
|
213
|
+
* Default settings for parsing and generating CSV files in Promptbook.
|
|
199
214
|
*
|
|
200
215
|
* @public exported from `@promptbook/core`
|
|
201
216
|
*/
|
|
@@ -206,19 +221,19 @@ const DEFAULT_CSV_SETTINGS = Object.freeze({
|
|
|
206
221
|
skipEmptyLines: true,
|
|
207
222
|
});
|
|
208
223
|
/**
|
|
209
|
-
*
|
|
224
|
+
* Controls whether verbose logging is enabled by default throughout the application.
|
|
210
225
|
*
|
|
211
226
|
* @public exported from `@promptbook/core`
|
|
212
227
|
*/
|
|
213
228
|
let DEFAULT_IS_VERBOSE = false;
|
|
214
229
|
/**
|
|
215
|
-
*
|
|
230
|
+
* Controls whether auto-installation of dependencies is enabled by default.
|
|
216
231
|
*
|
|
217
232
|
* @public exported from `@promptbook/core`
|
|
218
233
|
*/
|
|
219
234
|
const DEFAULT_IS_AUTO_INSTALLED = false;
|
|
220
235
|
/**
|
|
221
|
-
*
|
|
236
|
+
* Indicates whether pipeline logic validation is enabled. When true, the pipeline logic is checked for consistency.
|
|
222
237
|
*
|
|
223
238
|
* @private within the repository
|
|
224
239
|
*/
|
|
@@ -286,7 +301,7 @@ class UnexpectedError extends Error {
|
|
|
286
301
|
${block(message)}
|
|
287
302
|
|
|
288
303
|
Note: This error should not happen.
|
|
289
|
-
It's
|
|
304
|
+
It's probably a bug in the pipeline collection
|
|
290
305
|
|
|
291
306
|
Please report issue:
|
|
292
307
|
${block(getErrorReportUrl(new Error(message)).href)}
|
|
@@ -328,15 +343,12 @@ function jsonParse(value) {
|
|
|
328
343
|
}
|
|
329
344
|
throw new Error(spaceTrim((block) => `
|
|
330
345
|
${block(error.message)}
|
|
331
|
-
|
|
346
|
+
|
|
332
347
|
The JSON text:
|
|
333
348
|
${block(value)}
|
|
334
349
|
`));
|
|
335
350
|
}
|
|
336
351
|
}
|
|
337
|
-
/**
|
|
338
|
-
* TODO: !!!! Use in Promptbook.studio
|
|
339
|
-
*/
|
|
340
352
|
|
|
341
353
|
/**
|
|
342
354
|
* Orders JSON object by keys
|
|
@@ -567,8 +579,12 @@ function checkSerializableAsJson(options) {
|
|
|
567
579
|
*/
|
|
568
580
|
|
|
569
581
|
/**
|
|
570
|
-
*
|
|
582
|
+
* Creates a deep clone of the given object
|
|
583
|
+
*
|
|
584
|
+
* Note: This method only works for objects that are fully serializable to JSON and do not contain functions, Dates, or special types.
|
|
571
585
|
*
|
|
586
|
+
* @param objectValue The object to clone.
|
|
587
|
+
* @returns A deep, writable clone of the input object.
|
|
572
588
|
* @public exported from `@promptbook/utils`
|
|
573
589
|
*/
|
|
574
590
|
function deepClone(objectValue) {
|
|
@@ -650,13 +666,13 @@ const ORDER_OF_PIPELINE_JSON = [
|
|
|
650
666
|
*/
|
|
651
667
|
const REPLACING_NONCE = 'ptbkauk42kV2dzao34faw7FudQUHYPtW';
|
|
652
668
|
/**
|
|
653
|
-
*
|
|
669
|
+
* Placeholder value indicating a parameter is missing its value.
|
|
654
670
|
*
|
|
655
671
|
* @private within the repository
|
|
656
672
|
*/
|
|
657
673
|
const RESERVED_PARAMETER_MISSING_VALUE = 'MISSING-' + REPLACING_NONCE;
|
|
658
674
|
/**
|
|
659
|
-
*
|
|
675
|
+
* Placeholder value indicating a parameter is restricted and cannot be used directly.
|
|
660
676
|
*
|
|
661
677
|
* @private within the repository
|
|
662
678
|
*/
|
|
@@ -717,7 +733,7 @@ class PipelineLogicError extends Error {
|
|
|
717
733
|
/**
|
|
718
734
|
* Tests if given string is valid semantic version
|
|
719
735
|
*
|
|
720
|
-
* Note: There are two
|
|
736
|
+
* Note: There are two similar functions:
|
|
721
737
|
* - `isValidSemanticVersion` which tests any semantic version
|
|
722
738
|
* - `isValidPromptbookVersion` *(this one)* which tests just Promptbook versions
|
|
723
739
|
*
|
|
@@ -739,7 +755,7 @@ function isValidSemanticVersion(version) {
|
|
|
739
755
|
*
|
|
740
756
|
* @see https://www.npmjs.com/package/promptbook?activeTab=versions
|
|
741
757
|
* Note: When you are using for example promptbook 2.0.0 and there already is promptbook 3.0.0 it don`t know about it.
|
|
742
|
-
* Note: There are two
|
|
758
|
+
* Note: There are two similar functions:
|
|
743
759
|
* - `isValidSemanticVersion` which tests any semantic version
|
|
744
760
|
* - `isValidPromptbookVersion` *(this one)* which tests just Promptbook versions
|
|
745
761
|
*
|
|
@@ -760,7 +776,7 @@ function isValidPromptbookVersion(version) {
|
|
|
760
776
|
* Tests if given string is valid URL.
|
|
761
777
|
*
|
|
762
778
|
* Note: Dataurl are considered perfectly valid.
|
|
763
|
-
* Note: There are two
|
|
779
|
+
* Note: There are two similar functions:
|
|
764
780
|
* - `isValidUrl` which tests any URL
|
|
765
781
|
* - `isValidPipelineUrl` *(this one)* which tests just promptbook URL
|
|
766
782
|
*
|
|
@@ -788,7 +804,7 @@ function isValidUrl(url) {
|
|
|
788
804
|
/**
|
|
789
805
|
* Tests if given string is valid pipeline URL URL.
|
|
790
806
|
*
|
|
791
|
-
* Note: There are two
|
|
807
|
+
* Note: There are two similar functions:
|
|
792
808
|
* - `isValidUrl` which tests any URL
|
|
793
809
|
* - `isValidPipelineUrl` *(this one)* which tests just pipeline URL
|
|
794
810
|
*
|
|
@@ -885,7 +901,7 @@ function validatePipeline_InnerFunction(pipeline) {
|
|
|
885
901
|
${block(pipelineIdentification)}
|
|
886
902
|
`));
|
|
887
903
|
}
|
|
888
|
-
// TODO: [🧠] Maybe do here some
|
|
904
|
+
// TODO: [🧠] Maybe do here some proper JSON-schema / ZOD checking
|
|
889
905
|
if (!Array.isArray(pipeline.parameters)) {
|
|
890
906
|
// TODO: [🧠] what is the correct error tp throw - maybe PromptbookSchemaError
|
|
891
907
|
throw new ParseError(spaceTrim$1((block) => `
|
|
@@ -896,7 +912,7 @@ function validatePipeline_InnerFunction(pipeline) {
|
|
|
896
912
|
${block(pipelineIdentification)}
|
|
897
913
|
`));
|
|
898
914
|
}
|
|
899
|
-
// TODO: [🧠] Maybe do here some
|
|
915
|
+
// TODO: [🧠] Maybe do here some proper JSON-schema / ZOD checking
|
|
900
916
|
if (!Array.isArray(pipeline.tasks)) {
|
|
901
917
|
// TODO: [🧠] what is the correct error tp throw - maybe PromptbookSchemaError
|
|
902
918
|
throw new ParseError(spaceTrim$1((block) => `
|
|
@@ -1207,7 +1223,7 @@ function isValidFilePath(filename) {
|
|
|
1207
1223
|
* Function isValidJsonString will tell you if the string is valid JSON or not
|
|
1208
1224
|
*
|
|
1209
1225
|
* @param value The string to check
|
|
1210
|
-
* @returns
|
|
1226
|
+
* @returns `true` if the string is a valid JSON string, false otherwise
|
|
1211
1227
|
*
|
|
1212
1228
|
* @public exported from `@promptbook/utils`
|
|
1213
1229
|
*/
|
|
@@ -1319,7 +1335,7 @@ function pipelineJsonToString(pipelineJson) {
|
|
|
1319
1335
|
if (bookVersion !== `undefined`) {
|
|
1320
1336
|
commands.push(`BOOK VERSION ${bookVersion}`);
|
|
1321
1337
|
}
|
|
1322
|
-
// TODO: [main] !!5 This increases size of the bundle and is
|
|
1338
|
+
// TODO: [main] !!5 This increases size of the bundle and is probably not necessary
|
|
1323
1339
|
pipelineString = prettifyMarkdown(pipelineString);
|
|
1324
1340
|
for (const parameter of parameters.filter(({ isInput }) => isInput)) {
|
|
1325
1341
|
commands.push(`INPUT PARAMETER ${taskParameterJsonToString(parameter)}`);
|
|
@@ -1529,7 +1545,7 @@ class SimplePipelineCollection {
|
|
|
1529
1545
|
/**
|
|
1530
1546
|
* Constructs a pipeline collection from pipelines
|
|
1531
1547
|
*
|
|
1532
|
-
* @param pipelines
|
|
1548
|
+
* @param pipelines Array of pipeline JSON objects to include in the collection
|
|
1533
1549
|
*
|
|
1534
1550
|
* Note: During the construction logic of all pipelines are validated
|
|
1535
1551
|
* Note: It is not recommended to use this constructor directly, use `createCollectionFromJson` *(or other variant)* instead
|
|
@@ -1643,7 +1659,7 @@ class MissingToolsError extends Error {
|
|
|
1643
1659
|
super(spaceTrim$1((block) => `
|
|
1644
1660
|
${block(message)}
|
|
1645
1661
|
|
|
1646
|
-
Note: You have
|
|
1662
|
+
Note: You have probably forgot to provide some tools for pipeline execution or preparation
|
|
1647
1663
|
|
|
1648
1664
|
`));
|
|
1649
1665
|
this.name = 'MissingToolsError';
|
|
@@ -1659,15 +1675,18 @@ class MissingToolsError extends Error {
|
|
|
1659
1675
|
* @public exported from `@promptbook/core`
|
|
1660
1676
|
*/
|
|
1661
1677
|
function isPipelinePrepared(pipeline) {
|
|
1662
|
-
// Note: Ignoring `pipeline.preparations`
|
|
1663
|
-
// Note: Ignoring `pipeline.knowledgePieces`
|
|
1678
|
+
// Note: Ignoring `pipeline.preparations`
|
|
1679
|
+
// Note: Ignoring `pipeline.knowledgePieces`
|
|
1664
1680
|
if (pipeline.title === undefined || pipeline.title === '' || pipeline.title === DEFAULT_BOOK_TITLE) {
|
|
1681
|
+
// console.log('Pipeline is not prepared because title is undefined or empty', pipeline);
|
|
1665
1682
|
return false;
|
|
1666
1683
|
}
|
|
1667
1684
|
if (!pipeline.personas.every((persona) => persona.modelsRequirements !== undefined)) {
|
|
1685
|
+
// console.log('Pipeline is not prepared because personas are not prepared', pipeline.personas);
|
|
1668
1686
|
return false;
|
|
1669
1687
|
}
|
|
1670
1688
|
if (!pipeline.knowledgeSources.every((knowledgeSource) => knowledgeSource.preparationIds !== undefined)) {
|
|
1689
|
+
//console.log('Pipeline is not prepared because knowledge sources are not prepared', pipeline.knowledgeSources);
|
|
1671
1690
|
return false;
|
|
1672
1691
|
}
|
|
1673
1692
|
/*
|
|
@@ -1688,70 +1707,6 @@ function isPipelinePrepared(pipeline) {
|
|
|
1688
1707
|
* - [♨] Are tasks prepared
|
|
1689
1708
|
*/
|
|
1690
1709
|
|
|
1691
|
-
/**
|
|
1692
|
-
* Generates random token
|
|
1693
|
-
*
|
|
1694
|
-
* Note: This function is cryptographically secure (it uses crypto.randomBytes internally)
|
|
1695
|
-
*
|
|
1696
|
-
* @private internal helper function
|
|
1697
|
-
* @returns secure random token
|
|
1698
|
-
*/
|
|
1699
|
-
function $randomToken(randomness) {
|
|
1700
|
-
return randomBytes(randomness).toString('hex');
|
|
1701
|
-
}
|
|
1702
|
-
/**
|
|
1703
|
-
* TODO: Maybe use nanoid instead https://github.com/ai/nanoid
|
|
1704
|
-
*/
|
|
1705
|
-
|
|
1706
|
-
/**
|
|
1707
|
-
* Recursively converts JSON strings to JSON objects
|
|
1708
|
-
|
|
1709
|
-
* @public exported from `@promptbook/utils`
|
|
1710
|
-
*/
|
|
1711
|
-
function jsonStringsToJsons(object) {
|
|
1712
|
-
if (object === null) {
|
|
1713
|
-
return object;
|
|
1714
|
-
}
|
|
1715
|
-
if (Array.isArray(object)) {
|
|
1716
|
-
return object.map(jsonStringsToJsons);
|
|
1717
|
-
}
|
|
1718
|
-
if (typeof object !== 'object') {
|
|
1719
|
-
return object;
|
|
1720
|
-
}
|
|
1721
|
-
const newObject = { ...object };
|
|
1722
|
-
for (const [key, value] of Object.entries(object)) {
|
|
1723
|
-
if (typeof value === 'string' && isValidJsonString(value)) {
|
|
1724
|
-
newObject[key] = jsonParse(value);
|
|
1725
|
-
}
|
|
1726
|
-
else {
|
|
1727
|
-
newObject[key] = jsonStringsToJsons(value);
|
|
1728
|
-
}
|
|
1729
|
-
}
|
|
1730
|
-
return newObject;
|
|
1731
|
-
}
|
|
1732
|
-
/**
|
|
1733
|
-
* TODO: Type the return type correctly
|
|
1734
|
-
*/
|
|
1735
|
-
|
|
1736
|
-
/**
|
|
1737
|
-
* This error indicates errors during the execution of the pipeline
|
|
1738
|
-
*
|
|
1739
|
-
* @public exported from `@promptbook/core`
|
|
1740
|
-
*/
|
|
1741
|
-
class PipelineExecutionError extends Error {
|
|
1742
|
-
constructor(message) {
|
|
1743
|
-
// Added id parameter
|
|
1744
|
-
super(message);
|
|
1745
|
-
this.name = 'PipelineExecutionError';
|
|
1746
|
-
// TODO: [🐙] DRY - Maybe $randomId
|
|
1747
|
-
this.id = `error-${$randomToken(8 /* <- TODO: To global config + Use Base58 to avoid simmilar char conflicts */)}`;
|
|
1748
|
-
Object.setPrototypeOf(this, PipelineExecutionError.prototype);
|
|
1749
|
-
}
|
|
1750
|
-
}
|
|
1751
|
-
/**
|
|
1752
|
-
* TODO: [🧠][🌂] Add id to all errors
|
|
1753
|
-
*/
|
|
1754
|
-
|
|
1755
1710
|
/**
|
|
1756
1711
|
* This error indicates problems parsing the format value
|
|
1757
1712
|
*
|
|
@@ -1796,7 +1751,7 @@ class AuthenticationError extends Error {
|
|
|
1796
1751
|
}
|
|
1797
1752
|
|
|
1798
1753
|
/**
|
|
1799
|
-
* This error indicates that the pipeline collection cannot be
|
|
1754
|
+
* This error indicates that the pipeline collection cannot be properly loaded
|
|
1800
1755
|
*
|
|
1801
1756
|
* @public exported from `@promptbook/core`
|
|
1802
1757
|
*/
|
|
@@ -1886,6 +1841,40 @@ class NotYetImplementedError extends Error {
|
|
|
1886
1841
|
}
|
|
1887
1842
|
}
|
|
1888
1843
|
|
|
1844
|
+
/**
|
|
1845
|
+
* Generates random token
|
|
1846
|
+
*
|
|
1847
|
+
* Note: This function is cryptographically secure (it uses crypto.randomBytes internally)
|
|
1848
|
+
*
|
|
1849
|
+
* @private internal helper function
|
|
1850
|
+
* @returns secure random token
|
|
1851
|
+
*/
|
|
1852
|
+
function $randomToken(randomness) {
|
|
1853
|
+
return randomBytes(randomness).toString('hex');
|
|
1854
|
+
}
|
|
1855
|
+
/**
|
|
1856
|
+
* TODO: Maybe use nanoid instead https://github.com/ai/nanoid
|
|
1857
|
+
*/
|
|
1858
|
+
|
|
1859
|
+
/**
|
|
1860
|
+
* This error indicates errors during the execution of the pipeline
|
|
1861
|
+
*
|
|
1862
|
+
* @public exported from `@promptbook/core`
|
|
1863
|
+
*/
|
|
1864
|
+
class PipelineExecutionError extends Error {
|
|
1865
|
+
constructor(message) {
|
|
1866
|
+
// Added id parameter
|
|
1867
|
+
super(message);
|
|
1868
|
+
this.name = 'PipelineExecutionError';
|
|
1869
|
+
// TODO: [🐙] DRY - Maybe $randomId
|
|
1870
|
+
this.id = `error-${$randomToken(8 /* <- TODO: To global config + Use Base58 to avoid similar char conflicts */)}`;
|
|
1871
|
+
Object.setPrototypeOf(this, PipelineExecutionError.prototype);
|
|
1872
|
+
}
|
|
1873
|
+
}
|
|
1874
|
+
/**
|
|
1875
|
+
* TODO: [🧠][🌂] Add id to all errors
|
|
1876
|
+
*/
|
|
1877
|
+
|
|
1889
1878
|
/**
|
|
1890
1879
|
* Error thrown when a fetch request fails
|
|
1891
1880
|
*
|
|
@@ -1961,6 +1950,65 @@ const ALL_ERRORS = {
|
|
|
1961
1950
|
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
1962
1951
|
*/
|
|
1963
1952
|
|
|
1953
|
+
/**
|
|
1954
|
+
* Serializes an error into a [🚉] JSON-serializable object
|
|
1955
|
+
*
|
|
1956
|
+
* @public exported from `@promptbook/utils`
|
|
1957
|
+
*/
|
|
1958
|
+
function serializeError(error) {
|
|
1959
|
+
const { name, message, stack } = error;
|
|
1960
|
+
const { id } = error;
|
|
1961
|
+
if (!Object.keys(ALL_ERRORS).includes(name)) {
|
|
1962
|
+
console.error(spaceTrim((block) => `
|
|
1963
|
+
|
|
1964
|
+
Cannot serialize error with name "${name}"
|
|
1965
|
+
|
|
1966
|
+
Authors of Promptbook probably forgot to add this error into the list of errors:
|
|
1967
|
+
https://github.com/webgptorg/promptbook/blob/main/src/errors/0-index.ts
|
|
1968
|
+
|
|
1969
|
+
|
|
1970
|
+
${block(stack || message)}
|
|
1971
|
+
|
|
1972
|
+
`));
|
|
1973
|
+
}
|
|
1974
|
+
return {
|
|
1975
|
+
name: name,
|
|
1976
|
+
message,
|
|
1977
|
+
stack,
|
|
1978
|
+
id, // Include id in the serialized object
|
|
1979
|
+
};
|
|
1980
|
+
}
|
|
1981
|
+
|
|
1982
|
+
/**
|
|
1983
|
+
* Recursively converts JSON strings to JSON objects
|
|
1984
|
+
|
|
1985
|
+
* @public exported from `@promptbook/utils`
|
|
1986
|
+
*/
|
|
1987
|
+
function jsonStringsToJsons(object) {
|
|
1988
|
+
if (object === null) {
|
|
1989
|
+
return object;
|
|
1990
|
+
}
|
|
1991
|
+
if (Array.isArray(object)) {
|
|
1992
|
+
return object.map(jsonStringsToJsons);
|
|
1993
|
+
}
|
|
1994
|
+
if (typeof object !== 'object') {
|
|
1995
|
+
return object;
|
|
1996
|
+
}
|
|
1997
|
+
const newObject = { ...object };
|
|
1998
|
+
for (const [key, value] of Object.entries(object)) {
|
|
1999
|
+
if (typeof value === 'string' && isValidJsonString(value)) {
|
|
2000
|
+
newObject[key] = jsonParse(value);
|
|
2001
|
+
}
|
|
2002
|
+
else {
|
|
2003
|
+
newObject[key] = jsonStringsToJsons(value);
|
|
2004
|
+
}
|
|
2005
|
+
}
|
|
2006
|
+
return newObject;
|
|
2007
|
+
}
|
|
2008
|
+
/**
|
|
2009
|
+
* TODO: Type the return type correctly
|
|
2010
|
+
*/
|
|
2011
|
+
|
|
1964
2012
|
/**
|
|
1965
2013
|
* Deserializes the error object
|
|
1966
2014
|
*
|
|
@@ -2035,8 +2083,9 @@ function assertsTaskSuccessful(executionResult) {
|
|
|
2035
2083
|
*/
|
|
2036
2084
|
function createTask(options) {
|
|
2037
2085
|
const { taskType, taskProcessCallback } = options;
|
|
2086
|
+
let { title } = options;
|
|
2038
2087
|
// TODO: [🐙] DRY
|
|
2039
|
-
const taskId = `${taskType.toLowerCase().substring(0, 4)}-${$randomToken(8 /* <- TODO: To global config + Use Base58 to avoid
|
|
2088
|
+
const taskId = `${taskType.toLowerCase().substring(0, 4)}-${$randomToken(8 /* <- TODO: To global config + Use Base58 to avoid similar char conflicts */)}`;
|
|
2040
2089
|
let status = 'RUNNING';
|
|
2041
2090
|
const createdAt = new Date();
|
|
2042
2091
|
let updatedAt = createdAt;
|
|
@@ -2046,6 +2095,10 @@ function createTask(options) {
|
|
|
2046
2095
|
const partialResultSubject = new Subject();
|
|
2047
2096
|
// <- Note: Not using `BehaviorSubject` because on error we can't access the last value
|
|
2048
2097
|
const finalResultPromise = /* not await */ taskProcessCallback((newOngoingResult) => {
|
|
2098
|
+
if (newOngoingResult.title) {
|
|
2099
|
+
title = newOngoingResult.title;
|
|
2100
|
+
}
|
|
2101
|
+
updatedAt = new Date();
|
|
2049
2102
|
Object.assign(currentValue, newOngoingResult);
|
|
2050
2103
|
// <- TODO: assign deep
|
|
2051
2104
|
partialResultSubject.next(newOngoingResult);
|
|
@@ -2091,17 +2144,24 @@ function createTask(options) {
|
|
|
2091
2144
|
return {
|
|
2092
2145
|
taskType,
|
|
2093
2146
|
taskId,
|
|
2147
|
+
get promptbookVersion() {
|
|
2148
|
+
return PROMPTBOOK_ENGINE_VERSION;
|
|
2149
|
+
},
|
|
2150
|
+
get title() {
|
|
2151
|
+
return title;
|
|
2152
|
+
// <- Note: [1] These must be getters to allow changing the value in the future
|
|
2153
|
+
},
|
|
2094
2154
|
get status() {
|
|
2095
2155
|
return status;
|
|
2096
|
-
// <- Note: [1]
|
|
2156
|
+
// <- Note: [1] --||--
|
|
2097
2157
|
},
|
|
2098
2158
|
get createdAt() {
|
|
2099
2159
|
return createdAt;
|
|
2100
|
-
// <- Note: [1]
|
|
2160
|
+
// <- Note: [1] --||--
|
|
2101
2161
|
},
|
|
2102
2162
|
get updatedAt() {
|
|
2103
2163
|
return updatedAt;
|
|
2104
|
-
// <- Note: [1]
|
|
2164
|
+
// <- Note: [1] --||--
|
|
2105
2165
|
},
|
|
2106
2166
|
asPromise,
|
|
2107
2167
|
asObservable() {
|
|
@@ -2109,15 +2169,15 @@ function createTask(options) {
|
|
|
2109
2169
|
},
|
|
2110
2170
|
get errors() {
|
|
2111
2171
|
return errors;
|
|
2112
|
-
// <- Note: [1]
|
|
2172
|
+
// <- Note: [1] --||--
|
|
2113
2173
|
},
|
|
2114
2174
|
get warnings() {
|
|
2115
2175
|
return warnings;
|
|
2116
|
-
// <- Note: [1]
|
|
2176
|
+
// <- Note: [1] --||--
|
|
2117
2177
|
},
|
|
2118
2178
|
get currentValue() {
|
|
2119
2179
|
return currentValue;
|
|
2120
|
-
// <- Note: [1]
|
|
2180
|
+
// <- Note: [1] --||--
|
|
2121
2181
|
},
|
|
2122
2182
|
};
|
|
2123
2183
|
}
|
|
@@ -2127,33 +2187,72 @@ function createTask(options) {
|
|
|
2127
2187
|
*/
|
|
2128
2188
|
|
|
2129
2189
|
/**
|
|
2130
|
-
*
|
|
2190
|
+
* Represents the uncertain value
|
|
2131
2191
|
*
|
|
2132
|
-
* @public exported from `@promptbook/
|
|
2192
|
+
* @public exported from `@promptbook/core`
|
|
2193
|
+
*/
|
|
2194
|
+
const ZERO_VALUE = $deepFreeze({ value: 0 });
|
|
2195
|
+
/**
|
|
2196
|
+
* Represents the uncertain value
|
|
2197
|
+
*
|
|
2198
|
+
* @public exported from `@promptbook/core`
|
|
2199
|
+
*/
|
|
2200
|
+
const UNCERTAIN_ZERO_VALUE = $deepFreeze({ value: 0, isUncertain: true });
|
|
2201
|
+
/**
|
|
2202
|
+
* Represents the usage with no resources consumed
|
|
2203
|
+
*
|
|
2204
|
+
* @public exported from `@promptbook/core`
|
|
2205
|
+
*/
|
|
2206
|
+
const ZERO_USAGE = $deepFreeze({
|
|
2207
|
+
price: ZERO_VALUE,
|
|
2208
|
+
input: {
|
|
2209
|
+
tokensCount: ZERO_VALUE,
|
|
2210
|
+
charactersCount: ZERO_VALUE,
|
|
2211
|
+
wordsCount: ZERO_VALUE,
|
|
2212
|
+
sentencesCount: ZERO_VALUE,
|
|
2213
|
+
linesCount: ZERO_VALUE,
|
|
2214
|
+
paragraphsCount: ZERO_VALUE,
|
|
2215
|
+
pagesCount: ZERO_VALUE,
|
|
2216
|
+
},
|
|
2217
|
+
output: {
|
|
2218
|
+
tokensCount: ZERO_VALUE,
|
|
2219
|
+
charactersCount: ZERO_VALUE,
|
|
2220
|
+
wordsCount: ZERO_VALUE,
|
|
2221
|
+
sentencesCount: ZERO_VALUE,
|
|
2222
|
+
linesCount: ZERO_VALUE,
|
|
2223
|
+
paragraphsCount: ZERO_VALUE,
|
|
2224
|
+
pagesCount: ZERO_VALUE,
|
|
2225
|
+
},
|
|
2226
|
+
});
|
|
2227
|
+
/**
|
|
2228
|
+
* Represents the usage with unknown resources consumed
|
|
2229
|
+
*
|
|
2230
|
+
* @public exported from `@promptbook/core`
|
|
2231
|
+
*/
|
|
2232
|
+
const UNCERTAIN_USAGE = $deepFreeze({
|
|
2233
|
+
price: UNCERTAIN_ZERO_VALUE,
|
|
2234
|
+
input: {
|
|
2235
|
+
tokensCount: UNCERTAIN_ZERO_VALUE,
|
|
2236
|
+
charactersCount: UNCERTAIN_ZERO_VALUE,
|
|
2237
|
+
wordsCount: UNCERTAIN_ZERO_VALUE,
|
|
2238
|
+
sentencesCount: UNCERTAIN_ZERO_VALUE,
|
|
2239
|
+
linesCount: UNCERTAIN_ZERO_VALUE,
|
|
2240
|
+
paragraphsCount: UNCERTAIN_ZERO_VALUE,
|
|
2241
|
+
pagesCount: UNCERTAIN_ZERO_VALUE,
|
|
2242
|
+
},
|
|
2243
|
+
output: {
|
|
2244
|
+
tokensCount: UNCERTAIN_ZERO_VALUE,
|
|
2245
|
+
charactersCount: UNCERTAIN_ZERO_VALUE,
|
|
2246
|
+
wordsCount: UNCERTAIN_ZERO_VALUE,
|
|
2247
|
+
sentencesCount: UNCERTAIN_ZERO_VALUE,
|
|
2248
|
+
linesCount: UNCERTAIN_ZERO_VALUE,
|
|
2249
|
+
paragraphsCount: UNCERTAIN_ZERO_VALUE,
|
|
2250
|
+
pagesCount: UNCERTAIN_ZERO_VALUE,
|
|
2251
|
+
},
|
|
2252
|
+
});
|
|
2253
|
+
/**
|
|
2254
|
+
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
2133
2255
|
*/
|
|
2134
|
-
function serializeError(error) {
|
|
2135
|
-
const { name, message, stack } = error;
|
|
2136
|
-
const { id } = error;
|
|
2137
|
-
if (!Object.keys(ALL_ERRORS).includes(name)) {
|
|
2138
|
-
console.error(spaceTrim((block) => `
|
|
2139
|
-
|
|
2140
|
-
Cannot serialize error with name "${name}"
|
|
2141
|
-
|
|
2142
|
-
Authors of Promptbook probably forgot to add this error into the list of errors:
|
|
2143
|
-
https://github.com/webgptorg/promptbook/blob/main/src/errors/0-index.ts
|
|
2144
|
-
|
|
2145
|
-
|
|
2146
|
-
${block(stack || message)}
|
|
2147
|
-
|
|
2148
|
-
`));
|
|
2149
|
-
}
|
|
2150
|
-
return {
|
|
2151
|
-
name: name,
|
|
2152
|
-
message,
|
|
2153
|
-
stack,
|
|
2154
|
-
id, // Include id in the serialized object
|
|
2155
|
-
};
|
|
2156
|
-
}
|
|
2157
2256
|
|
|
2158
2257
|
/**
|
|
2159
2258
|
* Format either small or big number
|
|
@@ -2236,74 +2335,6 @@ function valueToString(value) {
|
|
|
2236
2335
|
}
|
|
2237
2336
|
}
|
|
2238
2337
|
|
|
2239
|
-
/**
|
|
2240
|
-
* Represents the uncertain value
|
|
2241
|
-
*
|
|
2242
|
-
* @public exported from `@promptbook/core`
|
|
2243
|
-
*/
|
|
2244
|
-
const ZERO_VALUE = $deepFreeze({ value: 0 });
|
|
2245
|
-
/**
|
|
2246
|
-
* Represents the uncertain value
|
|
2247
|
-
*
|
|
2248
|
-
* @public exported from `@promptbook/core`
|
|
2249
|
-
*/
|
|
2250
|
-
const UNCERTAIN_ZERO_VALUE = $deepFreeze({ value: 0, isUncertain: true });
|
|
2251
|
-
/**
|
|
2252
|
-
* Represents the usage with no resources consumed
|
|
2253
|
-
*
|
|
2254
|
-
* @public exported from `@promptbook/core`
|
|
2255
|
-
*/
|
|
2256
|
-
const ZERO_USAGE = $deepFreeze({
|
|
2257
|
-
price: ZERO_VALUE,
|
|
2258
|
-
input: {
|
|
2259
|
-
tokensCount: ZERO_VALUE,
|
|
2260
|
-
charactersCount: ZERO_VALUE,
|
|
2261
|
-
wordsCount: ZERO_VALUE,
|
|
2262
|
-
sentencesCount: ZERO_VALUE,
|
|
2263
|
-
linesCount: ZERO_VALUE,
|
|
2264
|
-
paragraphsCount: ZERO_VALUE,
|
|
2265
|
-
pagesCount: ZERO_VALUE,
|
|
2266
|
-
},
|
|
2267
|
-
output: {
|
|
2268
|
-
tokensCount: ZERO_VALUE,
|
|
2269
|
-
charactersCount: ZERO_VALUE,
|
|
2270
|
-
wordsCount: ZERO_VALUE,
|
|
2271
|
-
sentencesCount: ZERO_VALUE,
|
|
2272
|
-
linesCount: ZERO_VALUE,
|
|
2273
|
-
paragraphsCount: ZERO_VALUE,
|
|
2274
|
-
pagesCount: ZERO_VALUE,
|
|
2275
|
-
},
|
|
2276
|
-
});
|
|
2277
|
-
/**
|
|
2278
|
-
* Represents the usage with unknown resources consumed
|
|
2279
|
-
*
|
|
2280
|
-
* @public exported from `@promptbook/core`
|
|
2281
|
-
*/
|
|
2282
|
-
$deepFreeze({
|
|
2283
|
-
price: UNCERTAIN_ZERO_VALUE,
|
|
2284
|
-
input: {
|
|
2285
|
-
tokensCount: UNCERTAIN_ZERO_VALUE,
|
|
2286
|
-
charactersCount: UNCERTAIN_ZERO_VALUE,
|
|
2287
|
-
wordsCount: UNCERTAIN_ZERO_VALUE,
|
|
2288
|
-
sentencesCount: UNCERTAIN_ZERO_VALUE,
|
|
2289
|
-
linesCount: UNCERTAIN_ZERO_VALUE,
|
|
2290
|
-
paragraphsCount: UNCERTAIN_ZERO_VALUE,
|
|
2291
|
-
pagesCount: UNCERTAIN_ZERO_VALUE,
|
|
2292
|
-
},
|
|
2293
|
-
output: {
|
|
2294
|
-
tokensCount: UNCERTAIN_ZERO_VALUE,
|
|
2295
|
-
charactersCount: UNCERTAIN_ZERO_VALUE,
|
|
2296
|
-
wordsCount: UNCERTAIN_ZERO_VALUE,
|
|
2297
|
-
sentencesCount: UNCERTAIN_ZERO_VALUE,
|
|
2298
|
-
linesCount: UNCERTAIN_ZERO_VALUE,
|
|
2299
|
-
paragraphsCount: UNCERTAIN_ZERO_VALUE,
|
|
2300
|
-
pagesCount: UNCERTAIN_ZERO_VALUE,
|
|
2301
|
-
},
|
|
2302
|
-
});
|
|
2303
|
-
/**
|
|
2304
|
-
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
2305
|
-
*/
|
|
2306
|
-
|
|
2307
2338
|
/**
|
|
2308
2339
|
* Function `addUsage` will add multiple usages into one
|
|
2309
2340
|
*
|
|
@@ -2512,7 +2543,7 @@ function union(...sets) {
|
|
|
2512
2543
|
}
|
|
2513
2544
|
|
|
2514
2545
|
/**
|
|
2515
|
-
*
|
|
2546
|
+
* Contains configuration options for parsing and generating CSV files, such as delimiters and quoting rules.
|
|
2516
2547
|
*
|
|
2517
2548
|
* @public exported from `@promptbook/core`
|
|
2518
2549
|
*/
|
|
@@ -2521,11 +2552,29 @@ const MANDATORY_CSV_SETTINGS = Object.freeze({
|
|
|
2521
2552
|
// encoding: 'utf-8',
|
|
2522
2553
|
});
|
|
2523
2554
|
|
|
2555
|
+
/**
|
|
2556
|
+
* Converts a CSV string into an object
|
|
2557
|
+
*
|
|
2558
|
+
* Note: This is wrapper around `papaparse.parse()` with better autohealing
|
|
2559
|
+
*
|
|
2560
|
+
* @private - for now until `@promptbook/csv` is released
|
|
2561
|
+
*/
|
|
2562
|
+
function csvParse(value /* <- TODO: string_csv */, settings, schema /* <- TODO: Make CSV Schemas */) {
|
|
2563
|
+
settings = { ...settings, ...MANDATORY_CSV_SETTINGS };
|
|
2564
|
+
// Note: Autoheal invalid '\n' characters
|
|
2565
|
+
if (settings.newline && !settings.newline.includes('\r') && value.includes('\r')) {
|
|
2566
|
+
console.warn('CSV string contains carriage return characters, but in the CSV settings the `newline` setting does not include them. Autohealing the CSV string.');
|
|
2567
|
+
value = value.replace(/\r\n/g, '\n').replace(/\r/g, '\n');
|
|
2568
|
+
}
|
|
2569
|
+
const csv = parse(value, settings);
|
|
2570
|
+
return csv;
|
|
2571
|
+
}
|
|
2572
|
+
|
|
2524
2573
|
/**
|
|
2525
2574
|
* Function to check if a string is valid CSV
|
|
2526
2575
|
*
|
|
2527
2576
|
* @param value The string to check
|
|
2528
|
-
* @returns
|
|
2577
|
+
* @returns `true` if the string is a valid CSV string, false otherwise
|
|
2529
2578
|
*
|
|
2530
2579
|
* @public exported from `@promptbook/utils`
|
|
2531
2580
|
*/
|
|
@@ -2549,7 +2598,7 @@ function isValidCsvString(value) {
|
|
|
2549
2598
|
* @public exported from `@promptbook/core`
|
|
2550
2599
|
* <- TODO: [🏢] Export from package `@promptbook/csv`
|
|
2551
2600
|
*/
|
|
2552
|
-
const
|
|
2601
|
+
const CsvFormatParser = {
|
|
2553
2602
|
formatName: 'CSV',
|
|
2554
2603
|
aliases: ['SPREADSHEET', 'TABLE'],
|
|
2555
2604
|
isValid(value, settings, schema) {
|
|
@@ -2561,12 +2610,12 @@ const CsvFormatDefinition = {
|
|
|
2561
2610
|
heal(value, settings, schema) {
|
|
2562
2611
|
throw new Error('Not implemented');
|
|
2563
2612
|
},
|
|
2564
|
-
|
|
2613
|
+
subvalueParsers: [
|
|
2565
2614
|
{
|
|
2566
2615
|
subvalueName: 'ROW',
|
|
2567
|
-
async mapValues(
|
|
2568
|
-
|
|
2569
|
-
const csv =
|
|
2616
|
+
async mapValues(options) {
|
|
2617
|
+
const { value, outputParameterName, settings, mapCallback, onProgress } = options;
|
|
2618
|
+
const csv = csvParse(value, settings);
|
|
2570
2619
|
if (csv.errors.length !== 0) {
|
|
2571
2620
|
throw new CsvFormatError(spaceTrim((block) => `
|
|
2572
2621
|
CSV parsing error
|
|
@@ -2581,23 +2630,37 @@ const CsvFormatDefinition = {
|
|
|
2581
2630
|
${block(value)}
|
|
2582
2631
|
`));
|
|
2583
2632
|
}
|
|
2584
|
-
const mappedData =
|
|
2633
|
+
const mappedData = [];
|
|
2634
|
+
const length = csv.data.length;
|
|
2635
|
+
for (let index = 0; index < length; index++) {
|
|
2636
|
+
const row = csv.data[index];
|
|
2585
2637
|
if (row[outputParameterName]) {
|
|
2586
2638
|
throw new CsvFormatError(`Can not overwrite existing column "${outputParameterName}" in CSV row`);
|
|
2587
2639
|
}
|
|
2588
|
-
|
|
2640
|
+
const mappedRow = {
|
|
2589
2641
|
...row,
|
|
2590
|
-
[outputParameterName]: await mapCallback(row, index),
|
|
2642
|
+
[outputParameterName]: await mapCallback(row, index, length),
|
|
2591
2643
|
};
|
|
2592
|
-
|
|
2644
|
+
mappedData.push(mappedRow);
|
|
2645
|
+
if (onProgress) {
|
|
2646
|
+
// Note: Report the CSV with all rows mapped so far
|
|
2647
|
+
/*
|
|
2648
|
+
// TODO: [🛕] Report progress with all the rows including the pending ones
|
|
2649
|
+
const progressData = mappedData.map((row, i) =>
|
|
2650
|
+
i > index ? { ...row, [outputParameterName]: PENDING_VALUE_PLACEHOLDER } : row,
|
|
2651
|
+
);
|
|
2652
|
+
*/
|
|
2653
|
+
await onProgress(unparse(mappedData, { ...settings, ...MANDATORY_CSV_SETTINGS }));
|
|
2654
|
+
}
|
|
2655
|
+
}
|
|
2593
2656
|
return unparse(mappedData, { ...settings, ...MANDATORY_CSV_SETTINGS });
|
|
2594
2657
|
},
|
|
2595
2658
|
},
|
|
2596
2659
|
{
|
|
2597
2660
|
subvalueName: 'CELL',
|
|
2598
|
-
async mapValues(
|
|
2599
|
-
|
|
2600
|
-
const csv =
|
|
2661
|
+
async mapValues(options) {
|
|
2662
|
+
const { value, settings, mapCallback, onProgress } = options;
|
|
2663
|
+
const csv = csvParse(value, settings);
|
|
2601
2664
|
if (csv.errors.length !== 0) {
|
|
2602
2665
|
throw new CsvFormatError(spaceTrim((block) => `
|
|
2603
2666
|
CSV parsing error
|
|
@@ -2613,9 +2676,9 @@ const CsvFormatDefinition = {
|
|
|
2613
2676
|
`));
|
|
2614
2677
|
}
|
|
2615
2678
|
const mappedData = await Promise.all(csv.data.map(async (row, rowIndex) => {
|
|
2616
|
-
return /* not await */ Promise.all(Object.entries(row).map(async ([key, value], columnIndex) => {
|
|
2679
|
+
return /* not await */ Promise.all(Object.entries(row).map(async ([key, value], columnIndex, array) => {
|
|
2617
2680
|
const index = rowIndex * Object.keys(row).length + columnIndex;
|
|
2618
|
-
return /* not await */ mapCallback({ [key]: value }, index);
|
|
2681
|
+
return /* not await */ mapCallback({ [key]: value }, index, array.length);
|
|
2619
2682
|
}));
|
|
2620
2683
|
}));
|
|
2621
2684
|
return unparse(mappedData, { ...settings, ...MANDATORY_CSV_SETTINGS });
|
|
@@ -2624,10 +2687,10 @@ const CsvFormatDefinition = {
|
|
|
2624
2687
|
],
|
|
2625
2688
|
};
|
|
2626
2689
|
/**
|
|
2627
|
-
* TODO: [🍓] In `
|
|
2628
|
-
* TODO: [🍓] In `
|
|
2629
|
-
* TODO: [🍓] In `
|
|
2630
|
-
* TODO: [🍓] In `
|
|
2690
|
+
* TODO: [🍓] In `CsvFormatParser` implement simple `isValid`
|
|
2691
|
+
* TODO: [🍓] In `CsvFormatParser` implement partial `canBeValid`
|
|
2692
|
+
* TODO: [🍓] In `CsvFormatParser` implement `heal
|
|
2693
|
+
* TODO: [🍓] In `CsvFormatParser` implement `subvalueParsers`
|
|
2631
2694
|
* TODO: [🏢] Allow to expect something inside CSV objects and other formats
|
|
2632
2695
|
*/
|
|
2633
2696
|
|
|
@@ -2636,7 +2699,7 @@ const CsvFormatDefinition = {
|
|
|
2636
2699
|
*
|
|
2637
2700
|
* @private still in development [🏢]
|
|
2638
2701
|
*/
|
|
2639
|
-
const
|
|
2702
|
+
const JsonFormatParser = {
|
|
2640
2703
|
formatName: 'JSON',
|
|
2641
2704
|
mimeType: 'application/json',
|
|
2642
2705
|
isValid(value, settings, schema) {
|
|
@@ -2648,28 +2711,28 @@ const JsonFormatDefinition = {
|
|
|
2648
2711
|
heal(value, settings, schema) {
|
|
2649
2712
|
throw new Error('Not implemented');
|
|
2650
2713
|
},
|
|
2651
|
-
|
|
2714
|
+
subvalueParsers: [],
|
|
2652
2715
|
};
|
|
2653
2716
|
/**
|
|
2654
|
-
* TODO: [🧠] Maybe
|
|
2717
|
+
* TODO: [🧠] Maybe proper instance of object
|
|
2655
2718
|
* TODO: [0] Make string_serialized_json
|
|
2656
2719
|
* TODO: [1] Make type for JSON Settings and Schema
|
|
2657
2720
|
* TODO: [🧠] What to use for validating JSONs - JSON Schema, ZoD, typescript types/interfaces,...?
|
|
2658
|
-
* TODO: [🍓] In `
|
|
2659
|
-
* TODO: [🍓] In `
|
|
2660
|
-
* TODO: [🍓] In `
|
|
2661
|
-
* TODO: [🍓] In `
|
|
2721
|
+
* TODO: [🍓] In `JsonFormatParser` implement simple `isValid`
|
|
2722
|
+
* TODO: [🍓] In `JsonFormatParser` implement partial `canBeValid`
|
|
2723
|
+
* TODO: [🍓] In `JsonFormatParser` implement `heal
|
|
2724
|
+
* TODO: [🍓] In `JsonFormatParser` implement `subvalueParsers`
|
|
2662
2725
|
* TODO: [🏢] Allow to expect something inside JSON objects and other formats
|
|
2663
2726
|
*/
|
|
2664
2727
|
|
|
2665
2728
|
/**
|
|
2666
2729
|
* Definition for any text - this will be always valid
|
|
2667
2730
|
*
|
|
2668
|
-
* Note: This is not useful for validation, but for splitting and mapping with `
|
|
2731
|
+
* Note: This is not useful for validation, but for splitting and mapping with `subvalueParsers`
|
|
2669
2732
|
*
|
|
2670
2733
|
* @public exported from `@promptbook/core`
|
|
2671
2734
|
*/
|
|
2672
|
-
const
|
|
2735
|
+
const TextFormatParser = {
|
|
2673
2736
|
formatName: 'TEXT',
|
|
2674
2737
|
isValid(value) {
|
|
2675
2738
|
return typeof value === 'string';
|
|
@@ -2678,19 +2741,20 @@ const TextFormatDefinition = {
|
|
|
2678
2741
|
return typeof partialValue === 'string';
|
|
2679
2742
|
},
|
|
2680
2743
|
heal() {
|
|
2681
|
-
throw new UnexpectedError('It does not make sense to call `
|
|
2744
|
+
throw new UnexpectedError('It does not make sense to call `TextFormatParser.heal`');
|
|
2682
2745
|
},
|
|
2683
|
-
|
|
2746
|
+
subvalueParsers: [
|
|
2684
2747
|
{
|
|
2685
2748
|
subvalueName: 'LINE',
|
|
2686
|
-
async mapValues(
|
|
2749
|
+
async mapValues(options) {
|
|
2750
|
+
const { value, mapCallback, onProgress } = options;
|
|
2687
2751
|
const lines = value.split('\n');
|
|
2688
|
-
const mappedLines = await Promise.all(lines.map((lineContent, lineNumber) =>
|
|
2752
|
+
const mappedLines = await Promise.all(lines.map((lineContent, lineNumber, array) =>
|
|
2689
2753
|
// TODO: [🧠] Maybe option to skip empty line
|
|
2690
2754
|
/* not await */ mapCallback({
|
|
2691
2755
|
lineContent,
|
|
2692
2756
|
// TODO: [🧠] Maybe also put here `lineNumber`
|
|
2693
|
-
}, lineNumber)));
|
|
2757
|
+
}, lineNumber, array.length)));
|
|
2694
2758
|
return mappedLines.join('\n');
|
|
2695
2759
|
},
|
|
2696
2760
|
},
|
|
@@ -2700,10 +2764,10 @@ const TextFormatDefinition = {
|
|
|
2700
2764
|
/**
|
|
2701
2765
|
* TODO: [1] Make type for XML Text and Schema
|
|
2702
2766
|
* TODO: [🧠][🤠] Here should be all words, characters, lines, paragraphs, pages available as subvalues
|
|
2703
|
-
* TODO: [🍓] In `
|
|
2704
|
-
* TODO: [🍓] In `
|
|
2705
|
-
* TODO: [🍓] In `
|
|
2706
|
-
* TODO: [🍓] In `
|
|
2767
|
+
* TODO: [🍓] In `TextFormatParser` implement simple `isValid`
|
|
2768
|
+
* TODO: [🍓] In `TextFormatParser` implement partial `canBeValid`
|
|
2769
|
+
* TODO: [🍓] In `TextFormatParser` implement `heal
|
|
2770
|
+
* TODO: [🍓] In `TextFormatParser` implement `subvalueParsers`
|
|
2707
2771
|
* TODO: [🏢] Allow to expect something inside each item of list and other formats
|
|
2708
2772
|
*/
|
|
2709
2773
|
|
|
@@ -2711,7 +2775,7 @@ const TextFormatDefinition = {
|
|
|
2711
2775
|
* Function to check if a string is valid XML
|
|
2712
2776
|
*
|
|
2713
2777
|
* @param value
|
|
2714
|
-
* @returns
|
|
2778
|
+
* @returns `true` if the string is a valid XML string, false otherwise
|
|
2715
2779
|
*
|
|
2716
2780
|
* @public exported from `@promptbook/utils`
|
|
2717
2781
|
*/
|
|
@@ -2736,7 +2800,7 @@ function isValidXmlString(value) {
|
|
|
2736
2800
|
*
|
|
2737
2801
|
* @private still in development [🏢]
|
|
2738
2802
|
*/
|
|
2739
|
-
const
|
|
2803
|
+
const XmlFormatParser = {
|
|
2740
2804
|
formatName: 'XML',
|
|
2741
2805
|
mimeType: 'application/xml',
|
|
2742
2806
|
isValid(value, settings, schema) {
|
|
@@ -2748,17 +2812,17 @@ const XmlFormatDefinition = {
|
|
|
2748
2812
|
heal(value, settings, schema) {
|
|
2749
2813
|
throw new Error('Not implemented');
|
|
2750
2814
|
},
|
|
2751
|
-
|
|
2815
|
+
subvalueParsers: [],
|
|
2752
2816
|
};
|
|
2753
2817
|
/**
|
|
2754
|
-
* TODO: [🧠] Maybe
|
|
2818
|
+
* TODO: [🧠] Maybe proper instance of object
|
|
2755
2819
|
* TODO: [0] Make string_serialized_xml
|
|
2756
2820
|
* TODO: [1] Make type for XML Settings and Schema
|
|
2757
2821
|
* TODO: [🧠] What to use for validating XMLs - XSD,...
|
|
2758
|
-
* TODO: [🍓] In `
|
|
2759
|
-
* TODO: [🍓] In `
|
|
2760
|
-
* TODO: [🍓] In `
|
|
2761
|
-
* TODO: [🍓] In `
|
|
2822
|
+
* TODO: [🍓] In `XmlFormatParser` implement simple `isValid`
|
|
2823
|
+
* TODO: [🍓] In `XmlFormatParser` implement partial `canBeValid`
|
|
2824
|
+
* TODO: [🍓] In `XmlFormatParser` implement `heal
|
|
2825
|
+
* TODO: [🍓] In `XmlFormatParser` implement `subvalueParsers`
|
|
2762
2826
|
* TODO: [🏢] Allow to expect something inside XML and other formats
|
|
2763
2827
|
*/
|
|
2764
2828
|
|
|
@@ -2767,24 +2831,19 @@ const XmlFormatDefinition = {
|
|
|
2767
2831
|
*
|
|
2768
2832
|
* @private internal index of `...` <- TODO [🏢]
|
|
2769
2833
|
*/
|
|
2770
|
-
const FORMAT_DEFINITIONS = [
|
|
2771
|
-
JsonFormatDefinition,
|
|
2772
|
-
XmlFormatDefinition,
|
|
2773
|
-
TextFormatDefinition,
|
|
2774
|
-
CsvFormatDefinition,
|
|
2775
|
-
];
|
|
2834
|
+
const FORMAT_DEFINITIONS = [JsonFormatParser, XmlFormatParser, TextFormatParser, CsvFormatParser];
|
|
2776
2835
|
/**
|
|
2777
2836
|
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
2778
2837
|
*/
|
|
2779
2838
|
|
|
2780
2839
|
/**
|
|
2781
|
-
* Maps available parameters to expected parameters
|
|
2840
|
+
* Maps available parameters to expected parameters for a pipeline task.
|
|
2782
2841
|
*
|
|
2783
2842
|
* The strategy is:
|
|
2784
|
-
* 1)
|
|
2785
|
-
* 2)
|
|
2843
|
+
* 1) First, match parameters by name where both available and expected.
|
|
2844
|
+
* 2) Then, if there are unmatched expected and available parameters, map them by order.
|
|
2786
2845
|
*
|
|
2787
|
-
* @throws {PipelineExecutionError}
|
|
2846
|
+
* @throws {PipelineExecutionError} If the number of unmatched expected and available parameters does not match, or mapping is ambiguous.
|
|
2788
2847
|
* @private within the repository used in `createPipelineExecutor`
|
|
2789
2848
|
*/
|
|
2790
2849
|
function mapAvailableToExpectedParameters(options) {
|
|
@@ -2807,7 +2866,7 @@ function mapAvailableToExpectedParameters(options) {
|
|
|
2807
2866
|
else if (!availableParametersNames.has(parameterName) && expectedParameterNames.has(parameterName)) ;
|
|
2808
2867
|
}
|
|
2809
2868
|
if (expectedParameterNames.size === 0) {
|
|
2810
|
-
// Note: [👨👨👧] Now we can freeze `mappedParameters` to prevent
|
|
2869
|
+
// Note: [👨👨👧] Now we can freeze `mappedParameters` to prevent accidental modifications after mapping
|
|
2811
2870
|
Object.freeze(mappedParameters);
|
|
2812
2871
|
return mappedParameters;
|
|
2813
2872
|
}
|
|
@@ -2838,7 +2897,7 @@ function mapAvailableToExpectedParameters(options) {
|
|
|
2838
2897
|
for (let i = 0; i < expectedParameterNames.size; i++) {
|
|
2839
2898
|
mappedParameters[expectedParameterNamesArray[i]] = availableParameters[availableParametersNamesArray[i]];
|
|
2840
2899
|
}
|
|
2841
|
-
// Note: [👨👨👧] Now we can freeze `mappedParameters` to prevent
|
|
2900
|
+
// Note: [👨👨👧] Now we can freeze `mappedParameters` to prevent accidental modifications after mapping
|
|
2842
2901
|
Object.freeze(mappedParameters);
|
|
2843
2902
|
return mappedParameters;
|
|
2844
2903
|
}
|
|
@@ -2860,29 +2919,40 @@ class MultipleLlmExecutionTools {
|
|
|
2860
2919
|
return 'Multiple LLM Providers';
|
|
2861
2920
|
}
|
|
2862
2921
|
get description() {
|
|
2863
|
-
|
|
2922
|
+
const innerModelsTitlesAndDescriptions = this.llmExecutionTools
|
|
2923
|
+
.map(({ title, description }, index) => {
|
|
2924
|
+
const headLine = `${index + 1}) \`${title}\``;
|
|
2925
|
+
if (description === undefined) {
|
|
2926
|
+
return headLine;
|
|
2927
|
+
}
|
|
2928
|
+
return spaceTrim((block) => `
|
|
2929
|
+
${headLine}
|
|
2930
|
+
|
|
2931
|
+
${ /* <- Note: Indenting the description: */block(description)}
|
|
2932
|
+
`);
|
|
2933
|
+
})
|
|
2934
|
+
.join('\n\n');
|
|
2935
|
+
return spaceTrim((block) => `
|
|
2936
|
+
Multiple LLM Providers:
|
|
2937
|
+
|
|
2938
|
+
${block(innerModelsTitlesAndDescriptions)}
|
|
2939
|
+
`);
|
|
2864
2940
|
}
|
|
2865
2941
|
/**
|
|
2866
2942
|
* Check the configuration of all execution tools
|
|
2867
2943
|
*/
|
|
2868
2944
|
async checkConfiguration() {
|
|
2869
|
-
//
|
|
2870
|
-
|
|
2871
|
-
await llmExecutionTools.checkConfiguration();
|
|
2872
|
-
}
|
|
2945
|
+
// Note: Run checks in parallel
|
|
2946
|
+
await Promise.all(this.llmExecutionTools.map((tools) => tools.checkConfiguration()));
|
|
2873
2947
|
}
|
|
2874
2948
|
/**
|
|
2875
2949
|
* List all available models that can be used
|
|
2876
2950
|
* This lists is a combination of all available models from all execution tools
|
|
2877
2951
|
*/
|
|
2878
2952
|
async listModels() {
|
|
2879
|
-
|
|
2880
|
-
|
|
2881
|
-
|
|
2882
|
-
const models = await llmExecutionTools.listModels();
|
|
2883
|
-
availableModels.push(...models);
|
|
2884
|
-
}
|
|
2885
|
-
return availableModels;
|
|
2953
|
+
// Obtain all models in parallel and flatten
|
|
2954
|
+
const modelArrays = await Promise.all(this.llmExecutionTools.map((tools) => tools.listModels()));
|
|
2955
|
+
return modelArrays.flat();
|
|
2886
2956
|
}
|
|
2887
2957
|
/**
|
|
2888
2958
|
* Calls the best available chat model
|
|
@@ -3038,8 +3108,8 @@ function joinLlmExecutionTools(...llmExecutionTools) {
|
|
|
3038
3108
|
/**
|
|
3039
3109
|
* Extracts all code blocks from markdown.
|
|
3040
3110
|
*
|
|
3041
|
-
* Note: There are multiple
|
|
3042
|
-
* - `extractBlock` just extracts the content of the code block which is also used as
|
|
3111
|
+
* Note: There are multiple similar functions:
|
|
3112
|
+
* - `extractBlock` just extracts the content of the code block which is also used as built-in function for postprocessing
|
|
3043
3113
|
* - `extractJsonBlock` extracts exactly one valid JSON code block
|
|
3044
3114
|
* - `extractOneBlockFromMarkdown` extracts exactly one code block with language of the code block
|
|
3045
3115
|
* - `extractAllBlocksFromMarkdown` extracts all code blocks with language of the code block
|
|
@@ -3089,7 +3159,7 @@ function extractAllBlocksFromMarkdown(markdown) {
|
|
|
3089
3159
|
if (currentCodeBlock.content !== '') {
|
|
3090
3160
|
currentCodeBlock.content += '\n';
|
|
3091
3161
|
}
|
|
3092
|
-
currentCodeBlock.content += line.split('\\`\\`\\`').join('```') /* <- TODO: Maybe make
|
|
3162
|
+
currentCodeBlock.content += line.split('\\`\\`\\`').join('```') /* <- TODO: Maybe make proper unescape */;
|
|
3093
3163
|
}
|
|
3094
3164
|
}
|
|
3095
3165
|
if (currentCodeBlock !== null) {
|
|
@@ -3109,7 +3179,7 @@ function extractAllBlocksFromMarkdown(markdown) {
|
|
|
3109
3179
|
* - When there are multiple JSON code blocks the function throws a `ParseError`
|
|
3110
3180
|
*
|
|
3111
3181
|
* Note: It is not important if marked as ```json BUT if it is VALID JSON
|
|
3112
|
-
* Note: There are multiple
|
|
3182
|
+
* Note: There are multiple similar function:
|
|
3113
3183
|
* - `extractBlock` just extracts the content of the code block which is also used as build-in function for postprocessing
|
|
3114
3184
|
* - `extractJsonBlock` extracts exactly one valid JSON code block
|
|
3115
3185
|
* - `extractOneBlockFromMarkdown` extracts exactly one code block with language of the code block
|
|
@@ -3134,7 +3204,7 @@ function extractJsonBlock(markdown) {
|
|
|
3134
3204
|
}
|
|
3135
3205
|
/**
|
|
3136
3206
|
* TODO: Add some auto-healing logic + extract YAML, JSON5, TOML, etc.
|
|
3137
|
-
* TODO: [🏢] Make this logic part of `
|
|
3207
|
+
* TODO: [🏢] Make this logic part of `JsonFormatParser` or `isValidJsonString`
|
|
3138
3208
|
*/
|
|
3139
3209
|
|
|
3140
3210
|
/**
|
|
@@ -3160,7 +3230,7 @@ function arrayableToArray(input) {
|
|
|
3160
3230
|
* Just says that the variable is not used but should be kept
|
|
3161
3231
|
* No side effects.
|
|
3162
3232
|
*
|
|
3163
|
-
* Note: It can be
|
|
3233
|
+
* Note: It can be useful for:
|
|
3164
3234
|
*
|
|
3165
3235
|
* 1) Suppressing eager optimization of unused imports
|
|
3166
3236
|
* 2) Suppressing eslint errors of unused variables in the tests
|
|
@@ -3587,10 +3657,10 @@ for (let i = 0; i < defaultDiacriticsRemovalMap.length; i++) {
|
|
|
3587
3657
|
*/
|
|
3588
3658
|
|
|
3589
3659
|
/**
|
|
3590
|
-
*
|
|
3660
|
+
* Removes diacritic marks (accents) from characters in a string.
|
|
3591
3661
|
*
|
|
3592
|
-
* @param input
|
|
3593
|
-
* @returns
|
|
3662
|
+
* @param input The string containing diacritics to be normalized.
|
|
3663
|
+
* @returns The string with diacritics removed or normalized.
|
|
3594
3664
|
* @public exported from `@promptbook/utils`
|
|
3595
3665
|
*/
|
|
3596
3666
|
function removeDiacritics(input) {
|
|
@@ -3633,14 +3703,14 @@ const CountUtils = {
|
|
|
3633
3703
|
PAGES: countPages,
|
|
3634
3704
|
};
|
|
3635
3705
|
/**
|
|
3636
|
-
* TODO: [🧠][🤠] This should be
|
|
3706
|
+
* TODO: [🧠][🤠] This should be probably as part of `TextFormatParser`
|
|
3637
3707
|
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
3638
3708
|
*/
|
|
3639
3709
|
|
|
3640
3710
|
/**
|
|
3641
3711
|
* Function checkExpectations will check if the expectations on given value are met
|
|
3642
3712
|
*
|
|
3643
|
-
* Note: There are two
|
|
3713
|
+
* Note: There are two similar functions:
|
|
3644
3714
|
* - `checkExpectations` which throws an error if the expectations are not met
|
|
3645
3715
|
* - `isPassingExpectations` which returns a boolean
|
|
3646
3716
|
*
|
|
@@ -3661,13 +3731,17 @@ function checkExpectations(expectations, value) {
|
|
|
3661
3731
|
}
|
|
3662
3732
|
/**
|
|
3663
3733
|
* TODO: [💝] Unite object for expecting amount and format
|
|
3664
|
-
* TODO: [🧠][🤠] This should be part of `
|
|
3734
|
+
* TODO: [🧠][🤠] This should be part of `TextFormatParser`
|
|
3665
3735
|
* Note: [💝] and [🤠] are interconnected together
|
|
3666
3736
|
*/
|
|
3667
3737
|
|
|
3668
3738
|
/**
|
|
3669
|
-
*
|
|
3739
|
+
* Executes a pipeline task with multiple attempts, including joker and retry logic. Handles different task types
|
|
3740
|
+
* (prompt, script, dialog, etc.), applies postprocessing, checks expectations, and updates the execution report.
|
|
3741
|
+
* Throws errors if execution fails after all attempts.
|
|
3670
3742
|
*
|
|
3743
|
+
* @param options - The options for execution, including task, parameters, pipeline, and configuration.
|
|
3744
|
+
* @returns The result string of the executed task.
|
|
3671
3745
|
* @private internal utility of `createPipelineExecutor`
|
|
3672
3746
|
*/
|
|
3673
3747
|
async function executeAttempts(options) {
|
|
@@ -3889,7 +3963,7 @@ async function executeAttempts(options) {
|
|
|
3889
3963
|
if (task.format) {
|
|
3890
3964
|
if (task.format === 'JSON') {
|
|
3891
3965
|
if (!isValidJsonString($ongoingTaskResult.$resultString || '')) {
|
|
3892
|
-
// TODO: [🏢] Do more universally via `
|
|
3966
|
+
// TODO: [🏢] Do more universally via `FormatParser`
|
|
3893
3967
|
try {
|
|
3894
3968
|
$ongoingTaskResult.$resultString = extractJsonBlock($ongoingTaskResult.$resultString || '');
|
|
3895
3969
|
}
|
|
@@ -3991,12 +4065,16 @@ async function executeAttempts(options) {
|
|
|
3991
4065
|
*/
|
|
3992
4066
|
|
|
3993
4067
|
/**
|
|
3994
|
-
*
|
|
4068
|
+
* Executes a pipeline task that requires mapping or iterating over subvalues of a parameter (such as rows in a CSV).
|
|
4069
|
+
* Handles format and subformat resolution, error handling, and progress reporting.
|
|
4070
|
+
*
|
|
4071
|
+
* @param options - Options for execution, including task details and progress callback.
|
|
4072
|
+
* @returns The result of the subvalue mapping or execution attempts.
|
|
3995
4073
|
*
|
|
3996
4074
|
* @private internal utility of `createPipelineExecutor`
|
|
3997
4075
|
*/
|
|
3998
4076
|
async function executeFormatSubvalues(options) {
|
|
3999
|
-
const { task, jokerParameterNames, parameters, priority, csvSettings, pipelineIdentification } = options;
|
|
4077
|
+
const { task, jokerParameterNames, parameters, priority, csvSettings, onProgress, pipelineIdentification } = options;
|
|
4000
4078
|
if (task.foreach === undefined) {
|
|
4001
4079
|
return /* not await */ executeAttempts(options);
|
|
4002
4080
|
}
|
|
@@ -4027,16 +4105,16 @@ async function executeFormatSubvalues(options) {
|
|
|
4027
4105
|
${block(pipelineIdentification)}
|
|
4028
4106
|
`));
|
|
4029
4107
|
}
|
|
4030
|
-
const
|
|
4031
|
-
if (
|
|
4108
|
+
const subvalueParser = formatDefinition.subvalueParsers.find((subvalueParser) => [subvalueParser.subvalueName, ...(subvalueParser.aliases || [])].includes(task.foreach.subformatName));
|
|
4109
|
+
if (subvalueParser === undefined) {
|
|
4032
4110
|
throw new UnexpectedError(
|
|
4033
4111
|
// <- TODO: [🧠][🧐] Should be formats fixed per promptbook version or behave as plugins (=> change UnexpectedError)
|
|
4034
4112
|
spaceTrim((block) => `
|
|
4035
4113
|
Unsupported subformat name "${task.foreach.subformatName}" for format "${task.foreach.formatName}"
|
|
4036
4114
|
|
|
4037
4115
|
Available subformat names for format "${formatDefinition.formatName}":
|
|
4038
|
-
${block(formatDefinition.
|
|
4039
|
-
.map((
|
|
4116
|
+
${block(formatDefinition.subvalueParsers
|
|
4117
|
+
.map((subvalueParser) => subvalueParser.subvalueName)
|
|
4040
4118
|
.map((subvalueName) => `- ${subvalueName}`)
|
|
4041
4119
|
.join('\n'))}
|
|
4042
4120
|
|
|
@@ -4048,55 +4126,85 @@ async function executeFormatSubvalues(options) {
|
|
|
4048
4126
|
let formatSettings;
|
|
4049
4127
|
if (formatDefinition.formatName === 'CSV') {
|
|
4050
4128
|
formatSettings = csvSettings;
|
|
4051
|
-
// <- TODO: [🤹♂️] More universal, make
|
|
4052
|
-
}
|
|
4053
|
-
const resultString = await
|
|
4054
|
-
|
|
4055
|
-
|
|
4056
|
-
|
|
4057
|
-
|
|
4058
|
-
|
|
4059
|
-
|
|
4060
|
-
|
|
4061
|
-
|
|
4062
|
-
|
|
4063
|
-
|
|
4064
|
-
|
|
4065
|
-
|
|
4129
|
+
// <- TODO: [🤹♂️] More universal, make similar pattern for other formats for example \n vs \r\n in text
|
|
4130
|
+
}
|
|
4131
|
+
const resultString = await subvalueParser.mapValues({
|
|
4132
|
+
value: parameterValue,
|
|
4133
|
+
outputParameterName: task.foreach.outputSubparameterName,
|
|
4134
|
+
settings: formatSettings,
|
|
4135
|
+
onProgress(partialResultString) {
|
|
4136
|
+
return onProgress(Object.freeze({
|
|
4137
|
+
[task.resultingParameterName]: partialResultString,
|
|
4138
|
+
}));
|
|
4139
|
+
},
|
|
4140
|
+
async mapCallback(subparameters, index, length) {
|
|
4141
|
+
let mappedParameters;
|
|
4142
|
+
try {
|
|
4143
|
+
mappedParameters = mapAvailableToExpectedParameters({
|
|
4144
|
+
expectedParameters: Object.fromEntries(task.foreach.inputSubparameterNames.map((subparameterName) => [subparameterName, null])),
|
|
4145
|
+
availableParameters: subparameters,
|
|
4146
|
+
});
|
|
4066
4147
|
}
|
|
4067
|
-
|
|
4068
|
-
|
|
4148
|
+
catch (error) {
|
|
4149
|
+
if (!(error instanceof PipelineExecutionError)) {
|
|
4150
|
+
throw error;
|
|
4151
|
+
}
|
|
4152
|
+
const highLevelError = new PipelineExecutionError(spaceTrim((block) => `
|
|
4153
|
+
${error.message}
|
|
4069
4154
|
|
|
4070
|
-
|
|
4071
|
-
|
|
4155
|
+
This is error in FOREACH command when mapping ${formatDefinition.formatName} ${subvalueParser.subvalueName} data (${index + 1}/${length})
|
|
4156
|
+
You have probably passed wrong data to pipeline or wrong data was generated which are processed by FOREACH command
|
|
4072
4157
|
|
|
4073
|
-
|
|
4074
|
-
|
|
4075
|
-
|
|
4076
|
-
|
|
4077
|
-
|
|
4078
|
-
|
|
4079
|
-
|
|
4080
|
-
|
|
4081
|
-
|
|
4082
|
-
|
|
4083
|
-
|
|
4084
|
-
|
|
4085
|
-
|
|
4086
|
-
|
|
4087
|
-
|
|
4088
|
-
|
|
4089
|
-
|
|
4090
|
-
|
|
4091
|
-
|
|
4092
|
-
|
|
4158
|
+
${block(pipelineIdentification)}
|
|
4159
|
+
`));
|
|
4160
|
+
if (length > BIG_DATASET_TRESHOLD) {
|
|
4161
|
+
console.error(highLevelError);
|
|
4162
|
+
return FAILED_VALUE_PLACEHOLDER;
|
|
4163
|
+
}
|
|
4164
|
+
throw highLevelError;
|
|
4165
|
+
}
|
|
4166
|
+
const allSubparameters = {
|
|
4167
|
+
...parameters,
|
|
4168
|
+
...mappedParameters,
|
|
4169
|
+
};
|
|
4170
|
+
Object.freeze(allSubparameters);
|
|
4171
|
+
try {
|
|
4172
|
+
const subresultString = await executeAttempts({
|
|
4173
|
+
...options,
|
|
4174
|
+
priority: priority + index,
|
|
4175
|
+
parameters: allSubparameters,
|
|
4176
|
+
pipelineIdentification: spaceTrim((block) => `
|
|
4177
|
+
${block(pipelineIdentification)}
|
|
4178
|
+
Subparameter index: ${index}
|
|
4179
|
+
`),
|
|
4180
|
+
});
|
|
4181
|
+
return subresultString;
|
|
4182
|
+
}
|
|
4183
|
+
catch (error) {
|
|
4184
|
+
if (length > BIG_DATASET_TRESHOLD) {
|
|
4185
|
+
console.error(spaceTrim((block) => `
|
|
4186
|
+
${error.message}
|
|
4187
|
+
|
|
4188
|
+
This is error in FOREACH command when processing ${formatDefinition.formatName} ${subvalueParser.subvalueName} data (${index + 1}/${length})
|
|
4189
|
+
|
|
4190
|
+
${block(pipelineIdentification)}
|
|
4191
|
+
`));
|
|
4192
|
+
return FAILED_VALUE_PLACEHOLDER;
|
|
4193
|
+
}
|
|
4194
|
+
throw error;
|
|
4195
|
+
}
|
|
4196
|
+
},
|
|
4093
4197
|
});
|
|
4094
4198
|
return resultString;
|
|
4095
4199
|
}
|
|
4096
4200
|
|
|
4097
4201
|
/**
|
|
4098
|
-
*
|
|
4202
|
+
* Returns the context for a given task, typically used to provide additional information or variables
|
|
4203
|
+
* required for the execution of the task within a pipeline. The context is returned as a string value
|
|
4204
|
+
* that may include markdown formatting.
|
|
4099
4205
|
*
|
|
4206
|
+
* @param task - The task for which the context is being generated. This should be a deeply immutable TaskJson object.
|
|
4207
|
+
* @returns The context as a string, formatted as markdown and parameter value.
|
|
4100
4208
|
* @private internal utility of `createPipelineExecutor`
|
|
4101
4209
|
*/
|
|
4102
4210
|
async function getContextForTask(task) {
|
|
@@ -4104,7 +4212,7 @@ async function getContextForTask(task) {
|
|
|
4104
4212
|
}
|
|
4105
4213
|
|
|
4106
4214
|
/**
|
|
4107
|
-
*
|
|
4215
|
+
* Retrieves example values or templates for a given task, used to guide or validate pipeline execution.
|
|
4108
4216
|
*
|
|
4109
4217
|
* @private internal utility of `createPipelineExecutor`
|
|
4110
4218
|
*/
|
|
@@ -4113,91 +4221,128 @@ async function getExamplesForTask(task) {
|
|
|
4113
4221
|
}
|
|
4114
4222
|
|
|
4115
4223
|
/**
|
|
4116
|
-
*
|
|
4224
|
+
* Computes the cosine similarity between two embedding vectors
|
|
4225
|
+
*
|
|
4226
|
+
* Note: This is helping function for RAG (retrieval-augmented generation)
|
|
4227
|
+
*
|
|
4228
|
+
* @param embeddingVector1
|
|
4229
|
+
* @param embeddingVector2
|
|
4230
|
+
* @returns Cosine similarity between the two vectors
|
|
4231
|
+
*
|
|
4232
|
+
* @public exported from `@promptbook/core`
|
|
4233
|
+
*/
|
|
4234
|
+
function computeCosineSimilarity(embeddingVector1, embeddingVector2) {
|
|
4235
|
+
if (embeddingVector1.length !== embeddingVector2.length) {
|
|
4236
|
+
throw new TypeError('Embedding vectors must have the same length');
|
|
4237
|
+
}
|
|
4238
|
+
const dotProduct = embeddingVector1.reduce((sum, value, index) => sum + value * embeddingVector2[index], 0);
|
|
4239
|
+
const magnitude1 = Math.sqrt(embeddingVector1.reduce((sum, value) => sum + value * value, 0));
|
|
4240
|
+
const magnitude2 = Math.sqrt(embeddingVector2.reduce((sum, value) => sum + value * value, 0));
|
|
4241
|
+
return 1 - dotProduct / (magnitude1 * magnitude2);
|
|
4242
|
+
}
|
|
4243
|
+
|
|
4244
|
+
/**
|
|
4245
|
+
*
|
|
4246
|
+
* @param knowledgePieces
|
|
4247
|
+
* @returns
|
|
4117
4248
|
*
|
|
4118
|
-
*
|
|
4249
|
+
* @private internal utility of `createPipelineExecutor`
|
|
4250
|
+
*/
|
|
4251
|
+
function knowledgePiecesToString(knowledgePieces) {
|
|
4252
|
+
return knowledgePieces
|
|
4253
|
+
.map((knowledgePiece) => {
|
|
4254
|
+
const { content } = knowledgePiece;
|
|
4255
|
+
return `- ${content}`;
|
|
4256
|
+
})
|
|
4257
|
+
.join('\n');
|
|
4258
|
+
// <- TODO: [🧠] Some smarter aggregation of knowledge pieces, single-line vs multi-line vs mixed
|
|
4259
|
+
}
|
|
4260
|
+
|
|
4261
|
+
/**
|
|
4262
|
+
* Retrieves the most relevant knowledge pieces for a given task using embedding-based similarity search.
|
|
4263
|
+
* This is where retrieval-augmented generation (RAG) is performed to enhance the task with external knowledge.
|
|
4119
4264
|
*
|
|
4120
4265
|
* @private internal utility of `createPipelineExecutor`
|
|
4121
4266
|
*/
|
|
4122
4267
|
async function getKnowledgeForTask(options) {
|
|
4123
|
-
const { tools, preparedPipeline, task } = options;
|
|
4268
|
+
const { tools, preparedPipeline, task, parameters } = options;
|
|
4124
4269
|
const firstKnowlegePiece = preparedPipeline.knowledgePieces[0];
|
|
4125
4270
|
const firstKnowlegeIndex = firstKnowlegePiece === null || firstKnowlegePiece === void 0 ? void 0 : firstKnowlegePiece.index[0];
|
|
4126
4271
|
// <- TODO: Do not use just first knowledge piece and first index to determine embedding model, use also keyword search
|
|
4127
4272
|
if (firstKnowlegePiece === undefined || firstKnowlegeIndex === undefined) {
|
|
4128
|
-
return '
|
|
4273
|
+
return ''; // <- Note: Np knowledge present, return empty string
|
|
4129
4274
|
}
|
|
4130
|
-
|
|
4131
|
-
|
|
4132
|
-
|
|
4133
|
-
|
|
4134
|
-
|
|
4135
|
-
|
|
4136
|
-
|
|
4137
|
-
|
|
4138
|
-
|
|
4139
|
-
|
|
4140
|
-
|
|
4141
|
-
|
|
4142
|
-
}
|
|
4143
|
-
|
|
4144
|
-
|
|
4145
|
-
|
|
4146
|
-
|
|
4147
|
-
|
|
4148
|
-
|
|
4149
|
-
|
|
4275
|
+
try {
|
|
4276
|
+
// TODO: [🚐] Make arrayable LLMs -> single LLM DRY
|
|
4277
|
+
const _llms = arrayableToArray(tools.llm);
|
|
4278
|
+
const llmTools = _llms.length === 1 ? _llms[0] : joinLlmExecutionTools(..._llms);
|
|
4279
|
+
const taskEmbeddingPrompt = {
|
|
4280
|
+
title: 'Knowledge Search',
|
|
4281
|
+
modelRequirements: {
|
|
4282
|
+
modelVariant: 'EMBEDDING',
|
|
4283
|
+
modelName: firstKnowlegeIndex.modelName,
|
|
4284
|
+
},
|
|
4285
|
+
content: task.content,
|
|
4286
|
+
parameters,
|
|
4287
|
+
};
|
|
4288
|
+
const taskEmbeddingResult = await llmTools.callEmbeddingModel(taskEmbeddingPrompt);
|
|
4289
|
+
const knowledgePiecesWithRelevance = preparedPipeline.knowledgePieces.map((knowledgePiece) => {
|
|
4290
|
+
const { index } = knowledgePiece;
|
|
4291
|
+
const knowledgePieceIndex = index.find((i) => i.modelName === firstKnowlegeIndex.modelName);
|
|
4292
|
+
// <- TODO: Do not use just first knowledge piece and first index to determine embedding model
|
|
4293
|
+
if (knowledgePieceIndex === undefined) {
|
|
4294
|
+
return {
|
|
4295
|
+
content: knowledgePiece.content,
|
|
4296
|
+
relevance: 0,
|
|
4297
|
+
};
|
|
4298
|
+
}
|
|
4299
|
+
const relevance = computeCosineSimilarity(knowledgePieceIndex.position, taskEmbeddingResult.content);
|
|
4150
4300
|
return {
|
|
4151
4301
|
content: knowledgePiece.content,
|
|
4152
|
-
relevance
|
|
4302
|
+
relevance,
|
|
4153
4303
|
};
|
|
4154
|
-
}
|
|
4155
|
-
const
|
|
4156
|
-
|
|
4157
|
-
|
|
4158
|
-
|
|
4159
|
-
|
|
4160
|
-
|
|
4161
|
-
|
|
4162
|
-
|
|
4163
|
-
|
|
4164
|
-
|
|
4165
|
-
|
|
4166
|
-
|
|
4167
|
-
|
|
4168
|
-
|
|
4169
|
-
|
|
4170
|
-
|
|
4171
|
-
|
|
4172
|
-
|
|
4173
|
-
|
|
4174
|
-
|
|
4175
|
-
|
|
4176
|
-
// TODO: !!!!!! Annotate + to new file
|
|
4177
|
-
function computeCosineSimilarity(embeddingVector1, embeddingVector2) {
|
|
4178
|
-
if (embeddingVector1.length !== embeddingVector2.length) {
|
|
4179
|
-
throw new TypeError('Embedding vectors must have the same length');
|
|
4304
|
+
});
|
|
4305
|
+
const knowledgePiecesSorted = knowledgePiecesWithRelevance.sort((a, b) => a.relevance - b.relevance);
|
|
4306
|
+
const knowledgePiecesLimited = knowledgePiecesSorted.slice(0, 5);
|
|
4307
|
+
/*
|
|
4308
|
+
console.log('`getKnowledgeForTask` Embedding', {
|
|
4309
|
+
task,
|
|
4310
|
+
taskEmbeddingPrompt,
|
|
4311
|
+
taskEmbeddingResult,
|
|
4312
|
+
firstKnowlegePiece,
|
|
4313
|
+
firstKnowlegeIndex,
|
|
4314
|
+
knowledgePiecesWithRelevance,
|
|
4315
|
+
knowledgePiecesSorted,
|
|
4316
|
+
knowledgePiecesLimited,
|
|
4317
|
+
});
|
|
4318
|
+
*/
|
|
4319
|
+
return knowledgePiecesToString(knowledgePiecesLimited);
|
|
4320
|
+
}
|
|
4321
|
+
catch (error) {
|
|
4322
|
+
assertsError(error);
|
|
4323
|
+
console.error('Error in `getKnowledgeForTask`', error);
|
|
4324
|
+
// Note: If the LLM fails, just return all knowledge pieces
|
|
4325
|
+
return knowledgePiecesToString(preparedPipeline.knowledgePieces);
|
|
4180
4326
|
}
|
|
4181
|
-
const dotProduct = embeddingVector1.reduce((sum, value, index) => sum + value * embeddingVector2[index], 0);
|
|
4182
|
-
const magnitude1 = Math.sqrt(embeddingVector1.reduce((sum, value) => sum + value * value, 0));
|
|
4183
|
-
const magnitude2 = Math.sqrt(embeddingVector2.reduce((sum, value) => sum + value * value, 0));
|
|
4184
|
-
return 1 - dotProduct / (magnitude1 * magnitude2);
|
|
4185
4327
|
}
|
|
4186
4328
|
/**
|
|
4187
|
-
* TODO: !!!! Verify if this is working
|
|
4188
4329
|
* TODO: [♨] Implement Better - use keyword search
|
|
4189
4330
|
* TODO: [♨] Examples of values
|
|
4190
4331
|
*/
|
|
4191
4332
|
|
|
4192
4333
|
/**
|
|
4193
|
-
*
|
|
4334
|
+
* Retrieves all reserved parameters for a given pipeline task, including context, knowledge, examples, and metadata.
|
|
4335
|
+
* Ensures all reserved parameters are defined and throws if any are missing.
|
|
4336
|
+
*
|
|
4337
|
+
* @param options - Options including tools, pipeline, task, and context.
|
|
4338
|
+
* @returns An object containing all reserved parameters for the task.
|
|
4194
4339
|
*
|
|
4195
4340
|
* @private internal utility of `createPipelineExecutor`
|
|
4196
4341
|
*/
|
|
4197
4342
|
async function getReservedParametersForTask(options) {
|
|
4198
|
-
const { tools, preparedPipeline, task, pipelineIdentification } = options;
|
|
4343
|
+
const { tools, preparedPipeline, task, parameters, pipelineIdentification, isVerbose } = options;
|
|
4199
4344
|
const context = await getContextForTask(); // <- [🏍]
|
|
4200
|
-
const knowledge = await getKnowledgeForTask({ tools, preparedPipeline, task });
|
|
4345
|
+
const knowledge = await getKnowledgeForTask({ tools, preparedPipeline, task, parameters });
|
|
4201
4346
|
const examples = await getExamplesForTask();
|
|
4202
4347
|
const currentDate = new Date().toISOString(); // <- TODO: [🧠][💩] Better
|
|
4203
4348
|
const modelName = RESERVED_PARAMETER_MISSING_VALUE;
|
|
@@ -4209,6 +4354,9 @@ async function getReservedParametersForTask(options) {
|
|
|
4209
4354
|
currentDate,
|
|
4210
4355
|
modelName,
|
|
4211
4356
|
};
|
|
4357
|
+
if (isVerbose) {
|
|
4358
|
+
console.info('Reserved parameters for task:', { options, reservedParameters });
|
|
4359
|
+
}
|
|
4212
4360
|
// Note: Doublecheck that ALL reserved parameters are defined:
|
|
4213
4361
|
for (const parameterName of RESERVED_PARAMETER_NAMES) {
|
|
4214
4362
|
if (reservedParameters[parameterName] === undefined) {
|
|
@@ -4223,23 +4371,21 @@ async function getReservedParametersForTask(options) {
|
|
|
4223
4371
|
}
|
|
4224
4372
|
|
|
4225
4373
|
/**
|
|
4226
|
-
*
|
|
4374
|
+
* Executes a single task within a pipeline, handling parameter validation, error checking, and progress reporting.
|
|
4375
|
+
*
|
|
4376
|
+
* @param options - Options for execution, including the task, pipeline, parameters, and callbacks.
|
|
4377
|
+
* @returns The output parameters produced by the task.
|
|
4227
4378
|
*
|
|
4228
4379
|
* @private internal utility of `createPipelineExecutor`
|
|
4229
4380
|
*/
|
|
4230
4381
|
async function executeTask(options) {
|
|
4231
4382
|
const { currentTask, preparedPipeline, parametersToPass, tools, onProgress, $executionReport, pipelineIdentification, maxExecutionAttempts, maxParallelCount, csvSettings, isVerbose, rootDirname, cacheDirname, intermediateFilesStrategy, isAutoInstalled, isNotPreparedWarningSupressed, } = options;
|
|
4232
4383
|
const priority = preparedPipeline.tasks.length - preparedPipeline.tasks.indexOf(currentTask);
|
|
4233
|
-
await onProgress({
|
|
4234
|
-
outputParameters: {
|
|
4235
|
-
[currentTask.resultingParameterName]: '', // <- TODO: [🧠] What is the best value here?
|
|
4236
|
-
},
|
|
4237
|
-
});
|
|
4238
4384
|
// Note: Check consistency of used and dependent parameters which was also done in `validatePipeline`, but it’s good to doublecheck
|
|
4239
4385
|
const usedParameterNames = extractParameterNamesFromTask(currentTask);
|
|
4240
4386
|
const dependentParameterNames = new Set(currentTask.dependentParameterNames);
|
|
4241
4387
|
// TODO: [👩🏾🤝👩🏻] Use here `mapAvailableToExpectedParameters`
|
|
4242
|
-
if (union(difference(usedParameterNames, dependentParameterNames), difference(dependentParameterNames, usedParameterNames)).size !== 0) {
|
|
4388
|
+
if (difference(union(difference(usedParameterNames, dependentParameterNames), difference(dependentParameterNames, usedParameterNames)), new Set(RESERVED_PARAMETER_NAMES)).size !== 0) {
|
|
4243
4389
|
throw new UnexpectedError(spaceTrim$1((block) => `
|
|
4244
4390
|
Dependent parameters are not consistent with used parameters:
|
|
4245
4391
|
|
|
@@ -4257,13 +4403,16 @@ async function executeTask(options) {
|
|
|
4257
4403
|
|
|
4258
4404
|
`));
|
|
4259
4405
|
}
|
|
4406
|
+
const reservedParameters = await getReservedParametersForTask({
|
|
4407
|
+
tools,
|
|
4408
|
+
preparedPipeline,
|
|
4409
|
+
task: currentTask,
|
|
4410
|
+
pipelineIdentification,
|
|
4411
|
+
parameters: parametersToPass,
|
|
4412
|
+
isVerbose,
|
|
4413
|
+
});
|
|
4260
4414
|
const definedParameters = Object.freeze({
|
|
4261
|
-
...
|
|
4262
|
-
tools,
|
|
4263
|
-
preparedPipeline,
|
|
4264
|
-
task: currentTask,
|
|
4265
|
-
pipelineIdentification,
|
|
4266
|
-
})),
|
|
4415
|
+
...reservedParameters,
|
|
4267
4416
|
...parametersToPass,
|
|
4268
4417
|
});
|
|
4269
4418
|
const definedParameterNames = new Set(Object.keys(definedParameters));
|
|
@@ -4308,6 +4457,7 @@ async function executeTask(options) {
|
|
|
4308
4457
|
preparedPipeline,
|
|
4309
4458
|
tools,
|
|
4310
4459
|
$executionReport,
|
|
4460
|
+
onProgress,
|
|
4311
4461
|
pipelineIdentification,
|
|
4312
4462
|
maxExecutionAttempts,
|
|
4313
4463
|
maxParallelCount,
|
|
@@ -4335,7 +4485,8 @@ async function executeTask(options) {
|
|
|
4335
4485
|
*/
|
|
4336
4486
|
|
|
4337
4487
|
/**
|
|
4338
|
-
*
|
|
4488
|
+
* Filters and returns only the output parameters from the provided pipeline execution options.
|
|
4489
|
+
* Adds warnings for any expected output parameters that are missing.
|
|
4339
4490
|
*
|
|
4340
4491
|
* @private internal utility of `createPipelineExecutor`
|
|
4341
4492
|
*/
|
|
@@ -4360,9 +4511,12 @@ function filterJustOutputParameters(options) {
|
|
|
4360
4511
|
}
|
|
4361
4512
|
|
|
4362
4513
|
/**
|
|
4363
|
-
*
|
|
4514
|
+
* Executes an entire pipeline, resolving tasks in dependency order, handling errors, and reporting progress.
|
|
4515
|
+
*
|
|
4516
|
+
* Note: This is not a `PipelineExecutor` (which is bound to a single pipeline), but a utility function used by `createPipelineExecutor` to create a `PipelineExecutor`.
|
|
4364
4517
|
*
|
|
4365
|
-
*
|
|
4518
|
+
* @param options - Options for execution, including input parameters, pipeline, and callbacks.
|
|
4519
|
+
* @returns The result of the pipeline execution, including output parameters, errors, and usage statistics.
|
|
4366
4520
|
*
|
|
4367
4521
|
* @private internal utility of `createPipelineExecutor`
|
|
4368
4522
|
*/
|
|
@@ -4685,10 +4839,27 @@ function createPipelineExecutor(options) {
|
|
|
4685
4839
|
cacheDirname,
|
|
4686
4840
|
intermediateFilesStrategy,
|
|
4687
4841
|
isAutoInstalled,
|
|
4842
|
+
}).catch((error) => {
|
|
4843
|
+
assertsError(error);
|
|
4844
|
+
return exportJson({
|
|
4845
|
+
name: 'pipelineExecutorResult',
|
|
4846
|
+
message: `Unuccessful PipelineExecutorResult, last catch`,
|
|
4847
|
+
order: [],
|
|
4848
|
+
value: {
|
|
4849
|
+
isSuccessful: false,
|
|
4850
|
+
errors: [serializeError(error)],
|
|
4851
|
+
warnings: [],
|
|
4852
|
+
usage: UNCERTAIN_USAGE,
|
|
4853
|
+
executionReport: null,
|
|
4854
|
+
outputParameters: {},
|
|
4855
|
+
preparedPipeline,
|
|
4856
|
+
},
|
|
4857
|
+
});
|
|
4688
4858
|
});
|
|
4689
4859
|
};
|
|
4690
4860
|
const pipelineExecutor = (inputParameters) => createTask({
|
|
4691
4861
|
taskType: 'EXECUTION',
|
|
4862
|
+
title: pipeline.title,
|
|
4692
4863
|
taskProcessCallback(updateOngoingResult) {
|
|
4693
4864
|
return pipelineExecutorWithCallback(inputParameters, async (newOngoingResult) => {
|
|
4694
4865
|
updateOngoingResult(newOngoingResult);
|
|
@@ -4740,12 +4911,14 @@ function countUsage(llmTools) {
|
|
|
4740
4911
|
const spending = new Subject();
|
|
4741
4912
|
const proxyTools = {
|
|
4742
4913
|
get title() {
|
|
4743
|
-
|
|
4744
|
-
|
|
4914
|
+
return `${llmTools.title} (+usage)`;
|
|
4915
|
+
// <- TODO: [🧈] Maybe standartize the suffix when wrapping `LlmExecutionTools` up
|
|
4916
|
+
// <- TODO: [🧈][🧠] Does it make sence to suffix "(+usage)"?
|
|
4745
4917
|
},
|
|
4746
4918
|
get description() {
|
|
4747
|
-
|
|
4748
|
-
|
|
4919
|
+
return `${llmTools.description} (+usage)`;
|
|
4920
|
+
// <- TODO: [🧈] Maybe standartize the suffix when wrapping `LlmExecutionTools` up
|
|
4921
|
+
// <- TODO: [🧈][🧠] Does it make sence to suffix "(+usage)"?
|
|
4749
4922
|
},
|
|
4750
4923
|
checkConfiguration() {
|
|
4751
4924
|
return /* not await */ llmTools.checkConfiguration();
|
|
@@ -4868,7 +5041,8 @@ async function preparePersona(personaDescription, tools, options) {
|
|
|
4868
5041
|
*/
|
|
4869
5042
|
|
|
4870
5043
|
/**
|
|
4871
|
-
*
|
|
5044
|
+
* Safely retrieves the global scope object (window in browser, global in Node.js)
|
|
5045
|
+
* regardless of the JavaScript environment in which the code is running
|
|
4872
5046
|
*
|
|
4873
5047
|
* Note: `$` is used to indicate that this function is not a pure function - it access global scope
|
|
4874
5048
|
*
|
|
@@ -4879,10 +5053,10 @@ function $getGlobalScope() {
|
|
|
4879
5053
|
}
|
|
4880
5054
|
|
|
4881
5055
|
/**
|
|
4882
|
-
*
|
|
5056
|
+
* Normalizes a text string to SCREAMING_CASE (all uppercase with underscores).
|
|
4883
5057
|
*
|
|
4884
|
-
* @param text
|
|
4885
|
-
* @returns
|
|
5058
|
+
* @param text The text string to be converted to SCREAMING_CASE format.
|
|
5059
|
+
* @returns The normalized text in SCREAMING_CASE format.
|
|
4886
5060
|
* @example 'HELLO_WORLD'
|
|
4887
5061
|
* @example 'I_LOVE_PROMPTBOOK'
|
|
4888
5062
|
* @public exported from `@promptbook/utils`
|
|
@@ -4934,10 +5108,10 @@ function normalizeTo_SCREAMING_CASE(text) {
|
|
|
4934
5108
|
*/
|
|
4935
5109
|
|
|
4936
5110
|
/**
|
|
4937
|
-
*
|
|
5111
|
+
* Normalizes a text string to snake_case format.
|
|
4938
5112
|
*
|
|
4939
|
-
* @param text
|
|
4940
|
-
* @returns
|
|
5113
|
+
* @param text The text string to be converted to snake_case format.
|
|
5114
|
+
* @returns The normalized text in snake_case format.
|
|
4941
5115
|
* @example 'hello_world'
|
|
4942
5116
|
* @example 'i_love_promptbook'
|
|
4943
5117
|
* @public exported from `@promptbook/utils`
|
|
@@ -4947,11 +5121,11 @@ function normalizeTo_snake_case(text) {
|
|
|
4947
5121
|
}
|
|
4948
5122
|
|
|
4949
5123
|
/**
|
|
4950
|
-
*
|
|
5124
|
+
* Global registry for storing and managing registered entities of a given type.
|
|
4951
5125
|
*
|
|
4952
5126
|
* Note: `$` is used to indicate that this function is not a pure function - it accesses and adds variables in global scope.
|
|
4953
5127
|
*
|
|
4954
|
-
* @private internal utility, exported are only
|
|
5128
|
+
* @private internal utility, exported are only singleton instances of this class
|
|
4955
5129
|
*/
|
|
4956
5130
|
class $Register {
|
|
4957
5131
|
constructor(registerName) {
|
|
@@ -4995,10 +5169,10 @@ class $Register {
|
|
|
4995
5169
|
}
|
|
4996
5170
|
|
|
4997
5171
|
/**
|
|
4998
|
-
*
|
|
5172
|
+
* Global registry for storing metadata about all available scrapers and converters.
|
|
4999
5173
|
*
|
|
5000
|
-
* Note: `$` is used to indicate that this interacts with the global scope
|
|
5001
|
-
* @singleton Only one instance of each register is created per build, but
|
|
5174
|
+
* Note: `$` is used to indicate that this interacts with the global scope.
|
|
5175
|
+
* @singleton Only one instance of each register is created per build, but there can be more in different contexts (e.g., tests).
|
|
5002
5176
|
* @public exported from `@promptbook/core`
|
|
5003
5177
|
*/
|
|
5004
5178
|
const $scrapersMetadataRegister = new $Register('scrapers_metadata');
|
|
@@ -5007,10 +5181,11 @@ const $scrapersMetadataRegister = new $Register('scrapers_metadata');
|
|
|
5007
5181
|
*/
|
|
5008
5182
|
|
|
5009
5183
|
/**
|
|
5010
|
-
*
|
|
5184
|
+
* Registry for all available scrapers in the system.
|
|
5185
|
+
* Central point for registering and accessing different types of content scrapers.
|
|
5011
5186
|
*
|
|
5012
5187
|
* Note: `$` is used to indicate that this interacts with the global scope
|
|
5013
|
-
* @singleton Only one instance of each register is created per build, but
|
|
5188
|
+
* @singleton Only one instance of each register is created per build, but there can be more than one in different build modules
|
|
5014
5189
|
* @public exported from `@promptbook/core`
|
|
5015
5190
|
*/
|
|
5016
5191
|
const $scrapersRegister = new $Register('scraper_constructors');
|
|
@@ -5106,10 +5281,10 @@ function $registeredScrapersMessage(availableScrapers) {
|
|
|
5106
5281
|
*/
|
|
5107
5282
|
|
|
5108
5283
|
/**
|
|
5109
|
-
*
|
|
5284
|
+
* Converts a given text to kebab-case format.
|
|
5110
5285
|
*
|
|
5111
|
-
* @param text
|
|
5112
|
-
* @returns
|
|
5286
|
+
* @param text The text to be converted.
|
|
5287
|
+
* @returns The kebab-case formatted string.
|
|
5113
5288
|
* @example 'hello-world'
|
|
5114
5289
|
* @example 'i-love-promptbook'
|
|
5115
5290
|
* @public exported from `@promptbook/utils`
|
|
@@ -5178,7 +5353,8 @@ function knowledgeSourceContentToName(knowledgeSourceContent) {
|
|
|
5178
5353
|
*/
|
|
5179
5354
|
|
|
5180
5355
|
/**
|
|
5181
|
-
*
|
|
5356
|
+
* Converts a name to a properly formatted subfolder path for cache storage.
|
|
5357
|
+
* Handles normalization and path formatting to create consistent cache directory structures.
|
|
5182
5358
|
*
|
|
5183
5359
|
* @private for `FileCacheStorage`
|
|
5184
5360
|
*/
|
|
@@ -5258,11 +5434,11 @@ function removeEmojis(text) {
|
|
|
5258
5434
|
}
|
|
5259
5435
|
|
|
5260
5436
|
/**
|
|
5261
|
-
*
|
|
5437
|
+
* Converts a title string into a normalized name.
|
|
5262
5438
|
*
|
|
5263
|
-
* @param value
|
|
5264
|
-
* @returns
|
|
5265
|
-
* @example
|
|
5439
|
+
* @param value The title string to be converted to a name.
|
|
5440
|
+
* @returns A normalized name derived from the input title.
|
|
5441
|
+
* @example 'Hello World!' -> 'hello-world'
|
|
5266
5442
|
* @public exported from `@promptbook/utils`
|
|
5267
5443
|
*/
|
|
5268
5444
|
function titleToName(value) {
|
|
@@ -5313,7 +5489,9 @@ const promptbookFetch = async (urlOrRequest, init) => {
|
|
|
5313
5489
|
*/
|
|
5314
5490
|
|
|
5315
5491
|
/**
|
|
5316
|
-
*
|
|
5492
|
+
* Factory function that creates a handler for processing knowledge sources.
|
|
5493
|
+
* Provides standardized processing of different types of knowledge sources
|
|
5494
|
+
* across various scraper implementations.
|
|
5317
5495
|
*
|
|
5318
5496
|
* @public exported from `@promptbook/core`
|
|
5319
5497
|
*/
|
|
@@ -5452,7 +5630,7 @@ async function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
|
|
|
5452
5630
|
}
|
|
5453
5631
|
|
|
5454
5632
|
/**
|
|
5455
|
-
* Prepares the
|
|
5633
|
+
* Prepares the knowledge pieces
|
|
5456
5634
|
*
|
|
5457
5635
|
* @see https://github.com/webgptorg/promptbook/discussions/41
|
|
5458
5636
|
* @public exported from `@promptbook/core`
|
|
@@ -5548,15 +5726,18 @@ TODO: [🧊] This is how it can look in future
|
|
|
5548
5726
|
* TODO: [🧊] In future one preparation can take data from previous preparation and save tokens and time
|
|
5549
5727
|
* Put `knowledgePieces` into `PrepareKnowledgeOptions`
|
|
5550
5728
|
* TODO: [🪂] More than max things can run in parallel by acident [1,[2a,2b,_],[3a,3b,_]]
|
|
5551
|
-
* TODO: [🧠][❎] Do here
|
|
5729
|
+
* TODO: [🧠][❎] Do here proper M:N mapping
|
|
5552
5730
|
* [x] One source can make multiple pieces
|
|
5553
5731
|
* [ ] One piece can have multiple sources
|
|
5554
5732
|
*/
|
|
5555
5733
|
|
|
5556
5734
|
/**
|
|
5557
|
-
*
|
|
5735
|
+
* Prepares tasks by adding knowledge to the prompt and ensuring all necessary parameters are included.
|
|
5558
5736
|
*
|
|
5559
|
-
* @
|
|
5737
|
+
* @param tasks Sequence of tasks that are chained together to form a pipeline
|
|
5738
|
+
* @returns A promise that resolves to the prepared tasks.
|
|
5739
|
+
*
|
|
5740
|
+
* @private internal utility of `preparePipeline`
|
|
5560
5741
|
*/
|
|
5561
5742
|
async function prepareTasks(pipeline, tools, options) {
|
|
5562
5743
|
const { maxParallelCount = DEFAULT_MAX_PARALLEL_COUNT } = options;
|
|
@@ -5594,7 +5775,7 @@ async function prepareTasks(pipeline, tools, options) {
|
|
|
5594
5775
|
return { tasksPrepared };
|
|
5595
5776
|
}
|
|
5596
5777
|
/**
|
|
5597
|
-
* TODO: [😂] Adding knowledge should be convert to async high-level abstractions,
|
|
5778
|
+
* TODO: [😂] Adding knowledge should be convert to async high-level abstractions, similar thing with expectations to sync high-level abstractions
|
|
5598
5779
|
* TODO: [🧠] Add context to each task (if missing)
|
|
5599
5780
|
* TODO: [🧠] What is better name `prepareTask` or `prepareTaskAndParameters`
|
|
5600
5781
|
* TODO: [♨][main] !!3 Prepare index the examples and maybe tasks
|
|
@@ -5730,7 +5911,7 @@ async function preparePipeline(pipeline, tools, options) {
|
|
|
5730
5911
|
order: ORDER_OF_PIPELINE_JSON,
|
|
5731
5912
|
value: {
|
|
5732
5913
|
...pipeline,
|
|
5733
|
-
// <- TODO:
|
|
5914
|
+
// <- TODO: Probably deeply clone the pipeline because `$exportJson` freezes the subobjects
|
|
5734
5915
|
title,
|
|
5735
5916
|
knowledgeSources: knowledgeSourcesPrepared,
|
|
5736
5917
|
knowledgePieces: knowledgePiecesPrepared,
|
|
@@ -6006,7 +6187,7 @@ const sectionCommandParser = {
|
|
|
6006
6187
|
throw new ParseError(`Task section and example section must end with return statement -> {parameterName}`);
|
|
6007
6188
|
};
|
|
6008
6189
|
if ($taskJson.content === undefined) {
|
|
6009
|
-
throw new UnexpectedError(`Content is missing in the taskJson -
|
|
6190
|
+
throw new UnexpectedError(`Content is missing in the taskJson - probably commands are applied in wrong order`);
|
|
6010
6191
|
}
|
|
6011
6192
|
if (command.taskType === 'EXAMPLE') {
|
|
6012
6193
|
expectResultingParameterName();
|
|
@@ -6074,7 +6255,7 @@ const sectionCommandParser = {
|
|
|
6074
6255
|
/**
|
|
6075
6256
|
* Parses the boilerplate command
|
|
6076
6257
|
*
|
|
6077
|
-
* Note:
|
|
6258
|
+
* Note: @@ This command is used as boilerplate for new commands - it should NOT be used in any `.book` file
|
|
6078
6259
|
*
|
|
6079
6260
|
* @see `documentationUrl` for more details
|
|
6080
6261
|
* @private within the commands folder
|
|
@@ -6462,11 +6643,11 @@ const expectCommandParser = {
|
|
|
6462
6643
|
};
|
|
6463
6644
|
|
|
6464
6645
|
/**
|
|
6465
|
-
*
|
|
6646
|
+
* Normalizes a given text to camelCase format.
|
|
6466
6647
|
*
|
|
6467
|
-
* @param text
|
|
6468
|
-
* @param _isFirstLetterCapital
|
|
6469
|
-
* @returns
|
|
6648
|
+
* @param text The text to be normalized.
|
|
6649
|
+
* @param _isFirstLetterCapital Whether the first letter should be capitalized.
|
|
6650
|
+
* @returns The camelCase formatted string.
|
|
6470
6651
|
* @example 'helloWorld'
|
|
6471
6652
|
* @example 'iLovePromptbook'
|
|
6472
6653
|
* @public exported from `@promptbook/utils`
|
|
@@ -6516,9 +6697,9 @@ function normalizeTo_camelCase(text, _isFirstLetterCapital = false) {
|
|
|
6516
6697
|
/**
|
|
6517
6698
|
* Removes quotes from a string
|
|
6518
6699
|
*
|
|
6519
|
-
* Tip: This is very
|
|
6700
|
+
* Tip: This is very useful for post-processing of the result of the LLM model
|
|
6520
6701
|
* Note: This function removes only the same quotes from the beginning and the end of the string
|
|
6521
|
-
* Note: There are two
|
|
6702
|
+
* Note: There are two similar functions:
|
|
6522
6703
|
* - `removeQuotes` which removes only bounding quotes
|
|
6523
6704
|
* - `unwrapResult` which removes whole introduce sentence
|
|
6524
6705
|
*
|
|
@@ -6530,18 +6711,19 @@ function removeQuotes(text) {
|
|
|
6530
6711
|
if (text.startsWith('"') && text.endsWith('"')) {
|
|
6531
6712
|
return text.slice(1, -1);
|
|
6532
6713
|
}
|
|
6533
|
-
if (text.startsWith('
|
|
6714
|
+
if (text.startsWith("'") && text.endsWith("'")) {
|
|
6534
6715
|
return text.slice(1, -1);
|
|
6535
6716
|
}
|
|
6536
6717
|
return text;
|
|
6537
6718
|
}
|
|
6538
6719
|
|
|
6539
6720
|
/**
|
|
6540
|
-
* Function `validateParameterName` will
|
|
6721
|
+
* Function `validateParameterName` will normalize and validate a parameter name for use in pipelines.
|
|
6722
|
+
* It removes diacritics, emojis, and quotes, normalizes to camelCase, and checks for reserved names and invalid characters.
|
|
6541
6723
|
*
|
|
6542
|
-
* @param parameterName
|
|
6543
|
-
* @returns
|
|
6544
|
-
* @throws {ParseError}
|
|
6724
|
+
* @param parameterName The parameter name to validate and normalize.
|
|
6725
|
+
* @returns The validated and normalized parameter name.
|
|
6726
|
+
* @throws {ParseError} If the parameter name is empty, reserved, or contains invalid characters.
|
|
6545
6727
|
* @private within the repository
|
|
6546
6728
|
*/
|
|
6547
6729
|
function validateParameterName(parameterName) {
|
|
@@ -6611,8 +6793,6 @@ function validateParameterName(parameterName) {
|
|
|
6611
6793
|
/**
|
|
6612
6794
|
* Parses the foreach command
|
|
6613
6795
|
*
|
|
6614
|
-
* Note: @@@ This command is used as foreach for new commands - it should NOT be used in any `.book` file
|
|
6615
|
-
*
|
|
6616
6796
|
* @see `documentationUrl` for more details
|
|
6617
6797
|
* @public exported from `@promptbook/editable`
|
|
6618
6798
|
*/
|
|
@@ -6669,14 +6849,14 @@ const foreachCommandParser = {
|
|
|
6669
6849
|
`));
|
|
6670
6850
|
// <- TODO: [🏢] List all supported format names
|
|
6671
6851
|
}
|
|
6672
|
-
const
|
|
6673
|
-
if (
|
|
6852
|
+
const subvalueParser = formatDefinition.subvalueParsers.find((subvalueParser) => [subvalueParser.subvalueName, ...(subvalueParser.aliases || [])].includes(subformatName));
|
|
6853
|
+
if (subvalueParser === undefined) {
|
|
6674
6854
|
throw new ParseError(spaceTrim((block) => `
|
|
6675
6855
|
Unsupported subformat name "${subformatName}" for format "${formatName}"
|
|
6676
6856
|
|
|
6677
6857
|
Available subformat names for format "${formatDefinition.formatName}":
|
|
6678
|
-
${block(formatDefinition.
|
|
6679
|
-
.map((
|
|
6858
|
+
${block(formatDefinition.subvalueParsers
|
|
6859
|
+
.map((subvalueParser) => subvalueParser.subvalueName)
|
|
6680
6860
|
.map((subvalueName) => `- ${subvalueName}`)
|
|
6681
6861
|
.join('\n'))}
|
|
6682
6862
|
`));
|
|
@@ -6853,14 +7033,14 @@ const formatCommandParser = {
|
|
|
6853
7033
|
};
|
|
6854
7034
|
|
|
6855
7035
|
/**
|
|
6856
|
-
*
|
|
7036
|
+
* Chatbot form factor definition for conversational interfaces that interact with users in a chat-like manner.
|
|
6857
7037
|
*
|
|
6858
7038
|
* @public exported from `@promptbook/core`
|
|
6859
7039
|
*/
|
|
6860
7040
|
const ChatbotFormfactorDefinition = {
|
|
6861
7041
|
name: 'CHATBOT',
|
|
6862
7042
|
aliasNames: ['CHAT'],
|
|
6863
|
-
description:
|
|
7043
|
+
description: `A chatbot form factor for conversational user interfaces.`,
|
|
6864
7044
|
documentationUrl: `https://github.com/webgptorg/promptbook/discussions/174`,
|
|
6865
7045
|
pipelineInterface: {
|
|
6866
7046
|
inputParameters: [
|
|
@@ -6887,7 +7067,45 @@ const ChatbotFormfactorDefinition = {
|
|
|
6887
7067
|
};
|
|
6888
7068
|
|
|
6889
7069
|
/**
|
|
6890
|
-
*
|
|
7070
|
+
* Completion is formfactor that emulates completion models
|
|
7071
|
+
*
|
|
7072
|
+
* @public exported from `@promptbook/core`
|
|
7073
|
+
*/
|
|
7074
|
+
const CompletionFormfactorDefinition = {
|
|
7075
|
+
name: 'COMPLETION',
|
|
7076
|
+
description: `Completion is formfactor that emulates completion models`,
|
|
7077
|
+
documentationUrl: `https://github.com/webgptorg/promptbook/discussions/@@`,
|
|
7078
|
+
// <- TODO: https://github.com/webgptorg/promptbook/discussions/new?category=concepts
|
|
7079
|
+
// "🔠 Completion Formfactor"
|
|
7080
|
+
pipelineInterface: {
|
|
7081
|
+
inputParameters: [
|
|
7082
|
+
{
|
|
7083
|
+
name: 'inputText',
|
|
7084
|
+
description: `Input text to be completed`,
|
|
7085
|
+
isInput: true,
|
|
7086
|
+
isOutput: false,
|
|
7087
|
+
},
|
|
7088
|
+
{
|
|
7089
|
+
name: 'instructions',
|
|
7090
|
+
description: `Additional instructions for the model, for example the required length, empty by default`,
|
|
7091
|
+
isInput: true,
|
|
7092
|
+
isOutput: false,
|
|
7093
|
+
},
|
|
7094
|
+
],
|
|
7095
|
+
outputParameters: [
|
|
7096
|
+
{
|
|
7097
|
+
name: 'followingText',
|
|
7098
|
+
description: `Text that follows the input text`,
|
|
7099
|
+
isInput: false,
|
|
7100
|
+
isOutput: true,
|
|
7101
|
+
},
|
|
7102
|
+
],
|
|
7103
|
+
},
|
|
7104
|
+
};
|
|
7105
|
+
|
|
7106
|
+
/**
|
|
7107
|
+
* Generator form factor represents an application that generates content or data based on user input or predefined rules.
|
|
7108
|
+
* This form factor is used for apps that produce outputs, such as text, images, or other media, based on provided input.
|
|
6891
7109
|
*
|
|
6892
7110
|
* @public exported from `@promptbook/core`
|
|
6893
7111
|
*/
|
|
@@ -6916,7 +7134,7 @@ const GeneratorFormfactorDefinition = {
|
|
|
6916
7134
|
};
|
|
6917
7135
|
|
|
6918
7136
|
/**
|
|
6919
|
-
*
|
|
7137
|
+
* Pipeline interface which is equivalent to `any`
|
|
6920
7138
|
*
|
|
6921
7139
|
* @see https://github.com/webgptorg/promptbook/discussions/171
|
|
6922
7140
|
*
|
|
@@ -6931,13 +7149,13 @@ const GENERIC_PIPELINE_INTERFACE = {
|
|
|
6931
7149
|
*/
|
|
6932
7150
|
|
|
6933
7151
|
/**
|
|
6934
|
-
*
|
|
7152
|
+
* A generic pipeline
|
|
6935
7153
|
*
|
|
6936
7154
|
* @public exported from `@promptbook/core`
|
|
6937
7155
|
*/
|
|
6938
7156
|
const GenericFormfactorDefinition = {
|
|
6939
7157
|
name: 'GENERIC',
|
|
6940
|
-
description:
|
|
7158
|
+
description: `A generic pipeline`,
|
|
6941
7159
|
documentationUrl: `https://github.com/webgptorg/promptbook/discussions/173`,
|
|
6942
7160
|
pipelineInterface: GENERIC_PIPELINE_INTERFACE,
|
|
6943
7161
|
};
|
|
@@ -6972,17 +7190,20 @@ const ImageGeneratorFormfactorDefinition = {
|
|
|
6972
7190
|
};
|
|
6973
7191
|
|
|
6974
7192
|
/**
|
|
6975
|
-
* Matcher is form of app that
|
|
7193
|
+
* Matcher is form of app that evaluates (spreadsheet) content against defined criteria or patterns,
|
|
7194
|
+
* determining if it matches or meets specific requirements. Used for classification,
|
|
7195
|
+
* validation, filtering, and quality assessment of inputs.
|
|
6976
7196
|
*
|
|
6977
7197
|
* @public exported from `@promptbook/core`
|
|
6978
7198
|
*/
|
|
6979
7199
|
const MatcherFormfactorDefinition = {
|
|
6980
7200
|
name: 'EXPERIMENTAL_MATCHER',
|
|
6981
|
-
description:
|
|
7201
|
+
description: `An evaluation system that determines whether content meets specific criteria or patterns.
|
|
7202
|
+
Used for content validation, quality assessment, and intelligent filtering tasks. Currently in experimental phase.`,
|
|
6982
7203
|
documentationUrl: `https://github.com/webgptorg/promptbook/discussions/177`,
|
|
6983
7204
|
pipelineInterface: {
|
|
6984
7205
|
inputParameters: [
|
|
6985
|
-
/*
|
|
7206
|
+
/* Input parameters for content to be matched and criteria to match against */
|
|
6986
7207
|
{
|
|
6987
7208
|
name: 'nonce',
|
|
6988
7209
|
description: 'Just to prevent EXPERIMENTAL_MATCHER to be set as implicit formfactor',
|
|
@@ -6991,20 +7212,21 @@ const MatcherFormfactorDefinition = {
|
|
|
6991
7212
|
},
|
|
6992
7213
|
],
|
|
6993
7214
|
outputParameters: [
|
|
6994
|
-
/*
|
|
7215
|
+
/* Output parameters containing match results, confidence scores, and relevant metadata */
|
|
6995
7216
|
],
|
|
6996
7217
|
},
|
|
6997
7218
|
};
|
|
6998
7219
|
|
|
6999
7220
|
/**
|
|
7000
|
-
* Sheets is form of app that
|
|
7221
|
+
* Sheets is form of app that processes tabular data in CSV format, allowing transformation
|
|
7222
|
+
* and analysis of structured data through AI-powered operations
|
|
7001
7223
|
*
|
|
7002
7224
|
* @public exported from `@promptbook/core`
|
|
7003
7225
|
*/
|
|
7004
7226
|
const SheetsFormfactorDefinition = {
|
|
7005
7227
|
name: 'SHEETS',
|
|
7006
7228
|
aliasNames: ['SHEETS', 'SHEET'],
|
|
7007
|
-
description:
|
|
7229
|
+
description: `A formfactor for processing spreadsheet-like data in CSV format, enabling AI transformations on tabular data`,
|
|
7008
7230
|
documentationUrl: `https://github.com/webgptorg/promptbook/discussions/176`,
|
|
7009
7231
|
pipelineInterface: {
|
|
7010
7232
|
inputParameters: [
|
|
@@ -7027,13 +7249,16 @@ const SheetsFormfactorDefinition = {
|
|
|
7027
7249
|
};
|
|
7028
7250
|
|
|
7029
7251
|
/**
|
|
7030
|
-
* Translator is form of app that
|
|
7252
|
+
* Translator is form of app that transforms input text from one form to another,
|
|
7253
|
+
* such as language translation, style conversion, tone modification, or other text transformations.
|
|
7031
7254
|
*
|
|
7032
7255
|
* @public exported from `@promptbook/core`
|
|
7033
7256
|
*/
|
|
7034
7257
|
const TranslatorFormfactorDefinition = {
|
|
7035
7258
|
name: 'TRANSLATOR',
|
|
7036
|
-
description:
|
|
7259
|
+
description: `A text transformation system that converts input content into different forms,
|
|
7260
|
+
including language translations, paraphrasing, style conversions, and tone adjustments.
|
|
7261
|
+
This form factor takes one input and produces one transformed output.`,
|
|
7037
7262
|
documentationUrl: `https://github.com/webgptorg/promptbook/discussions/175`,
|
|
7038
7263
|
pipelineInterface: {
|
|
7039
7264
|
inputParameters: [
|
|
@@ -7070,6 +7295,8 @@ const FORMFACTOR_DEFINITIONS = [
|
|
|
7070
7295
|
MatcherFormfactorDefinition,
|
|
7071
7296
|
GeneratorFormfactorDefinition,
|
|
7072
7297
|
ImageGeneratorFormfactorDefinition,
|
|
7298
|
+
CompletionFormfactorDefinition,
|
|
7299
|
+
// <- [🛬] When making new formfactor, copy the _boilerplate and link it here
|
|
7073
7300
|
];
|
|
7074
7301
|
/**
|
|
7075
7302
|
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
@@ -7078,7 +7305,7 @@ const FORMFACTOR_DEFINITIONS = [
|
|
|
7078
7305
|
/**
|
|
7079
7306
|
* Parses the formfactor command
|
|
7080
7307
|
*
|
|
7081
|
-
* Note:
|
|
7308
|
+
* Note: This command is used as a formfactor for new commands and defines the app type format - it should NOT be used in any `.book` file
|
|
7082
7309
|
*
|
|
7083
7310
|
* @see `documentationUrl` for more details
|
|
7084
7311
|
* @public exported from `@promptbook/editable`
|
|
@@ -7100,7 +7327,7 @@ const formfactorCommandParser = {
|
|
|
7100
7327
|
/**
|
|
7101
7328
|
* Description of the FORMFACTOR command
|
|
7102
7329
|
*/
|
|
7103
|
-
description:
|
|
7330
|
+
description: `Specifies the application type and interface requirements that this promptbook should conform to`,
|
|
7104
7331
|
/**
|
|
7105
7332
|
* Link to documentation
|
|
7106
7333
|
*/
|
|
@@ -7243,8 +7470,7 @@ const jokerCommandParser = {
|
|
|
7243
7470
|
};
|
|
7244
7471
|
|
|
7245
7472
|
/**
|
|
7246
|
-
*
|
|
7247
|
-
*
|
|
7473
|
+
* @see {@link ModelVariant}
|
|
7248
7474
|
* @public exported from `@promptbook/core`
|
|
7249
7475
|
*/
|
|
7250
7476
|
const MODEL_VARIANTS = ['COMPLETION', 'CHAT', 'EMBEDDING' /* <- TODO [🏳] */ /* <- [🤖] */];
|
|
@@ -7676,10 +7902,10 @@ function $applyToTaskJson(command, $taskJson, $pipelineJson) {
|
|
|
7676
7902
|
}
|
|
7677
7903
|
|
|
7678
7904
|
/**
|
|
7679
|
-
*
|
|
7905
|
+
* Checks if the given value is a valid JavaScript identifier name.
|
|
7680
7906
|
*
|
|
7681
|
-
* @param javascriptName
|
|
7682
|
-
* @returns
|
|
7907
|
+
* @param javascriptName The value to check for JavaScript identifier validity.
|
|
7908
|
+
* @returns `true` if the value is a valid JavaScript name, false otherwise.
|
|
7683
7909
|
* @public exported from `@promptbook/utils`
|
|
7684
7910
|
*/
|
|
7685
7911
|
function isValidJavascriptName(javascriptName) {
|
|
@@ -8140,7 +8366,7 @@ function parseCommand(raw, usagePlace) {
|
|
|
8140
8366
|
// Arg1 Arg2 Arg3 | FOO
|
|
8141
8367
|
{
|
|
8142
8368
|
const commandNameRaw = items.slice(-1).join('_');
|
|
8143
|
-
const args = items.slice(0, -1); // <- Note: This is
|
|
8369
|
+
const args = items.slice(0, -1); // <- Note: This is probably not correct
|
|
8144
8370
|
const rawArgs = raw
|
|
8145
8371
|
.substring(0, raw.length - commandNameRaw.length)
|
|
8146
8372
|
.trim();
|
|
@@ -8159,7 +8385,10 @@ function parseCommand(raw, usagePlace) {
|
|
|
8159
8385
|
`));
|
|
8160
8386
|
}
|
|
8161
8387
|
/**
|
|
8162
|
-
*
|
|
8388
|
+
* Generates a markdown-formatted message listing all supported commands
|
|
8389
|
+
* with their descriptions and documentation links
|
|
8390
|
+
*
|
|
8391
|
+
* @returns A formatted markdown string containing all available commands and their details
|
|
8163
8392
|
*/
|
|
8164
8393
|
function getSupportedCommandsMessage() {
|
|
8165
8394
|
return COMMANDS.flatMap(({ name, aliasNames, description, documentationUrl }) =>
|
|
@@ -8170,7 +8399,10 @@ function getSupportedCommandsMessage() {
|
|
|
8170
8399
|
]).join('\n');
|
|
8171
8400
|
}
|
|
8172
8401
|
/**
|
|
8173
|
-
*
|
|
8402
|
+
* Attempts to parse a command variant using the provided input parameters
|
|
8403
|
+
*
|
|
8404
|
+
* @param input Object containing command parsing information including raw command text and normalized values
|
|
8405
|
+
* @returns A parsed Command object if successful, or null if the command cannot be parsed
|
|
8174
8406
|
*/
|
|
8175
8407
|
function parseCommandVariant(input) {
|
|
8176
8408
|
const { commandNameRaw, usagePlace, normalized, args, raw, rawArgs } = input;
|
|
@@ -8217,7 +8449,7 @@ function parseCommandVariant(input) {
|
|
|
8217
8449
|
}
|
|
8218
8450
|
|
|
8219
8451
|
/**
|
|
8220
|
-
*
|
|
8452
|
+
* Extracts the interface (input and output parameters) from a pipeline.
|
|
8221
8453
|
*
|
|
8222
8454
|
* @deprecated https://github.com/webgptorg/promptbook/pull/186
|
|
8223
8455
|
* @see https://github.com/webgptorg/promptbook/discussions/171
|
|
@@ -8250,7 +8482,7 @@ function getPipelineInterface(pipeline) {
|
|
|
8250
8482
|
}
|
|
8251
8483
|
|
|
8252
8484
|
/**
|
|
8253
|
-
*
|
|
8485
|
+
* Checks if two pipeline interfaces are structurally identical.
|
|
8254
8486
|
*
|
|
8255
8487
|
* @deprecated https://github.com/webgptorg/promptbook/pull/186
|
|
8256
8488
|
* @see https://github.com/webgptorg/promptbook/discussions/171
|
|
@@ -8282,10 +8514,11 @@ function isPipelineInterfacesEqual(pipelineInterface1, pipelineInterface2) {
|
|
|
8282
8514
|
}
|
|
8283
8515
|
|
|
8284
8516
|
/**
|
|
8285
|
-
*
|
|
8517
|
+
* Checks if a given pipeline satisfies the requirements of a specified pipeline interface.
|
|
8286
8518
|
*
|
|
8287
8519
|
* @deprecated https://github.com/webgptorg/promptbook/pull/186
|
|
8288
8520
|
* @see https://github.com/webgptorg/promptbook/discussions/171
|
|
8521
|
+
* @returns `true` if the pipeline implements the interface, `false` otherwise.
|
|
8289
8522
|
*
|
|
8290
8523
|
* @public exported from `@promptbook/core`
|
|
8291
8524
|
*/
|
|
@@ -8471,7 +8704,8 @@ function removeMarkdownComments(content) {
|
|
|
8471
8704
|
}
|
|
8472
8705
|
|
|
8473
8706
|
/**
|
|
8474
|
-
*
|
|
8707
|
+
* Utility to determine if a pipeline string is in flat format.
|
|
8708
|
+
* A flat pipeline is a simple text without proper structure (headers, blocks, etc).
|
|
8475
8709
|
*
|
|
8476
8710
|
* @public exported from `@promptbook/editable`
|
|
8477
8711
|
*/
|
|
@@ -8492,7 +8726,10 @@ function isFlatPipeline(pipelineString) {
|
|
|
8492
8726
|
}
|
|
8493
8727
|
|
|
8494
8728
|
/**
|
|
8495
|
-
*
|
|
8729
|
+
* Converts a pipeline structure to its string representation.
|
|
8730
|
+
*
|
|
8731
|
+
* Transforms a flat, simple pipeline into a properly formatted pipeline string
|
|
8732
|
+
* with sections for title, prompt, and return statement.
|
|
8496
8733
|
*
|
|
8497
8734
|
* @public exported from `@promptbook/editable`
|
|
8498
8735
|
*/
|
|
@@ -8549,7 +8786,7 @@ function deflatePipeline(pipelineString) {
|
|
|
8549
8786
|
* Note: It can not work with html syntax and comments
|
|
8550
8787
|
*
|
|
8551
8788
|
* @param markdown any valid markdown
|
|
8552
|
-
* @returns
|
|
8789
|
+
* @returns An array of strings, each representing an individual list item found in the markdown
|
|
8553
8790
|
* @public exported from `@promptbook/markdown-utils`
|
|
8554
8791
|
*/
|
|
8555
8792
|
function extractAllListItemsFromMarkdown(markdown) {
|
|
@@ -8574,8 +8811,8 @@ function extractAllListItemsFromMarkdown(markdown) {
|
|
|
8574
8811
|
*
|
|
8575
8812
|
* - When there are multiple or no code blocks the function throws a `ParseError`
|
|
8576
8813
|
*
|
|
8577
|
-
* Note: There are multiple
|
|
8578
|
-
* - `extractBlock` just extracts the content of the code block which is also used as
|
|
8814
|
+
* Note: There are multiple similar functions:
|
|
8815
|
+
* - `extractBlock` just extracts the content of the code block which is also used as built-in function for postprocessing
|
|
8579
8816
|
* - `extractJsonBlock` extracts exactly one valid JSON code block
|
|
8580
8817
|
* - `extractOneBlockFromMarkdown` extracts exactly one code block with language of the code block
|
|
8581
8818
|
* - `extractAllBlocksFromMarkdown` extracts all code blocks with language of the code block
|
|
@@ -8730,7 +8967,7 @@ function flattenMarkdown(markdown) {
|
|
|
8730
8967
|
* Compile pipeline from string (markdown) format to JSON format synchronously
|
|
8731
8968
|
*
|
|
8732
8969
|
* Note: There are 3 similar functions:
|
|
8733
|
-
* - `compilePipeline` **(preferred)** - which
|
|
8970
|
+
* - `compilePipeline` **(preferred)** - which properly compiles the promptbook and uses embedding for external knowledge
|
|
8734
8971
|
* - `parsePipeline` - use only if you need to compile promptbook synchronously and it contains NO external knowledge
|
|
8735
8972
|
* - `preparePipeline` - just one step in the compilation process
|
|
8736
8973
|
*
|
|
@@ -9167,7 +9404,7 @@ function parsePipeline(pipelineString) {
|
|
|
9167
9404
|
* TODO: Use spaceTrim more effectively
|
|
9168
9405
|
* TODO: [🧠] Parameter flags - isInput, isOutput, isInternal
|
|
9169
9406
|
* TODO: [🥞] Not optimal parsing because `splitMarkdownIntoSections` is executed twice with same string, once through `flattenMarkdown` and second directly here
|
|
9170
|
-
* TODO: [♈]
|
|
9407
|
+
* TODO: [♈] Probably move expectations from tasks to parameters
|
|
9171
9408
|
* TODO: [🛠] Actions, instruments (and maybe knowledge) => Functions and tools
|
|
9172
9409
|
* TODO: [🍙] Make some standard order of json properties
|
|
9173
9410
|
*/
|
|
@@ -9384,7 +9621,8 @@ function $execCommand(options) {
|
|
|
9384
9621
|
*/
|
|
9385
9622
|
|
|
9386
9623
|
/**
|
|
9387
|
-
*
|
|
9624
|
+
* Attempts to locate the specified application on a Linux system using the 'which' command.
|
|
9625
|
+
* Returns the path to the executable if found, or null otherwise.
|
|
9388
9626
|
*
|
|
9389
9627
|
* @private within the repository
|
|
9390
9628
|
*/
|
|
@@ -9404,7 +9642,8 @@ async function locateAppOnLinux({ linuxWhich, }) {
|
|
|
9404
9642
|
*/
|
|
9405
9643
|
|
|
9406
9644
|
/**
|
|
9407
|
-
*
|
|
9645
|
+
* Provides filesystem access (for example for Node.js-based scrapers)
|
|
9646
|
+
* Creates a standardized filesystem interface that scrapers can use for file operations.
|
|
9408
9647
|
*
|
|
9409
9648
|
* @public exported from `@promptbook/node`
|
|
9410
9649
|
*/
|
|
@@ -9450,7 +9689,8 @@ async function isExecutable(path, fs) {
|
|
|
9450
9689
|
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
|
9451
9690
|
const userhome = require('userhome');
|
|
9452
9691
|
/**
|
|
9453
|
-
*
|
|
9692
|
+
* Attempts to locate the specified application on a macOS system by checking standard application paths and using mdfind.
|
|
9693
|
+
* Returns the path to the executable if found, or null otherwise.
|
|
9454
9694
|
*
|
|
9455
9695
|
* @private within the repository
|
|
9456
9696
|
*/
|
|
@@ -9482,7 +9722,8 @@ async function locateAppOnMacOs({ macOsName, }) {
|
|
|
9482
9722
|
*/
|
|
9483
9723
|
|
|
9484
9724
|
/**
|
|
9485
|
-
*
|
|
9725
|
+
* Attempts to locate the specified application on a Windows system by searching common installation directories.
|
|
9726
|
+
* Returns the path to the executable if found, or null otherwise.
|
|
9486
9727
|
*
|
|
9487
9728
|
* @private within the repository
|
|
9488
9729
|
*/
|
|
@@ -9553,7 +9794,8 @@ function locateApp(options) {
|
|
|
9553
9794
|
*/
|
|
9554
9795
|
|
|
9555
9796
|
/**
|
|
9556
|
-
*
|
|
9797
|
+
* Locates the LibreOffice executable on the current system by searching platform-specific paths.
|
|
9798
|
+
* Returns the path to the executable if found, or null otherwise.
|
|
9557
9799
|
*
|
|
9558
9800
|
* @private within the repository
|
|
9559
9801
|
*/
|
|
@@ -9571,7 +9813,8 @@ function locateLibreoffice() {
|
|
|
9571
9813
|
*/
|
|
9572
9814
|
|
|
9573
9815
|
/**
|
|
9574
|
-
*
|
|
9816
|
+
* Locates the Pandoc executable on the current system by searching platform-specific paths.
|
|
9817
|
+
* Returns the path to the executable if found, or null otherwise.
|
|
9575
9818
|
*
|
|
9576
9819
|
* @private within the repository
|
|
9577
9820
|
*/
|
|
@@ -9589,7 +9832,7 @@ function locatePandoc() {
|
|
|
9589
9832
|
*/
|
|
9590
9833
|
|
|
9591
9834
|
/**
|
|
9592
|
-
*
|
|
9835
|
+
* Provides paths to required executables (i.e. as Pandoc and LibreOffice) for Node.js environments.
|
|
9593
9836
|
*
|
|
9594
9837
|
* @public exported from `@promptbook/node`
|
|
9595
9838
|
*/
|
|
@@ -9609,10 +9852,10 @@ async function $provideExecutablesForNode(options) {
|
|
|
9609
9852
|
*/
|
|
9610
9853
|
|
|
9611
9854
|
/**
|
|
9612
|
-
*
|
|
9855
|
+
* Register for LLM tools metadata.
|
|
9613
9856
|
*
|
|
9614
9857
|
* Note: `$` is used to indicate that this interacts with the global scope
|
|
9615
|
-
* @singleton Only one instance of each register is created per build, but
|
|
9858
|
+
* @singleton Only one instance of each register is created per build, but there can be more instances across different builds or environments.
|
|
9616
9859
|
* @public exported from `@promptbook/core`
|
|
9617
9860
|
*/
|
|
9618
9861
|
const $llmToolsMetadataRegister = new $Register('llm_tools_metadata');
|
|
@@ -9640,10 +9883,10 @@ function isRootPath(value) {
|
|
|
9640
9883
|
*/
|
|
9641
9884
|
|
|
9642
9885
|
/**
|
|
9643
|
-
*
|
|
9886
|
+
* Register for LLM tools.
|
|
9644
9887
|
*
|
|
9645
9888
|
* Note: `$` is used to indicate that this interacts with the global scope
|
|
9646
|
-
* @singleton Only one instance of each register is created per build, but
|
|
9889
|
+
* @singleton Only one instance of each register is created per build, but there can be more instances across different builds or environments.
|
|
9647
9890
|
* @public exported from `@promptbook/core`
|
|
9648
9891
|
*/
|
|
9649
9892
|
const $llmToolsRegister = new $Register('llm_execution_tools_constructors');
|
|
@@ -9836,9 +10079,8 @@ async function $provideEnvFilename() {
|
|
|
9836
10079
|
*/
|
|
9837
10080
|
|
|
9838
10081
|
/**
|
|
9839
|
-
*
|
|
10082
|
+
* Provides LLM tools configuration by reading environment variables.
|
|
9840
10083
|
*
|
|
9841
|
-
* @@@ .env
|
|
9842
10084
|
* Note: `$` is used to indicate that this function is not a pure function - it uses filesystem to access `.env` file
|
|
9843
10085
|
*
|
|
9844
10086
|
* It looks for environment variables:
|
|
@@ -9846,7 +10088,8 @@ async function $provideEnvFilename() {
|
|
|
9846
10088
|
* - `process.env.ANTHROPIC_CLAUDE_API_KEY`
|
|
9847
10089
|
* - ...
|
|
9848
10090
|
*
|
|
9849
|
-
* @
|
|
10091
|
+
* @see Environment variables documentation or .env file for required variables.
|
|
10092
|
+
* @returns A promise that resolves to the LLM tools configuration, or null if configuration is incomplete or missing.
|
|
9850
10093
|
* @public exported from `@promptbook/node`
|
|
9851
10094
|
*/
|
|
9852
10095
|
async function $provideLlmToolsConfigurationFromEnv() {
|
|
@@ -9868,11 +10111,16 @@ async function $provideLlmToolsConfigurationFromEnv() {
|
|
|
9868
10111
|
*/
|
|
9869
10112
|
|
|
9870
10113
|
/**
|
|
9871
|
-
*
|
|
10114
|
+
* Creates LLM execution tools from provided configuration objects
|
|
10115
|
+
*
|
|
10116
|
+
* Instantiates and configures LLM tool instances for each configuration entry,
|
|
10117
|
+
* combining them into a unified interface via MultipleLlmExecutionTools.
|
|
9872
10118
|
*
|
|
9873
10119
|
* Note: This function is not cached, every call creates new instance of `MultipleLlmExecutionTools`
|
|
9874
10120
|
*
|
|
9875
|
-
* @
|
|
10121
|
+
* @param configuration Array of LLM tool configurations to instantiate
|
|
10122
|
+
* @param options Additional options for configuring the LLM tools
|
|
10123
|
+
* @returns A unified interface combining all successfully instantiated LLM tools
|
|
9876
10124
|
* @public exported from `@promptbook/core`
|
|
9877
10125
|
*/
|
|
9878
10126
|
function createLlmToolsFromConfiguration(configuration, options = {}) {
|
|
@@ -9911,7 +10159,11 @@ function createLlmToolsFromConfiguration(configuration, options = {}) {
|
|
|
9911
10159
|
/**
|
|
9912
10160
|
* TODO: [🎌] Together with `createLlmToolsFromConfiguration` + 'EXECUTION_TOOLS_CLASSES' gets to `@promptbook/core` ALL model providers, make this more efficient
|
|
9913
10161
|
* TODO: [🧠][🎌] Dynamically install required providers
|
|
9914
|
-
* TODO:
|
|
10162
|
+
* TODO: We should implement an interactive configuration wizard that would:
|
|
10163
|
+
* 1. Detect which LLM providers are available in the environment
|
|
10164
|
+
* 2. Guide users through required configuration settings for each provider
|
|
10165
|
+
* 3. Allow testing connections before completing setup
|
|
10166
|
+
* 4. Generate appropriate configuration code for application integration
|
|
9915
10167
|
* TODO: [🧠][🍛] Which name is better `createLlmToolsFromConfig` or `createLlmToolsFromConfiguration`?
|
|
9916
10168
|
* TODO: [🧠] Is there some meaningfull way how to test this util
|
|
9917
10169
|
* TODO: This should be maybe not under `_common` but under `utils`
|
|
@@ -9919,11 +10171,14 @@ function createLlmToolsFromConfiguration(configuration, options = {}) {
|
|
|
9919
10171
|
*/
|
|
9920
10172
|
|
|
9921
10173
|
/**
|
|
9922
|
-
*
|
|
10174
|
+
* Automatically configures LLM tools from environment variables in Node.js
|
|
10175
|
+
*
|
|
10176
|
+
* This utility function detects available LLM providers based on environment variables
|
|
10177
|
+
* and creates properly configured LLM execution tools for each detected provider.
|
|
9923
10178
|
*
|
|
9924
10179
|
* Note: This function is not cached, every call creates new instance of `MultipleLlmExecutionTools`
|
|
9925
10180
|
*
|
|
9926
|
-
*
|
|
10181
|
+
* Supports environment variables from .env files when dotenv is configured
|
|
9927
10182
|
* Note: `$` is used to indicate that this function is not a pure function - it uses filesystem to access `.env` file
|
|
9928
10183
|
*
|
|
9929
10184
|
* It looks for environment variables:
|
|
@@ -9931,7 +10186,8 @@ function createLlmToolsFromConfiguration(configuration, options = {}) {
|
|
|
9931
10186
|
* - `process.env.ANTHROPIC_CLAUDE_API_KEY`
|
|
9932
10187
|
* - ...
|
|
9933
10188
|
*
|
|
9934
|
-
* @
|
|
10189
|
+
* @param options Configuration options for the LLM tools
|
|
10190
|
+
* @returns A unified interface containing all detected and configured LLM tools
|
|
9935
10191
|
* @public exported from `@promptbook/node`
|
|
9936
10192
|
*/
|
|
9937
10193
|
async function $provideLlmToolsFromEnv(options = {}) {
|
|
@@ -9957,7 +10213,16 @@ async function $provideLlmToolsFromEnv(options = {}) {
|
|
|
9957
10213
|
return createLlmToolsFromConfiguration(configuration, options);
|
|
9958
10214
|
}
|
|
9959
10215
|
/**
|
|
9960
|
-
* TODO:
|
|
10216
|
+
* TODO: The architecture for LLM tools configuration consists of three key functions:
|
|
10217
|
+
* 1. `$provideLlmToolsFromEnv` - High-level function that detects available providers from env vars and returns ready-to-use LLM tools
|
|
10218
|
+
* 2. `$provideLlmToolsConfigurationFromEnv` - Middle layer that extracts configuration objects from environment variables
|
|
10219
|
+
* 3. `createLlmToolsFromConfiguration` - Low-level function that instantiates LLM tools from explicit configuration
|
|
10220
|
+
*
|
|
10221
|
+
* This layered approach allows flexibility in how tools are configured:
|
|
10222
|
+
* - Use $provideLlmToolsFromEnv for automatic detection and setup in Node.js environments
|
|
10223
|
+
* - Use $provideLlmToolsConfigurationFromEnv to extract config objects for modification before instantiation
|
|
10224
|
+
* - Use createLlmToolsFromConfiguration for explicit control over tool configurations
|
|
10225
|
+
*
|
|
9961
10226
|
* TODO: [🧠][🍛] Which name is better `$provideLlmToolsFromEnv` or `$provideLlmToolsFromEnvironment`?
|
|
9962
10227
|
* TODO: [🧠] Is there some meaningfull way how to test this util
|
|
9963
10228
|
* Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
|
|
@@ -9967,11 +10232,9 @@ async function $provideLlmToolsFromEnv(options = {}) {
|
|
|
9967
10232
|
*/
|
|
9968
10233
|
|
|
9969
10234
|
/**
|
|
9970
|
-
*
|
|
9971
|
-
*
|
|
9972
|
-
*
|
|
9973
|
-
* 2) @@@
|
|
9974
|
-
*
|
|
10235
|
+
* Provides a collection of scrapers optimized for Node.js environment.
|
|
10236
|
+
* 1) `provideScrapersForNode` use as default
|
|
10237
|
+
* 2) `provideScrapersForBrowser` use in limited browser environment *
|
|
9975
10238
|
* @public exported from `@promptbook/node`
|
|
9976
10239
|
*/
|
|
9977
10240
|
async function $provideScrapersForNode(tools, options) {
|
|
@@ -10078,11 +10341,11 @@ function parseKeywordsFromString(input) {
|
|
|
10078
10341
|
}
|
|
10079
10342
|
|
|
10080
10343
|
/**
|
|
10081
|
-
*
|
|
10344
|
+
* Converts a name string into a URI-compatible format.
|
|
10082
10345
|
*
|
|
10083
|
-
* @param name
|
|
10084
|
-
* @returns
|
|
10085
|
-
* @example
|
|
10346
|
+
* @param name The string to be converted to a URI-compatible format.
|
|
10347
|
+
* @returns A URI-compatible string derived from the input name.
|
|
10348
|
+
* @example 'Hello World' -> 'hello-world'
|
|
10086
10349
|
* @public exported from `@promptbook/utils`
|
|
10087
10350
|
*/
|
|
10088
10351
|
function nameToUriPart(name) {
|
|
@@ -10096,11 +10359,11 @@ function nameToUriPart(name) {
|
|
|
10096
10359
|
}
|
|
10097
10360
|
|
|
10098
10361
|
/**
|
|
10099
|
-
*
|
|
10362
|
+
* Converts a given name into URI-compatible parts.
|
|
10100
10363
|
*
|
|
10101
|
-
* @param name
|
|
10102
|
-
* @returns
|
|
10103
|
-
* @example
|
|
10364
|
+
* @param name The name to be converted into URI parts.
|
|
10365
|
+
* @returns An array of URI-compatible parts derived from the name.
|
|
10366
|
+
* @example 'Example Name' -> ['example', 'name']
|
|
10104
10367
|
* @public exported from `@promptbook/utils`
|
|
10105
10368
|
*/
|
|
10106
10369
|
function nameToUriParts(name) {
|
|
@@ -10167,7 +10430,7 @@ function isSerializableAsJson(value) {
|
|
|
10167
10430
|
/**
|
|
10168
10431
|
* Function trimCodeBlock will trim starting and ending code block from the string if it is present.
|
|
10169
10432
|
*
|
|
10170
|
-
* Note: This is
|
|
10433
|
+
* Note: This is useful for post-processing of the result of the chat LLM model
|
|
10171
10434
|
* when the model wraps the result in the (markdown) code block.
|
|
10172
10435
|
*
|
|
10173
10436
|
* @public exported from `@promptbook/utils`
|
|
@@ -10186,7 +10449,7 @@ function trimCodeBlock(value) {
|
|
|
10186
10449
|
/**
|
|
10187
10450
|
* Function trimEndOfCodeBlock will remove ending code block from the string if it is present.
|
|
10188
10451
|
*
|
|
10189
|
-
* Note: This is
|
|
10452
|
+
* Note: This is useful for post-processing of the result of the completion LLM model
|
|
10190
10453
|
* if you want to start code block in the prompt but you don't want to end it in the result.
|
|
10191
10454
|
*
|
|
10192
10455
|
* @public exported from `@promptbook/utils`
|
|
@@ -10201,9 +10464,9 @@ function trimEndOfCodeBlock(value) {
|
|
|
10201
10464
|
/**
|
|
10202
10465
|
* Removes quotes and optional introduce text from a string
|
|
10203
10466
|
*
|
|
10204
|
-
* Tip: This is very
|
|
10467
|
+
* Tip: This is very useful for post-processing of the result of the LLM model
|
|
10205
10468
|
* Note: This function trims the text and removes whole introduce sentence if it is present
|
|
10206
|
-
* Note: There are two
|
|
10469
|
+
* Note: There are two similar functions:
|
|
10207
10470
|
* - `removeQuotes` which removes only bounding quotes
|
|
10208
10471
|
* - `unwrapResult` which removes whole introduce sentence
|
|
10209
10472
|
*
|
|
@@ -10274,7 +10537,7 @@ function unwrapResult(text, options) {
|
|
|
10274
10537
|
*
|
|
10275
10538
|
* - When there are multiple or no code blocks the function throws a `ParseError`
|
|
10276
10539
|
*
|
|
10277
|
-
* Note: There are multiple
|
|
10540
|
+
* Note: There are multiple similar function:
|
|
10278
10541
|
* - `extractBlock` just extracts the content of the code block which is also used as build-in function for postprocessing
|
|
10279
10542
|
* - `extractJsonBlock` extracts exactly one valid JSON code block
|
|
10280
10543
|
* - `extractOneBlockFromMarkdown` extracts exactly one code block with language of the code block
|
|
@@ -10311,7 +10574,7 @@ function preserve(func) {
|
|
|
10311
10574
|
})();
|
|
10312
10575
|
}
|
|
10313
10576
|
/**
|
|
10314
|
-
* TODO:
|
|
10577
|
+
* TODO: Probably remove in favour of `keepImported`
|
|
10315
10578
|
* TODO: [1] This maybe does memory leak
|
|
10316
10579
|
*/
|
|
10317
10580
|
|
|
@@ -10515,7 +10778,7 @@ class JavascriptEvalExecutionTools {
|
|
|
10515
10778
|
*/
|
|
10516
10779
|
|
|
10517
10780
|
/**
|
|
10518
|
-
* Placeholder for better implementation of JavascriptExecutionTools - some
|
|
10781
|
+
* Placeholder for better implementation of JavascriptExecutionTools - some proper sandboxing
|
|
10519
10782
|
*
|
|
10520
10783
|
* @alias JavascriptExecutionTools
|
|
10521
10784
|
* @public exported from `@promptbook/javascript`
|
|
@@ -10900,14 +11163,17 @@ function stringifyPipelineJson(pipeline) {
|
|
|
10900
11163
|
return pipelineJsonStringified;
|
|
10901
11164
|
}
|
|
10902
11165
|
/**
|
|
10903
|
-
* TODO: [🐝] Not Working
|
|
11166
|
+
* TODO: [🐝] Not Working properly @see https://promptbook.studio/examples/mixed-knowledge.book
|
|
10904
11167
|
* TODO: [🧠][0] Maybe rename to `stringifyPipelineJson`, `stringifyIndexedJson`,...
|
|
10905
11168
|
* TODO: [🧠] Maybe more elegant solution than replacing via regex
|
|
10906
11169
|
* TODO: [🍙] Make some standard order of json properties
|
|
10907
11170
|
*/
|
|
10908
11171
|
|
|
10909
11172
|
/**
|
|
10910
|
-
*
|
|
11173
|
+
* A storage implementation that caches data in files organized in a directory structure.
|
|
11174
|
+
* Provides methods for retrieving, storing, and managing cached data on the filesystem.
|
|
11175
|
+
*
|
|
11176
|
+
* This class implements the PromptbookStorage interface for filesystem-based caching.
|
|
10911
11177
|
*
|
|
10912
11178
|
* @public exported from `@promptbook/node`
|
|
10913
11179
|
*/
|
|
@@ -10920,7 +11186,8 @@ class FileCacheStorage {
|
|
|
10920
11186
|
}
|
|
10921
11187
|
}
|
|
10922
11188
|
/**
|
|
10923
|
-
*
|
|
11189
|
+
* Converts a storage key to a filesystem path where the data should be stored.
|
|
11190
|
+
* Creates a consistent, deterministic file path based on the key string.
|
|
10924
11191
|
*/
|
|
10925
11192
|
getFilenameForKey(key) {
|
|
10926
11193
|
// TODO: [👬] DRY
|
|
@@ -10932,7 +11199,8 @@ class FileCacheStorage {
|
|
|
10932
11199
|
...nameToSubfolderPath(hash /* <- TODO: [🎎] Maybe add some SHA256 prefix */), `${name.substring(0, MAX_FILENAME_LENGTH)}.json`);
|
|
10933
11200
|
}
|
|
10934
11201
|
/**
|
|
10935
|
-
*
|
|
11202
|
+
* Returns the current value associated with the given key, or null if the given key does not exist.
|
|
11203
|
+
* Retrieves the cached data from the file system storage.
|
|
10936
11204
|
*/
|
|
10937
11205
|
async getItem(key) {
|
|
10938
11206
|
const filename = this.getFilenameForKey(key);
|
|
@@ -10945,7 +11213,8 @@ class FileCacheStorage {
|
|
|
10945
11213
|
return value;
|
|
10946
11214
|
}
|
|
10947
11215
|
/**
|
|
10948
|
-
*
|
|
11216
|
+
* Sets the value of the pair identified by key to value, creating a new key/value pair if none existed for key previously.
|
|
11217
|
+
* Persists data to the file system, creating necessary directory structure if it doesn't exist.
|
|
10949
11218
|
*/
|
|
10950
11219
|
async setItem(key, value) {
|
|
10951
11220
|
const filename = this.getFilenameForKey(key);
|
|
@@ -10957,7 +11226,8 @@ class FileCacheStorage {
|
|
|
10957
11226
|
await writeFile(filename, fileContent, 'utf-8');
|
|
10958
11227
|
}
|
|
10959
11228
|
/**
|
|
10960
|
-
*
|
|
11229
|
+
* Removes the key/value pair with the given key from the storage, if a key/value pair with the given key exists.
|
|
11230
|
+
* Deletes the corresponding file from the filesystem.
|
|
10961
11231
|
*/
|
|
10962
11232
|
async removeItem(key) {
|
|
10963
11233
|
const filename = this.getFilenameForKey(key);
|