@promptbook/cli 0.92.0-9 → 0.93.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +141 -89
- package/esm/index.es.js +1010 -674
- package/esm/index.es.js.map +1 -1
- package/esm/typings/src/_packages/browser.index.d.ts +2 -0
- package/esm/typings/src/_packages/core.index.d.ts +26 -14
- package/esm/typings/src/_packages/types.index.d.ts +6 -2
- package/esm/typings/src/collection/PipelineCollection.d.ts +0 -2
- package/esm/typings/src/collection/SimplePipelineCollection.d.ts +1 -1
- package/esm/typings/src/commands/FOREACH/ForeachJson.d.ts +6 -6
- package/esm/typings/src/commands/FOREACH/foreachCommandParser.d.ts +0 -2
- package/esm/typings/src/commands/FORMFACTOR/formfactorCommandParser.d.ts +1 -1
- package/esm/typings/src/commands/_BOILERPLATE/boilerplateCommandParser.d.ts +1 -1
- package/esm/typings/src/commands/_common/types/CommandParser.d.ts +36 -28
- package/esm/typings/src/config.d.ts +41 -11
- package/esm/typings/src/constants.d.ts +43 -2
- package/esm/typings/src/conversion/parsePipeline.d.ts +2 -2
- package/esm/typings/src/errors/0-BoilerplateError.d.ts +2 -2
- package/esm/typings/src/errors/CollectionError.d.ts +1 -1
- package/esm/typings/src/executables/$provideExecutablesForNode.d.ts +1 -1
- package/esm/typings/src/executables/apps/locateLibreoffice.d.ts +2 -1
- package/esm/typings/src/executables/apps/locatePandoc.d.ts +2 -1
- package/esm/typings/src/executables/locateApp.d.ts +2 -2
- package/esm/typings/src/executables/platforms/locateAppOnLinux.d.ts +2 -1
- package/esm/typings/src/executables/platforms/locateAppOnMacOs.d.ts +2 -1
- package/esm/typings/src/executables/platforms/locateAppOnWindows.d.ts +2 -1
- package/esm/typings/src/execution/AbstractTaskResult.d.ts +1 -1
- package/esm/typings/src/execution/CommonToolsOptions.d.ts +3 -3
- package/esm/typings/src/execution/ExecutionTask.d.ts +19 -1
- package/esm/typings/src/execution/LlmExecutionToolsConstructor.d.ts +2 -1
- package/esm/typings/src/execution/PipelineExecutorResult.d.ts +4 -2
- package/esm/typings/src/execution/PromptbookFetch.d.ts +1 -1
- package/esm/typings/src/execution/ScriptExecutionTools.d.ts +1 -1
- package/esm/typings/src/execution/createPipelineExecutor/$OngoingTaskResult.d.ts +12 -9
- package/esm/typings/src/execution/createPipelineExecutor/10-executePipeline.d.ts +13 -10
- package/esm/typings/src/execution/createPipelineExecutor/20-executeTask.d.ts +12 -9
- package/esm/typings/src/execution/createPipelineExecutor/30-executeFormatSubvalues.d.ts +15 -3
- package/esm/typings/src/execution/createPipelineExecutor/40-executeAttempts.d.ts +21 -15
- package/esm/typings/src/execution/createPipelineExecutor/computeCosineSimilarity.d.ts +13 -0
- package/esm/typings/src/execution/createPipelineExecutor/filterJustOutputParameters.d.ts +7 -6
- package/esm/typings/src/execution/createPipelineExecutor/getContextForTask.d.ts +5 -1
- package/esm/typings/src/execution/createPipelineExecutor/getExamplesForTask.d.ts +1 -1
- package/esm/typings/src/execution/createPipelineExecutor/getKnowledgeForTask.d.ts +12 -9
- package/esm/typings/src/execution/createPipelineExecutor/getReservedParametersForTask.d.ts +18 -5
- package/esm/typings/src/execution/createPipelineExecutor/knowledgePiecesToString.d.ts +9 -0
- package/esm/typings/src/execution/execution-report/ExecutionReportJson.d.ts +1 -1
- package/esm/typings/src/execution/execution-report/ExecutionReportString.d.ts +1 -1
- package/esm/typings/src/execution/translation/automatic-translate/automatic-translators/LindatAutomaticTranslator.d.ts +4 -4
- package/esm/typings/src/execution/utils/checkExpectations.d.ts +3 -3
- package/esm/typings/src/execution/utils/uncertainNumber.d.ts +3 -2
- package/esm/typings/src/execution/utils/usageToWorktime.d.ts +1 -1
- package/esm/typings/src/formats/_common/{FormatDefinition.d.ts → FormatParser.d.ts} +8 -6
- package/esm/typings/src/formats/_common/FormatSubvalueParser.d.ts +66 -0
- package/esm/typings/src/formats/csv/CsvFormatParser.d.ts +17 -0
- package/esm/typings/src/formats/csv/CsvSettings.d.ts +2 -2
- package/esm/typings/src/formats/csv/utils/csvParse.d.ts +12 -0
- package/esm/typings/src/formats/csv/utils/isValidCsvString.d.ts +1 -1
- package/esm/typings/src/formats/index.d.ts +2 -2
- package/esm/typings/src/formats/json/JsonFormatParser.d.ts +19 -0
- package/esm/typings/src/formats/json/utils/isValidJsonString.d.ts +1 -1
- package/esm/typings/src/formats/json/utils/jsonParse.d.ts +0 -3
- package/esm/typings/src/formats/text/{TextFormatDefinition.d.ts → TextFormatParser.d.ts} +7 -7
- package/esm/typings/src/formats/xml/XmlFormatParser.d.ts +19 -0
- package/esm/typings/src/formats/xml/utils/isValidXmlString.d.ts +1 -1
- package/esm/typings/src/formfactors/_boilerplate/BoilerplateFormfactorDefinition.d.ts +3 -2
- package/esm/typings/src/formfactors/_common/AbstractFormfactorDefinition.d.ts +16 -7
- package/esm/typings/src/formfactors/_common/FormfactorDefinition.d.ts +3 -1
- package/esm/typings/src/formfactors/_common/string_formfactor_name.d.ts +2 -1
- package/esm/typings/src/formfactors/chatbot/ChatbotFormfactorDefinition.d.ts +2 -2
- package/esm/typings/src/formfactors/completion/CompletionFormfactorDefinition.d.ts +29 -0
- package/esm/typings/src/formfactors/generator/GeneratorFormfactorDefinition.d.ts +2 -1
- package/esm/typings/src/formfactors/generic/GenericFormfactorDefinition.d.ts +2 -2
- package/esm/typings/src/formfactors/index.d.ts +33 -8
- package/esm/typings/src/formfactors/matcher/MatcherFormfactorDefinition.d.ts +4 -2
- package/esm/typings/src/formfactors/sheets/SheetsFormfactorDefinition.d.ts +3 -2
- package/esm/typings/src/formfactors/translator/TranslatorFormfactorDefinition.d.ts +3 -2
- package/esm/typings/src/high-level-abstractions/index.d.ts +2 -2
- package/esm/typings/src/llm-providers/_common/register/$llmToolsMetadataRegister.d.ts +3 -3
- package/esm/typings/src/llm-providers/_common/register/$llmToolsRegister.d.ts +3 -3
- package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsConfigurationFromEnv.d.ts +4 -4
- package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground.d.ts +4 -3
- package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsFromEnv.d.ts +18 -5
- package/esm/typings/src/llm-providers/_common/register/LlmToolsConfiguration.d.ts +11 -4
- package/esm/typings/src/llm-providers/_common/register/LlmToolsMetadata.d.ts +21 -42
- package/esm/typings/src/llm-providers/_common/register/LlmToolsOptions.d.ts +9 -2
- package/esm/typings/src/llm-providers/_common/register/createLlmToolsFromConfiguration.d.ts +13 -4
- package/esm/typings/src/llm-providers/_common/utils/cache/CacheItem.d.ts +10 -5
- package/esm/typings/src/llm-providers/_common/utils/cache/CacheLlmToolsOptions.d.ts +11 -3
- package/esm/typings/src/llm-providers/_common/utils/cache/cacheLlmTools.d.ts +3 -3
- package/esm/typings/src/llm-providers/_common/utils/count-total-usage/limitTotalUsage.d.ts +5 -5
- package/esm/typings/src/llm-providers/anthropic-claude/AnthropicClaudeExecutionTools.d.ts +6 -0
- package/esm/typings/src/llm-providers/anthropic-claude/anthropic-claude-models.d.ts +1 -1
- package/esm/typings/src/llm-providers/azure-openai/AzureOpenAiExecutionTools.d.ts +1 -1
- package/esm/typings/src/llm-providers/azure-openai/AzureOpenAiExecutionToolsOptions.d.ts +4 -4
- package/esm/typings/src/llm-providers/deepseek/deepseek-models.d.ts +1 -1
- package/esm/typings/src/llm-providers/google/google-models.d.ts +1 -1
- package/esm/typings/src/llm-providers/openai/OpenAiAssistantExecutionTools.d.ts +1 -1
- package/esm/typings/src/llm-providers/openai/openai-models.d.ts +1 -1
- package/esm/typings/src/llm-providers/openai/register-configuration.d.ts +2 -2
- package/esm/typings/src/llm-providers/openai/register-constructor.d.ts +2 -2
- package/esm/typings/src/migrations/migratePipeline.d.ts +9 -0
- package/esm/typings/src/other/templates/getBookTemplates.d.ts +2 -2
- package/esm/typings/src/pipeline/PipelineInterface/PipelineInterface.d.ts +3 -3
- package/esm/typings/src/pipeline/PipelineInterface/constants.d.ts +1 -1
- package/esm/typings/src/pipeline/PipelineInterface/getPipelineInterface.d.ts +1 -1
- package/esm/typings/src/pipeline/PipelineInterface/isPipelineImplementingInterface.d.ts +5 -4
- package/esm/typings/src/pipeline/PipelineInterface/isPipelineInterfacesEqual.d.ts +1 -1
- package/esm/typings/src/pipeline/PipelineJson/CommonTaskJson.d.ts +10 -7
- package/esm/typings/src/pipeline/PipelineJson/ParameterJson.d.ts +1 -1
- package/esm/typings/src/pipeline/PipelineJson/PipelineJson.d.ts +3 -2
- package/esm/typings/src/pipeline/PipelineString.d.ts +3 -1
- package/esm/typings/src/pipeline/book-notation.d.ts +2 -2
- package/esm/typings/src/postprocessing/utils/extractBlock.d.ts +1 -1
- package/esm/typings/src/postprocessing/utils/extractJsonBlock.d.ts +2 -2
- package/esm/typings/src/prepare/prepareTasks.d.ts +8 -5
- package/esm/typings/src/remote-server/openapi.d.ts +1 -1
- package/esm/typings/src/remote-server/socket-types/listModels/PromptbookServer_ListModels_Response.d.ts +1 -1
- package/esm/typings/src/remote-server/startRemoteServer.d.ts +1 -1
- package/esm/typings/src/remote-server/types/RemoteServerOptions.d.ts +7 -6
- package/esm/typings/src/scrapers/_boilerplate/BoilerplateScraper.d.ts +3 -3
- package/esm/typings/src/scrapers/_boilerplate/createBoilerplateScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/_boilerplate/register-metadata.d.ts +1 -1
- package/esm/typings/src/scrapers/_common/Converter.d.ts +3 -1
- package/esm/typings/src/scrapers/_common/Scraper.d.ts +4 -3
- package/esm/typings/src/scrapers/_common/ScraperIntermediateSource.d.ts +4 -2
- package/esm/typings/src/scrapers/_common/prepareKnowledgePieces.d.ts +2 -2
- package/esm/typings/src/scrapers/_common/register/$provideFilesystemForNode.d.ts +2 -1
- package/esm/typings/src/scrapers/_common/register/$provideScrapersForBrowser.d.ts +6 -3
- package/esm/typings/src/scrapers/_common/register/$provideScrapersForNode.d.ts +3 -5
- package/esm/typings/src/scrapers/_common/register/$scrapersMetadataRegister.d.ts +3 -3
- package/esm/typings/src/scrapers/_common/register/$scrapersRegister.d.ts +3 -2
- package/esm/typings/src/scrapers/_common/register/ScraperAndConverterMetadata.d.ts +8 -5
- package/esm/typings/src/scrapers/_common/register/ScraperConstructor.d.ts +2 -1
- package/esm/typings/src/scrapers/_common/utils/getScraperIntermediateSource.d.ts +6 -5
- package/esm/typings/src/scrapers/_common/utils/makeKnowledgeSourceHandler.d.ts +3 -1
- package/esm/typings/src/scrapers/document/createDocumentScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/document-legacy/createLegacyDocumentScraper.d.ts +2 -1
- package/esm/typings/src/scrapers/markdown/createMarkdownScraper.d.ts +4 -1
- package/esm/typings/src/scrapers/markitdown/MarkitdownScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/pdf/createPdfScraper.d.ts +2 -1
- package/esm/typings/src/scrapers/website/createWebsiteScraper.d.ts +3 -4
- package/esm/typings/src/scripting/javascript/JavascriptExecutionTools.d.ts +1 -1
- package/esm/typings/src/scripting/javascript/postprocessing-functions.d.ts +5 -1
- package/esm/typings/src/scripting/javascript/utils/preserve.d.ts +1 -1
- package/esm/typings/src/storage/_common/PromptbookStorage.d.ts +1 -1
- package/esm/typings/src/storage/file-cache-storage/FileCacheStorage.d.ts +12 -5
- package/esm/typings/src/storage/file-cache-storage/FileCacheStorageOptions.d.ts +4 -2
- package/esm/typings/src/storage/file-cache-storage/utils/nameToSubfolderPath.d.ts +2 -1
- package/esm/typings/src/storage/local-storage/getIndexedDbStorage.d.ts +11 -0
- package/esm/typings/src/storage/local-storage/utils/IndexedDbStorageOptions.d.ts +14 -0
- package/esm/typings/src/storage/local-storage/utils/makePromptbookStorageFromIndexedDb.d.ts +8 -0
- package/esm/typings/src/storage/local-storage/utils/makePromptbookStorageFromWebStorage.d.ts +2 -1
- package/esm/typings/src/types/IntermediateFilesStrategy.d.ts +2 -1
- package/esm/typings/src/types/ModelRequirements.d.ts +2 -2
- package/esm/typings/src/types/ModelVariant.d.ts +5 -5
- package/esm/typings/src/types/typeAliases.d.ts +22 -19
- package/esm/typings/src/utils/$Register.d.ts +8 -7
- package/esm/typings/src/utils/editable/edit-pipeline-string/addPipelineCommand.d.ts +2 -2
- package/esm/typings/src/utils/editable/edit-pipeline-string/deflatePipeline.d.ts +4 -1
- package/esm/typings/src/utils/editable/utils/isFlatPipeline.d.ts +2 -1
- package/esm/typings/src/utils/editable/utils/stringifyPipelineJson.d.ts +1 -1
- package/esm/typings/src/utils/environment/$getGlobalScope.d.ts +2 -1
- package/esm/typings/src/utils/expectation-counters/index.d.ts +1 -1
- package/esm/typings/src/utils/markdown/extractAllBlocksFromMarkdown.d.ts +2 -2
- package/esm/typings/src/utils/markdown/extractAllListItemsFromMarkdown.d.ts +1 -1
- package/esm/typings/src/utils/markdown/extractOneBlockFromMarkdown.d.ts +2 -2
- package/esm/typings/src/utils/normalization/nameToUriPart.d.ts +4 -4
- package/esm/typings/src/utils/normalization/nameToUriParts.d.ts +4 -4
- package/esm/typings/src/utils/normalization/normalize-to-kebab-case.d.ts +3 -3
- package/esm/typings/src/utils/normalization/normalizeTo_SCREAMING_CASE.d.ts +3 -3
- package/esm/typings/src/utils/normalization/normalizeTo_camelCase.d.ts +4 -4
- package/esm/typings/src/utils/normalization/normalizeTo_snake_case.d.ts +3 -3
- package/esm/typings/src/utils/normalization/removeDiacritics.d.ts +3 -3
- package/esm/typings/src/utils/normalization/searchKeywords.d.ts +4 -1
- package/esm/typings/src/utils/normalization/titleToName.d.ts +4 -4
- package/esm/typings/src/utils/organization/TODO_USE.d.ts +1 -1
- package/esm/typings/src/utils/organization/empty_object.d.ts +2 -2
- package/esm/typings/src/utils/organization/just.d.ts +1 -1
- package/esm/typings/src/utils/organization/just_empty_object.d.ts +4 -4
- package/esm/typings/src/utils/organization/keepUnused.d.ts +1 -1
- package/esm/typings/src/utils/parameters/mapAvailableToExpectedParameters.d.ts +7 -7
- package/esm/typings/src/utils/removeQuotes.d.ts +2 -2
- package/esm/typings/src/utils/serialization/clonePipeline.d.ts +4 -3
- package/esm/typings/src/utils/serialization/deepClone.d.ts +5 -1
- package/esm/typings/src/utils/trimCodeBlock.d.ts +1 -1
- package/esm/typings/src/utils/trimEndOfCodeBlock.d.ts +1 -1
- package/esm/typings/src/utils/unwrapResult.d.ts +2 -2
- package/esm/typings/src/utils/validators/javascriptName/isValidJavascriptName.d.ts +3 -3
- package/esm/typings/src/utils/validators/parameterName/validateParameterName.d.ts +5 -4
- package/esm/typings/src/utils/validators/semanticVersion/isValidPromptbookVersion.d.ts +1 -1
- package/esm/typings/src/utils/validators/semanticVersion/isValidSemanticVersion.d.ts +1 -1
- package/esm/typings/src/utils/validators/url/isHostnameOnPrivateNetwork.d.ts +1 -1
- package/esm/typings/src/utils/validators/url/isUrlOnPrivateNetwork.d.ts +1 -1
- package/esm/typings/src/utils/validators/url/isValidPipelineUrl.d.ts +1 -1
- package/esm/typings/src/utils/validators/url/isValidUrl.d.ts +1 -1
- package/esm/typings/src/version.d.ts +2 -1
- package/esm/typings/src/wizzard/wizzard.d.ts +1 -1
- package/package.json +14 -2
- package/umd/index.umd.js +1018 -682
- package/umd/index.umd.js.map +1 -1
- package/esm/typings/src/formats/_common/FormatSubvalueDefinition.d.ts +0 -31
- package/esm/typings/src/formats/csv/CsvFormatDefinition.d.ts +0 -17
- package/esm/typings/src/formats/json/JsonFormatDefinition.d.ts +0 -19
- package/esm/typings/src/formats/xml/XmlFormatDefinition.d.ts +0 -19
- /package/esm/typings/src/llm-providers/{multiple → _multiple}/MultipleLlmExecutionTools.d.ts +0 -0
- /package/esm/typings/src/llm-providers/{multiple → _multiple}/joinLlmExecutionTools.d.ts +0 -0
- /package/esm/typings/src/llm-providers/{multiple → _multiple}/playground/playground.d.ts +0 -0
package/esm/index.es.js
CHANGED
|
@@ -21,13 +21,13 @@ import { lookup, extension } from 'mime-types';
|
|
|
21
21
|
import glob from 'glob-promise';
|
|
22
22
|
import moment from 'moment';
|
|
23
23
|
import express from 'express';
|
|
24
|
+
import * as OpenApiValidator from 'express-openapi-validator';
|
|
24
25
|
import http from 'http';
|
|
25
26
|
import { Server } from 'socket.io';
|
|
26
|
-
import * as OpenApiValidator from 'express-openapi-validator';
|
|
27
27
|
import swaggerUi from 'swagger-ui-express';
|
|
28
28
|
import Anthropic from '@anthropic-ai/sdk';
|
|
29
|
-
import { OpenAIClient, AzureKeyCredential } from '@azure/openai';
|
|
30
29
|
import Bottleneck from 'bottleneck';
|
|
30
|
+
import { OpenAIClient, AzureKeyCredential } from '@azure/openai';
|
|
31
31
|
import OpenAI from 'openai';
|
|
32
32
|
import { Readability } from '@mozilla/readability';
|
|
33
33
|
import { JSDOM } from 'jsdom';
|
|
@@ -47,7 +47,7 @@ const BOOK_LANGUAGE_VERSION = '1.0.0';
|
|
|
47
47
|
* @generated
|
|
48
48
|
* @see https://github.com/webgptorg/promptbook
|
|
49
49
|
*/
|
|
50
|
-
const PROMPTBOOK_ENGINE_VERSION = '0.
|
|
50
|
+
const PROMPTBOOK_ENGINE_VERSION = '0.93.0';
|
|
51
51
|
/**
|
|
52
52
|
* TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
|
|
53
53
|
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
@@ -88,7 +88,7 @@ const REMOTE_SERVER_URLS = [
|
|
|
88
88
|
* Returns the same value that is passed as argument.
|
|
89
89
|
* No side effects.
|
|
90
90
|
*
|
|
91
|
-
* Note: It can be
|
|
91
|
+
* Note: It can be useful for:
|
|
92
92
|
*
|
|
93
93
|
* 1) Leveling indentation
|
|
94
94
|
* 2) Putting always-true or always-false conditions without getting eslint errors
|
|
@@ -163,6 +163,21 @@ const DEFAULT_BOOK_OUTPUT_PARAMETER_NAME = 'result';
|
|
|
163
163
|
* @public exported from `@promptbook/core`
|
|
164
164
|
*/
|
|
165
165
|
const DEFAULT_MAX_FILE_SIZE = 100 * 1024 * 1024; // 100MB
|
|
166
|
+
/**
|
|
167
|
+
* Threshold value that determines when a dataset is considered "big"
|
|
168
|
+
* and may require special handling or optimizations
|
|
169
|
+
*
|
|
170
|
+
* For example, when error occurs in one item of the big dataset, it will not fail the whole pipeline
|
|
171
|
+
*
|
|
172
|
+
* @public exported from `@promptbook/core`
|
|
173
|
+
*/
|
|
174
|
+
const BIG_DATASET_TRESHOLD = 50;
|
|
175
|
+
/**
|
|
176
|
+
* Placeholder text used to represent a placeholder value of failed operation
|
|
177
|
+
*
|
|
178
|
+
* @public exported from `@promptbook/core`
|
|
179
|
+
*/
|
|
180
|
+
const FAILED_VALUE_PLACEHOLDER = '!?';
|
|
166
181
|
/**
|
|
167
182
|
* Warning message for the generated sections and files files
|
|
168
183
|
*
|
|
@@ -247,6 +262,7 @@ const DEFAULT_MAX_PARALLEL_COUNT = 5; // <- TODO: [🤹♂️]
|
|
|
247
262
|
* @public exported from `@promptbook/core`
|
|
248
263
|
*/
|
|
249
264
|
const DEFAULT_MAX_EXECUTION_ATTEMPTS = 10; // <- TODO: [🤹♂️]
|
|
265
|
+
// <- TODO: [🐝]
|
|
250
266
|
/**
|
|
251
267
|
* Where to store your books
|
|
252
268
|
* This is kind of a "src" for your books
|
|
@@ -318,7 +334,7 @@ const MOMENT_ARG_THRESHOLDS = {
|
|
|
318
334
|
const DEFAULT_REMOTE_SERVER_URL = REMOTE_SERVER_URLS[0].urls[0];
|
|
319
335
|
// <- TODO: [🧜♂️]
|
|
320
336
|
/**
|
|
321
|
-
*
|
|
337
|
+
* Default settings for parsing and generating CSV files in Promptbook.
|
|
322
338
|
*
|
|
323
339
|
* @public exported from `@promptbook/core`
|
|
324
340
|
*/
|
|
@@ -329,13 +345,13 @@ const DEFAULT_CSV_SETTINGS = Object.freeze({
|
|
|
329
345
|
skipEmptyLines: true,
|
|
330
346
|
});
|
|
331
347
|
/**
|
|
332
|
-
*
|
|
348
|
+
* Controls whether verbose logging is enabled by default throughout the application.
|
|
333
349
|
*
|
|
334
350
|
* @public exported from `@promptbook/core`
|
|
335
351
|
*/
|
|
336
352
|
let DEFAULT_IS_VERBOSE = false;
|
|
337
353
|
/**
|
|
338
|
-
*
|
|
354
|
+
* Controls whether auto-installation of dependencies is enabled by default.
|
|
339
355
|
*
|
|
340
356
|
* @public exported from `@promptbook/core`
|
|
341
357
|
*/
|
|
@@ -347,7 +363,15 @@ const DEFAULT_IS_AUTO_INSTALLED = false;
|
|
|
347
363
|
*/
|
|
348
364
|
const DEFAULT_GET_PIPELINE_COLLECTION_FUNCTION_NAME = `getPipelineCollection`;
|
|
349
365
|
/**
|
|
350
|
-
*
|
|
366
|
+
* Default rate limits (requests per minute)
|
|
367
|
+
*
|
|
368
|
+
* Note: Adjust based on the provider tier you are have
|
|
369
|
+
*
|
|
370
|
+
* @public exported from `@promptbook/core`
|
|
371
|
+
*/
|
|
372
|
+
const DEFAULT_MAX_REQUESTS_PER_MINUTE = 60;
|
|
373
|
+
/**
|
|
374
|
+
* Indicates whether pipeline logic validation is enabled. When true, the pipeline logic is checked for consistency.
|
|
351
375
|
*
|
|
352
376
|
* @private within the repository
|
|
353
377
|
*/
|
|
@@ -504,7 +528,7 @@ class UnexpectedError extends Error {
|
|
|
504
528
|
${block(message)}
|
|
505
529
|
|
|
506
530
|
Note: This error should not happen.
|
|
507
|
-
It's
|
|
531
|
+
It's probably a bug in the pipeline collection
|
|
508
532
|
|
|
509
533
|
Please report issue:
|
|
510
534
|
${block(getErrorReportUrl(new Error(message)).href)}
|
|
@@ -687,7 +711,8 @@ class NotYetImplementedError extends Error {
|
|
|
687
711
|
}
|
|
688
712
|
|
|
689
713
|
/**
|
|
690
|
-
*
|
|
714
|
+
* Safely retrieves the global scope object (window in browser, global in Node.js)
|
|
715
|
+
* regardless of the JavaScript environment in which the code is running
|
|
691
716
|
*
|
|
692
717
|
* Note: `$` is used to indicate that this function is not a pure function - it access global scope
|
|
693
718
|
*
|
|
@@ -698,10 +723,10 @@ function $getGlobalScope() {
|
|
|
698
723
|
}
|
|
699
724
|
|
|
700
725
|
/**
|
|
701
|
-
*
|
|
726
|
+
* Normalizes a text string to SCREAMING_CASE (all uppercase with underscores).
|
|
702
727
|
*
|
|
703
|
-
* @param text
|
|
704
|
-
* @returns
|
|
728
|
+
* @param text The text string to be converted to SCREAMING_CASE format.
|
|
729
|
+
* @returns The normalized text in SCREAMING_CASE format.
|
|
705
730
|
* @example 'HELLO_WORLD'
|
|
706
731
|
* @example 'I_LOVE_PROMPTBOOK'
|
|
707
732
|
* @public exported from `@promptbook/utils`
|
|
@@ -753,10 +778,10 @@ function normalizeTo_SCREAMING_CASE(text) {
|
|
|
753
778
|
*/
|
|
754
779
|
|
|
755
780
|
/**
|
|
756
|
-
*
|
|
781
|
+
* Normalizes a text string to snake_case format.
|
|
757
782
|
*
|
|
758
|
-
* @param text
|
|
759
|
-
* @returns
|
|
783
|
+
* @param text The text string to be converted to snake_case format.
|
|
784
|
+
* @returns The normalized text in snake_case format.
|
|
760
785
|
* @example 'hello_world'
|
|
761
786
|
* @example 'i_love_promptbook'
|
|
762
787
|
* @public exported from `@promptbook/utils`
|
|
@@ -766,11 +791,11 @@ function normalizeTo_snake_case(text) {
|
|
|
766
791
|
}
|
|
767
792
|
|
|
768
793
|
/**
|
|
769
|
-
*
|
|
794
|
+
* Global registry for storing and managing registered entities of a given type.
|
|
770
795
|
*
|
|
771
796
|
* Note: `$` is used to indicate that this function is not a pure function - it accesses and adds variables in global scope.
|
|
772
797
|
*
|
|
773
|
-
* @private internal utility, exported are only
|
|
798
|
+
* @private internal utility, exported are only singleton instances of this class
|
|
774
799
|
*/
|
|
775
800
|
class $Register {
|
|
776
801
|
constructor(registerName) {
|
|
@@ -814,10 +839,10 @@ class $Register {
|
|
|
814
839
|
}
|
|
815
840
|
|
|
816
841
|
/**
|
|
817
|
-
*
|
|
842
|
+
* Register for LLM tools metadata.
|
|
818
843
|
*
|
|
819
844
|
* Note: `$` is used to indicate that this interacts with the global scope
|
|
820
|
-
* @singleton Only one instance of each register is created per build, but
|
|
845
|
+
* @singleton Only one instance of each register is created per build, but there can be more instances across different builds or environments.
|
|
821
846
|
* @public exported from `@promptbook/core`
|
|
822
847
|
*/
|
|
823
848
|
const $llmToolsMetadataRegister = new $Register('llm_tools_metadata');
|
|
@@ -826,10 +851,10 @@ const $llmToolsMetadataRegister = new $Register('llm_tools_metadata');
|
|
|
826
851
|
*/
|
|
827
852
|
|
|
828
853
|
/**
|
|
829
|
-
*
|
|
854
|
+
* Register for LLM tools.
|
|
830
855
|
*
|
|
831
856
|
* Note: `$` is used to indicate that this interacts with the global scope
|
|
832
|
-
* @singleton Only one instance of each register is created per build, but
|
|
857
|
+
* @singleton Only one instance of each register is created per build, but there can be more instances across different builds or environments.
|
|
833
858
|
* @public exported from `@promptbook/core`
|
|
834
859
|
*/
|
|
835
860
|
const $llmToolsRegister = new $Register('llm_execution_tools_constructors');
|
|
@@ -981,7 +1006,7 @@ function $registeredLlmToolsMessage() {
|
|
|
981
1006
|
* Just says that the variable is not used but should be kept
|
|
982
1007
|
* No side effects.
|
|
983
1008
|
*
|
|
984
|
-
* Note: It can be
|
|
1009
|
+
* Note: It can be useful for:
|
|
985
1010
|
*
|
|
986
1011
|
* 1) Suppressing eager optimization of unused imports
|
|
987
1012
|
* 2) Suppressing eslint errors of unused variables in the tests
|
|
@@ -1034,15 +1059,12 @@ function jsonParse(value) {
|
|
|
1034
1059
|
}
|
|
1035
1060
|
throw new Error(spaceTrim((block) => `
|
|
1036
1061
|
${block(error.message)}
|
|
1037
|
-
|
|
1062
|
+
|
|
1038
1063
|
The JSON text:
|
|
1039
1064
|
${block(value)}
|
|
1040
1065
|
`));
|
|
1041
1066
|
}
|
|
1042
1067
|
}
|
|
1043
|
-
/**
|
|
1044
|
-
* TODO: !!!! Use in Promptbook.studio
|
|
1045
|
-
*/
|
|
1046
1068
|
|
|
1047
1069
|
/**
|
|
1048
1070
|
* Convert identification to Promptbook token
|
|
@@ -1082,7 +1104,7 @@ function promptbookTokenToIdentification(promptbookToken) {
|
|
|
1082
1104
|
* Just marks a place of place where should be something implemented
|
|
1083
1105
|
* No side effects.
|
|
1084
1106
|
*
|
|
1085
|
-
* Note: It can be
|
|
1107
|
+
* Note: It can be useful suppressing eslint errors of unused variables
|
|
1086
1108
|
*
|
|
1087
1109
|
* @param value any values
|
|
1088
1110
|
* @returns void
|
|
@@ -1092,7 +1114,8 @@ function TODO_USE(...value) {
|
|
|
1092
1114
|
}
|
|
1093
1115
|
|
|
1094
1116
|
/**
|
|
1095
|
-
*
|
|
1117
|
+
* Provides filesystem access (for example for Node.js-based scrapers)
|
|
1118
|
+
* Creates a standardized filesystem interface that scrapers can use for file operations.
|
|
1096
1119
|
*
|
|
1097
1120
|
* @public exported from `@promptbook/node`
|
|
1098
1121
|
*/
|
|
@@ -1467,8 +1490,12 @@ function checkSerializableAsJson(options) {
|
|
|
1467
1490
|
*/
|
|
1468
1491
|
|
|
1469
1492
|
/**
|
|
1470
|
-
*
|
|
1493
|
+
* Creates a deep clone of the given object
|
|
1471
1494
|
*
|
|
1495
|
+
* Note: This method only works for objects that are fully serializable to JSON and do not contain functions, Dates, or special types.
|
|
1496
|
+
*
|
|
1497
|
+
* @param objectValue The object to clone.
|
|
1498
|
+
* @returns A deep, writable clone of the input object.
|
|
1472
1499
|
* @public exported from `@promptbook/utils`
|
|
1473
1500
|
*/
|
|
1474
1501
|
function deepClone(objectValue) {
|
|
@@ -1523,6 +1550,26 @@ function exportJson(options) {
|
|
|
1523
1550
|
* TODO: [🧠] Is there a way how to meaningfully test this utility
|
|
1524
1551
|
*/
|
|
1525
1552
|
|
|
1553
|
+
// <- TODO: Maybe do better levels of trust
|
|
1554
|
+
/**
|
|
1555
|
+
* How is the model provider important?
|
|
1556
|
+
*
|
|
1557
|
+
* @public exported from `@promptbook/core`
|
|
1558
|
+
*/
|
|
1559
|
+
const MODEL_ORDERS = {
|
|
1560
|
+
/**
|
|
1561
|
+
* Top-tier models, e.g. OpenAI, Anthropic,...
|
|
1562
|
+
*/
|
|
1563
|
+
TOP_TIER: 333,
|
|
1564
|
+
/**
|
|
1565
|
+
* Mid-tier models, e.g. Llama, Mistral, etc.
|
|
1566
|
+
*/
|
|
1567
|
+
NORMAL: 100,
|
|
1568
|
+
/**
|
|
1569
|
+
* Low-tier models, e.g. Phi, Tiny, etc.
|
|
1570
|
+
*/
|
|
1571
|
+
LOW_TIER: 0,
|
|
1572
|
+
};
|
|
1526
1573
|
/**
|
|
1527
1574
|
* Order of keys in the pipeline JSON
|
|
1528
1575
|
*
|
|
@@ -1550,13 +1597,19 @@ const ORDER_OF_PIPELINE_JSON = [
|
|
|
1550
1597
|
*/
|
|
1551
1598
|
const REPLACING_NONCE = 'ptbkauk42kV2dzao34faw7FudQUHYPtW';
|
|
1552
1599
|
/**
|
|
1553
|
-
*
|
|
1600
|
+
* Nonce which is used as string which is not occurring in normal text
|
|
1601
|
+
*
|
|
1602
|
+
* @private within the repository
|
|
1603
|
+
*/
|
|
1604
|
+
const SALT_NONCE = 'ptbkghhewbvruets21t54et5';
|
|
1605
|
+
/**
|
|
1606
|
+
* Placeholder value indicating a parameter is missing its value.
|
|
1554
1607
|
*
|
|
1555
1608
|
* @private within the repository
|
|
1556
1609
|
*/
|
|
1557
1610
|
const RESERVED_PARAMETER_MISSING_VALUE = 'MISSING-' + REPLACING_NONCE;
|
|
1558
1611
|
/**
|
|
1559
|
-
*
|
|
1612
|
+
* Placeholder value indicating a parameter is restricted and cannot be used directly.
|
|
1560
1613
|
*
|
|
1561
1614
|
* @private within the repository
|
|
1562
1615
|
*/
|
|
@@ -1646,7 +1699,7 @@ function stringifyPipelineJson(pipeline) {
|
|
|
1646
1699
|
return pipelineJsonStringified;
|
|
1647
1700
|
}
|
|
1648
1701
|
/**
|
|
1649
|
-
* TODO: [🐝] Not Working
|
|
1702
|
+
* TODO: [🐝] Not Working properly @see https://promptbook.studio/examples/mixed-knowledge.book
|
|
1650
1703
|
* TODO: [🧠][0] Maybe rename to `stringifyPipelineJson`, `stringifyIndexedJson`,...
|
|
1651
1704
|
* TODO: [🧠] Maybe more elegant solution than replacing via regex
|
|
1652
1705
|
* TODO: [🍙] Make some standard order of json properties
|
|
@@ -1721,7 +1774,7 @@ function isValidFilePath(filename) {
|
|
|
1721
1774
|
* Tests if given string is valid URL.
|
|
1722
1775
|
*
|
|
1723
1776
|
* Note: Dataurl are considered perfectly valid.
|
|
1724
|
-
* Note: There are two
|
|
1777
|
+
* Note: There are two similar functions:
|
|
1725
1778
|
* - `isValidUrl` which tests any URL
|
|
1726
1779
|
* - `isValidPipelineUrl` *(this one)* which tests just promptbook URL
|
|
1727
1780
|
*
|
|
@@ -1991,10 +2044,10 @@ for (let i = 0; i < defaultDiacriticsRemovalMap.length; i++) {
|
|
|
1991
2044
|
*/
|
|
1992
2045
|
|
|
1993
2046
|
/**
|
|
1994
|
-
*
|
|
2047
|
+
* Removes diacritic marks (accents) from characters in a string.
|
|
1995
2048
|
*
|
|
1996
|
-
* @param input
|
|
1997
|
-
* @returns
|
|
2049
|
+
* @param input The string containing diacritics to be normalized.
|
|
2050
|
+
* @returns The string with diacritics removed or normalized.
|
|
1998
2051
|
* @public exported from `@promptbook/utils`
|
|
1999
2052
|
*/
|
|
2000
2053
|
function removeDiacritics(input) {
|
|
@@ -2008,10 +2061,10 @@ function removeDiacritics(input) {
|
|
|
2008
2061
|
*/
|
|
2009
2062
|
|
|
2010
2063
|
/**
|
|
2011
|
-
*
|
|
2064
|
+
* Converts a given text to kebab-case format.
|
|
2012
2065
|
*
|
|
2013
|
-
* @param text
|
|
2014
|
-
* @returns
|
|
2066
|
+
* @param text The text to be converted.
|
|
2067
|
+
* @returns The kebab-case formatted string.
|
|
2015
2068
|
* @example 'hello-world'
|
|
2016
2069
|
* @example 'i-love-promptbook'
|
|
2017
2070
|
* @public exported from `@promptbook/utils`
|
|
@@ -2059,11 +2112,11 @@ function normalizeToKebabCase(text) {
|
|
|
2059
2112
|
*/
|
|
2060
2113
|
|
|
2061
2114
|
/**
|
|
2062
|
-
*
|
|
2115
|
+
* Converts a title string into a normalized name.
|
|
2063
2116
|
*
|
|
2064
|
-
* @param value
|
|
2065
|
-
* @returns
|
|
2066
|
-
* @example
|
|
2117
|
+
* @param value The title string to be converted to a name.
|
|
2118
|
+
* @returns A normalized name derived from the input title.
|
|
2119
|
+
* @example 'Hello World!' -> 'hello-world'
|
|
2067
2120
|
* @public exported from `@promptbook/utils`
|
|
2068
2121
|
*/
|
|
2069
2122
|
function titleToName(value) {
|
|
@@ -2083,7 +2136,8 @@ function titleToName(value) {
|
|
|
2083
2136
|
}
|
|
2084
2137
|
|
|
2085
2138
|
/**
|
|
2086
|
-
*
|
|
2139
|
+
* Converts a name to a properly formatted subfolder path for cache storage.
|
|
2140
|
+
* Handles normalization and path formatting to create consistent cache directory structures.
|
|
2087
2141
|
*
|
|
2088
2142
|
* @private for `FileCacheStorage`
|
|
2089
2143
|
*/
|
|
@@ -2092,7 +2146,10 @@ function nameToSubfolderPath(name) {
|
|
|
2092
2146
|
}
|
|
2093
2147
|
|
|
2094
2148
|
/**
|
|
2095
|
-
*
|
|
2149
|
+
* A storage implementation that caches data in files organized in a directory structure.
|
|
2150
|
+
* Provides methods for retrieving, storing, and managing cached data on the filesystem.
|
|
2151
|
+
*
|
|
2152
|
+
* This class implements the PromptbookStorage interface for filesystem-based caching.
|
|
2096
2153
|
*
|
|
2097
2154
|
* @public exported from `@promptbook/node`
|
|
2098
2155
|
*/
|
|
@@ -2105,7 +2162,8 @@ class FileCacheStorage {
|
|
|
2105
2162
|
}
|
|
2106
2163
|
}
|
|
2107
2164
|
/**
|
|
2108
|
-
*
|
|
2165
|
+
* Converts a storage key to a filesystem path where the data should be stored.
|
|
2166
|
+
* Creates a consistent, deterministic file path based on the key string.
|
|
2109
2167
|
*/
|
|
2110
2168
|
getFilenameForKey(key) {
|
|
2111
2169
|
// TODO: [👬] DRY
|
|
@@ -2117,7 +2175,8 @@ class FileCacheStorage {
|
|
|
2117
2175
|
...nameToSubfolderPath(hash /* <- TODO: [🎎] Maybe add some SHA256 prefix */), `${name.substring(0, MAX_FILENAME_LENGTH)}.json`);
|
|
2118
2176
|
}
|
|
2119
2177
|
/**
|
|
2120
|
-
*
|
|
2178
|
+
* Returns the current value associated with the given key, or null if the given key does not exist.
|
|
2179
|
+
* Retrieves the cached data from the file system storage.
|
|
2121
2180
|
*/
|
|
2122
2181
|
async getItem(key) {
|
|
2123
2182
|
const filename = this.getFilenameForKey(key);
|
|
@@ -2130,7 +2189,8 @@ class FileCacheStorage {
|
|
|
2130
2189
|
return value;
|
|
2131
2190
|
}
|
|
2132
2191
|
/**
|
|
2133
|
-
*
|
|
2192
|
+
* Sets the value of the pair identified by key to value, creating a new key/value pair if none existed for key previously.
|
|
2193
|
+
* Persists data to the file system, creating necessary directory structure if it doesn't exist.
|
|
2134
2194
|
*/
|
|
2135
2195
|
async setItem(key, value) {
|
|
2136
2196
|
const filename = this.getFilenameForKey(key);
|
|
@@ -2142,7 +2202,8 @@ class FileCacheStorage {
|
|
|
2142
2202
|
await writeFile(filename, fileContent, 'utf-8');
|
|
2143
2203
|
}
|
|
2144
2204
|
/**
|
|
2145
|
-
*
|
|
2205
|
+
* Removes the key/value pair with the given key from the storage, if a key/value pair with the given key exists.
|
|
2206
|
+
* Deletes the corresponding file from the filesystem.
|
|
2146
2207
|
*/
|
|
2147
2208
|
async removeItem(key) {
|
|
2148
2209
|
const filename = this.getFilenameForKey(key);
|
|
@@ -2201,7 +2262,7 @@ class AuthenticationError extends Error {
|
|
|
2201
2262
|
}
|
|
2202
2263
|
|
|
2203
2264
|
/**
|
|
2204
|
-
* This error indicates that the pipeline collection cannot be
|
|
2265
|
+
* This error indicates that the pipeline collection cannot be properly loaded
|
|
2205
2266
|
*
|
|
2206
2267
|
* @public exported from `@promptbook/core`
|
|
2207
2268
|
*/
|
|
@@ -2265,7 +2326,7 @@ class MissingToolsError extends Error {
|
|
|
2265
2326
|
super(spaceTrim$1((block) => `
|
|
2266
2327
|
${block(message)}
|
|
2267
2328
|
|
|
2268
|
-
Note: You have
|
|
2329
|
+
Note: You have probably forgot to provide some tools for pipeline execution or preparation
|
|
2269
2330
|
|
|
2270
2331
|
`));
|
|
2271
2332
|
this.name = 'MissingToolsError';
|
|
@@ -2328,7 +2389,7 @@ class PipelineExecutionError extends Error {
|
|
|
2328
2389
|
super(message);
|
|
2329
2390
|
this.name = 'PipelineExecutionError';
|
|
2330
2391
|
// TODO: [🐙] DRY - Maybe $randomId
|
|
2331
|
-
this.id = `error-${$randomToken(8 /* <- TODO: To global config + Use Base58 to avoid
|
|
2392
|
+
this.id = `error-${$randomToken(8 /* <- TODO: To global config + Use Base58 to avoid similar char conflicts */)}`;
|
|
2332
2393
|
Object.setPrototypeOf(this, PipelineExecutionError.prototype);
|
|
2333
2394
|
}
|
|
2334
2395
|
}
|
|
@@ -2525,10 +2586,10 @@ class RemoteLlmExecutionTools {
|
|
|
2525
2586
|
}
|
|
2526
2587
|
get title() {
|
|
2527
2588
|
// TODO: [🧠] Maybe fetch title+description from the remote server (as well as if model methods are defined)
|
|
2528
|
-
return '
|
|
2589
|
+
return 'Promptbook remote server';
|
|
2529
2590
|
}
|
|
2530
2591
|
get description() {
|
|
2531
|
-
return
|
|
2592
|
+
return `Models from Promptbook remote server ${this.options.remoteServerUrl}`;
|
|
2532
2593
|
}
|
|
2533
2594
|
/**
|
|
2534
2595
|
* Check the configuration of all execution tools
|
|
@@ -2543,7 +2604,7 @@ class RemoteLlmExecutionTools {
|
|
|
2543
2604
|
* List all available models that can be used
|
|
2544
2605
|
*/
|
|
2545
2606
|
async listModels() {
|
|
2546
|
-
// TODO: [👒] Listing models (and checking configuration)
|
|
2607
|
+
// TODO: [👒] Listing models (and checking configuration) probably should go through REST API not Socket.io
|
|
2547
2608
|
const socket = await createRemoteClient(this.options);
|
|
2548
2609
|
socket.emit('listModels-request', {
|
|
2549
2610
|
identification: this.options.identification,
|
|
@@ -2680,6 +2741,23 @@ function $getCurrentDate() {
|
|
|
2680
2741
|
return new Date().toISOString();
|
|
2681
2742
|
}
|
|
2682
2743
|
|
|
2744
|
+
/**
|
|
2745
|
+
* Parses the task and returns the list of all parameter names
|
|
2746
|
+
*
|
|
2747
|
+
* @param template the string template with parameters in {curly} braces
|
|
2748
|
+
* @returns the list of parameter names
|
|
2749
|
+
* @public exported from `@promptbook/utils`
|
|
2750
|
+
*/
|
|
2751
|
+
function extractParameterNames(template) {
|
|
2752
|
+
const matches = template.matchAll(/{\w+}/g);
|
|
2753
|
+
const parameterNames = new Set();
|
|
2754
|
+
for (const match of matches) {
|
|
2755
|
+
const parameterName = match[0].slice(1, -1);
|
|
2756
|
+
parameterNames.add(parameterName);
|
|
2757
|
+
}
|
|
2758
|
+
return parameterNames;
|
|
2759
|
+
}
|
|
2760
|
+
|
|
2683
2761
|
/**
|
|
2684
2762
|
* Intercepts LLM tools and counts total usage of the tools
|
|
2685
2763
|
*
|
|
@@ -2690,17 +2768,19 @@ function $getCurrentDate() {
|
|
|
2690
2768
|
* @public exported from `@promptbook/core`
|
|
2691
2769
|
*/
|
|
2692
2770
|
function cacheLlmTools(llmTools, options = {}) {
|
|
2693
|
-
const { storage = new MemoryStorage(), isCacheReloaded = false } = options;
|
|
2771
|
+
const { storage = new MemoryStorage(), isCacheReloaded = false, isVerbose = DEFAULT_IS_VERBOSE } = options;
|
|
2694
2772
|
const proxyTools = {
|
|
2695
2773
|
...llmTools,
|
|
2696
2774
|
// <- Note: [🥫]
|
|
2697
2775
|
get title() {
|
|
2698
|
-
|
|
2699
|
-
|
|
2776
|
+
return `${llmTools.title} (cached)`;
|
|
2777
|
+
// <- TODO: [🧈] Maybe standartize the suffix when wrapping `LlmExecutionTools` up
|
|
2778
|
+
// <- TODO: [🧈][🧠] Does it make sence to suffix "(cached)"?
|
|
2700
2779
|
},
|
|
2701
2780
|
get description() {
|
|
2702
|
-
|
|
2703
|
-
|
|
2781
|
+
return `${llmTools.description} (cached)`;
|
|
2782
|
+
// <- TODO: [🧈] Maybe standartize the suffix when wrapping `LlmExecutionTools` up
|
|
2783
|
+
// <- TODO: [🧈][🧠] Does it make sence to suffix "(cached)"?
|
|
2704
2784
|
},
|
|
2705
2785
|
listModels() {
|
|
2706
2786
|
// TODO: [🧠] Should be model listing also cached?
|
|
@@ -2710,13 +2790,37 @@ function cacheLlmTools(llmTools, options = {}) {
|
|
|
2710
2790
|
const callCommonModel = async (prompt) => {
|
|
2711
2791
|
const { parameters, content, modelRequirements } = prompt;
|
|
2712
2792
|
// <- Note: These are relevant things from the prompt that the cache key should depend on.
|
|
2793
|
+
// TODO: Maybe some standalone function for normalization of content for cache
|
|
2794
|
+
let normalizedContent = content;
|
|
2795
|
+
normalizedContent = normalizedContent.replace(/\s+/g, ' ');
|
|
2796
|
+
normalizedContent = normalizedContent.split('\r\n').join('\n');
|
|
2797
|
+
normalizedContent = spaceTrim(normalizedContent);
|
|
2798
|
+
// Note: Do not need to save everything in the cache, just the relevant parameters
|
|
2799
|
+
const relevantParameterNames = extractParameterNames(content);
|
|
2800
|
+
const relevantParameters = Object.fromEntries(Object.entries(parameters).filter(([key]) => relevantParameterNames.has(key)));
|
|
2801
|
+
const keyHashBase = { relevantParameters, normalizedContent, modelRequirements };
|
|
2713
2802
|
const key = titleToName(prompt.title.substring(0, MAX_FILENAME_LENGTH - 10) +
|
|
2714
2803
|
'-' +
|
|
2715
|
-
sha256(hexEncoder.parse(JSON.stringify(
|
|
2804
|
+
sha256(hexEncoder.parse(JSON.stringify(keyHashBase)))
|
|
2805
|
+
.toString( /* hex */)
|
|
2806
|
+
.substring(0, 10 - 1));
|
|
2716
2807
|
const cacheItem = !isCacheReloaded ? await storage.getItem(key) : null;
|
|
2717
2808
|
if (cacheItem) {
|
|
2718
2809
|
return cacheItem.promptResult;
|
|
2719
2810
|
}
|
|
2811
|
+
if (isVerbose) {
|
|
2812
|
+
console.info('Cache miss for key:', key, {
|
|
2813
|
+
prompt,
|
|
2814
|
+
'prompt.title': prompt.title,
|
|
2815
|
+
MAX_FILENAME_LENGTH,
|
|
2816
|
+
keyHashBase,
|
|
2817
|
+
parameters,
|
|
2818
|
+
relevantParameters,
|
|
2819
|
+
content,
|
|
2820
|
+
normalizedContent,
|
|
2821
|
+
modelRequirements,
|
|
2822
|
+
});
|
|
2823
|
+
}
|
|
2720
2824
|
let promptResult;
|
|
2721
2825
|
variant: switch (prompt.modelRequirements.modelVariant) {
|
|
2722
2826
|
case 'CHAT':
|
|
@@ -2737,7 +2841,16 @@ function cacheLlmTools(llmTools, options = {}) {
|
|
|
2737
2841
|
await storage.setItem(key, {
|
|
2738
2842
|
date: $getCurrentDate(),
|
|
2739
2843
|
promptbookVersion: PROMPTBOOK_ENGINE_VERSION,
|
|
2740
|
-
|
|
2844
|
+
bookVersion: BOOK_LANGUAGE_VERSION,
|
|
2845
|
+
prompt: {
|
|
2846
|
+
...prompt,
|
|
2847
|
+
parameters: Object.entries(parameters).length === Object.entries(relevantParameters).length
|
|
2848
|
+
? parameters
|
|
2849
|
+
: {
|
|
2850
|
+
...relevantParameters,
|
|
2851
|
+
note: `<- Note: Only relevant parameters are stored in the cache`,
|
|
2852
|
+
},
|
|
2853
|
+
},
|
|
2741
2854
|
promptResult,
|
|
2742
2855
|
});
|
|
2743
2856
|
return promptResult;
|
|
@@ -2763,9 +2876,9 @@ function cacheLlmTools(llmTools, options = {}) {
|
|
|
2763
2876
|
/**
|
|
2764
2877
|
* TODO: [🧠][💸] Maybe make some common abstraction `interceptLlmTools` and use here (or use javascript Proxy?)
|
|
2765
2878
|
* TODO: [🧠] Is there some meaningfull way how to test this util
|
|
2766
|
-
* TODO: [👷♂️]
|
|
2767
|
-
*
|
|
2768
|
-
*
|
|
2879
|
+
* TODO: [👷♂️] Comprehensive manual about construction of llmTools
|
|
2880
|
+
* Detailed explanation about caching strategies and appropriate storage selection for different use cases
|
|
2881
|
+
* Examples of how to combine multiple interceptors for advanced caching, logging, and usage tracking
|
|
2769
2882
|
*/
|
|
2770
2883
|
|
|
2771
2884
|
/**
|
|
@@ -2895,12 +3008,14 @@ function countUsage(llmTools) {
|
|
|
2895
3008
|
const spending = new Subject();
|
|
2896
3009
|
const proxyTools = {
|
|
2897
3010
|
get title() {
|
|
2898
|
-
|
|
2899
|
-
|
|
3011
|
+
return `${llmTools.title} (+usage)`;
|
|
3012
|
+
// <- TODO: [🧈] Maybe standartize the suffix when wrapping `LlmExecutionTools` up
|
|
3013
|
+
// <- TODO: [🧈][🧠] Does it make sence to suffix "(+usage)"?
|
|
2900
3014
|
},
|
|
2901
3015
|
get description() {
|
|
2902
|
-
|
|
2903
|
-
|
|
3016
|
+
return `${llmTools.description} (+usage)`;
|
|
3017
|
+
// <- TODO: [🧈] Maybe standartize the suffix when wrapping `LlmExecutionTools` up
|
|
3018
|
+
// <- TODO: [🧈][🧠] Does it make sence to suffix "(+usage)"?
|
|
2904
3019
|
},
|
|
2905
3020
|
checkConfiguration() {
|
|
2906
3021
|
return /* not await */ llmTools.checkConfiguration();
|
|
@@ -2955,9 +3070,8 @@ function countUsage(llmTools) {
|
|
|
2955
3070
|
*/
|
|
2956
3071
|
|
|
2957
3072
|
/**
|
|
2958
|
-
*
|
|
3073
|
+
* Provides LLM tools configuration by reading environment variables.
|
|
2959
3074
|
*
|
|
2960
|
-
* @@@ .env
|
|
2961
3075
|
* Note: `$` is used to indicate that this function is not a pure function - it uses filesystem to access `.env` file
|
|
2962
3076
|
*
|
|
2963
3077
|
* It looks for environment variables:
|
|
@@ -2965,7 +3079,8 @@ function countUsage(llmTools) {
|
|
|
2965
3079
|
* - `process.env.ANTHROPIC_CLAUDE_API_KEY`
|
|
2966
3080
|
* - ...
|
|
2967
3081
|
*
|
|
2968
|
-
* @
|
|
3082
|
+
* @see Environment variables documentation or .env file for required variables.
|
|
3083
|
+
* @returns A promise that resolves to the LLM tools configuration, or null if configuration is incomplete or missing.
|
|
2969
3084
|
* @public exported from `@promptbook/node`
|
|
2970
3085
|
*/
|
|
2971
3086
|
async function $provideLlmToolsConfigurationFromEnv() {
|
|
@@ -3003,29 +3118,40 @@ class MultipleLlmExecutionTools {
|
|
|
3003
3118
|
return 'Multiple LLM Providers';
|
|
3004
3119
|
}
|
|
3005
3120
|
get description() {
|
|
3006
|
-
|
|
3121
|
+
const innerModelsTitlesAndDescriptions = this.llmExecutionTools
|
|
3122
|
+
.map(({ title, description }, index) => {
|
|
3123
|
+
const headLine = `${index + 1}) \`${title}\``;
|
|
3124
|
+
if (description === undefined) {
|
|
3125
|
+
return headLine;
|
|
3126
|
+
}
|
|
3127
|
+
return spaceTrim((block) => `
|
|
3128
|
+
${headLine}
|
|
3129
|
+
|
|
3130
|
+
${ /* <- Note: Indenting the description: */block(description)}
|
|
3131
|
+
`);
|
|
3132
|
+
})
|
|
3133
|
+
.join('\n\n');
|
|
3134
|
+
return spaceTrim((block) => `
|
|
3135
|
+
Multiple LLM Providers:
|
|
3136
|
+
|
|
3137
|
+
${block(innerModelsTitlesAndDescriptions)}
|
|
3138
|
+
`);
|
|
3007
3139
|
}
|
|
3008
3140
|
/**
|
|
3009
3141
|
* Check the configuration of all execution tools
|
|
3010
3142
|
*/
|
|
3011
3143
|
async checkConfiguration() {
|
|
3012
|
-
//
|
|
3013
|
-
|
|
3014
|
-
await llmExecutionTools.checkConfiguration();
|
|
3015
|
-
}
|
|
3144
|
+
// Note: Run checks in parallel
|
|
3145
|
+
await Promise.all(this.llmExecutionTools.map((tools) => tools.checkConfiguration()));
|
|
3016
3146
|
}
|
|
3017
3147
|
/**
|
|
3018
3148
|
* List all available models that can be used
|
|
3019
3149
|
* This lists is a combination of all available models from all execution tools
|
|
3020
3150
|
*/
|
|
3021
3151
|
async listModels() {
|
|
3022
|
-
|
|
3023
|
-
|
|
3024
|
-
|
|
3025
|
-
const models = await llmExecutionTools.listModels();
|
|
3026
|
-
availableModels.push(...models);
|
|
3027
|
-
}
|
|
3028
|
-
return availableModels;
|
|
3152
|
+
// Obtain all models in parallel and flatten
|
|
3153
|
+
const modelArrays = await Promise.all(this.llmExecutionTools.map((tools) => tools.listModels()));
|
|
3154
|
+
return modelArrays.flat();
|
|
3029
3155
|
}
|
|
3030
3156
|
/**
|
|
3031
3157
|
* Calls the best available chat model
|
|
@@ -3179,11 +3305,16 @@ function joinLlmExecutionTools(...llmExecutionTools) {
|
|
|
3179
3305
|
*/
|
|
3180
3306
|
|
|
3181
3307
|
/**
|
|
3182
|
-
*
|
|
3308
|
+
* Creates LLM execution tools from provided configuration objects
|
|
3309
|
+
*
|
|
3310
|
+
* Instantiates and configures LLM tool instances for each configuration entry,
|
|
3311
|
+
* combining them into a unified interface via MultipleLlmExecutionTools.
|
|
3183
3312
|
*
|
|
3184
3313
|
* Note: This function is not cached, every call creates new instance of `MultipleLlmExecutionTools`
|
|
3185
3314
|
*
|
|
3186
|
-
* @
|
|
3315
|
+
* @param configuration Array of LLM tool configurations to instantiate
|
|
3316
|
+
* @param options Additional options for configuring the LLM tools
|
|
3317
|
+
* @returns A unified interface combining all successfully instantiated LLM tools
|
|
3187
3318
|
* @public exported from `@promptbook/core`
|
|
3188
3319
|
*/
|
|
3189
3320
|
function createLlmToolsFromConfiguration(configuration, options = {}) {
|
|
@@ -3222,7 +3353,11 @@ function createLlmToolsFromConfiguration(configuration, options = {}) {
|
|
|
3222
3353
|
/**
|
|
3223
3354
|
* TODO: [🎌] Together with `createLlmToolsFromConfiguration` + 'EXECUTION_TOOLS_CLASSES' gets to `@promptbook/core` ALL model providers, make this more efficient
|
|
3224
3355
|
* TODO: [🧠][🎌] Dynamically install required providers
|
|
3225
|
-
* TODO:
|
|
3356
|
+
* TODO: We should implement an interactive configuration wizard that would:
|
|
3357
|
+
* 1. Detect which LLM providers are available in the environment
|
|
3358
|
+
* 2. Guide users through required configuration settings for each provider
|
|
3359
|
+
* 3. Allow testing connections before completing setup
|
|
3360
|
+
* 4. Generate appropriate configuration code for application integration
|
|
3226
3361
|
* TODO: [🧠][🍛] Which name is better `createLlmToolsFromConfig` or `createLlmToolsFromConfiguration`?
|
|
3227
3362
|
* TODO: [🧠] Is there some meaningfull way how to test this util
|
|
3228
3363
|
* TODO: This should be maybe not under `_common` but under `utils`
|
|
@@ -3230,11 +3365,14 @@ function createLlmToolsFromConfiguration(configuration, options = {}) {
|
|
|
3230
3365
|
*/
|
|
3231
3366
|
|
|
3232
3367
|
/**
|
|
3233
|
-
*
|
|
3368
|
+
* Automatically configures LLM tools from environment variables in Node.js
|
|
3369
|
+
*
|
|
3370
|
+
* This utility function detects available LLM providers based on environment variables
|
|
3371
|
+
* and creates properly configured LLM execution tools for each detected provider.
|
|
3234
3372
|
*
|
|
3235
3373
|
* Note: This function is not cached, every call creates new instance of `MultipleLlmExecutionTools`
|
|
3236
3374
|
*
|
|
3237
|
-
*
|
|
3375
|
+
* Supports environment variables from .env files when dotenv is configured
|
|
3238
3376
|
* Note: `$` is used to indicate that this function is not a pure function - it uses filesystem to access `.env` file
|
|
3239
3377
|
*
|
|
3240
3378
|
* It looks for environment variables:
|
|
@@ -3242,7 +3380,8 @@ function createLlmToolsFromConfiguration(configuration, options = {}) {
|
|
|
3242
3380
|
* - `process.env.ANTHROPIC_CLAUDE_API_KEY`
|
|
3243
3381
|
* - ...
|
|
3244
3382
|
*
|
|
3245
|
-
* @
|
|
3383
|
+
* @param options Configuration options for the LLM tools
|
|
3384
|
+
* @returns A unified interface containing all detected and configured LLM tools
|
|
3246
3385
|
* @public exported from `@promptbook/node`
|
|
3247
3386
|
*/
|
|
3248
3387
|
async function $provideLlmToolsFromEnv(options = {}) {
|
|
@@ -3268,7 +3407,16 @@ async function $provideLlmToolsFromEnv(options = {}) {
|
|
|
3268
3407
|
return createLlmToolsFromConfiguration(configuration, options);
|
|
3269
3408
|
}
|
|
3270
3409
|
/**
|
|
3271
|
-
* TODO:
|
|
3410
|
+
* TODO: The architecture for LLM tools configuration consists of three key functions:
|
|
3411
|
+
* 1. `$provideLlmToolsFromEnv` - High-level function that detects available providers from env vars and returns ready-to-use LLM tools
|
|
3412
|
+
* 2. `$provideLlmToolsConfigurationFromEnv` - Middle layer that extracts configuration objects from environment variables
|
|
3413
|
+
* 3. `createLlmToolsFromConfiguration` - Low-level function that instantiates LLM tools from explicit configuration
|
|
3414
|
+
*
|
|
3415
|
+
* This layered approach allows flexibility in how tools are configured:
|
|
3416
|
+
* - Use $provideLlmToolsFromEnv for automatic detection and setup in Node.js environments
|
|
3417
|
+
* - Use $provideLlmToolsConfigurationFromEnv to extract config objects for modification before instantiation
|
|
3418
|
+
* - Use createLlmToolsFromConfiguration for explicit control over tool configurations
|
|
3419
|
+
*
|
|
3272
3420
|
* TODO: [🧠][🍛] Which name is better `$provideLlmToolsFromEnv` or `$provideLlmToolsFromEnvironment`?
|
|
3273
3421
|
* TODO: [🧠] Is there some meaningfull way how to test this util
|
|
3274
3422
|
* Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
|
|
@@ -3436,15 +3584,19 @@ async function $provideLlmToolsForCli(options) {
|
|
|
3436
3584
|
type: 'text',
|
|
3437
3585
|
name: 'username',
|
|
3438
3586
|
message: 'Enter your email:',
|
|
3439
|
-
validate
|
|
3587
|
+
validate(value) {
|
|
3588
|
+
return isValidEmail(value) ? true : 'Valid email is required';
|
|
3589
|
+
},
|
|
3440
3590
|
},
|
|
3441
3591
|
{
|
|
3442
3592
|
type: 'password',
|
|
3443
3593
|
name: 'password',
|
|
3444
3594
|
message: 'Enter your password:',
|
|
3445
|
-
validate
|
|
3446
|
-
|
|
3447
|
-
|
|
3595
|
+
validate(value) {
|
|
3596
|
+
return value.length /* <- TODO: [🧠] Better password validation */ > 0
|
|
3597
|
+
? true
|
|
3598
|
+
: 'Password is required';
|
|
3599
|
+
},
|
|
3448
3600
|
},
|
|
3449
3601
|
]);
|
|
3450
3602
|
const loginUrl = `${remoteServerUrl}/login`;
|
|
@@ -3681,7 +3833,8 @@ function $execCommand(options) {
|
|
|
3681
3833
|
*/
|
|
3682
3834
|
|
|
3683
3835
|
/**
|
|
3684
|
-
*
|
|
3836
|
+
* Attempts to locate the specified application on a Linux system using the 'which' command.
|
|
3837
|
+
* Returns the path to the executable if found, or null otherwise.
|
|
3685
3838
|
*
|
|
3686
3839
|
* @private within the repository
|
|
3687
3840
|
*/
|
|
@@ -3724,7 +3877,8 @@ async function isExecutable(path, fs) {
|
|
|
3724
3877
|
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
|
3725
3878
|
const userhome = require('userhome');
|
|
3726
3879
|
/**
|
|
3727
|
-
*
|
|
3880
|
+
* Attempts to locate the specified application on a macOS system by checking standard application paths and using mdfind.
|
|
3881
|
+
* Returns the path to the executable if found, or null otherwise.
|
|
3728
3882
|
*
|
|
3729
3883
|
* @private within the repository
|
|
3730
3884
|
*/
|
|
@@ -3756,7 +3910,8 @@ async function locateAppOnMacOs({ macOsName, }) {
|
|
|
3756
3910
|
*/
|
|
3757
3911
|
|
|
3758
3912
|
/**
|
|
3759
|
-
*
|
|
3913
|
+
* Attempts to locate the specified application on a Windows system by searching common installation directories.
|
|
3914
|
+
* Returns the path to the executable if found, or null otherwise.
|
|
3760
3915
|
*
|
|
3761
3916
|
* @private within the repository
|
|
3762
3917
|
*/
|
|
@@ -3827,7 +3982,8 @@ function locateApp(options) {
|
|
|
3827
3982
|
*/
|
|
3828
3983
|
|
|
3829
3984
|
/**
|
|
3830
|
-
*
|
|
3985
|
+
* Locates the LibreOffice executable on the current system by searching platform-specific paths.
|
|
3986
|
+
* Returns the path to the executable if found, or null otherwise.
|
|
3831
3987
|
*
|
|
3832
3988
|
* @private within the repository
|
|
3833
3989
|
*/
|
|
@@ -3845,7 +4001,8 @@ function locateLibreoffice() {
|
|
|
3845
4001
|
*/
|
|
3846
4002
|
|
|
3847
4003
|
/**
|
|
3848
|
-
*
|
|
4004
|
+
* Locates the Pandoc executable on the current system by searching platform-specific paths.
|
|
4005
|
+
* Returns the path to the executable if found, or null otherwise.
|
|
3849
4006
|
*
|
|
3850
4007
|
* @private within the repository
|
|
3851
4008
|
*/
|
|
@@ -3863,7 +4020,7 @@ function locatePandoc() {
|
|
|
3863
4020
|
*/
|
|
3864
4021
|
|
|
3865
4022
|
/**
|
|
3866
|
-
*
|
|
4023
|
+
* Provides paths to required executables (i.e. as Pandoc and LibreOffice) for Node.js environments.
|
|
3867
4024
|
*
|
|
3868
4025
|
* @public exported from `@promptbook/node`
|
|
3869
4026
|
*/
|
|
@@ -3883,10 +4040,11 @@ async function $provideExecutablesForNode(options) {
|
|
|
3883
4040
|
*/
|
|
3884
4041
|
|
|
3885
4042
|
/**
|
|
3886
|
-
*
|
|
4043
|
+
* Registry for all available scrapers in the system.
|
|
4044
|
+
* Central point for registering and accessing different types of content scrapers.
|
|
3887
4045
|
*
|
|
3888
4046
|
* Note: `$` is used to indicate that this interacts with the global scope
|
|
3889
|
-
* @singleton Only one instance of each register is created per build, but
|
|
4047
|
+
* @singleton Only one instance of each register is created per build, but there can be more than one in different build modules
|
|
3890
4048
|
* @public exported from `@promptbook/core`
|
|
3891
4049
|
*/
|
|
3892
4050
|
const $scrapersRegister = new $Register('scraper_constructors');
|
|
@@ -3895,11 +4053,9 @@ const $scrapersRegister = new $Register('scraper_constructors');
|
|
|
3895
4053
|
*/
|
|
3896
4054
|
|
|
3897
4055
|
/**
|
|
3898
|
-
*
|
|
3899
|
-
*
|
|
3900
|
-
*
|
|
3901
|
-
* 2) @@@
|
|
3902
|
-
*
|
|
4056
|
+
* Provides a collection of scrapers optimized for Node.js environment.
|
|
4057
|
+
* 1) `provideScrapersForNode` use as default
|
|
4058
|
+
* 2) `provideScrapersForBrowser` use in limited browser environment *
|
|
3903
4059
|
* @public exported from `@promptbook/node`
|
|
3904
4060
|
*/
|
|
3905
4061
|
async function $provideScrapersForNode(tools, options) {
|
|
@@ -3924,10 +4080,10 @@ async function $provideScrapersForNode(tools, options) {
|
|
|
3924
4080
|
*/
|
|
3925
4081
|
|
|
3926
4082
|
/**
|
|
3927
|
-
*
|
|
4083
|
+
* Global registry for storing metadata about all available scrapers and converters.
|
|
3928
4084
|
*
|
|
3929
|
-
* Note: `$` is used to indicate that this interacts with the global scope
|
|
3930
|
-
* @singleton Only one instance of each register is created per build, but
|
|
4085
|
+
* Note: `$` is used to indicate that this interacts with the global scope.
|
|
4086
|
+
* @singleton Only one instance of each register is created per build, but there can be more in different contexts (e.g., tests).
|
|
3931
4087
|
* @public exported from `@promptbook/core`
|
|
3932
4088
|
*/
|
|
3933
4089
|
const $scrapersMetadataRegister = new $Register('scrapers_metadata');
|
|
@@ -4110,7 +4266,7 @@ async function collectionToJson(collection) {
|
|
|
4110
4266
|
/**
|
|
4111
4267
|
* Tests if given string is valid semantic version
|
|
4112
4268
|
*
|
|
4113
|
-
* Note: There are two
|
|
4269
|
+
* Note: There are two similar functions:
|
|
4114
4270
|
* - `isValidSemanticVersion` which tests any semantic version
|
|
4115
4271
|
* - `isValidPromptbookVersion` *(this one)* which tests just Promptbook versions
|
|
4116
4272
|
*
|
|
@@ -4132,7 +4288,7 @@ function isValidSemanticVersion(version) {
|
|
|
4132
4288
|
*
|
|
4133
4289
|
* @see https://www.npmjs.com/package/promptbook?activeTab=versions
|
|
4134
4290
|
* Note: When you are using for example promptbook 2.0.0 and there already is promptbook 3.0.0 it don`t know about it.
|
|
4135
|
-
* Note: There are two
|
|
4291
|
+
* Note: There are two similar functions:
|
|
4136
4292
|
* - `isValidSemanticVersion` which tests any semantic version
|
|
4137
4293
|
* - `isValidPromptbookVersion` *(this one)* which tests just Promptbook versions
|
|
4138
4294
|
*
|
|
@@ -4152,7 +4308,7 @@ function isValidPromptbookVersion(version) {
|
|
|
4152
4308
|
/**
|
|
4153
4309
|
* Tests if given string is valid pipeline URL URL.
|
|
4154
4310
|
*
|
|
4155
|
-
* Note: There are two
|
|
4311
|
+
* Note: There are two similar functions:
|
|
4156
4312
|
* - `isValidUrl` which tests any URL
|
|
4157
4313
|
* - `isValidPipelineUrl` *(this one)* which tests just pipeline URL
|
|
4158
4314
|
*
|
|
@@ -4249,7 +4405,7 @@ function validatePipeline_InnerFunction(pipeline) {
|
|
|
4249
4405
|
${block(pipelineIdentification)}
|
|
4250
4406
|
`));
|
|
4251
4407
|
}
|
|
4252
|
-
// TODO: [🧠] Maybe do here some
|
|
4408
|
+
// TODO: [🧠] Maybe do here some proper JSON-schema / ZOD checking
|
|
4253
4409
|
if (!Array.isArray(pipeline.parameters)) {
|
|
4254
4410
|
// TODO: [🧠] what is the correct error tp throw - maybe PromptbookSchemaError
|
|
4255
4411
|
throw new ParseError(spaceTrim$1((block) => `
|
|
@@ -4260,7 +4416,7 @@ function validatePipeline_InnerFunction(pipeline) {
|
|
|
4260
4416
|
${block(pipelineIdentification)}
|
|
4261
4417
|
`));
|
|
4262
4418
|
}
|
|
4263
|
-
// TODO: [🧠] Maybe do here some
|
|
4419
|
+
// TODO: [🧠] Maybe do here some proper JSON-schema / ZOD checking
|
|
4264
4420
|
if (!Array.isArray(pipeline.tasks)) {
|
|
4265
4421
|
// TODO: [🧠] what is the correct error tp throw - maybe PromptbookSchemaError
|
|
4266
4422
|
throw new ParseError(spaceTrim$1((block) => `
|
|
@@ -4507,7 +4663,7 @@ var PipelineCollection = [{title:"Prepare Knowledge from Markdown",pipelineUrl:"
|
|
|
4507
4663
|
* Function isValidJsonString will tell you if the string is valid JSON or not
|
|
4508
4664
|
*
|
|
4509
4665
|
* @param value The string to check
|
|
4510
|
-
* @returns
|
|
4666
|
+
* @returns `true` if the string is a valid JSON string, false otherwise
|
|
4511
4667
|
*
|
|
4512
4668
|
* @public exported from `@promptbook/utils`
|
|
4513
4669
|
*/
|
|
@@ -4619,7 +4775,7 @@ function pipelineJsonToString(pipelineJson) {
|
|
|
4619
4775
|
if (bookVersion !== `undefined`) {
|
|
4620
4776
|
commands.push(`BOOK VERSION ${bookVersion}`);
|
|
4621
4777
|
}
|
|
4622
|
-
// TODO: [main] !!5 This increases size of the bundle and is
|
|
4778
|
+
// TODO: [main] !!5 This increases size of the bundle and is probably not necessary
|
|
4623
4779
|
pipelineString = prettifyMarkdown(pipelineString);
|
|
4624
4780
|
for (const parameter of parameters.filter(({ isInput }) => isInput)) {
|
|
4625
4781
|
commands.push(`INPUT PARAMETER ${taskParameterJsonToString(parameter)}`);
|
|
@@ -4737,23 +4893,6 @@ function taskParameterJsonToString(taskParameterJson) {
|
|
|
4737
4893
|
* TODO: [🧠] Should be in generated .book.md file GENERATOR_WARNING
|
|
4738
4894
|
*/
|
|
4739
4895
|
|
|
4740
|
-
/**
|
|
4741
|
-
* Parses the task and returns the list of all parameter names
|
|
4742
|
-
*
|
|
4743
|
-
* @param template the string template with parameters in {curly} braces
|
|
4744
|
-
* @returns the list of parameter names
|
|
4745
|
-
* @public exported from `@promptbook/utils`
|
|
4746
|
-
*/
|
|
4747
|
-
function extractParameterNames(template) {
|
|
4748
|
-
const matches = template.matchAll(/{\w+}/g);
|
|
4749
|
-
const parameterNames = new Set();
|
|
4750
|
-
for (const match of matches) {
|
|
4751
|
-
const parameterName = match[0].slice(1, -1);
|
|
4752
|
-
parameterNames.add(parameterName);
|
|
4753
|
-
}
|
|
4754
|
-
return parameterNames;
|
|
4755
|
-
}
|
|
4756
|
-
|
|
4757
4896
|
/**
|
|
4758
4897
|
* Unprepare just strips the preparation data of the pipeline
|
|
4759
4898
|
*
|
|
@@ -4803,7 +4942,7 @@ class SimplePipelineCollection {
|
|
|
4803
4942
|
/**
|
|
4804
4943
|
* Constructs a pipeline collection from pipelines
|
|
4805
4944
|
*
|
|
4806
|
-
* @param pipelines
|
|
4945
|
+
* @param pipelines Array of pipeline JSON objects to include in the collection
|
|
4807
4946
|
*
|
|
4808
4947
|
* Note: During the construction logic of all pipelines are validated
|
|
4809
4948
|
* Note: It is not recommended to use this constructor directly, use `createCollectionFromJson` *(or other variant)* instead
|
|
@@ -4915,15 +5054,18 @@ function createCollectionFromJson(...promptbooks) {
|
|
|
4915
5054
|
* @public exported from `@promptbook/core`
|
|
4916
5055
|
*/
|
|
4917
5056
|
function isPipelinePrepared(pipeline) {
|
|
4918
|
-
// Note: Ignoring `pipeline.preparations`
|
|
4919
|
-
// Note: Ignoring `pipeline.knowledgePieces`
|
|
5057
|
+
// Note: Ignoring `pipeline.preparations`
|
|
5058
|
+
// Note: Ignoring `pipeline.knowledgePieces`
|
|
4920
5059
|
if (pipeline.title === undefined || pipeline.title === '' || pipeline.title === DEFAULT_BOOK_TITLE) {
|
|
5060
|
+
// console.log('Pipeline is not prepared because title is undefined or empty', pipeline);
|
|
4921
5061
|
return false;
|
|
4922
5062
|
}
|
|
4923
5063
|
if (!pipeline.personas.every((persona) => persona.modelsRequirements !== undefined)) {
|
|
5064
|
+
// console.log('Pipeline is not prepared because personas are not prepared', pipeline.personas);
|
|
4924
5065
|
return false;
|
|
4925
5066
|
}
|
|
4926
5067
|
if (!pipeline.knowledgeSources.every((knowledgeSource) => knowledgeSource.preparationIds !== undefined)) {
|
|
5068
|
+
//console.log('Pipeline is not prepared because knowledge sources are not prepared', pipeline.knowledgeSources);
|
|
4927
5069
|
return false;
|
|
4928
5070
|
}
|
|
4929
5071
|
/*
|
|
@@ -4944,6 +5086,35 @@ function isPipelinePrepared(pipeline) {
|
|
|
4944
5086
|
* - [♨] Are tasks prepared
|
|
4945
5087
|
*/
|
|
4946
5088
|
|
|
5089
|
+
/**
|
|
5090
|
+
* Serializes an error into a [🚉] JSON-serializable object
|
|
5091
|
+
*
|
|
5092
|
+
* @public exported from `@promptbook/utils`
|
|
5093
|
+
*/
|
|
5094
|
+
function serializeError(error) {
|
|
5095
|
+
const { name, message, stack } = error;
|
|
5096
|
+
const { id } = error;
|
|
5097
|
+
if (!Object.keys(ALL_ERRORS).includes(name)) {
|
|
5098
|
+
console.error(spaceTrim((block) => `
|
|
5099
|
+
|
|
5100
|
+
Cannot serialize error with name "${name}"
|
|
5101
|
+
|
|
5102
|
+
Authors of Promptbook probably forgot to add this error into the list of errors:
|
|
5103
|
+
https://github.com/webgptorg/promptbook/blob/main/src/errors/0-index.ts
|
|
5104
|
+
|
|
5105
|
+
|
|
5106
|
+
${block(stack || message)}
|
|
5107
|
+
|
|
5108
|
+
`));
|
|
5109
|
+
}
|
|
5110
|
+
return {
|
|
5111
|
+
name: name,
|
|
5112
|
+
message,
|
|
5113
|
+
stack,
|
|
5114
|
+
id, // Include id in the serialized object
|
|
5115
|
+
};
|
|
5116
|
+
}
|
|
5117
|
+
|
|
4947
5118
|
/**
|
|
4948
5119
|
* Recursively converts JSON strings to JSON objects
|
|
4949
5120
|
|
|
@@ -5022,8 +5193,9 @@ function assertsTaskSuccessful(executionResult) {
|
|
|
5022
5193
|
*/
|
|
5023
5194
|
function createTask(options) {
|
|
5024
5195
|
const { taskType, taskProcessCallback } = options;
|
|
5196
|
+
let { title } = options;
|
|
5025
5197
|
// TODO: [🐙] DRY
|
|
5026
|
-
const taskId = `${taskType.toLowerCase().substring(0, 4)}-${$randomToken(8 /* <- TODO: To global config + Use Base58 to avoid
|
|
5198
|
+
const taskId = `${taskType.toLowerCase().substring(0, 4)}-${$randomToken(8 /* <- TODO: To global config + Use Base58 to avoid similar char conflicts */)}`;
|
|
5027
5199
|
let status = 'RUNNING';
|
|
5028
5200
|
const createdAt = new Date();
|
|
5029
5201
|
let updatedAt = createdAt;
|
|
@@ -5033,6 +5205,10 @@ function createTask(options) {
|
|
|
5033
5205
|
const partialResultSubject = new Subject();
|
|
5034
5206
|
// <- Note: Not using `BehaviorSubject` because on error we can't access the last value
|
|
5035
5207
|
const finalResultPromise = /* not await */ taskProcessCallback((newOngoingResult) => {
|
|
5208
|
+
if (newOngoingResult.title) {
|
|
5209
|
+
title = newOngoingResult.title;
|
|
5210
|
+
}
|
|
5211
|
+
updatedAt = new Date();
|
|
5036
5212
|
Object.assign(currentValue, newOngoingResult);
|
|
5037
5213
|
// <- TODO: assign deep
|
|
5038
5214
|
partialResultSubject.next(newOngoingResult);
|
|
@@ -5078,17 +5254,24 @@ function createTask(options) {
|
|
|
5078
5254
|
return {
|
|
5079
5255
|
taskType,
|
|
5080
5256
|
taskId,
|
|
5257
|
+
get promptbookVersion() {
|
|
5258
|
+
return PROMPTBOOK_ENGINE_VERSION;
|
|
5259
|
+
},
|
|
5260
|
+
get title() {
|
|
5261
|
+
return title;
|
|
5262
|
+
// <- Note: [1] These must be getters to allow changing the value in the future
|
|
5263
|
+
},
|
|
5081
5264
|
get status() {
|
|
5082
5265
|
return status;
|
|
5083
|
-
// <- Note: [1]
|
|
5266
|
+
// <- Note: [1] --||--
|
|
5084
5267
|
},
|
|
5085
5268
|
get createdAt() {
|
|
5086
5269
|
return createdAt;
|
|
5087
|
-
// <- Note: [1]
|
|
5270
|
+
// <- Note: [1] --||--
|
|
5088
5271
|
},
|
|
5089
5272
|
get updatedAt() {
|
|
5090
5273
|
return updatedAt;
|
|
5091
|
-
// <- Note: [1]
|
|
5274
|
+
// <- Note: [1] --||--
|
|
5092
5275
|
},
|
|
5093
5276
|
asPromise,
|
|
5094
5277
|
asObservable() {
|
|
@@ -5096,15 +5279,15 @@ function createTask(options) {
|
|
|
5096
5279
|
},
|
|
5097
5280
|
get errors() {
|
|
5098
5281
|
return errors;
|
|
5099
|
-
// <- Note: [1]
|
|
5282
|
+
// <- Note: [1] --||--
|
|
5100
5283
|
},
|
|
5101
5284
|
get warnings() {
|
|
5102
5285
|
return warnings;
|
|
5103
|
-
// <- Note: [1]
|
|
5286
|
+
// <- Note: [1] --||--
|
|
5104
5287
|
},
|
|
5105
5288
|
get currentValue() {
|
|
5106
5289
|
return currentValue;
|
|
5107
|
-
// <- Note: [1]
|
|
5290
|
+
// <- Note: [1] --||--
|
|
5108
5291
|
},
|
|
5109
5292
|
};
|
|
5110
5293
|
}
|
|
@@ -5113,35 +5296,6 @@ function createTask(options) {
|
|
|
5113
5296
|
* TODO: [🐚] Split into more files and make `PrepareTask` & `RemoteTask` + split the function
|
|
5114
5297
|
*/
|
|
5115
5298
|
|
|
5116
|
-
/**
|
|
5117
|
-
* Serializes an error into a [🚉] JSON-serializable object
|
|
5118
|
-
*
|
|
5119
|
-
* @public exported from `@promptbook/utils`
|
|
5120
|
-
*/
|
|
5121
|
-
function serializeError(error) {
|
|
5122
|
-
const { name, message, stack } = error;
|
|
5123
|
-
const { id } = error;
|
|
5124
|
-
if (!Object.keys(ALL_ERRORS).includes(name)) {
|
|
5125
|
-
console.error(spaceTrim((block) => `
|
|
5126
|
-
|
|
5127
|
-
Cannot serialize error with name "${name}"
|
|
5128
|
-
|
|
5129
|
-
Authors of Promptbook probably forgot to add this error into the list of errors:
|
|
5130
|
-
https://github.com/webgptorg/promptbook/blob/main/src/errors/0-index.ts
|
|
5131
|
-
|
|
5132
|
-
|
|
5133
|
-
${block(stack || message)}
|
|
5134
|
-
|
|
5135
|
-
`));
|
|
5136
|
-
}
|
|
5137
|
-
return {
|
|
5138
|
-
name: name,
|
|
5139
|
-
message,
|
|
5140
|
-
stack,
|
|
5141
|
-
id, // Include id in the serialized object
|
|
5142
|
-
};
|
|
5143
|
-
}
|
|
5144
|
-
|
|
5145
5299
|
/**
|
|
5146
5300
|
* Format either small or big number
|
|
5147
5301
|
*
|
|
@@ -5384,7 +5538,7 @@ function union(...sets) {
|
|
|
5384
5538
|
}
|
|
5385
5539
|
|
|
5386
5540
|
/**
|
|
5387
|
-
*
|
|
5541
|
+
* Contains configuration options for parsing and generating CSV files, such as delimiters and quoting rules.
|
|
5388
5542
|
*
|
|
5389
5543
|
* @public exported from `@promptbook/core`
|
|
5390
5544
|
*/
|
|
@@ -5393,11 +5547,29 @@ const MANDATORY_CSV_SETTINGS = Object.freeze({
|
|
|
5393
5547
|
// encoding: 'utf-8',
|
|
5394
5548
|
});
|
|
5395
5549
|
|
|
5550
|
+
/**
|
|
5551
|
+
* Converts a CSV string into an object
|
|
5552
|
+
*
|
|
5553
|
+
* Note: This is wrapper around `papaparse.parse()` with better autohealing
|
|
5554
|
+
*
|
|
5555
|
+
* @private - for now until `@promptbook/csv` is released
|
|
5556
|
+
*/
|
|
5557
|
+
function csvParse(value /* <- TODO: string_csv */, settings, schema /* <- TODO: Make CSV Schemas */) {
|
|
5558
|
+
settings = { ...settings, ...MANDATORY_CSV_SETTINGS };
|
|
5559
|
+
// Note: Autoheal invalid '\n' characters
|
|
5560
|
+
if (settings.newline && !settings.newline.includes('\r') && value.includes('\r')) {
|
|
5561
|
+
console.warn('CSV string contains carriage return characters, but in the CSV settings the `newline` setting does not include them. Autohealing the CSV string.');
|
|
5562
|
+
value = value.replace(/\r\n/g, '\n').replace(/\r/g, '\n');
|
|
5563
|
+
}
|
|
5564
|
+
const csv = parse(value, settings);
|
|
5565
|
+
return csv;
|
|
5566
|
+
}
|
|
5567
|
+
|
|
5396
5568
|
/**
|
|
5397
5569
|
* Function to check if a string is valid CSV
|
|
5398
5570
|
*
|
|
5399
5571
|
* @param value The string to check
|
|
5400
|
-
* @returns
|
|
5572
|
+
* @returns `true` if the string is a valid CSV string, false otherwise
|
|
5401
5573
|
*
|
|
5402
5574
|
* @public exported from `@promptbook/utils`
|
|
5403
5575
|
*/
|
|
@@ -5421,7 +5593,7 @@ function isValidCsvString(value) {
|
|
|
5421
5593
|
* @public exported from `@promptbook/core`
|
|
5422
5594
|
* <- TODO: [🏢] Export from package `@promptbook/csv`
|
|
5423
5595
|
*/
|
|
5424
|
-
const
|
|
5596
|
+
const CsvFormatParser = {
|
|
5425
5597
|
formatName: 'CSV',
|
|
5426
5598
|
aliases: ['SPREADSHEET', 'TABLE'],
|
|
5427
5599
|
isValid(value, settings, schema) {
|
|
@@ -5433,12 +5605,12 @@ const CsvFormatDefinition = {
|
|
|
5433
5605
|
heal(value, settings, schema) {
|
|
5434
5606
|
throw new Error('Not implemented');
|
|
5435
5607
|
},
|
|
5436
|
-
|
|
5608
|
+
subvalueParsers: [
|
|
5437
5609
|
{
|
|
5438
5610
|
subvalueName: 'ROW',
|
|
5439
|
-
async mapValues(
|
|
5440
|
-
|
|
5441
|
-
const csv =
|
|
5611
|
+
async mapValues(options) {
|
|
5612
|
+
const { value, outputParameterName, settings, mapCallback, onProgress } = options;
|
|
5613
|
+
const csv = csvParse(value, settings);
|
|
5442
5614
|
if (csv.errors.length !== 0) {
|
|
5443
5615
|
throw new CsvFormatError(spaceTrim((block) => `
|
|
5444
5616
|
CSV parsing error
|
|
@@ -5453,23 +5625,37 @@ const CsvFormatDefinition = {
|
|
|
5453
5625
|
${block(value)}
|
|
5454
5626
|
`));
|
|
5455
5627
|
}
|
|
5456
|
-
const mappedData =
|
|
5628
|
+
const mappedData = [];
|
|
5629
|
+
const length = csv.data.length;
|
|
5630
|
+
for (let index = 0; index < length; index++) {
|
|
5631
|
+
const row = csv.data[index];
|
|
5457
5632
|
if (row[outputParameterName]) {
|
|
5458
5633
|
throw new CsvFormatError(`Can not overwrite existing column "${outputParameterName}" in CSV row`);
|
|
5459
5634
|
}
|
|
5460
|
-
|
|
5635
|
+
const mappedRow = {
|
|
5461
5636
|
...row,
|
|
5462
|
-
[outputParameterName]: await mapCallback(row, index),
|
|
5637
|
+
[outputParameterName]: await mapCallback(row, index, length),
|
|
5463
5638
|
};
|
|
5464
|
-
|
|
5639
|
+
mappedData.push(mappedRow);
|
|
5640
|
+
if (onProgress) {
|
|
5641
|
+
// Note: Report the CSV with all rows mapped so far
|
|
5642
|
+
/*
|
|
5643
|
+
// TODO: [🛕] Report progress with all the rows including the pending ones
|
|
5644
|
+
const progressData = mappedData.map((row, i) =>
|
|
5645
|
+
i > index ? { ...row, [outputParameterName]: PENDING_VALUE_PLACEHOLDER } : row,
|
|
5646
|
+
);
|
|
5647
|
+
*/
|
|
5648
|
+
await onProgress(unparse(mappedData, { ...settings, ...MANDATORY_CSV_SETTINGS }));
|
|
5649
|
+
}
|
|
5650
|
+
}
|
|
5465
5651
|
return unparse(mappedData, { ...settings, ...MANDATORY_CSV_SETTINGS });
|
|
5466
5652
|
},
|
|
5467
5653
|
},
|
|
5468
5654
|
{
|
|
5469
5655
|
subvalueName: 'CELL',
|
|
5470
|
-
async mapValues(
|
|
5471
|
-
|
|
5472
|
-
const csv =
|
|
5656
|
+
async mapValues(options) {
|
|
5657
|
+
const { value, settings, mapCallback, onProgress } = options;
|
|
5658
|
+
const csv = csvParse(value, settings);
|
|
5473
5659
|
if (csv.errors.length !== 0) {
|
|
5474
5660
|
throw new CsvFormatError(spaceTrim((block) => `
|
|
5475
5661
|
CSV parsing error
|
|
@@ -5485,9 +5671,9 @@ const CsvFormatDefinition = {
|
|
|
5485
5671
|
`));
|
|
5486
5672
|
}
|
|
5487
5673
|
const mappedData = await Promise.all(csv.data.map(async (row, rowIndex) => {
|
|
5488
|
-
return /* not await */ Promise.all(Object.entries(row).map(async ([key, value], columnIndex) => {
|
|
5674
|
+
return /* not await */ Promise.all(Object.entries(row).map(async ([key, value], columnIndex, array) => {
|
|
5489
5675
|
const index = rowIndex * Object.keys(row).length + columnIndex;
|
|
5490
|
-
return /* not await */ mapCallback({ [key]: value }, index);
|
|
5676
|
+
return /* not await */ mapCallback({ [key]: value }, index, array.length);
|
|
5491
5677
|
}));
|
|
5492
5678
|
}));
|
|
5493
5679
|
return unparse(mappedData, { ...settings, ...MANDATORY_CSV_SETTINGS });
|
|
@@ -5496,10 +5682,10 @@ const CsvFormatDefinition = {
|
|
|
5496
5682
|
],
|
|
5497
5683
|
};
|
|
5498
5684
|
/**
|
|
5499
|
-
* TODO: [🍓] In `
|
|
5500
|
-
* TODO: [🍓] In `
|
|
5501
|
-
* TODO: [🍓] In `
|
|
5502
|
-
* TODO: [🍓] In `
|
|
5685
|
+
* TODO: [🍓] In `CsvFormatParser` implement simple `isValid`
|
|
5686
|
+
* TODO: [🍓] In `CsvFormatParser` implement partial `canBeValid`
|
|
5687
|
+
* TODO: [🍓] In `CsvFormatParser` implement `heal
|
|
5688
|
+
* TODO: [🍓] In `CsvFormatParser` implement `subvalueParsers`
|
|
5503
5689
|
* TODO: [🏢] Allow to expect something inside CSV objects and other formats
|
|
5504
5690
|
*/
|
|
5505
5691
|
|
|
@@ -5508,7 +5694,7 @@ const CsvFormatDefinition = {
|
|
|
5508
5694
|
*
|
|
5509
5695
|
* @private still in development [🏢]
|
|
5510
5696
|
*/
|
|
5511
|
-
const
|
|
5697
|
+
const JsonFormatParser = {
|
|
5512
5698
|
formatName: 'JSON',
|
|
5513
5699
|
mimeType: 'application/json',
|
|
5514
5700
|
isValid(value, settings, schema) {
|
|
@@ -5520,28 +5706,28 @@ const JsonFormatDefinition = {
|
|
|
5520
5706
|
heal(value, settings, schema) {
|
|
5521
5707
|
throw new Error('Not implemented');
|
|
5522
5708
|
},
|
|
5523
|
-
|
|
5709
|
+
subvalueParsers: [],
|
|
5524
5710
|
};
|
|
5525
5711
|
/**
|
|
5526
|
-
* TODO: [🧠] Maybe
|
|
5712
|
+
* TODO: [🧠] Maybe proper instance of object
|
|
5527
5713
|
* TODO: [0] Make string_serialized_json
|
|
5528
5714
|
* TODO: [1] Make type for JSON Settings and Schema
|
|
5529
5715
|
* TODO: [🧠] What to use for validating JSONs - JSON Schema, ZoD, typescript types/interfaces,...?
|
|
5530
|
-
* TODO: [🍓] In `
|
|
5531
|
-
* TODO: [🍓] In `
|
|
5532
|
-
* TODO: [🍓] In `
|
|
5533
|
-
* TODO: [🍓] In `
|
|
5716
|
+
* TODO: [🍓] In `JsonFormatParser` implement simple `isValid`
|
|
5717
|
+
* TODO: [🍓] In `JsonFormatParser` implement partial `canBeValid`
|
|
5718
|
+
* TODO: [🍓] In `JsonFormatParser` implement `heal
|
|
5719
|
+
* TODO: [🍓] In `JsonFormatParser` implement `subvalueParsers`
|
|
5534
5720
|
* TODO: [🏢] Allow to expect something inside JSON objects and other formats
|
|
5535
5721
|
*/
|
|
5536
5722
|
|
|
5537
5723
|
/**
|
|
5538
5724
|
* Definition for any text - this will be always valid
|
|
5539
5725
|
*
|
|
5540
|
-
* Note: This is not useful for validation, but for splitting and mapping with `
|
|
5726
|
+
* Note: This is not useful for validation, but for splitting and mapping with `subvalueParsers`
|
|
5541
5727
|
*
|
|
5542
5728
|
* @public exported from `@promptbook/core`
|
|
5543
5729
|
*/
|
|
5544
|
-
const
|
|
5730
|
+
const TextFormatParser = {
|
|
5545
5731
|
formatName: 'TEXT',
|
|
5546
5732
|
isValid(value) {
|
|
5547
5733
|
return typeof value === 'string';
|
|
@@ -5550,19 +5736,20 @@ const TextFormatDefinition = {
|
|
|
5550
5736
|
return typeof partialValue === 'string';
|
|
5551
5737
|
},
|
|
5552
5738
|
heal() {
|
|
5553
|
-
throw new UnexpectedError('It does not make sense to call `
|
|
5739
|
+
throw new UnexpectedError('It does not make sense to call `TextFormatParser.heal`');
|
|
5554
5740
|
},
|
|
5555
|
-
|
|
5741
|
+
subvalueParsers: [
|
|
5556
5742
|
{
|
|
5557
5743
|
subvalueName: 'LINE',
|
|
5558
|
-
async mapValues(
|
|
5744
|
+
async mapValues(options) {
|
|
5745
|
+
const { value, mapCallback, onProgress } = options;
|
|
5559
5746
|
const lines = value.split('\n');
|
|
5560
|
-
const mappedLines = await Promise.all(lines.map((lineContent, lineNumber) =>
|
|
5747
|
+
const mappedLines = await Promise.all(lines.map((lineContent, lineNumber, array) =>
|
|
5561
5748
|
// TODO: [🧠] Maybe option to skip empty line
|
|
5562
5749
|
/* not await */ mapCallback({
|
|
5563
5750
|
lineContent,
|
|
5564
5751
|
// TODO: [🧠] Maybe also put here `lineNumber`
|
|
5565
|
-
}, lineNumber)));
|
|
5752
|
+
}, lineNumber, array.length)));
|
|
5566
5753
|
return mappedLines.join('\n');
|
|
5567
5754
|
},
|
|
5568
5755
|
},
|
|
@@ -5572,10 +5759,10 @@ const TextFormatDefinition = {
|
|
|
5572
5759
|
/**
|
|
5573
5760
|
* TODO: [1] Make type for XML Text and Schema
|
|
5574
5761
|
* TODO: [🧠][🤠] Here should be all words, characters, lines, paragraphs, pages available as subvalues
|
|
5575
|
-
* TODO: [🍓] In `
|
|
5576
|
-
* TODO: [🍓] In `
|
|
5577
|
-
* TODO: [🍓] In `
|
|
5578
|
-
* TODO: [🍓] In `
|
|
5762
|
+
* TODO: [🍓] In `TextFormatParser` implement simple `isValid`
|
|
5763
|
+
* TODO: [🍓] In `TextFormatParser` implement partial `canBeValid`
|
|
5764
|
+
* TODO: [🍓] In `TextFormatParser` implement `heal
|
|
5765
|
+
* TODO: [🍓] In `TextFormatParser` implement `subvalueParsers`
|
|
5579
5766
|
* TODO: [🏢] Allow to expect something inside each item of list and other formats
|
|
5580
5767
|
*/
|
|
5581
5768
|
|
|
@@ -5583,7 +5770,7 @@ const TextFormatDefinition = {
|
|
|
5583
5770
|
* Function to check if a string is valid XML
|
|
5584
5771
|
*
|
|
5585
5772
|
* @param value
|
|
5586
|
-
* @returns
|
|
5773
|
+
* @returns `true` if the string is a valid XML string, false otherwise
|
|
5587
5774
|
*
|
|
5588
5775
|
* @public exported from `@promptbook/utils`
|
|
5589
5776
|
*/
|
|
@@ -5608,7 +5795,7 @@ function isValidXmlString(value) {
|
|
|
5608
5795
|
*
|
|
5609
5796
|
* @private still in development [🏢]
|
|
5610
5797
|
*/
|
|
5611
|
-
const
|
|
5798
|
+
const XmlFormatParser = {
|
|
5612
5799
|
formatName: 'XML',
|
|
5613
5800
|
mimeType: 'application/xml',
|
|
5614
5801
|
isValid(value, settings, schema) {
|
|
@@ -5620,17 +5807,17 @@ const XmlFormatDefinition = {
|
|
|
5620
5807
|
heal(value, settings, schema) {
|
|
5621
5808
|
throw new Error('Not implemented');
|
|
5622
5809
|
},
|
|
5623
|
-
|
|
5810
|
+
subvalueParsers: [],
|
|
5624
5811
|
};
|
|
5625
5812
|
/**
|
|
5626
|
-
* TODO: [🧠] Maybe
|
|
5813
|
+
* TODO: [🧠] Maybe proper instance of object
|
|
5627
5814
|
* TODO: [0] Make string_serialized_xml
|
|
5628
5815
|
* TODO: [1] Make type for XML Settings and Schema
|
|
5629
5816
|
* TODO: [🧠] What to use for validating XMLs - XSD,...
|
|
5630
|
-
* TODO: [🍓] In `
|
|
5631
|
-
* TODO: [🍓] In `
|
|
5632
|
-
* TODO: [🍓] In `
|
|
5633
|
-
* TODO: [🍓] In `
|
|
5817
|
+
* TODO: [🍓] In `XmlFormatParser` implement simple `isValid`
|
|
5818
|
+
* TODO: [🍓] In `XmlFormatParser` implement partial `canBeValid`
|
|
5819
|
+
* TODO: [🍓] In `XmlFormatParser` implement `heal
|
|
5820
|
+
* TODO: [🍓] In `XmlFormatParser` implement `subvalueParsers`
|
|
5634
5821
|
* TODO: [🏢] Allow to expect something inside XML and other formats
|
|
5635
5822
|
*/
|
|
5636
5823
|
|
|
@@ -5639,24 +5826,19 @@ const XmlFormatDefinition = {
|
|
|
5639
5826
|
*
|
|
5640
5827
|
* @private internal index of `...` <- TODO [🏢]
|
|
5641
5828
|
*/
|
|
5642
|
-
const FORMAT_DEFINITIONS = [
|
|
5643
|
-
JsonFormatDefinition,
|
|
5644
|
-
XmlFormatDefinition,
|
|
5645
|
-
TextFormatDefinition,
|
|
5646
|
-
CsvFormatDefinition,
|
|
5647
|
-
];
|
|
5829
|
+
const FORMAT_DEFINITIONS = [JsonFormatParser, XmlFormatParser, TextFormatParser, CsvFormatParser];
|
|
5648
5830
|
/**
|
|
5649
5831
|
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
5650
5832
|
*/
|
|
5651
5833
|
|
|
5652
5834
|
/**
|
|
5653
|
-
* Maps available parameters to expected parameters
|
|
5835
|
+
* Maps available parameters to expected parameters for a pipeline task.
|
|
5654
5836
|
*
|
|
5655
5837
|
* The strategy is:
|
|
5656
|
-
* 1)
|
|
5657
|
-
* 2)
|
|
5838
|
+
* 1) First, match parameters by name where both available and expected.
|
|
5839
|
+
* 2) Then, if there are unmatched expected and available parameters, map them by order.
|
|
5658
5840
|
*
|
|
5659
|
-
* @throws {PipelineExecutionError}
|
|
5841
|
+
* @throws {PipelineExecutionError} If the number of unmatched expected and available parameters does not match, or mapping is ambiguous.
|
|
5660
5842
|
* @private within the repository used in `createPipelineExecutor`
|
|
5661
5843
|
*/
|
|
5662
5844
|
function mapAvailableToExpectedParameters(options) {
|
|
@@ -5679,7 +5861,7 @@ function mapAvailableToExpectedParameters(options) {
|
|
|
5679
5861
|
else if (!availableParametersNames.has(parameterName) && expectedParameterNames.has(parameterName)) ;
|
|
5680
5862
|
}
|
|
5681
5863
|
if (expectedParameterNames.size === 0) {
|
|
5682
|
-
// Note: [👨👨👧] Now we can freeze `mappedParameters` to prevent
|
|
5864
|
+
// Note: [👨👨👧] Now we can freeze `mappedParameters` to prevent accidental modifications after mapping
|
|
5683
5865
|
Object.freeze(mappedParameters);
|
|
5684
5866
|
return mappedParameters;
|
|
5685
5867
|
}
|
|
@@ -5710,7 +5892,7 @@ function mapAvailableToExpectedParameters(options) {
|
|
|
5710
5892
|
for (let i = 0; i < expectedParameterNames.size; i++) {
|
|
5711
5893
|
mappedParameters[expectedParameterNamesArray[i]] = availableParameters[availableParametersNamesArray[i]];
|
|
5712
5894
|
}
|
|
5713
|
-
// Note: [👨👨👧] Now we can freeze `mappedParameters` to prevent
|
|
5895
|
+
// Note: [👨👨👧] Now we can freeze `mappedParameters` to prevent accidental modifications after mapping
|
|
5714
5896
|
Object.freeze(mappedParameters);
|
|
5715
5897
|
return mappedParameters;
|
|
5716
5898
|
}
|
|
@@ -5718,8 +5900,8 @@ function mapAvailableToExpectedParameters(options) {
|
|
|
5718
5900
|
/**
|
|
5719
5901
|
* Extracts all code blocks from markdown.
|
|
5720
5902
|
*
|
|
5721
|
-
* Note: There are multiple
|
|
5722
|
-
* - `extractBlock` just extracts the content of the code block which is also used as
|
|
5903
|
+
* Note: There are multiple similar functions:
|
|
5904
|
+
* - `extractBlock` just extracts the content of the code block which is also used as built-in function for postprocessing
|
|
5723
5905
|
* - `extractJsonBlock` extracts exactly one valid JSON code block
|
|
5724
5906
|
* - `extractOneBlockFromMarkdown` extracts exactly one code block with language of the code block
|
|
5725
5907
|
* - `extractAllBlocksFromMarkdown` extracts all code blocks with language of the code block
|
|
@@ -5769,7 +5951,7 @@ function extractAllBlocksFromMarkdown(markdown) {
|
|
|
5769
5951
|
if (currentCodeBlock.content !== '') {
|
|
5770
5952
|
currentCodeBlock.content += '\n';
|
|
5771
5953
|
}
|
|
5772
|
-
currentCodeBlock.content += line.split('\\`\\`\\`').join('```') /* <- TODO: Maybe make
|
|
5954
|
+
currentCodeBlock.content += line.split('\\`\\`\\`').join('```') /* <- TODO: Maybe make proper unescape */;
|
|
5773
5955
|
}
|
|
5774
5956
|
}
|
|
5775
5957
|
if (currentCodeBlock !== null) {
|
|
@@ -5789,7 +5971,7 @@ function extractAllBlocksFromMarkdown(markdown) {
|
|
|
5789
5971
|
* - When there are multiple JSON code blocks the function throws a `ParseError`
|
|
5790
5972
|
*
|
|
5791
5973
|
* Note: It is not important if marked as ```json BUT if it is VALID JSON
|
|
5792
|
-
* Note: There are multiple
|
|
5974
|
+
* Note: There are multiple similar function:
|
|
5793
5975
|
* - `extractBlock` just extracts the content of the code block which is also used as build-in function for postprocessing
|
|
5794
5976
|
* - `extractJsonBlock` extracts exactly one valid JSON code block
|
|
5795
5977
|
* - `extractOneBlockFromMarkdown` extracts exactly one code block with language of the code block
|
|
@@ -5814,7 +5996,7 @@ function extractJsonBlock(markdown) {
|
|
|
5814
5996
|
}
|
|
5815
5997
|
/**
|
|
5816
5998
|
* TODO: Add some auto-healing logic + extract YAML, JSON5, TOML, etc.
|
|
5817
|
-
* TODO: [🏢] Make this logic part of `
|
|
5999
|
+
* TODO: [🏢] Make this logic part of `JsonFormatParser` or `isValidJsonString`
|
|
5818
6000
|
*/
|
|
5819
6001
|
|
|
5820
6002
|
/**
|
|
@@ -6035,14 +6217,14 @@ const CountUtils = {
|
|
|
6035
6217
|
PAGES: countPages,
|
|
6036
6218
|
};
|
|
6037
6219
|
/**
|
|
6038
|
-
* TODO: [🧠][🤠] This should be
|
|
6220
|
+
* TODO: [🧠][🤠] This should be probably as part of `TextFormatParser`
|
|
6039
6221
|
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
6040
6222
|
*/
|
|
6041
6223
|
|
|
6042
6224
|
/**
|
|
6043
6225
|
* Function checkExpectations will check if the expectations on given value are met
|
|
6044
6226
|
*
|
|
6045
|
-
* Note: There are two
|
|
6227
|
+
* Note: There are two similar functions:
|
|
6046
6228
|
* - `checkExpectations` which throws an error if the expectations are not met
|
|
6047
6229
|
* - `isPassingExpectations` which returns a boolean
|
|
6048
6230
|
*
|
|
@@ -6063,13 +6245,17 @@ function checkExpectations(expectations, value) {
|
|
|
6063
6245
|
}
|
|
6064
6246
|
/**
|
|
6065
6247
|
* TODO: [💝] Unite object for expecting amount and format
|
|
6066
|
-
* TODO: [🧠][🤠] This should be part of `
|
|
6248
|
+
* TODO: [🧠][🤠] This should be part of `TextFormatParser`
|
|
6067
6249
|
* Note: [💝] and [🤠] are interconnected together
|
|
6068
6250
|
*/
|
|
6069
6251
|
|
|
6070
6252
|
/**
|
|
6071
|
-
*
|
|
6253
|
+
* Executes a pipeline task with multiple attempts, including joker and retry logic. Handles different task types
|
|
6254
|
+
* (prompt, script, dialog, etc.), applies postprocessing, checks expectations, and updates the execution report.
|
|
6255
|
+
* Throws errors if execution fails after all attempts.
|
|
6072
6256
|
*
|
|
6257
|
+
* @param options - The options for execution, including task, parameters, pipeline, and configuration.
|
|
6258
|
+
* @returns The result string of the executed task.
|
|
6073
6259
|
* @private internal utility of `createPipelineExecutor`
|
|
6074
6260
|
*/
|
|
6075
6261
|
async function executeAttempts(options) {
|
|
@@ -6291,7 +6477,7 @@ async function executeAttempts(options) {
|
|
|
6291
6477
|
if (task.format) {
|
|
6292
6478
|
if (task.format === 'JSON') {
|
|
6293
6479
|
if (!isValidJsonString($ongoingTaskResult.$resultString || '')) {
|
|
6294
|
-
// TODO: [🏢] Do more universally via `
|
|
6480
|
+
// TODO: [🏢] Do more universally via `FormatParser`
|
|
6295
6481
|
try {
|
|
6296
6482
|
$ongoingTaskResult.$resultString = extractJsonBlock($ongoingTaskResult.$resultString || '');
|
|
6297
6483
|
}
|
|
@@ -6393,12 +6579,16 @@ async function executeAttempts(options) {
|
|
|
6393
6579
|
*/
|
|
6394
6580
|
|
|
6395
6581
|
/**
|
|
6396
|
-
*
|
|
6582
|
+
* Executes a pipeline task that requires mapping or iterating over subvalues of a parameter (such as rows in a CSV).
|
|
6583
|
+
* Handles format and subformat resolution, error handling, and progress reporting.
|
|
6584
|
+
*
|
|
6585
|
+
* @param options - Options for execution, including task details and progress callback.
|
|
6586
|
+
* @returns The result of the subvalue mapping or execution attempts.
|
|
6397
6587
|
*
|
|
6398
6588
|
* @private internal utility of `createPipelineExecutor`
|
|
6399
6589
|
*/
|
|
6400
6590
|
async function executeFormatSubvalues(options) {
|
|
6401
|
-
const { task, jokerParameterNames, parameters, priority, csvSettings, pipelineIdentification } = options;
|
|
6591
|
+
const { task, jokerParameterNames, parameters, priority, csvSettings, onProgress, pipelineIdentification } = options;
|
|
6402
6592
|
if (task.foreach === undefined) {
|
|
6403
6593
|
return /* not await */ executeAttempts(options);
|
|
6404
6594
|
}
|
|
@@ -6429,16 +6619,16 @@ async function executeFormatSubvalues(options) {
|
|
|
6429
6619
|
${block(pipelineIdentification)}
|
|
6430
6620
|
`));
|
|
6431
6621
|
}
|
|
6432
|
-
const
|
|
6433
|
-
if (
|
|
6622
|
+
const subvalueParser = formatDefinition.subvalueParsers.find((subvalueParser) => [subvalueParser.subvalueName, ...(subvalueParser.aliases || [])].includes(task.foreach.subformatName));
|
|
6623
|
+
if (subvalueParser === undefined) {
|
|
6434
6624
|
throw new UnexpectedError(
|
|
6435
6625
|
// <- TODO: [🧠][🧐] Should be formats fixed per promptbook version or behave as plugins (=> change UnexpectedError)
|
|
6436
6626
|
spaceTrim((block) => `
|
|
6437
6627
|
Unsupported subformat name "${task.foreach.subformatName}" for format "${task.foreach.formatName}"
|
|
6438
6628
|
|
|
6439
6629
|
Available subformat names for format "${formatDefinition.formatName}":
|
|
6440
|
-
${block(formatDefinition.
|
|
6441
|
-
.map((
|
|
6630
|
+
${block(formatDefinition.subvalueParsers
|
|
6631
|
+
.map((subvalueParser) => subvalueParser.subvalueName)
|
|
6442
6632
|
.map((subvalueName) => `- ${subvalueName}`)
|
|
6443
6633
|
.join('\n'))}
|
|
6444
6634
|
|
|
@@ -6450,55 +6640,85 @@ async function executeFormatSubvalues(options) {
|
|
|
6450
6640
|
let formatSettings;
|
|
6451
6641
|
if (formatDefinition.formatName === 'CSV') {
|
|
6452
6642
|
formatSettings = csvSettings;
|
|
6453
|
-
// <- TODO: [🤹♂️] More universal, make
|
|
6454
|
-
}
|
|
6455
|
-
const resultString = await
|
|
6456
|
-
|
|
6457
|
-
|
|
6458
|
-
|
|
6459
|
-
|
|
6460
|
-
|
|
6461
|
-
|
|
6462
|
-
|
|
6463
|
-
|
|
6464
|
-
|
|
6465
|
-
|
|
6466
|
-
|
|
6467
|
-
|
|
6643
|
+
// <- TODO: [🤹♂️] More universal, make similar pattern for other formats for example \n vs \r\n in text
|
|
6644
|
+
}
|
|
6645
|
+
const resultString = await subvalueParser.mapValues({
|
|
6646
|
+
value: parameterValue,
|
|
6647
|
+
outputParameterName: task.foreach.outputSubparameterName,
|
|
6648
|
+
settings: formatSettings,
|
|
6649
|
+
onProgress(partialResultString) {
|
|
6650
|
+
return onProgress(Object.freeze({
|
|
6651
|
+
[task.resultingParameterName]: partialResultString,
|
|
6652
|
+
}));
|
|
6653
|
+
},
|
|
6654
|
+
async mapCallback(subparameters, index, length) {
|
|
6655
|
+
let mappedParameters;
|
|
6656
|
+
try {
|
|
6657
|
+
mappedParameters = mapAvailableToExpectedParameters({
|
|
6658
|
+
expectedParameters: Object.fromEntries(task.foreach.inputSubparameterNames.map((subparameterName) => [subparameterName, null])),
|
|
6659
|
+
availableParameters: subparameters,
|
|
6660
|
+
});
|
|
6468
6661
|
}
|
|
6469
|
-
|
|
6470
|
-
|
|
6662
|
+
catch (error) {
|
|
6663
|
+
if (!(error instanceof PipelineExecutionError)) {
|
|
6664
|
+
throw error;
|
|
6665
|
+
}
|
|
6666
|
+
const highLevelError = new PipelineExecutionError(spaceTrim((block) => `
|
|
6667
|
+
${error.message}
|
|
6471
6668
|
|
|
6472
|
-
|
|
6473
|
-
|
|
6669
|
+
This is error in FOREACH command when mapping ${formatDefinition.formatName} ${subvalueParser.subvalueName} data (${index + 1}/${length})
|
|
6670
|
+
You have probably passed wrong data to pipeline or wrong data was generated which are processed by FOREACH command
|
|
6474
6671
|
|
|
6475
|
-
|
|
6476
|
-
|
|
6477
|
-
|
|
6478
|
-
|
|
6479
|
-
|
|
6480
|
-
|
|
6481
|
-
|
|
6482
|
-
|
|
6483
|
-
|
|
6484
|
-
|
|
6485
|
-
|
|
6486
|
-
|
|
6487
|
-
|
|
6488
|
-
|
|
6489
|
-
|
|
6490
|
-
|
|
6491
|
-
|
|
6492
|
-
|
|
6493
|
-
|
|
6494
|
-
|
|
6672
|
+
${block(pipelineIdentification)}
|
|
6673
|
+
`));
|
|
6674
|
+
if (length > BIG_DATASET_TRESHOLD) {
|
|
6675
|
+
console.error(highLevelError);
|
|
6676
|
+
return FAILED_VALUE_PLACEHOLDER;
|
|
6677
|
+
}
|
|
6678
|
+
throw highLevelError;
|
|
6679
|
+
}
|
|
6680
|
+
const allSubparameters = {
|
|
6681
|
+
...parameters,
|
|
6682
|
+
...mappedParameters,
|
|
6683
|
+
};
|
|
6684
|
+
Object.freeze(allSubparameters);
|
|
6685
|
+
try {
|
|
6686
|
+
const subresultString = await executeAttempts({
|
|
6687
|
+
...options,
|
|
6688
|
+
priority: priority + index,
|
|
6689
|
+
parameters: allSubparameters,
|
|
6690
|
+
pipelineIdentification: spaceTrim((block) => `
|
|
6691
|
+
${block(pipelineIdentification)}
|
|
6692
|
+
Subparameter index: ${index}
|
|
6693
|
+
`),
|
|
6694
|
+
});
|
|
6695
|
+
return subresultString;
|
|
6696
|
+
}
|
|
6697
|
+
catch (error) {
|
|
6698
|
+
if (length > BIG_DATASET_TRESHOLD) {
|
|
6699
|
+
console.error(spaceTrim((block) => `
|
|
6700
|
+
${error.message}
|
|
6701
|
+
|
|
6702
|
+
This is error in FOREACH command when processing ${formatDefinition.formatName} ${subvalueParser.subvalueName} data (${index + 1}/${length})
|
|
6703
|
+
|
|
6704
|
+
${block(pipelineIdentification)}
|
|
6705
|
+
`));
|
|
6706
|
+
return FAILED_VALUE_PLACEHOLDER;
|
|
6707
|
+
}
|
|
6708
|
+
throw error;
|
|
6709
|
+
}
|
|
6710
|
+
},
|
|
6495
6711
|
});
|
|
6496
6712
|
return resultString;
|
|
6497
6713
|
}
|
|
6498
6714
|
|
|
6499
6715
|
/**
|
|
6500
|
-
*
|
|
6716
|
+
* Returns the context for a given task, typically used to provide additional information or variables
|
|
6717
|
+
* required for the execution of the task within a pipeline. The context is returned as a string value
|
|
6718
|
+
* that may include markdown formatting.
|
|
6501
6719
|
*
|
|
6720
|
+
* @param task - The task for which the context is being generated. This should be a deeply immutable TaskJson object.
|
|
6721
|
+
* @returns The context as a string, formatted as markdown and parameter value.
|
|
6502
6722
|
* @private internal utility of `createPipelineExecutor`
|
|
6503
6723
|
*/
|
|
6504
6724
|
async function getContextForTask(task) {
|
|
@@ -6506,7 +6726,7 @@ async function getContextForTask(task) {
|
|
|
6506
6726
|
}
|
|
6507
6727
|
|
|
6508
6728
|
/**
|
|
6509
|
-
*
|
|
6729
|
+
* Retrieves example values or templates for a given task, used to guide or validate pipeline execution.
|
|
6510
6730
|
*
|
|
6511
6731
|
* @private internal utility of `createPipelineExecutor`
|
|
6512
6732
|
*/
|
|
@@ -6515,91 +6735,128 @@ async function getExamplesForTask(task) {
|
|
|
6515
6735
|
}
|
|
6516
6736
|
|
|
6517
6737
|
/**
|
|
6518
|
-
*
|
|
6738
|
+
* Computes the cosine similarity between two embedding vectors
|
|
6739
|
+
*
|
|
6740
|
+
* Note: This is helping function for RAG (retrieval-augmented generation)
|
|
6741
|
+
*
|
|
6742
|
+
* @param embeddingVector1
|
|
6743
|
+
* @param embeddingVector2
|
|
6744
|
+
* @returns Cosine similarity between the two vectors
|
|
6745
|
+
*
|
|
6746
|
+
* @public exported from `@promptbook/core`
|
|
6747
|
+
*/
|
|
6748
|
+
function computeCosineSimilarity(embeddingVector1, embeddingVector2) {
|
|
6749
|
+
if (embeddingVector1.length !== embeddingVector2.length) {
|
|
6750
|
+
throw new TypeError('Embedding vectors must have the same length');
|
|
6751
|
+
}
|
|
6752
|
+
const dotProduct = embeddingVector1.reduce((sum, value, index) => sum + value * embeddingVector2[index], 0);
|
|
6753
|
+
const magnitude1 = Math.sqrt(embeddingVector1.reduce((sum, value) => sum + value * value, 0));
|
|
6754
|
+
const magnitude2 = Math.sqrt(embeddingVector2.reduce((sum, value) => sum + value * value, 0));
|
|
6755
|
+
return 1 - dotProduct / (magnitude1 * magnitude2);
|
|
6756
|
+
}
|
|
6757
|
+
|
|
6758
|
+
/**
|
|
6759
|
+
*
|
|
6760
|
+
* @param knowledgePieces
|
|
6761
|
+
* @returns
|
|
6519
6762
|
*
|
|
6520
|
-
*
|
|
6763
|
+
* @private internal utility of `createPipelineExecutor`
|
|
6764
|
+
*/
|
|
6765
|
+
function knowledgePiecesToString(knowledgePieces) {
|
|
6766
|
+
return knowledgePieces
|
|
6767
|
+
.map((knowledgePiece) => {
|
|
6768
|
+
const { content } = knowledgePiece;
|
|
6769
|
+
return `- ${content}`;
|
|
6770
|
+
})
|
|
6771
|
+
.join('\n');
|
|
6772
|
+
// <- TODO: [🧠] Some smarter aggregation of knowledge pieces, single-line vs multi-line vs mixed
|
|
6773
|
+
}
|
|
6774
|
+
|
|
6775
|
+
/**
|
|
6776
|
+
* Retrieves the most relevant knowledge pieces for a given task using embedding-based similarity search.
|
|
6777
|
+
* This is where retrieval-augmented generation (RAG) is performed to enhance the task with external knowledge.
|
|
6521
6778
|
*
|
|
6522
6779
|
* @private internal utility of `createPipelineExecutor`
|
|
6523
6780
|
*/
|
|
6524
6781
|
async function getKnowledgeForTask(options) {
|
|
6525
|
-
const { tools, preparedPipeline, task } = options;
|
|
6782
|
+
const { tools, preparedPipeline, task, parameters } = options;
|
|
6526
6783
|
const firstKnowlegePiece = preparedPipeline.knowledgePieces[0];
|
|
6527
6784
|
const firstKnowlegeIndex = firstKnowlegePiece === null || firstKnowlegePiece === void 0 ? void 0 : firstKnowlegePiece.index[0];
|
|
6528
6785
|
// <- TODO: Do not use just first knowledge piece and first index to determine embedding model, use also keyword search
|
|
6529
6786
|
if (firstKnowlegePiece === undefined || firstKnowlegeIndex === undefined) {
|
|
6530
|
-
return '
|
|
6787
|
+
return ''; // <- Note: Np knowledge present, return empty string
|
|
6531
6788
|
}
|
|
6532
|
-
|
|
6533
|
-
|
|
6534
|
-
|
|
6535
|
-
|
|
6536
|
-
|
|
6537
|
-
|
|
6538
|
-
|
|
6539
|
-
|
|
6540
|
-
|
|
6541
|
-
|
|
6542
|
-
|
|
6543
|
-
|
|
6544
|
-
}
|
|
6545
|
-
|
|
6546
|
-
|
|
6547
|
-
|
|
6548
|
-
|
|
6549
|
-
|
|
6550
|
-
|
|
6551
|
-
|
|
6789
|
+
try {
|
|
6790
|
+
// TODO: [🚐] Make arrayable LLMs -> single LLM DRY
|
|
6791
|
+
const _llms = arrayableToArray(tools.llm);
|
|
6792
|
+
const llmTools = _llms.length === 1 ? _llms[0] : joinLlmExecutionTools(..._llms);
|
|
6793
|
+
const taskEmbeddingPrompt = {
|
|
6794
|
+
title: 'Knowledge Search',
|
|
6795
|
+
modelRequirements: {
|
|
6796
|
+
modelVariant: 'EMBEDDING',
|
|
6797
|
+
modelName: firstKnowlegeIndex.modelName,
|
|
6798
|
+
},
|
|
6799
|
+
content: task.content,
|
|
6800
|
+
parameters,
|
|
6801
|
+
};
|
|
6802
|
+
const taskEmbeddingResult = await llmTools.callEmbeddingModel(taskEmbeddingPrompt);
|
|
6803
|
+
const knowledgePiecesWithRelevance = preparedPipeline.knowledgePieces.map((knowledgePiece) => {
|
|
6804
|
+
const { index } = knowledgePiece;
|
|
6805
|
+
const knowledgePieceIndex = index.find((i) => i.modelName === firstKnowlegeIndex.modelName);
|
|
6806
|
+
// <- TODO: Do not use just first knowledge piece and first index to determine embedding model
|
|
6807
|
+
if (knowledgePieceIndex === undefined) {
|
|
6808
|
+
return {
|
|
6809
|
+
content: knowledgePiece.content,
|
|
6810
|
+
relevance: 0,
|
|
6811
|
+
};
|
|
6812
|
+
}
|
|
6813
|
+
const relevance = computeCosineSimilarity(knowledgePieceIndex.position, taskEmbeddingResult.content);
|
|
6552
6814
|
return {
|
|
6553
6815
|
content: knowledgePiece.content,
|
|
6554
|
-
relevance
|
|
6816
|
+
relevance,
|
|
6555
6817
|
};
|
|
6556
|
-
}
|
|
6557
|
-
const
|
|
6558
|
-
|
|
6559
|
-
|
|
6560
|
-
|
|
6561
|
-
|
|
6562
|
-
|
|
6563
|
-
|
|
6564
|
-
|
|
6565
|
-
|
|
6566
|
-
|
|
6567
|
-
|
|
6568
|
-
|
|
6569
|
-
|
|
6570
|
-
|
|
6571
|
-
|
|
6572
|
-
|
|
6573
|
-
|
|
6574
|
-
|
|
6575
|
-
|
|
6576
|
-
|
|
6577
|
-
|
|
6578
|
-
// TODO: !!!!!! Annotate + to new file
|
|
6579
|
-
function computeCosineSimilarity(embeddingVector1, embeddingVector2) {
|
|
6580
|
-
if (embeddingVector1.length !== embeddingVector2.length) {
|
|
6581
|
-
throw new TypeError('Embedding vectors must have the same length');
|
|
6818
|
+
});
|
|
6819
|
+
const knowledgePiecesSorted = knowledgePiecesWithRelevance.sort((a, b) => a.relevance - b.relevance);
|
|
6820
|
+
const knowledgePiecesLimited = knowledgePiecesSorted.slice(0, 5);
|
|
6821
|
+
/*
|
|
6822
|
+
console.log('`getKnowledgeForTask` Embedding', {
|
|
6823
|
+
task,
|
|
6824
|
+
taskEmbeddingPrompt,
|
|
6825
|
+
taskEmbeddingResult,
|
|
6826
|
+
firstKnowlegePiece,
|
|
6827
|
+
firstKnowlegeIndex,
|
|
6828
|
+
knowledgePiecesWithRelevance,
|
|
6829
|
+
knowledgePiecesSorted,
|
|
6830
|
+
knowledgePiecesLimited,
|
|
6831
|
+
});
|
|
6832
|
+
*/
|
|
6833
|
+
return knowledgePiecesToString(knowledgePiecesLimited);
|
|
6834
|
+
}
|
|
6835
|
+
catch (error) {
|
|
6836
|
+
assertsError(error);
|
|
6837
|
+
console.error('Error in `getKnowledgeForTask`', error);
|
|
6838
|
+
// Note: If the LLM fails, just return all knowledge pieces
|
|
6839
|
+
return knowledgePiecesToString(preparedPipeline.knowledgePieces);
|
|
6582
6840
|
}
|
|
6583
|
-
const dotProduct = embeddingVector1.reduce((sum, value, index) => sum + value * embeddingVector2[index], 0);
|
|
6584
|
-
const magnitude1 = Math.sqrt(embeddingVector1.reduce((sum, value) => sum + value * value, 0));
|
|
6585
|
-
const magnitude2 = Math.sqrt(embeddingVector2.reduce((sum, value) => sum + value * value, 0));
|
|
6586
|
-
return 1 - dotProduct / (magnitude1 * magnitude2);
|
|
6587
6841
|
}
|
|
6588
6842
|
/**
|
|
6589
|
-
* TODO: !!!! Verify if this is working
|
|
6590
6843
|
* TODO: [♨] Implement Better - use keyword search
|
|
6591
6844
|
* TODO: [♨] Examples of values
|
|
6592
6845
|
*/
|
|
6593
6846
|
|
|
6594
6847
|
/**
|
|
6595
|
-
*
|
|
6848
|
+
* Retrieves all reserved parameters for a given pipeline task, including context, knowledge, examples, and metadata.
|
|
6849
|
+
* Ensures all reserved parameters are defined and throws if any are missing.
|
|
6850
|
+
*
|
|
6851
|
+
* @param options - Options including tools, pipeline, task, and context.
|
|
6852
|
+
* @returns An object containing all reserved parameters for the task.
|
|
6596
6853
|
*
|
|
6597
6854
|
* @private internal utility of `createPipelineExecutor`
|
|
6598
6855
|
*/
|
|
6599
6856
|
async function getReservedParametersForTask(options) {
|
|
6600
|
-
const { tools, preparedPipeline, task, pipelineIdentification } = options;
|
|
6857
|
+
const { tools, preparedPipeline, task, parameters, pipelineIdentification, isVerbose } = options;
|
|
6601
6858
|
const context = await getContextForTask(); // <- [🏍]
|
|
6602
|
-
const knowledge = await getKnowledgeForTask({ tools, preparedPipeline, task });
|
|
6859
|
+
const knowledge = await getKnowledgeForTask({ tools, preparedPipeline, task, parameters });
|
|
6603
6860
|
const examples = await getExamplesForTask();
|
|
6604
6861
|
const currentDate = new Date().toISOString(); // <- TODO: [🧠][💩] Better
|
|
6605
6862
|
const modelName = RESERVED_PARAMETER_MISSING_VALUE;
|
|
@@ -6611,6 +6868,9 @@ async function getReservedParametersForTask(options) {
|
|
|
6611
6868
|
currentDate,
|
|
6612
6869
|
modelName,
|
|
6613
6870
|
};
|
|
6871
|
+
if (isVerbose) {
|
|
6872
|
+
console.info('Reserved parameters for task:', { options, reservedParameters });
|
|
6873
|
+
}
|
|
6614
6874
|
// Note: Doublecheck that ALL reserved parameters are defined:
|
|
6615
6875
|
for (const parameterName of RESERVED_PARAMETER_NAMES) {
|
|
6616
6876
|
if (reservedParameters[parameterName] === undefined) {
|
|
@@ -6625,23 +6885,21 @@ async function getReservedParametersForTask(options) {
|
|
|
6625
6885
|
}
|
|
6626
6886
|
|
|
6627
6887
|
/**
|
|
6628
|
-
*
|
|
6888
|
+
* Executes a single task within a pipeline, handling parameter validation, error checking, and progress reporting.
|
|
6889
|
+
*
|
|
6890
|
+
* @param options - Options for execution, including the task, pipeline, parameters, and callbacks.
|
|
6891
|
+
* @returns The output parameters produced by the task.
|
|
6629
6892
|
*
|
|
6630
6893
|
* @private internal utility of `createPipelineExecutor`
|
|
6631
6894
|
*/
|
|
6632
6895
|
async function executeTask(options) {
|
|
6633
6896
|
const { currentTask, preparedPipeline, parametersToPass, tools, onProgress, $executionReport, pipelineIdentification, maxExecutionAttempts, maxParallelCount, csvSettings, isVerbose, rootDirname, cacheDirname, intermediateFilesStrategy, isAutoInstalled, isNotPreparedWarningSupressed, } = options;
|
|
6634
6897
|
const priority = preparedPipeline.tasks.length - preparedPipeline.tasks.indexOf(currentTask);
|
|
6635
|
-
await onProgress({
|
|
6636
|
-
outputParameters: {
|
|
6637
|
-
[currentTask.resultingParameterName]: '', // <- TODO: [🧠] What is the best value here?
|
|
6638
|
-
},
|
|
6639
|
-
});
|
|
6640
6898
|
// Note: Check consistency of used and dependent parameters which was also done in `validatePipeline`, but it’s good to doublecheck
|
|
6641
6899
|
const usedParameterNames = extractParameterNamesFromTask(currentTask);
|
|
6642
6900
|
const dependentParameterNames = new Set(currentTask.dependentParameterNames);
|
|
6643
6901
|
// TODO: [👩🏾🤝👩🏻] Use here `mapAvailableToExpectedParameters`
|
|
6644
|
-
if (union(difference(usedParameterNames, dependentParameterNames), difference(dependentParameterNames, usedParameterNames)).size !== 0) {
|
|
6902
|
+
if (difference(union(difference(usedParameterNames, dependentParameterNames), difference(dependentParameterNames, usedParameterNames)), new Set(RESERVED_PARAMETER_NAMES)).size !== 0) {
|
|
6645
6903
|
throw new UnexpectedError(spaceTrim$1((block) => `
|
|
6646
6904
|
Dependent parameters are not consistent with used parameters:
|
|
6647
6905
|
|
|
@@ -6659,13 +6917,16 @@ async function executeTask(options) {
|
|
|
6659
6917
|
|
|
6660
6918
|
`));
|
|
6661
6919
|
}
|
|
6920
|
+
const reservedParameters = await getReservedParametersForTask({
|
|
6921
|
+
tools,
|
|
6922
|
+
preparedPipeline,
|
|
6923
|
+
task: currentTask,
|
|
6924
|
+
pipelineIdentification,
|
|
6925
|
+
parameters: parametersToPass,
|
|
6926
|
+
isVerbose,
|
|
6927
|
+
});
|
|
6662
6928
|
const definedParameters = Object.freeze({
|
|
6663
|
-
...
|
|
6664
|
-
tools,
|
|
6665
|
-
preparedPipeline,
|
|
6666
|
-
task: currentTask,
|
|
6667
|
-
pipelineIdentification,
|
|
6668
|
-
})),
|
|
6929
|
+
...reservedParameters,
|
|
6669
6930
|
...parametersToPass,
|
|
6670
6931
|
});
|
|
6671
6932
|
const definedParameterNames = new Set(Object.keys(definedParameters));
|
|
@@ -6710,6 +6971,7 @@ async function executeTask(options) {
|
|
|
6710
6971
|
preparedPipeline,
|
|
6711
6972
|
tools,
|
|
6712
6973
|
$executionReport,
|
|
6974
|
+
onProgress,
|
|
6713
6975
|
pipelineIdentification,
|
|
6714
6976
|
maxExecutionAttempts,
|
|
6715
6977
|
maxParallelCount,
|
|
@@ -6737,7 +6999,8 @@ async function executeTask(options) {
|
|
|
6737
6999
|
*/
|
|
6738
7000
|
|
|
6739
7001
|
/**
|
|
6740
|
-
*
|
|
7002
|
+
* Filters and returns only the output parameters from the provided pipeline execution options.
|
|
7003
|
+
* Adds warnings for any expected output parameters that are missing.
|
|
6741
7004
|
*
|
|
6742
7005
|
* @private internal utility of `createPipelineExecutor`
|
|
6743
7006
|
*/
|
|
@@ -6762,9 +7025,12 @@ function filterJustOutputParameters(options) {
|
|
|
6762
7025
|
}
|
|
6763
7026
|
|
|
6764
7027
|
/**
|
|
6765
|
-
*
|
|
7028
|
+
* Executes an entire pipeline, resolving tasks in dependency order, handling errors, and reporting progress.
|
|
7029
|
+
*
|
|
7030
|
+
* Note: This is not a `PipelineExecutor` (which is bound to a single pipeline), but a utility function used by `createPipelineExecutor` to create a `PipelineExecutor`.
|
|
6766
7031
|
*
|
|
6767
|
-
*
|
|
7032
|
+
* @param options - Options for execution, including input parameters, pipeline, and callbacks.
|
|
7033
|
+
* @returns The result of the pipeline execution, including output parameters, errors, and usage statistics.
|
|
6768
7034
|
*
|
|
6769
7035
|
* @private internal utility of `createPipelineExecutor`
|
|
6770
7036
|
*/
|
|
@@ -7087,10 +7353,27 @@ function createPipelineExecutor(options) {
|
|
|
7087
7353
|
cacheDirname,
|
|
7088
7354
|
intermediateFilesStrategy,
|
|
7089
7355
|
isAutoInstalled,
|
|
7356
|
+
}).catch((error) => {
|
|
7357
|
+
assertsError(error);
|
|
7358
|
+
return exportJson({
|
|
7359
|
+
name: 'pipelineExecutorResult',
|
|
7360
|
+
message: `Unuccessful PipelineExecutorResult, last catch`,
|
|
7361
|
+
order: [],
|
|
7362
|
+
value: {
|
|
7363
|
+
isSuccessful: false,
|
|
7364
|
+
errors: [serializeError(error)],
|
|
7365
|
+
warnings: [],
|
|
7366
|
+
usage: UNCERTAIN_USAGE,
|
|
7367
|
+
executionReport: null,
|
|
7368
|
+
outputParameters: {},
|
|
7369
|
+
preparedPipeline,
|
|
7370
|
+
},
|
|
7371
|
+
});
|
|
7090
7372
|
});
|
|
7091
7373
|
};
|
|
7092
7374
|
const pipelineExecutor = (inputParameters) => createTask({
|
|
7093
7375
|
taskType: 'EXECUTION',
|
|
7376
|
+
title: pipeline.title,
|
|
7094
7377
|
taskProcessCallback(updateOngoingResult) {
|
|
7095
7378
|
return pipelineExecutorWithCallback(inputParameters, async (newOngoingResult) => {
|
|
7096
7379
|
updateOngoingResult(newOngoingResult);
|
|
@@ -7250,7 +7533,9 @@ function mimeTypeToExtension(value) {
|
|
|
7250
7533
|
}
|
|
7251
7534
|
|
|
7252
7535
|
/**
|
|
7253
|
-
*
|
|
7536
|
+
* Factory function that creates a handler for processing knowledge sources.
|
|
7537
|
+
* Provides standardized processing of different types of knowledge sources
|
|
7538
|
+
* across various scraper implementations.
|
|
7254
7539
|
*
|
|
7255
7540
|
* @public exported from `@promptbook/core`
|
|
7256
7541
|
*/
|
|
@@ -7389,7 +7674,7 @@ async function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
|
|
|
7389
7674
|
}
|
|
7390
7675
|
|
|
7391
7676
|
/**
|
|
7392
|
-
* Prepares the
|
|
7677
|
+
* Prepares the knowledge pieces
|
|
7393
7678
|
*
|
|
7394
7679
|
* @see https://github.com/webgptorg/promptbook/discussions/41
|
|
7395
7680
|
* @public exported from `@promptbook/core`
|
|
@@ -7485,15 +7770,18 @@ TODO: [🧊] This is how it can look in future
|
|
|
7485
7770
|
* TODO: [🧊] In future one preparation can take data from previous preparation and save tokens and time
|
|
7486
7771
|
* Put `knowledgePieces` into `PrepareKnowledgeOptions`
|
|
7487
7772
|
* TODO: [🪂] More than max things can run in parallel by acident [1,[2a,2b,_],[3a,3b,_]]
|
|
7488
|
-
* TODO: [🧠][❎] Do here
|
|
7773
|
+
* TODO: [🧠][❎] Do here proper M:N mapping
|
|
7489
7774
|
* [x] One source can make multiple pieces
|
|
7490
7775
|
* [ ] One piece can have multiple sources
|
|
7491
7776
|
*/
|
|
7492
7777
|
|
|
7493
7778
|
/**
|
|
7494
|
-
*
|
|
7779
|
+
* Prepares tasks by adding knowledge to the prompt and ensuring all necessary parameters are included.
|
|
7495
7780
|
*
|
|
7496
|
-
* @
|
|
7781
|
+
* @param tasks Sequence of tasks that are chained together to form a pipeline
|
|
7782
|
+
* @returns A promise that resolves to the prepared tasks.
|
|
7783
|
+
*
|
|
7784
|
+
* @private internal utility of `preparePipeline`
|
|
7497
7785
|
*/
|
|
7498
7786
|
async function prepareTasks(pipeline, tools, options) {
|
|
7499
7787
|
const { maxParallelCount = DEFAULT_MAX_PARALLEL_COUNT } = options;
|
|
@@ -7531,7 +7819,7 @@ async function prepareTasks(pipeline, tools, options) {
|
|
|
7531
7819
|
return { tasksPrepared };
|
|
7532
7820
|
}
|
|
7533
7821
|
/**
|
|
7534
|
-
* TODO: [😂] Adding knowledge should be convert to async high-level abstractions,
|
|
7822
|
+
* TODO: [😂] Adding knowledge should be convert to async high-level abstractions, similar thing with expectations to sync high-level abstractions
|
|
7535
7823
|
* TODO: [🧠] Add context to each task (if missing)
|
|
7536
7824
|
* TODO: [🧠] What is better name `prepareTask` or `prepareTaskAndParameters`
|
|
7537
7825
|
* TODO: [♨][main] !!3 Prepare index the examples and maybe tasks
|
|
@@ -7667,7 +7955,7 @@ async function preparePipeline(pipeline, tools, options) {
|
|
|
7667
7955
|
order: ORDER_OF_PIPELINE_JSON,
|
|
7668
7956
|
value: {
|
|
7669
7957
|
...pipeline,
|
|
7670
|
-
// <- TODO:
|
|
7958
|
+
// <- TODO: Probably deeply clone the pipeline because `$exportJson` freezes the subobjects
|
|
7671
7959
|
title,
|
|
7672
7960
|
knowledgeSources: knowledgeSourcesPrepared,
|
|
7673
7961
|
knowledgePieces: knowledgePiecesPrepared,
|
|
@@ -7943,7 +8231,7 @@ const sectionCommandParser = {
|
|
|
7943
8231
|
throw new ParseError(`Task section and example section must end with return statement -> {parameterName}`);
|
|
7944
8232
|
};
|
|
7945
8233
|
if ($taskJson.content === undefined) {
|
|
7946
|
-
throw new UnexpectedError(`Content is missing in the taskJson -
|
|
8234
|
+
throw new UnexpectedError(`Content is missing in the taskJson - probably commands are applied in wrong order`);
|
|
7947
8235
|
}
|
|
7948
8236
|
if (command.taskType === 'EXAMPLE') {
|
|
7949
8237
|
expectResultingParameterName();
|
|
@@ -8011,7 +8299,7 @@ const sectionCommandParser = {
|
|
|
8011
8299
|
/**
|
|
8012
8300
|
* Parses the boilerplate command
|
|
8013
8301
|
*
|
|
8014
|
-
* Note:
|
|
8302
|
+
* Note: @@ This command is used as boilerplate for new commands - it should NOT be used in any `.book` file
|
|
8015
8303
|
*
|
|
8016
8304
|
* @see `documentationUrl` for more details
|
|
8017
8305
|
* @private within the commands folder
|
|
@@ -8399,11 +8687,11 @@ const expectCommandParser = {
|
|
|
8399
8687
|
};
|
|
8400
8688
|
|
|
8401
8689
|
/**
|
|
8402
|
-
*
|
|
8690
|
+
* Normalizes a given text to camelCase format.
|
|
8403
8691
|
*
|
|
8404
|
-
* @param text
|
|
8405
|
-
* @param _isFirstLetterCapital
|
|
8406
|
-
* @returns
|
|
8692
|
+
* @param text The text to be normalized.
|
|
8693
|
+
* @param _isFirstLetterCapital Whether the first letter should be capitalized.
|
|
8694
|
+
* @returns The camelCase formatted string.
|
|
8407
8695
|
* @example 'helloWorld'
|
|
8408
8696
|
* @example 'iLovePromptbook'
|
|
8409
8697
|
* @public exported from `@promptbook/utils`
|
|
@@ -8453,9 +8741,9 @@ function normalizeTo_camelCase(text, _isFirstLetterCapital = false) {
|
|
|
8453
8741
|
/**
|
|
8454
8742
|
* Removes quotes from a string
|
|
8455
8743
|
*
|
|
8456
|
-
* Tip: This is very
|
|
8744
|
+
* Tip: This is very useful for post-processing of the result of the LLM model
|
|
8457
8745
|
* Note: This function removes only the same quotes from the beginning and the end of the string
|
|
8458
|
-
* Note: There are two
|
|
8746
|
+
* Note: There are two similar functions:
|
|
8459
8747
|
* - `removeQuotes` which removes only bounding quotes
|
|
8460
8748
|
* - `unwrapResult` which removes whole introduce sentence
|
|
8461
8749
|
*
|
|
@@ -8467,18 +8755,19 @@ function removeQuotes(text) {
|
|
|
8467
8755
|
if (text.startsWith('"') && text.endsWith('"')) {
|
|
8468
8756
|
return text.slice(1, -1);
|
|
8469
8757
|
}
|
|
8470
|
-
if (text.startsWith('
|
|
8758
|
+
if (text.startsWith("'") && text.endsWith("'")) {
|
|
8471
8759
|
return text.slice(1, -1);
|
|
8472
8760
|
}
|
|
8473
8761
|
return text;
|
|
8474
8762
|
}
|
|
8475
8763
|
|
|
8476
8764
|
/**
|
|
8477
|
-
* Function `validateParameterName` will
|
|
8765
|
+
* Function `validateParameterName` will normalize and validate a parameter name for use in pipelines.
|
|
8766
|
+
* It removes diacritics, emojis, and quotes, normalizes to camelCase, and checks for reserved names and invalid characters.
|
|
8478
8767
|
*
|
|
8479
|
-
* @param parameterName
|
|
8480
|
-
* @returns
|
|
8481
|
-
* @throws {ParseError}
|
|
8768
|
+
* @param parameterName The parameter name to validate and normalize.
|
|
8769
|
+
* @returns The validated and normalized parameter name.
|
|
8770
|
+
* @throws {ParseError} If the parameter name is empty, reserved, or contains invalid characters.
|
|
8482
8771
|
* @private within the repository
|
|
8483
8772
|
*/
|
|
8484
8773
|
function validateParameterName(parameterName) {
|
|
@@ -8548,8 +8837,6 @@ function validateParameterName(parameterName) {
|
|
|
8548
8837
|
/**
|
|
8549
8838
|
* Parses the foreach command
|
|
8550
8839
|
*
|
|
8551
|
-
* Note: @@@ This command is used as foreach for new commands - it should NOT be used in any `.book` file
|
|
8552
|
-
*
|
|
8553
8840
|
* @see `documentationUrl` for more details
|
|
8554
8841
|
* @public exported from `@promptbook/editable`
|
|
8555
8842
|
*/
|
|
@@ -8606,14 +8893,14 @@ const foreachCommandParser = {
|
|
|
8606
8893
|
`));
|
|
8607
8894
|
// <- TODO: [🏢] List all supported format names
|
|
8608
8895
|
}
|
|
8609
|
-
const
|
|
8610
|
-
if (
|
|
8896
|
+
const subvalueParser = formatDefinition.subvalueParsers.find((subvalueParser) => [subvalueParser.subvalueName, ...(subvalueParser.aliases || [])].includes(subformatName));
|
|
8897
|
+
if (subvalueParser === undefined) {
|
|
8611
8898
|
throw new ParseError(spaceTrim((block) => `
|
|
8612
8899
|
Unsupported subformat name "${subformatName}" for format "${formatName}"
|
|
8613
8900
|
|
|
8614
8901
|
Available subformat names for format "${formatDefinition.formatName}":
|
|
8615
|
-
${block(formatDefinition.
|
|
8616
|
-
.map((
|
|
8902
|
+
${block(formatDefinition.subvalueParsers
|
|
8903
|
+
.map((subvalueParser) => subvalueParser.subvalueName)
|
|
8617
8904
|
.map((subvalueName) => `- ${subvalueName}`)
|
|
8618
8905
|
.join('\n'))}
|
|
8619
8906
|
`));
|
|
@@ -8790,14 +9077,14 @@ const formatCommandParser = {
|
|
|
8790
9077
|
};
|
|
8791
9078
|
|
|
8792
9079
|
/**
|
|
8793
|
-
*
|
|
9080
|
+
* Chatbot form factor definition for conversational interfaces that interact with users in a chat-like manner.
|
|
8794
9081
|
*
|
|
8795
9082
|
* @public exported from `@promptbook/core`
|
|
8796
9083
|
*/
|
|
8797
9084
|
const ChatbotFormfactorDefinition = {
|
|
8798
9085
|
name: 'CHATBOT',
|
|
8799
9086
|
aliasNames: ['CHAT'],
|
|
8800
|
-
description:
|
|
9087
|
+
description: `A chatbot form factor for conversational user interfaces.`,
|
|
8801
9088
|
documentationUrl: `https://github.com/webgptorg/promptbook/discussions/174`,
|
|
8802
9089
|
pipelineInterface: {
|
|
8803
9090
|
inputParameters: [
|
|
@@ -8824,7 +9111,45 @@ const ChatbotFormfactorDefinition = {
|
|
|
8824
9111
|
};
|
|
8825
9112
|
|
|
8826
9113
|
/**
|
|
8827
|
-
*
|
|
9114
|
+
* Completion is formfactor that emulates completion models
|
|
9115
|
+
*
|
|
9116
|
+
* @public exported from `@promptbook/core`
|
|
9117
|
+
*/
|
|
9118
|
+
const CompletionFormfactorDefinition = {
|
|
9119
|
+
name: 'COMPLETION',
|
|
9120
|
+
description: `Completion is formfactor that emulates completion models`,
|
|
9121
|
+
documentationUrl: `https://github.com/webgptorg/promptbook/discussions/@@`,
|
|
9122
|
+
// <- TODO: https://github.com/webgptorg/promptbook/discussions/new?category=concepts
|
|
9123
|
+
// "🔠 Completion Formfactor"
|
|
9124
|
+
pipelineInterface: {
|
|
9125
|
+
inputParameters: [
|
|
9126
|
+
{
|
|
9127
|
+
name: 'inputText',
|
|
9128
|
+
description: `Input text to be completed`,
|
|
9129
|
+
isInput: true,
|
|
9130
|
+
isOutput: false,
|
|
9131
|
+
},
|
|
9132
|
+
{
|
|
9133
|
+
name: 'instructions',
|
|
9134
|
+
description: `Additional instructions for the model, for example the required length, empty by default`,
|
|
9135
|
+
isInput: true,
|
|
9136
|
+
isOutput: false,
|
|
9137
|
+
},
|
|
9138
|
+
],
|
|
9139
|
+
outputParameters: [
|
|
9140
|
+
{
|
|
9141
|
+
name: 'followingText',
|
|
9142
|
+
description: `Text that follows the input text`,
|
|
9143
|
+
isInput: false,
|
|
9144
|
+
isOutput: true,
|
|
9145
|
+
},
|
|
9146
|
+
],
|
|
9147
|
+
},
|
|
9148
|
+
};
|
|
9149
|
+
|
|
9150
|
+
/**
|
|
9151
|
+
* Generator form factor represents an application that generates content or data based on user input or predefined rules.
|
|
9152
|
+
* This form factor is used for apps that produce outputs, such as text, images, or other media, based on provided input.
|
|
8828
9153
|
*
|
|
8829
9154
|
* @public exported from `@promptbook/core`
|
|
8830
9155
|
*/
|
|
@@ -8853,7 +9178,7 @@ const GeneratorFormfactorDefinition = {
|
|
|
8853
9178
|
};
|
|
8854
9179
|
|
|
8855
9180
|
/**
|
|
8856
|
-
*
|
|
9181
|
+
* Pipeline interface which is equivalent to `any`
|
|
8857
9182
|
*
|
|
8858
9183
|
* @see https://github.com/webgptorg/promptbook/discussions/171
|
|
8859
9184
|
*
|
|
@@ -8868,13 +9193,13 @@ const GENERIC_PIPELINE_INTERFACE = {
|
|
|
8868
9193
|
*/
|
|
8869
9194
|
|
|
8870
9195
|
/**
|
|
8871
|
-
*
|
|
9196
|
+
* A generic pipeline
|
|
8872
9197
|
*
|
|
8873
9198
|
* @public exported from `@promptbook/core`
|
|
8874
9199
|
*/
|
|
8875
9200
|
const GenericFormfactorDefinition = {
|
|
8876
9201
|
name: 'GENERIC',
|
|
8877
|
-
description:
|
|
9202
|
+
description: `A generic pipeline`,
|
|
8878
9203
|
documentationUrl: `https://github.com/webgptorg/promptbook/discussions/173`,
|
|
8879
9204
|
pipelineInterface: GENERIC_PIPELINE_INTERFACE,
|
|
8880
9205
|
};
|
|
@@ -8909,17 +9234,20 @@ const ImageGeneratorFormfactorDefinition = {
|
|
|
8909
9234
|
};
|
|
8910
9235
|
|
|
8911
9236
|
/**
|
|
8912
|
-
* Matcher is form of app that
|
|
9237
|
+
* Matcher is form of app that evaluates (spreadsheet) content against defined criteria or patterns,
|
|
9238
|
+
* determining if it matches or meets specific requirements. Used for classification,
|
|
9239
|
+
* validation, filtering, and quality assessment of inputs.
|
|
8913
9240
|
*
|
|
8914
9241
|
* @public exported from `@promptbook/core`
|
|
8915
9242
|
*/
|
|
8916
9243
|
const MatcherFormfactorDefinition = {
|
|
8917
9244
|
name: 'EXPERIMENTAL_MATCHER',
|
|
8918
|
-
description:
|
|
9245
|
+
description: `An evaluation system that determines whether content meets specific criteria or patterns.
|
|
9246
|
+
Used for content validation, quality assessment, and intelligent filtering tasks. Currently in experimental phase.`,
|
|
8919
9247
|
documentationUrl: `https://github.com/webgptorg/promptbook/discussions/177`,
|
|
8920
9248
|
pipelineInterface: {
|
|
8921
9249
|
inputParameters: [
|
|
8922
|
-
/*
|
|
9250
|
+
/* Input parameters for content to be matched and criteria to match against */
|
|
8923
9251
|
{
|
|
8924
9252
|
name: 'nonce',
|
|
8925
9253
|
description: 'Just to prevent EXPERIMENTAL_MATCHER to be set as implicit formfactor',
|
|
@@ -8928,20 +9256,21 @@ const MatcherFormfactorDefinition = {
|
|
|
8928
9256
|
},
|
|
8929
9257
|
],
|
|
8930
9258
|
outputParameters: [
|
|
8931
|
-
/*
|
|
9259
|
+
/* Output parameters containing match results, confidence scores, and relevant metadata */
|
|
8932
9260
|
],
|
|
8933
9261
|
},
|
|
8934
9262
|
};
|
|
8935
9263
|
|
|
8936
9264
|
/**
|
|
8937
|
-
* Sheets is form of app that
|
|
9265
|
+
* Sheets is form of app that processes tabular data in CSV format, allowing transformation
|
|
9266
|
+
* and analysis of structured data through AI-powered operations
|
|
8938
9267
|
*
|
|
8939
9268
|
* @public exported from `@promptbook/core`
|
|
8940
9269
|
*/
|
|
8941
9270
|
const SheetsFormfactorDefinition = {
|
|
8942
9271
|
name: 'SHEETS',
|
|
8943
9272
|
aliasNames: ['SHEETS', 'SHEET'],
|
|
8944
|
-
description:
|
|
9273
|
+
description: `A formfactor for processing spreadsheet-like data in CSV format, enabling AI transformations on tabular data`,
|
|
8945
9274
|
documentationUrl: `https://github.com/webgptorg/promptbook/discussions/176`,
|
|
8946
9275
|
pipelineInterface: {
|
|
8947
9276
|
inputParameters: [
|
|
@@ -8964,13 +9293,16 @@ const SheetsFormfactorDefinition = {
|
|
|
8964
9293
|
};
|
|
8965
9294
|
|
|
8966
9295
|
/**
|
|
8967
|
-
* Translator is form of app that
|
|
9296
|
+
* Translator is form of app that transforms input text from one form to another,
|
|
9297
|
+
* such as language translation, style conversion, tone modification, or other text transformations.
|
|
8968
9298
|
*
|
|
8969
9299
|
* @public exported from `@promptbook/core`
|
|
8970
9300
|
*/
|
|
8971
9301
|
const TranslatorFormfactorDefinition = {
|
|
8972
9302
|
name: 'TRANSLATOR',
|
|
8973
|
-
description:
|
|
9303
|
+
description: `A text transformation system that converts input content into different forms,
|
|
9304
|
+
including language translations, paraphrasing, style conversions, and tone adjustments.
|
|
9305
|
+
This form factor takes one input and produces one transformed output.`,
|
|
8974
9306
|
documentationUrl: `https://github.com/webgptorg/promptbook/discussions/175`,
|
|
8975
9307
|
pipelineInterface: {
|
|
8976
9308
|
inputParameters: [
|
|
@@ -9007,6 +9339,8 @@ const FORMFACTOR_DEFINITIONS = [
|
|
|
9007
9339
|
MatcherFormfactorDefinition,
|
|
9008
9340
|
GeneratorFormfactorDefinition,
|
|
9009
9341
|
ImageGeneratorFormfactorDefinition,
|
|
9342
|
+
CompletionFormfactorDefinition,
|
|
9343
|
+
// <- [🛬] When making new formfactor, copy the _boilerplate and link it here
|
|
9010
9344
|
];
|
|
9011
9345
|
/**
|
|
9012
9346
|
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
@@ -9015,7 +9349,7 @@ const FORMFACTOR_DEFINITIONS = [
|
|
|
9015
9349
|
/**
|
|
9016
9350
|
* Parses the formfactor command
|
|
9017
9351
|
*
|
|
9018
|
-
* Note:
|
|
9352
|
+
* Note: This command is used as a formfactor for new commands and defines the app type format - it should NOT be used in any `.book` file
|
|
9019
9353
|
*
|
|
9020
9354
|
* @see `documentationUrl` for more details
|
|
9021
9355
|
* @public exported from `@promptbook/editable`
|
|
@@ -9037,7 +9371,7 @@ const formfactorCommandParser = {
|
|
|
9037
9371
|
/**
|
|
9038
9372
|
* Description of the FORMFACTOR command
|
|
9039
9373
|
*/
|
|
9040
|
-
description:
|
|
9374
|
+
description: `Specifies the application type and interface requirements that this promptbook should conform to`,
|
|
9041
9375
|
/**
|
|
9042
9376
|
* Link to documentation
|
|
9043
9377
|
*/
|
|
@@ -9180,8 +9514,7 @@ const jokerCommandParser = {
|
|
|
9180
9514
|
};
|
|
9181
9515
|
|
|
9182
9516
|
/**
|
|
9183
|
-
*
|
|
9184
|
-
*
|
|
9517
|
+
* @see {@link ModelVariant}
|
|
9185
9518
|
* @public exported from `@promptbook/core`
|
|
9186
9519
|
*/
|
|
9187
9520
|
const MODEL_VARIANTS = ['COMPLETION', 'CHAT', 'EMBEDDING' /* <- TODO [🏳] */ /* <- [🤖] */];
|
|
@@ -9613,10 +9946,10 @@ function $applyToTaskJson(command, $taskJson, $pipelineJson) {
|
|
|
9613
9946
|
}
|
|
9614
9947
|
|
|
9615
9948
|
/**
|
|
9616
|
-
*
|
|
9949
|
+
* Checks if the given value is a valid JavaScript identifier name.
|
|
9617
9950
|
*
|
|
9618
|
-
* @param javascriptName
|
|
9619
|
-
* @returns
|
|
9951
|
+
* @param javascriptName The value to check for JavaScript identifier validity.
|
|
9952
|
+
* @returns `true` if the value is a valid JavaScript name, false otherwise.
|
|
9620
9953
|
* @public exported from `@promptbook/utils`
|
|
9621
9954
|
*/
|
|
9622
9955
|
function isValidJavascriptName(javascriptName) {
|
|
@@ -10077,7 +10410,7 @@ function parseCommand(raw, usagePlace) {
|
|
|
10077
10410
|
// Arg1 Arg2 Arg3 | FOO
|
|
10078
10411
|
{
|
|
10079
10412
|
const commandNameRaw = items.slice(-1).join('_');
|
|
10080
|
-
const args = items.slice(0, -1); // <- Note: This is
|
|
10413
|
+
const args = items.slice(0, -1); // <- Note: This is probably not correct
|
|
10081
10414
|
const rawArgs = raw
|
|
10082
10415
|
.substring(0, raw.length - commandNameRaw.length)
|
|
10083
10416
|
.trim();
|
|
@@ -10096,7 +10429,10 @@ function parseCommand(raw, usagePlace) {
|
|
|
10096
10429
|
`));
|
|
10097
10430
|
}
|
|
10098
10431
|
/**
|
|
10099
|
-
*
|
|
10432
|
+
* Generates a markdown-formatted message listing all supported commands
|
|
10433
|
+
* with their descriptions and documentation links
|
|
10434
|
+
*
|
|
10435
|
+
* @returns A formatted markdown string containing all available commands and their details
|
|
10100
10436
|
*/
|
|
10101
10437
|
function getSupportedCommandsMessage() {
|
|
10102
10438
|
return COMMANDS.flatMap(({ name, aliasNames, description, documentationUrl }) =>
|
|
@@ -10107,7 +10443,10 @@ function getSupportedCommandsMessage() {
|
|
|
10107
10443
|
]).join('\n');
|
|
10108
10444
|
}
|
|
10109
10445
|
/**
|
|
10110
|
-
*
|
|
10446
|
+
* Attempts to parse a command variant using the provided input parameters
|
|
10447
|
+
*
|
|
10448
|
+
* @param input Object containing command parsing information including raw command text and normalized values
|
|
10449
|
+
* @returns A parsed Command object if successful, or null if the command cannot be parsed
|
|
10111
10450
|
*/
|
|
10112
10451
|
function parseCommandVariant(input) {
|
|
10113
10452
|
const { commandNameRaw, usagePlace, normalized, args, raw, rawArgs } = input;
|
|
@@ -10154,7 +10493,7 @@ function parseCommandVariant(input) {
|
|
|
10154
10493
|
}
|
|
10155
10494
|
|
|
10156
10495
|
/**
|
|
10157
|
-
*
|
|
10496
|
+
* Extracts the interface (input and output parameters) from a pipeline.
|
|
10158
10497
|
*
|
|
10159
10498
|
* @deprecated https://github.com/webgptorg/promptbook/pull/186
|
|
10160
10499
|
* @see https://github.com/webgptorg/promptbook/discussions/171
|
|
@@ -10187,7 +10526,7 @@ function getPipelineInterface(pipeline) {
|
|
|
10187
10526
|
}
|
|
10188
10527
|
|
|
10189
10528
|
/**
|
|
10190
|
-
*
|
|
10529
|
+
* Checks if two pipeline interfaces are structurally identical.
|
|
10191
10530
|
*
|
|
10192
10531
|
* @deprecated https://github.com/webgptorg/promptbook/pull/186
|
|
10193
10532
|
* @see https://github.com/webgptorg/promptbook/discussions/171
|
|
@@ -10219,10 +10558,11 @@ function isPipelineInterfacesEqual(pipelineInterface1, pipelineInterface2) {
|
|
|
10219
10558
|
}
|
|
10220
10559
|
|
|
10221
10560
|
/**
|
|
10222
|
-
*
|
|
10561
|
+
* Checks if a given pipeline satisfies the requirements of a specified pipeline interface.
|
|
10223
10562
|
*
|
|
10224
10563
|
* @deprecated https://github.com/webgptorg/promptbook/pull/186
|
|
10225
10564
|
* @see https://github.com/webgptorg/promptbook/discussions/171
|
|
10565
|
+
* @returns `true` if the pipeline implements the interface, `false` otherwise.
|
|
10226
10566
|
*
|
|
10227
10567
|
* @public exported from `@promptbook/core`
|
|
10228
10568
|
*/
|
|
@@ -10408,7 +10748,8 @@ function removeMarkdownComments(content) {
|
|
|
10408
10748
|
}
|
|
10409
10749
|
|
|
10410
10750
|
/**
|
|
10411
|
-
*
|
|
10751
|
+
* Utility to determine if a pipeline string is in flat format.
|
|
10752
|
+
* A flat pipeline is a simple text without proper structure (headers, blocks, etc).
|
|
10412
10753
|
*
|
|
10413
10754
|
* @public exported from `@promptbook/editable`
|
|
10414
10755
|
*/
|
|
@@ -10429,7 +10770,10 @@ function isFlatPipeline(pipelineString) {
|
|
|
10429
10770
|
}
|
|
10430
10771
|
|
|
10431
10772
|
/**
|
|
10432
|
-
*
|
|
10773
|
+
* Converts a pipeline structure to its string representation.
|
|
10774
|
+
*
|
|
10775
|
+
* Transforms a flat, simple pipeline into a properly formatted pipeline string
|
|
10776
|
+
* with sections for title, prompt, and return statement.
|
|
10433
10777
|
*
|
|
10434
10778
|
* @public exported from `@promptbook/editable`
|
|
10435
10779
|
*/
|
|
@@ -10486,7 +10830,7 @@ function deflatePipeline(pipelineString) {
|
|
|
10486
10830
|
* Note: It can not work with html syntax and comments
|
|
10487
10831
|
*
|
|
10488
10832
|
* @param markdown any valid markdown
|
|
10489
|
-
* @returns
|
|
10833
|
+
* @returns An array of strings, each representing an individual list item found in the markdown
|
|
10490
10834
|
* @public exported from `@promptbook/markdown-utils`
|
|
10491
10835
|
*/
|
|
10492
10836
|
function extractAllListItemsFromMarkdown(markdown) {
|
|
@@ -10511,8 +10855,8 @@ function extractAllListItemsFromMarkdown(markdown) {
|
|
|
10511
10855
|
*
|
|
10512
10856
|
* - When there are multiple or no code blocks the function throws a `ParseError`
|
|
10513
10857
|
*
|
|
10514
|
-
* Note: There are multiple
|
|
10515
|
-
* - `extractBlock` just extracts the content of the code block which is also used as
|
|
10858
|
+
* Note: There are multiple similar functions:
|
|
10859
|
+
* - `extractBlock` just extracts the content of the code block which is also used as built-in function for postprocessing
|
|
10516
10860
|
* - `extractJsonBlock` extracts exactly one valid JSON code block
|
|
10517
10861
|
* - `extractOneBlockFromMarkdown` extracts exactly one code block with language of the code block
|
|
10518
10862
|
* - `extractAllBlocksFromMarkdown` extracts all code blocks with language of the code block
|
|
@@ -10667,7 +11011,7 @@ function flattenMarkdown(markdown) {
|
|
|
10667
11011
|
* Compile pipeline from string (markdown) format to JSON format synchronously
|
|
10668
11012
|
*
|
|
10669
11013
|
* Note: There are 3 similar functions:
|
|
10670
|
-
* - `compilePipeline` **(preferred)** - which
|
|
11014
|
+
* - `compilePipeline` **(preferred)** - which properly compiles the promptbook and uses embedding for external knowledge
|
|
10671
11015
|
* - `parsePipeline` - use only if you need to compile promptbook synchronously and it contains NO external knowledge
|
|
10672
11016
|
* - `preparePipeline` - just one step in the compilation process
|
|
10673
11017
|
*
|
|
@@ -11104,7 +11448,7 @@ function parsePipeline(pipelineString) {
|
|
|
11104
11448
|
* TODO: Use spaceTrim more effectively
|
|
11105
11449
|
* TODO: [🧠] Parameter flags - isInput, isOutput, isInternal
|
|
11106
11450
|
* TODO: [🥞] Not optimal parsing because `splitMarkdownIntoSections` is executed twice with same string, once through `flattenMarkdown` and second directly here
|
|
11107
|
-
* TODO: [♈]
|
|
11451
|
+
* TODO: [♈] Probably move expectations from tasks to parameters
|
|
11108
11452
|
* TODO: [🛠] Actions, instruments (and maybe knowledge) => Functions and tools
|
|
11109
11453
|
* TODO: [🍙] Make some standard order of json properties
|
|
11110
11454
|
*/
|
|
@@ -11149,45 +11493,43 @@ async function compilePipeline(pipelineString, tools, options) {
|
|
|
11149
11493
|
*/
|
|
11150
11494
|
function renderPromptbookMermaid(pipelineJson, options) {
|
|
11151
11495
|
const { linkTask = () => null } = options || {};
|
|
11496
|
+
const MERMAID_PREFIX = 'pipeline_';
|
|
11497
|
+
const MERMAID_KNOWLEDGE_NAME = MERMAID_PREFIX + 'knowledge';
|
|
11498
|
+
const MERMAID_RESERVED_NAME = MERMAID_PREFIX + 'reserved';
|
|
11499
|
+
const MERMAID_INPUT_NAME = MERMAID_PREFIX + 'input';
|
|
11500
|
+
const MERMAID_OUTPUT_NAME = MERMAID_PREFIX + 'output';
|
|
11152
11501
|
const parameterNameToTaskName = (parameterName) => {
|
|
11502
|
+
if (parameterName === 'knowledge') {
|
|
11503
|
+
return MERMAID_KNOWLEDGE_NAME;
|
|
11504
|
+
}
|
|
11505
|
+
else if (RESERVED_PARAMETER_NAMES.includes(parameterName)) {
|
|
11506
|
+
return MERMAID_RESERVED_NAME;
|
|
11507
|
+
}
|
|
11153
11508
|
const parameter = pipelineJson.parameters.find((parameter) => parameter.name === parameterName);
|
|
11154
11509
|
if (!parameter) {
|
|
11155
11510
|
throw new UnexpectedError(`Could not find {${parameterName}}`);
|
|
11156
|
-
// <- TODO:
|
|
11511
|
+
// <- TODO: This causes problems when {knowledge} and other reserved parameters are used
|
|
11157
11512
|
}
|
|
11158
11513
|
if (parameter.isInput) {
|
|
11159
|
-
return
|
|
11514
|
+
return MERMAID_INPUT_NAME;
|
|
11160
11515
|
}
|
|
11161
11516
|
const task = pipelineJson.tasks.find((task) => task.resultingParameterName === parameterName);
|
|
11162
11517
|
if (!task) {
|
|
11163
11518
|
throw new Error(`Could not find task for {${parameterName}}`);
|
|
11164
11519
|
}
|
|
11165
|
-
return task.name || normalizeTo_camelCase('task-' + titleToName(task.title));
|
|
11520
|
+
return MERMAID_PREFIX + (task.name || normalizeTo_camelCase('task-' + titleToName(task.title)));
|
|
11166
11521
|
};
|
|
11167
|
-
const
|
|
11168
|
-
|
|
11169
|
-
%% 🔮 Tip: Open this on GitHub or in the VSCode website to see the Mermaid graph visually
|
|
11170
|
-
|
|
11171
|
-
flowchart LR
|
|
11172
|
-
subgraph "${pipelineJson.title}"
|
|
11173
|
-
|
|
11174
|
-
direction TB
|
|
11175
|
-
|
|
11176
|
-
input((Input)):::input
|
|
11177
|
-
${block(pipelineJson.tasks
|
|
11522
|
+
const inputAndIntermediateParametersMermaid = pipelineJson.tasks
|
|
11178
11523
|
.flatMap(({ title, dependentParameterNames, resultingParameterName }) => [
|
|
11179
11524
|
`${parameterNameToTaskName(resultingParameterName)}("${title}")`,
|
|
11180
11525
|
...dependentParameterNames.map((dependentParameterName) => `${parameterNameToTaskName(dependentParameterName)}--"{${dependentParameterName}}"-->${parameterNameToTaskName(resultingParameterName)}`),
|
|
11181
11526
|
])
|
|
11182
|
-
.join('\n')
|
|
11183
|
-
|
|
11184
|
-
${block(pipelineJson.parameters
|
|
11527
|
+
.join('\n');
|
|
11528
|
+
const outputParametersMermaid = pipelineJson.parameters
|
|
11185
11529
|
.filter(({ isOutput }) => isOutput)
|
|
11186
|
-
.map(({ name }) => `${parameterNameToTaskName(name)}--"{${name}}"
|
|
11187
|
-
.join('\n')
|
|
11188
|
-
|
|
11189
|
-
|
|
11190
|
-
${block(pipelineJson.tasks
|
|
11530
|
+
.map(({ name }) => `${parameterNameToTaskName(name)}--"{${name}}"-->${MERMAID_OUTPUT_NAME}`)
|
|
11531
|
+
.join('\n');
|
|
11532
|
+
const linksMermaid = pipelineJson.tasks
|
|
11191
11533
|
.map((task) => {
|
|
11192
11534
|
const link = linkTask(task);
|
|
11193
11535
|
if (link === null) {
|
|
@@ -11198,10 +11540,44 @@ function renderPromptbookMermaid(pipelineJson, options) {
|
|
|
11198
11540
|
return `click ${taskName} href "${href}" "${title}";`;
|
|
11199
11541
|
})
|
|
11200
11542
|
.filter((line) => line !== '')
|
|
11201
|
-
.join('\n')
|
|
11543
|
+
.join('\n');
|
|
11544
|
+
const interactionPointsMermaid = Object.entries({
|
|
11545
|
+
[MERMAID_INPUT_NAME]: 'Input',
|
|
11546
|
+
[MERMAID_OUTPUT_NAME]: 'Output',
|
|
11547
|
+
[MERMAID_RESERVED_NAME]: 'Other',
|
|
11548
|
+
[MERMAID_KNOWLEDGE_NAME]: 'Knowledge',
|
|
11549
|
+
})
|
|
11550
|
+
.filter(([MERMAID_NAME]) => (inputAndIntermediateParametersMermaid + outputParametersMermaid).includes(MERMAID_NAME))
|
|
11551
|
+
.map(([MERMAID_NAME, title]) => `${MERMAID_NAME}((${title})):::${MERMAID_NAME}`)
|
|
11552
|
+
.join('\n');
|
|
11553
|
+
const promptbookMermaid = spaceTrim$1((block) => `
|
|
11554
|
+
|
|
11555
|
+
%% 🔮 Tip: Open this on GitHub or in the VSCode website to see the Mermaid graph visually
|
|
11556
|
+
|
|
11557
|
+
flowchart LR
|
|
11558
|
+
subgraph "${pipelineJson.title}"
|
|
11559
|
+
|
|
11560
|
+
%% Basic configuration
|
|
11561
|
+
direction TB
|
|
11562
|
+
|
|
11563
|
+
%% Interaction points from pipeline to outside
|
|
11564
|
+
${block(interactionPointsMermaid)}
|
|
11565
|
+
|
|
11566
|
+
%% Input and intermediate parameters
|
|
11567
|
+
${block(inputAndIntermediateParametersMermaid)}
|
|
11568
|
+
|
|
11202
11569
|
|
|
11203
|
-
|
|
11204
|
-
|
|
11570
|
+
%% Output parameters
|
|
11571
|
+
${block(outputParametersMermaid)}
|
|
11572
|
+
|
|
11573
|
+
%% Links
|
|
11574
|
+
${block(linksMermaid)}
|
|
11575
|
+
|
|
11576
|
+
%% Styles
|
|
11577
|
+
classDef ${MERMAID_INPUT_NAME} color: grey;
|
|
11578
|
+
classDef ${MERMAID_OUTPUT_NAME} color: grey;
|
|
11579
|
+
classDef ${MERMAID_RESERVED_NAME} color: grey;
|
|
11580
|
+
classDef ${MERMAID_KNOWLEDGE_NAME} color: grey;
|
|
11205
11581
|
|
|
11206
11582
|
end;
|
|
11207
11583
|
|
|
@@ -11241,11 +11617,11 @@ function parseKeywordsFromString(input) {
|
|
|
11241
11617
|
}
|
|
11242
11618
|
|
|
11243
11619
|
/**
|
|
11244
|
-
*
|
|
11620
|
+
* Converts a name string into a URI-compatible format.
|
|
11245
11621
|
*
|
|
11246
|
-
* @param name
|
|
11247
|
-
* @returns
|
|
11248
|
-
* @example
|
|
11622
|
+
* @param name The string to be converted to a URI-compatible format.
|
|
11623
|
+
* @returns A URI-compatible string derived from the input name.
|
|
11624
|
+
* @example 'Hello World' -> 'hello-world'
|
|
11249
11625
|
* @public exported from `@promptbook/utils`
|
|
11250
11626
|
*/
|
|
11251
11627
|
function nameToUriPart(name) {
|
|
@@ -11259,11 +11635,11 @@ function nameToUriPart(name) {
|
|
|
11259
11635
|
}
|
|
11260
11636
|
|
|
11261
11637
|
/**
|
|
11262
|
-
*
|
|
11638
|
+
* Converts a given name into URI-compatible parts.
|
|
11263
11639
|
*
|
|
11264
|
-
* @param name
|
|
11265
|
-
* @returns
|
|
11266
|
-
* @example
|
|
11640
|
+
* @param name The name to be converted into URI parts.
|
|
11641
|
+
* @returns An array of URI-compatible parts derived from the name.
|
|
11642
|
+
* @example 'Example Name' -> ['example', 'name']
|
|
11267
11643
|
* @public exported from `@promptbook/utils`
|
|
11268
11644
|
*/
|
|
11269
11645
|
function nameToUriParts(name) {
|
|
@@ -11307,7 +11683,7 @@ function suffixUrl(value, suffix) {
|
|
|
11307
11683
|
/**
|
|
11308
11684
|
* Function trimCodeBlock will trim starting and ending code block from the string if it is present.
|
|
11309
11685
|
*
|
|
11310
|
-
* Note: This is
|
|
11686
|
+
* Note: This is useful for post-processing of the result of the chat LLM model
|
|
11311
11687
|
* when the model wraps the result in the (markdown) code block.
|
|
11312
11688
|
*
|
|
11313
11689
|
* @public exported from `@promptbook/utils`
|
|
@@ -11326,7 +11702,7 @@ function trimCodeBlock(value) {
|
|
|
11326
11702
|
/**
|
|
11327
11703
|
* Function trimEndOfCodeBlock will remove ending code block from the string if it is present.
|
|
11328
11704
|
*
|
|
11329
|
-
* Note: This is
|
|
11705
|
+
* Note: This is useful for post-processing of the result of the completion LLM model
|
|
11330
11706
|
* if you want to start code block in the prompt but you don't want to end it in the result.
|
|
11331
11707
|
*
|
|
11332
11708
|
* @public exported from `@promptbook/utils`
|
|
@@ -11341,9 +11717,9 @@ function trimEndOfCodeBlock(value) {
|
|
|
11341
11717
|
/**
|
|
11342
11718
|
* Removes quotes and optional introduce text from a string
|
|
11343
11719
|
*
|
|
11344
|
-
* Tip: This is very
|
|
11720
|
+
* Tip: This is very useful for post-processing of the result of the LLM model
|
|
11345
11721
|
* Note: This function trims the text and removes whole introduce sentence if it is present
|
|
11346
|
-
* Note: There are two
|
|
11722
|
+
* Note: There are two similar functions:
|
|
11347
11723
|
* - `removeQuotes` which removes only bounding quotes
|
|
11348
11724
|
* - `unwrapResult` which removes whole introduce sentence
|
|
11349
11725
|
*
|
|
@@ -11414,7 +11790,7 @@ function unwrapResult(text, options) {
|
|
|
11414
11790
|
*
|
|
11415
11791
|
* - When there are multiple or no code blocks the function throws a `ParseError`
|
|
11416
11792
|
*
|
|
11417
|
-
* Note: There are multiple
|
|
11793
|
+
* Note: There are multiple similar function:
|
|
11418
11794
|
* - `extractBlock` just extracts the content of the code block which is also used as build-in function for postprocessing
|
|
11419
11795
|
* - `extractJsonBlock` extracts exactly one valid JSON code block
|
|
11420
11796
|
* - `extractOneBlockFromMarkdown` extracts exactly one code block with language of the code block
|
|
@@ -11451,7 +11827,7 @@ function preserve(func) {
|
|
|
11451
11827
|
})();
|
|
11452
11828
|
}
|
|
11453
11829
|
/**
|
|
11454
|
-
* TODO:
|
|
11830
|
+
* TODO: Probably remove in favour of `keepImported`
|
|
11455
11831
|
* TODO: [1] This maybe does memory leak
|
|
11456
11832
|
*/
|
|
11457
11833
|
|
|
@@ -11655,7 +12031,7 @@ class JavascriptEvalExecutionTools {
|
|
|
11655
12031
|
*/
|
|
11656
12032
|
|
|
11657
12033
|
/**
|
|
11658
|
-
* Placeholder for better implementation of JavascriptExecutionTools - some
|
|
12034
|
+
* Placeholder for better implementation of JavascriptExecutionTools - some proper sandboxing
|
|
11659
12035
|
*
|
|
11660
12036
|
* @alias JavascriptExecutionTools
|
|
11661
12037
|
* @public exported from `@promptbook/javascript`
|
|
@@ -12029,7 +12405,7 @@ async function saveArchive(filePath, collectionJson, fs) {
|
|
|
12029
12405
|
/**
|
|
12030
12406
|
* Function usageToWorktime will take usage and estimate saved worktime in hours of reading / writing
|
|
12031
12407
|
*
|
|
12032
|
-
* Note: This is an estimate based of
|
|
12408
|
+
* Note: This is an estimate based of these sources:
|
|
12033
12409
|
* - https://jecas.cz/doba-cteni
|
|
12034
12410
|
* - https://www.originalnitonery.cz/blog/psani-vsemi-deseti-se-muzete-naucit-i-sami-doma
|
|
12035
12411
|
*
|
|
@@ -13114,7 +13490,9 @@ function $initializeRunCommand(program) {
|
|
|
13114
13490
|
type: 'text',
|
|
13115
13491
|
name: 'pipelineSource',
|
|
13116
13492
|
message: '',
|
|
13117
|
-
validate
|
|
13493
|
+
validate(value) {
|
|
13494
|
+
return value.length > 0 ? true : 'Pipeline source is required';
|
|
13495
|
+
},
|
|
13118
13496
|
});
|
|
13119
13497
|
if (!response.pipelineSource) {
|
|
13120
13498
|
console.error(colors.red('Pipeline source is required'));
|
|
@@ -13239,10 +13617,10 @@ function $initializeRunCommand(program) {
|
|
|
13239
13617
|
console.info(colors.gray('--- Detailed Result ---'));
|
|
13240
13618
|
console.info({ isSuccessful, errors, warnings, outputParameters, executionReport });
|
|
13241
13619
|
}
|
|
13242
|
-
if (saveReport && saveReport.endsWith('.json')) {
|
|
13620
|
+
if (executionReport !== null && saveReport && saveReport.endsWith('.json')) {
|
|
13243
13621
|
await writeFile(saveReport, JSON.stringify(executionReport, null, 4) + '\n', 'utf-8');
|
|
13244
13622
|
}
|
|
13245
|
-
else if (saveReport && saveReport.endsWith('.md')) {
|
|
13623
|
+
else if (executionReport !== null && saveReport && saveReport.endsWith('.md')) {
|
|
13246
13624
|
const executionReportString = executionReportJsonToString(executionReport);
|
|
13247
13625
|
await writeFile(saveReport, executionReportString, 'utf-8');
|
|
13248
13626
|
}
|
|
@@ -13285,15 +13663,15 @@ function $initializeRunCommand(program) {
|
|
|
13285
13663
|
* TODO: [🖇] What about symlinks? Maybe flag --follow-symlinks
|
|
13286
13664
|
*/
|
|
13287
13665
|
|
|
13288
|
-
// TODO:
|
|
13289
|
-
// TODO:
|
|
13666
|
+
// TODO: [🥺] List running services from REMOTE_SERVER_URLS
|
|
13667
|
+
// TODO: [🥺] Import directly from YML
|
|
13290
13668
|
/**
|
|
13291
|
-
* @private
|
|
13669
|
+
* @private [🥺] Decide how to expose this
|
|
13292
13670
|
*/
|
|
13293
13671
|
const openapiJson = {
|
|
13294
13672
|
openapi: '3.0.0',
|
|
13295
13673
|
info: {
|
|
13296
|
-
title: 'Promptbook Remote Server API (
|
|
13674
|
+
title: 'Promptbook Remote Server API ([🥺] From YML)',
|
|
13297
13675
|
version: '1.0.0',
|
|
13298
13676
|
description: 'API documentation for the Promptbook Remote Server',
|
|
13299
13677
|
},
|
|
@@ -13916,7 +14294,7 @@ function startRemoteServer(options) {
|
|
|
13916
14294
|
// TODO: [main] !!4 Validate here userId (pass validator as dependency)
|
|
13917
14295
|
let llm;
|
|
13918
14296
|
if (isAnonymous === true) {
|
|
13919
|
-
// Note:
|
|
14297
|
+
// Note: Anonymous mode
|
|
13920
14298
|
// TODO: Maybe check that configuration is not empty
|
|
13921
14299
|
const { llmToolsConfiguration } = identification;
|
|
13922
14300
|
llm = createLlmToolsFromConfiguration(llmToolsConfiguration, { isVerbose });
|
|
@@ -13926,7 +14304,7 @@ function startRemoteServer(options) {
|
|
|
13926
14304
|
llm = await createLlmExecutionTools(identification);
|
|
13927
14305
|
}
|
|
13928
14306
|
else {
|
|
13929
|
-
throw new PipelineExecutionError(`You must provide either llmToolsConfiguration or non-anonymous mode must be
|
|
14307
|
+
throw new PipelineExecutionError(`You must provide either llmToolsConfiguration or non-anonymous mode must be properly configured`);
|
|
13930
14308
|
}
|
|
13931
14309
|
const customExecutionTools = createExecutionTools ? await createExecutionTools(identification) : {};
|
|
13932
14310
|
const fs = customExecutionTools.fs || $provideFilesystemForNode();
|
|
@@ -13951,7 +14329,7 @@ function startRemoteServer(options) {
|
|
|
13951
14329
|
response.setHeader('X-Powered-By', 'Promptbook engine');
|
|
13952
14330
|
next();
|
|
13953
14331
|
});
|
|
13954
|
-
// TODO:
|
|
14332
|
+
// TODO: [🥺] Expose openapiJson to consumer and also allow to add new routes
|
|
13955
14333
|
app.use(OpenApiValidator.middleware({
|
|
13956
14334
|
apiSpec: openapiJson,
|
|
13957
14335
|
ignorePaths(path) {
|
|
@@ -13991,7 +14369,7 @@ function startRemoteServer(options) {
|
|
|
13991
14369
|
|
|
13992
14370
|
**Server port:** ${port}
|
|
13993
14371
|
**Startup date:** ${startupDate.toISOString()}
|
|
13994
|
-
**
|
|
14372
|
+
**Anonymous mode:** ${isAnonymousModeAllowed ? 'enabled' : 'disabled'}
|
|
13995
14373
|
**Application mode:** ${isApplicationModeAllowed ? 'enabled' : 'disabled'}
|
|
13996
14374
|
${block(!isApplicationModeAllowed || collection === null
|
|
13997
14375
|
? ''
|
|
@@ -14021,7 +14399,7 @@ function startRemoteServer(options) {
|
|
|
14021
14399
|
To connect to this server use:
|
|
14022
14400
|
|
|
14023
14401
|
1) The client https://www.npmjs.com/package/@promptbook/remote-client
|
|
14024
|
-
2) OpenAI compatible client *(Not
|
|
14402
|
+
2) OpenAI compatible client *(Not working yet)*
|
|
14025
14403
|
3) REST API
|
|
14026
14404
|
|
|
14027
14405
|
For more information look at:
|
|
@@ -14105,12 +14483,13 @@ function startRemoteServer(options) {
|
|
|
14105
14483
|
});
|
|
14106
14484
|
function exportExecutionTask(executionTask, isFull) {
|
|
14107
14485
|
// <- TODO: [🧠] This should be maybe method of `ExecutionTask` itself
|
|
14108
|
-
const { taskType, taskId, status, errors, warnings, createdAt, updatedAt, currentValue } = executionTask;
|
|
14486
|
+
const { taskType, promptbookVersion, taskId, title, status, errors, warnings, createdAt, updatedAt, currentValue, } = executionTask;
|
|
14109
14487
|
if (isFull) {
|
|
14110
14488
|
return {
|
|
14111
|
-
nonce: '✨',
|
|
14112
14489
|
taskId,
|
|
14490
|
+
title,
|
|
14113
14491
|
taskType,
|
|
14492
|
+
promptbookVersion,
|
|
14114
14493
|
status,
|
|
14115
14494
|
errors: errors.map(serializeError),
|
|
14116
14495
|
warnings: warnings.map(serializeError),
|
|
@@ -14121,9 +14500,10 @@ function startRemoteServer(options) {
|
|
|
14121
14500
|
}
|
|
14122
14501
|
else {
|
|
14123
14502
|
return {
|
|
14124
|
-
nonce: '✨',
|
|
14125
14503
|
taskId,
|
|
14504
|
+
title,
|
|
14126
14505
|
taskType,
|
|
14506
|
+
promptbookVersion,
|
|
14127
14507
|
status,
|
|
14128
14508
|
createdAt,
|
|
14129
14509
|
updatedAt,
|
|
@@ -14273,7 +14653,7 @@ function startRemoteServer(options) {
|
|
|
14273
14653
|
}
|
|
14274
14654
|
});
|
|
14275
14655
|
// -----------
|
|
14276
|
-
// TODO: [👒] Listing models (and checking configuration)
|
|
14656
|
+
// TODO: [👒] Listing models (and checking configuration) probably should go through REST API not Socket.io
|
|
14277
14657
|
socket.on('listModels-request', async (request) => {
|
|
14278
14658
|
const { identification } = request;
|
|
14279
14659
|
if (isVerbose) {
|
|
@@ -14295,7 +14675,7 @@ function startRemoteServer(options) {
|
|
|
14295
14675
|
}
|
|
14296
14676
|
});
|
|
14297
14677
|
// -----------
|
|
14298
|
-
// TODO: [👒] Listing models (and checking configuration)
|
|
14678
|
+
// TODO: [👒] Listing models (and checking configuration) probably should go through REST API not Socket.io
|
|
14299
14679
|
socket.on('preparePipeline-request', async (request) => {
|
|
14300
14680
|
const { identification, pipeline } = request;
|
|
14301
14681
|
if (isVerbose) {
|
|
@@ -14355,7 +14735,7 @@ function startRemoteServer(options) {
|
|
|
14355
14735
|
};
|
|
14356
14736
|
}
|
|
14357
14737
|
/**
|
|
14358
|
-
* TODO: [🌡] Add CORS and security -
|
|
14738
|
+
* TODO: [🌡] Add CORS and security - probably via `helmet`
|
|
14359
14739
|
* TODO: Split this file into multiple functions - handler for each request
|
|
14360
14740
|
* TODO: Maybe use `$exportJson`
|
|
14361
14741
|
* TODO: [🧠][🛍] Maybe not `isAnonymous: boolean` BUT `mode: 'ANONYMOUS'|'COLLECTION'`
|
|
@@ -14591,7 +14971,7 @@ async function promptbookCli() {
|
|
|
14591
14971
|
program.alias('ptbk');
|
|
14592
14972
|
program.version(PROMPTBOOK_ENGINE_VERSION);
|
|
14593
14973
|
program.description(CLAIM);
|
|
14594
|
-
// Note:
|
|
14974
|
+
// Note: These options are valid for all commands
|
|
14595
14975
|
$initializeAboutCommand(program);
|
|
14596
14976
|
$initializeRunCommand(program);
|
|
14597
14977
|
$initializeLoginCommand(program);
|
|
@@ -14631,37 +15011,6 @@ const _CLI = {
|
|
|
14631
15011
|
* Note: [🟡] Code in this file should never be published outside of `@promptbook/cli`
|
|
14632
15012
|
*/
|
|
14633
15013
|
|
|
14634
|
-
/**
|
|
14635
|
-
* How is the model provider trusted?
|
|
14636
|
-
*
|
|
14637
|
-
* @public exported from `@promptbook/core`
|
|
14638
|
-
*/
|
|
14639
|
-
// <- TODO: Maybe do better levels of trust
|
|
14640
|
-
/**
|
|
14641
|
-
* How is the model provider important?
|
|
14642
|
-
*
|
|
14643
|
-
* @public exported from `@promptbook/core`
|
|
14644
|
-
*/
|
|
14645
|
-
const MODEL_ORDER = {
|
|
14646
|
-
/**
|
|
14647
|
-
* Top-tier models, e.g. OpenAI, Anthropic,...
|
|
14648
|
-
*/
|
|
14649
|
-
TOP_TIER: 333,
|
|
14650
|
-
/**
|
|
14651
|
-
* Mid-tier models, e.g. Llama, Mistral, etc.
|
|
14652
|
-
*/
|
|
14653
|
-
NORMAL: 100,
|
|
14654
|
-
/**
|
|
14655
|
-
* Low-tier models, e.g. Phi, Tiny, etc.
|
|
14656
|
-
*/
|
|
14657
|
-
LOW_TIER: 0,
|
|
14658
|
-
};
|
|
14659
|
-
/**
|
|
14660
|
-
* TODO: Add configuration schema and maybe some documentation link
|
|
14661
|
-
* TODO: Maybe constrain LlmToolsConfiguration[number] by generic to ensure that `createConfigurationFromEnv` and `getBoilerplateConfiguration` always create same `packageName` and `className`
|
|
14662
|
-
* TODO: [®] DRY Register logic
|
|
14663
|
-
*/
|
|
14664
|
-
|
|
14665
15014
|
/**
|
|
14666
15015
|
* Registration of LLM provider metadata
|
|
14667
15016
|
*
|
|
@@ -14677,7 +15026,7 @@ const _AnthropicClaudeMetadataRegistration = $llmToolsMetadataRegister.register(
|
|
|
14677
15026
|
className: 'AnthropicClaudeExecutionTools',
|
|
14678
15027
|
envVariables: ['ANTHROPIC_CLAUDE_API_KEY'],
|
|
14679
15028
|
trustLevel: 'CLOSED',
|
|
14680
|
-
order:
|
|
15029
|
+
order: MODEL_ORDERS.TOP_TIER,
|
|
14681
15030
|
getBoilerplateConfiguration() {
|
|
14682
15031
|
return {
|
|
14683
15032
|
title: 'Anthropic Claude',
|
|
@@ -14687,6 +15036,7 @@ const _AnthropicClaudeMetadataRegistration = $llmToolsMetadataRegister.register(
|
|
|
14687
15036
|
apiKey: 'sk-ant-api03-',
|
|
14688
15037
|
isProxied: true,
|
|
14689
15038
|
remoteServerUrl: DEFAULT_REMOTE_SERVER_URL,
|
|
15039
|
+
maxRequestsPerMinute: DEFAULT_MAX_REQUESTS_PER_MINUTE,
|
|
14690
15040
|
},
|
|
14691
15041
|
};
|
|
14692
15042
|
},
|
|
@@ -14722,7 +15072,7 @@ function computeUsage(value) {
|
|
|
14722
15072
|
/**
|
|
14723
15073
|
* List of available Anthropic Claude models with pricing
|
|
14724
15074
|
*
|
|
14725
|
-
* Note: Done at
|
|
15075
|
+
* Note: Done at 2025-05-06
|
|
14726
15076
|
*
|
|
14727
15077
|
* @see https://docs.anthropic.com/en/docs/models-overview
|
|
14728
15078
|
* @public exported from `@promptbook/anthropic-claude`
|
|
@@ -14736,8 +15086,8 @@ const ANTHROPIC_CLAUDE_MODELS = exportJson({
|
|
|
14736
15086
|
modelName: 'claude-3-5-sonnet-20240620',
|
|
14737
15087
|
modelDescription: 'Latest Claude model with great reasoning, coding, and language understanding capabilities. 200K context window. Optimized balance of intelligence and speed.',
|
|
14738
15088
|
pricing: {
|
|
14739
|
-
prompt: computeUsage(`$
|
|
14740
|
-
output: computeUsage(`$
|
|
15089
|
+
prompt: computeUsage(`$2.50 / 1M tokens`),
|
|
15090
|
+
output: computeUsage(`$12.50 / 1M tokens`),
|
|
14741
15091
|
},
|
|
14742
15092
|
},
|
|
14743
15093
|
{
|
|
@@ -14746,8 +15096,8 @@ const ANTHROPIC_CLAUDE_MODELS = exportJson({
|
|
|
14746
15096
|
modelName: 'claude-3-opus-20240229',
|
|
14747
15097
|
modelDescription: 'Most capable Claude model excelling at complex reasoning, coding, and detailed instruction following. 200K context window. Best for sophisticated tasks requiring nuanced understanding.',
|
|
14748
15098
|
pricing: {
|
|
14749
|
-
prompt: computeUsage(`$
|
|
14750
|
-
output: computeUsage(`$
|
|
15099
|
+
prompt: computeUsage(`$12.00 / 1M tokens`),
|
|
15100
|
+
output: computeUsage(`$60.00 / 1M tokens`),
|
|
14751
15101
|
},
|
|
14752
15102
|
},
|
|
14753
15103
|
{
|
|
@@ -14806,8 +15156,8 @@ const ANTHROPIC_CLAUDE_MODELS = exportJson({
|
|
|
14806
15156
|
modelName: 'claude-3-7-sonnet-20250219',
|
|
14807
15157
|
modelDescription: 'Latest generation Claude model with advanced reasoning and language understanding. Enhanced capabilities over 3.5 with improved domain knowledge. 200K context window.',
|
|
14808
15158
|
pricing: {
|
|
14809
|
-
prompt: computeUsage(`$
|
|
14810
|
-
output: computeUsage(`$
|
|
15159
|
+
prompt: computeUsage(`$2.50 / 1M tokens`),
|
|
15160
|
+
output: computeUsage(`$12.50 / 1M tokens`),
|
|
14811
15161
|
},
|
|
14812
15162
|
},
|
|
14813
15163
|
{
|
|
@@ -14854,14 +15204,18 @@ function computeUsageCounts(content) {
|
|
|
14854
15204
|
/**
|
|
14855
15205
|
* Make UncertainNumber
|
|
14856
15206
|
*
|
|
14857
|
-
* @param value
|
|
15207
|
+
* @param value value of the uncertain number, if `NaN` or `undefined`, it will be set to 0 and `isUncertain=true`
|
|
15208
|
+
* @param isUncertain if `true`, the value is uncertain, otherwise depends on the value
|
|
14858
15209
|
*
|
|
14859
15210
|
* @private utility for initializating UncertainNumber
|
|
14860
15211
|
*/
|
|
14861
|
-
function uncertainNumber(value) {
|
|
15212
|
+
function uncertainNumber(value, isUncertain) {
|
|
14862
15213
|
if (value === null || value === undefined || Number.isNaN(value)) {
|
|
14863
15214
|
return UNCERTAIN_ZERO_VALUE;
|
|
14864
15215
|
}
|
|
15216
|
+
if (isUncertain === true) {
|
|
15217
|
+
return { value, isUncertain };
|
|
15218
|
+
}
|
|
14865
15219
|
return { value };
|
|
14866
15220
|
}
|
|
14867
15221
|
|
|
@@ -14927,6 +15281,8 @@ class AnthropicClaudeExecutionTools {
|
|
|
14927
15281
|
* Anthropic Claude API client.
|
|
14928
15282
|
*/
|
|
14929
15283
|
this.client = null;
|
|
15284
|
+
const rate = this.options.maxRequestsPerMinute || DEFAULT_MAX_REQUESTS_PER_MINUTE;
|
|
15285
|
+
this.limiter = new Bottleneck({ minTime: 60000 / rate });
|
|
14930
15286
|
}
|
|
14931
15287
|
get title() {
|
|
14932
15288
|
return 'Anthropic Claude';
|
|
@@ -14978,8 +15334,6 @@ class AnthropicClaudeExecutionTools {
|
|
|
14978
15334
|
// <- TODO: [🌾] Make some global max cap for maxTokens
|
|
14979
15335
|
temperature: modelRequirements.temperature,
|
|
14980
15336
|
system: modelRequirements.systemMessage,
|
|
14981
|
-
// <- TODO: [🈁] Use `seed` here AND/OR use is `isDeterministic` for entire execution tools
|
|
14982
|
-
// <- Note: [🧆]
|
|
14983
15337
|
messages: [
|
|
14984
15338
|
{
|
|
14985
15339
|
role: 'user',
|
|
@@ -14988,14 +15342,14 @@ class AnthropicClaudeExecutionTools {
|
|
|
14988
15342
|
// @see https://docs.anthropic.com/en/docs/test-and-evaluate/strengthen-guardrails/increase-consistency#specify-the-desired-output-format
|
|
14989
15343
|
},
|
|
14990
15344
|
],
|
|
14991
|
-
// TODO: Is here some equivalent of user identification?> user: this.options.user,
|
|
14992
15345
|
};
|
|
14993
15346
|
const start = $getCurrentDate();
|
|
14994
|
-
let complete;
|
|
14995
15347
|
if (this.options.isVerbose) {
|
|
14996
15348
|
console.info(colors.bgWhite('rawRequest'), JSON.stringify(rawRequest, null, 4));
|
|
14997
15349
|
}
|
|
14998
|
-
const rawResponse = await
|
|
15350
|
+
const rawResponse = await this.limiter
|
|
15351
|
+
.schedule(() => client.messages.create(rawRequest))
|
|
15352
|
+
.catch((error) => {
|
|
14999
15353
|
if (this.options.isVerbose) {
|
|
15000
15354
|
console.info(colors.bgRed('error'), error);
|
|
15001
15355
|
}
|
|
@@ -15015,12 +15369,11 @@ class AnthropicClaudeExecutionTools {
|
|
|
15015
15369
|
throw new PipelineExecutionError(`Returned content is not "text" type but "${contentBlock.type}"`);
|
|
15016
15370
|
}
|
|
15017
15371
|
const resultContent = contentBlock.text;
|
|
15018
|
-
|
|
15019
|
-
complete = $getCurrentDate();
|
|
15372
|
+
const complete = $getCurrentDate();
|
|
15020
15373
|
const usage = computeAnthropicClaudeUsage(rawPromptContent || '', resultContent || '', rawResponse);
|
|
15021
15374
|
return exportJson({
|
|
15022
15375
|
name: 'promptResult',
|
|
15023
|
-
message: `Result of \`
|
|
15376
|
+
message: `Result of \`AnthropicClaudeExecutionTools.callChatModel\``,
|
|
15024
15377
|
order: [],
|
|
15025
15378
|
value: {
|
|
15026
15379
|
content: resultContent,
|
|
@@ -15037,83 +15390,59 @@ class AnthropicClaudeExecutionTools {
|
|
|
15037
15390
|
},
|
|
15038
15391
|
});
|
|
15039
15392
|
}
|
|
15040
|
-
|
|
15041
|
-
|
|
15042
|
-
|
|
15043
|
-
|
|
15044
|
-
): Promise<CompletionPromptResult> {
|
|
15045
|
-
|
|
15393
|
+
/**
|
|
15394
|
+
* Calls Anthropic Claude API to use a completion model.
|
|
15395
|
+
*/
|
|
15396
|
+
async callCompletionModel(prompt) {
|
|
15046
15397
|
if (this.options.isVerbose) {
|
|
15047
15398
|
console.info('🖋 Anthropic Claude callCompletionModel call');
|
|
15048
15399
|
}
|
|
15049
|
-
|
|
15050
15400
|
const { content, parameters, modelRequirements } = prompt;
|
|
15051
|
-
|
|
15052
|
-
// TODO: [☂] Use here more modelRequirements
|
|
15053
15401
|
if (modelRequirements.modelVariant !== 'COMPLETION') {
|
|
15054
15402
|
throw new PipelineExecutionError('Use callCompletionModel only for COMPLETION variant');
|
|
15055
15403
|
}
|
|
15056
|
-
|
|
15404
|
+
const client = await this.getClient();
|
|
15057
15405
|
const modelName = modelRequirements.modelName || this.getDefaultChatModel().modelName;
|
|
15058
|
-
const
|
|
15406
|
+
const rawPromptContent = templateParameters(content, { ...parameters, modelName });
|
|
15407
|
+
const rawRequest = {
|
|
15059
15408
|
model: modelName,
|
|
15060
|
-
|
|
15061
|
-
|
|
15062
|
-
// <- TODO: Use here `systemMessage`, `temperature` and `seed`
|
|
15063
|
-
};
|
|
15064
|
-
|
|
15065
|
-
const rawRequest: xxxx.Completions.CompletionCreateParamsNonStreaming = {
|
|
15066
|
-
...modelSettings,
|
|
15409
|
+
max_tokens_to_sample: modelRequirements.maxTokens || 2000,
|
|
15410
|
+
temperature: modelRequirements.temperature,
|
|
15067
15411
|
prompt: rawPromptContent,
|
|
15068
|
-
user: this.options.user,
|
|
15069
15412
|
};
|
|
15070
|
-
const start
|
|
15071
|
-
|
|
15072
|
-
|
|
15073
|
-
|
|
15074
|
-
|
|
15075
|
-
|
|
15076
|
-
|
|
15077
|
-
|
|
15078
|
-
|
|
15079
|
-
}
|
|
15080
|
-
throw error;
|
|
15081
|
-
});
|
|
15082
|
-
|
|
15083
|
-
|
|
15413
|
+
const start = $getCurrentDate();
|
|
15414
|
+
const rawResponse = await this.limiter
|
|
15415
|
+
.schedule(() => client.completions.create(rawRequest))
|
|
15416
|
+
.catch((error) => {
|
|
15417
|
+
if (this.options.isVerbose) {
|
|
15418
|
+
console.info(colors.bgRed('error'), error);
|
|
15419
|
+
}
|
|
15420
|
+
throw error;
|
|
15421
|
+
});
|
|
15084
15422
|
if (this.options.isVerbose) {
|
|
15085
15423
|
console.info(colors.bgWhite('rawResponse'), JSON.stringify(rawResponse, null, 4));
|
|
15086
15424
|
}
|
|
15087
|
-
|
|
15088
|
-
|
|
15089
|
-
throw new PipelineExecutionError('No choises from Anthropic Claude');
|
|
15090
|
-
}
|
|
15091
|
-
|
|
15092
|
-
if (rawResponse.choices.length > 1) {
|
|
15093
|
-
// TODO: This should be maybe only warning
|
|
15094
|
-
throw new PipelineExecutionError('More than one choise from Anthropic Claude');
|
|
15425
|
+
if (!rawResponse.completion) {
|
|
15426
|
+
throw new PipelineExecutionError('No completion from Anthropic Claude');
|
|
15095
15427
|
}
|
|
15096
|
-
|
|
15097
|
-
const
|
|
15098
|
-
|
|
15099
|
-
|
|
15100
|
-
|
|
15101
|
-
|
|
15102
|
-
|
|
15103
|
-
|
|
15104
|
-
|
|
15105
|
-
|
|
15106
|
-
|
|
15107
|
-
|
|
15108
|
-
|
|
15109
|
-
|
|
15428
|
+
const resultContent = rawResponse.completion;
|
|
15429
|
+
const complete = $getCurrentDate();
|
|
15430
|
+
const usage = computeAnthropicClaudeUsage(rawPromptContent, resultContent, rawResponse);
|
|
15431
|
+
return exportJson({
|
|
15432
|
+
name: 'promptResult',
|
|
15433
|
+
message: `Result of \`AnthropicClaudeExecutionTools.callCompletionModel\``,
|
|
15434
|
+
order: [],
|
|
15435
|
+
value: {
|
|
15436
|
+
content: resultContent,
|
|
15437
|
+
modelName: rawResponse.model || modelName,
|
|
15438
|
+
timing: { start, complete },
|
|
15439
|
+
usage,
|
|
15440
|
+
rawPromptContent,
|
|
15441
|
+
rawRequest,
|
|
15442
|
+
rawResponse,
|
|
15110
15443
|
},
|
|
15111
|
-
usage,
|
|
15112
|
-
rawResponse,
|
|
15113
|
-
// <- [🗯]
|
|
15114
15444
|
});
|
|
15115
15445
|
}
|
|
15116
|
-
*/
|
|
15117
15446
|
// <- Note: [🤖] callXxxModel
|
|
15118
15447
|
/**
|
|
15119
15448
|
* Get the model that should be used as default
|
|
@@ -15122,7 +15451,7 @@ class AnthropicClaudeExecutionTools {
|
|
|
15122
15451
|
const model = ANTHROPIC_CLAUDE_MODELS.find(({ modelName }) => modelName.startsWith(defaultModelName));
|
|
15123
15452
|
if (model === undefined) {
|
|
15124
15453
|
throw new UnexpectedError(spaceTrim((block) => `
|
|
15125
|
-
Cannot find model in
|
|
15454
|
+
Cannot find model in Anthropic Claude models with name "${defaultModelName}" which should be used as default.
|
|
15126
15455
|
|
|
15127
15456
|
Available models:
|
|
15128
15457
|
${block(ANTHROPIC_CLAUDE_MODELS.map(({ modelName }) => `- "${modelName}"`).join('\n'))}
|
|
@@ -15215,9 +15544,9 @@ const _AzureOpenAiMetadataRegistration = $llmToolsMetadataRegister.register({
|
|
|
15215
15544
|
title: 'Azure Open AI',
|
|
15216
15545
|
packageName: '@promptbook/azure-openai',
|
|
15217
15546
|
className: 'AzureOpenAiExecutionTools',
|
|
15218
|
-
envVariables: ['
|
|
15547
|
+
envVariables: ['AZUREOPENAI_API_KEY', 'AZUREOPENAI_RESOURCE_NAME', 'AZUREOPENAI_DEPLOYMENT_NAME'],
|
|
15219
15548
|
trustLevel: 'CLOSED_BUSINESS',
|
|
15220
|
-
order:
|
|
15549
|
+
order: MODEL_ORDERS.NORMAL,
|
|
15221
15550
|
getBoilerplateConfiguration() {
|
|
15222
15551
|
return {
|
|
15223
15552
|
title: 'Azure Open AI',
|
|
@@ -15225,6 +15554,9 @@ const _AzureOpenAiMetadataRegistration = $llmToolsMetadataRegister.register({
|
|
|
15225
15554
|
className: 'AzureOpenAiExecutionTools',
|
|
15226
15555
|
options: {
|
|
15227
15556
|
apiKey: 'sk-',
|
|
15557
|
+
resourceName: 'my-resource-name',
|
|
15558
|
+
deploymentName: 'my-deployment-name',
|
|
15559
|
+
maxRequestsPerMinute: DEFAULT_MAX_REQUESTS_PER_MINUTE,
|
|
15228
15560
|
},
|
|
15229
15561
|
};
|
|
15230
15562
|
},
|
|
@@ -15238,15 +15570,15 @@ const _AzureOpenAiMetadataRegistration = $llmToolsMetadataRegister.register({
|
|
|
15238
15570
|
packageName: '@promptbook/azure-openai',
|
|
15239
15571
|
className: 'AzureOpenAiExecutionTools',
|
|
15240
15572
|
options: {
|
|
15573
|
+
apiKey: env.AZUREOPENAI_API_KEY,
|
|
15241
15574
|
resourceName: env.AZUREOPENAI_RESOURCE_NAME,
|
|
15242
15575
|
deploymentName: env.AZUREOPENAI_DEPLOYMENT_NAME,
|
|
15243
|
-
apiKey: env.AZUREOPENAI_API_KEY,
|
|
15244
15576
|
},
|
|
15245
15577
|
};
|
|
15246
15578
|
}
|
|
15247
|
-
else if (typeof env.
|
|
15248
|
-
typeof env.
|
|
15249
|
-
typeof env.
|
|
15579
|
+
else if (typeof env.AZUREOPENAI_API_KEY === 'string' ||
|
|
15580
|
+
typeof env.AZUREOPENAI_RESOURCE_NAME === 'string' ||
|
|
15581
|
+
typeof env.AZUREOPENAI_DEPLOYMENT_NAME === 'string') {
|
|
15250
15582
|
return null;
|
|
15251
15583
|
/*
|
|
15252
15584
|
Note: [🗨] Partial configuration is handled more gracefully elsewhere
|
|
@@ -15275,7 +15607,7 @@ const _AzureOpenAiMetadataRegistration = $llmToolsMetadataRegister.register({
|
|
|
15275
15607
|
/**
|
|
15276
15608
|
* List of available OpenAI models with pricing
|
|
15277
15609
|
*
|
|
15278
|
-
* Note: Done at
|
|
15610
|
+
* Note: Done at 2025-05-06
|
|
15279
15611
|
*
|
|
15280
15612
|
* @see https://platform.openai.com/docs/models/
|
|
15281
15613
|
* @see https://openai.com/api/pricing/
|
|
@@ -15536,7 +15868,7 @@ const OPENAI_MODELS = exportJson({
|
|
|
15536
15868
|
modelName: 'text-embedding-3-large',
|
|
15537
15869
|
modelDescription: "OpenAI's most capable text embedding model designed for high-quality embeddings for complex similarity tasks and information retrieval.",
|
|
15538
15870
|
pricing: {
|
|
15539
|
-
prompt: computeUsage(`$0.13
|
|
15871
|
+
prompt: computeUsage(`$0.13 / 1M tokens`),
|
|
15540
15872
|
// TODO: [🏏] Leverage the batch API @see https://platform.openai.com/docs/guides/batch
|
|
15541
15873
|
output: 0, // <- Note: [🆖] In Embedding models you dont pay for output
|
|
15542
15874
|
},
|
|
@@ -15629,8 +15961,8 @@ const OPENAI_MODELS = exportJson({
|
|
|
15629
15961
|
modelName: 'gpt-4o-mini',
|
|
15630
15962
|
modelDescription: 'Smaller, more cost-effective version of GPT-4o with good performance across text, vision, and audio tasks at reduced complexity.',
|
|
15631
15963
|
pricing: {
|
|
15632
|
-
prompt: computeUsage(`$
|
|
15633
|
-
output: computeUsage(`$
|
|
15964
|
+
prompt: computeUsage(`$0.15 / 1M tokens`),
|
|
15965
|
+
output: computeUsage(`$0.60 / 1M tokens`),
|
|
15634
15966
|
},
|
|
15635
15967
|
},
|
|
15636
15968
|
/**/
|
|
@@ -15652,7 +15984,7 @@ const OPENAI_MODELS = exportJson({
|
|
|
15652
15984
|
modelTitle: 'o1-preview-2024-09-12',
|
|
15653
15985
|
modelName: 'o1-preview-2024-09-12',
|
|
15654
15986
|
modelDescription: 'September 2024 version of O1 preview with specialized reasoning capabilities for complex tasks requiring precise analytical thinking.',
|
|
15655
|
-
// <- TODO: [💩] Some better system to organize
|
|
15987
|
+
// <- TODO: [💩] Some better system to organize these date suffixes and versions
|
|
15656
15988
|
pricing: {
|
|
15657
15989
|
prompt: computeUsage(`$15.00 / 1M tokens`),
|
|
15658
15990
|
output: computeUsage(`$60.00 / 1M tokens`),
|
|
@@ -15738,9 +16070,6 @@ const OPENAI_MODELS = exportJson({
|
|
|
15738
16070
|
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
15739
16071
|
*/
|
|
15740
16072
|
|
|
15741
|
-
// Default rate limits (requests per minute) - adjust as needed based on Azure OpenAI tier
|
|
15742
|
-
const DEFAULT_RPM$1 = 60;
|
|
15743
|
-
// <- TODO: !!! Put in some better place
|
|
15744
16073
|
/**
|
|
15745
16074
|
* Execution Tools for calling Azure OpenAI API.
|
|
15746
16075
|
*
|
|
@@ -15760,7 +16089,7 @@ class AzureOpenAiExecutionTools {
|
|
|
15760
16089
|
this.client = null;
|
|
15761
16090
|
// TODO: Allow configuring rate limits via options
|
|
15762
16091
|
this.limiter = new Bottleneck({
|
|
15763
|
-
minTime: 60000 / (this.options.maxRequestsPerMinute ||
|
|
16092
|
+
minTime: 60000 / (this.options.maxRequestsPerMinute || DEFAULT_MAX_REQUESTS_PER_MINUTE),
|
|
15764
16093
|
});
|
|
15765
16094
|
}
|
|
15766
16095
|
get title() {
|
|
@@ -16014,7 +16343,7 @@ class AzureOpenAiExecutionTools {
|
|
|
16014
16343
|
});
|
|
16015
16344
|
}
|
|
16016
16345
|
/**
|
|
16017
|
-
* Changes Azure error (which is not
|
|
16346
|
+
* Changes Azure error (which is not proper Error but object) to proper Error
|
|
16018
16347
|
*/
|
|
16019
16348
|
transformAzureError(azureError) {
|
|
16020
16349
|
if (azureError instanceof UnexpectedError) {
|
|
@@ -16080,7 +16409,7 @@ const _DeepseekMetadataRegistration = $llmToolsMetadataRegister.register({
|
|
|
16080
16409
|
className: 'DeepseekExecutionTools',
|
|
16081
16410
|
envVariables: ['DEEPSEEK_GENERATIVE_AI_API_KEY'],
|
|
16082
16411
|
trustLevel: 'UNTRUSTED',
|
|
16083
|
-
order:
|
|
16412
|
+
order: MODEL_ORDERS.NORMAL,
|
|
16084
16413
|
getBoilerplateConfiguration() {
|
|
16085
16414
|
return {
|
|
16086
16415
|
title: 'Deepseek',
|
|
@@ -16160,12 +16489,14 @@ function createExecutionToolsFromVercelProvider(options) {
|
|
|
16160
16489
|
const { vercelProvider, availableModels, userId, additionalChatSettings = {} } = options;
|
|
16161
16490
|
if (!/Vercel/i.test(title)) {
|
|
16162
16491
|
title = `${title} (through Vercel)`;
|
|
16492
|
+
// <- TODO: [🧈] Maybe standartize the suffix
|
|
16163
16493
|
} /* not else */
|
|
16164
16494
|
if (description === undefined) {
|
|
16165
16495
|
description = `Implementation of ${title} through Vercel`;
|
|
16166
16496
|
} /* not else */
|
|
16167
16497
|
if (!/Vercel/i.test(description)) {
|
|
16168
16498
|
description = `${description} (through Vercel)`;
|
|
16499
|
+
// <- TODO: [🧈] Maybe standartize the suffix
|
|
16169
16500
|
} /* not else */
|
|
16170
16501
|
return {
|
|
16171
16502
|
title,
|
|
@@ -16283,7 +16614,7 @@ function createExecutionToolsFromVercelProvider(options) {
|
|
|
16283
16614
|
/**
|
|
16284
16615
|
* List of available Deepseek models with descriptions
|
|
16285
16616
|
*
|
|
16286
|
-
* Note: Done at 2025-
|
|
16617
|
+
* Note: Done at 2025-05-06
|
|
16287
16618
|
*
|
|
16288
16619
|
* @see https://www.deepseek.com/models
|
|
16289
16620
|
* @public exported from `@promptbook/deepseek`
|
|
@@ -16297,8 +16628,8 @@ const DEEPSEEK_MODELS = exportJson({
|
|
|
16297
16628
|
modelName: 'deepseek-chat',
|
|
16298
16629
|
modelDescription: 'General-purpose language model with strong performance across conversation, reasoning, and content generation. 128K context window with excellent instruction following capabilities.',
|
|
16299
16630
|
pricing: {
|
|
16300
|
-
prompt: computeUsage(`$
|
|
16301
|
-
output: computeUsage(`$
|
|
16631
|
+
prompt: computeUsage(`$0.80 / 1M tokens`),
|
|
16632
|
+
output: computeUsage(`$1.60 / 1M tokens`),
|
|
16302
16633
|
},
|
|
16303
16634
|
},
|
|
16304
16635
|
{
|
|
@@ -16307,8 +16638,8 @@ const DEEPSEEK_MODELS = exportJson({
|
|
|
16307
16638
|
modelName: 'deepseek-reasoner',
|
|
16308
16639
|
modelDescription: 'Specialized model focused on complex reasoning tasks like mathematical problem-solving and logical analysis. Enhanced step-by-step reasoning with explicit chain-of-thought processes. 128K context window.',
|
|
16309
16640
|
pricing: {
|
|
16310
|
-
prompt: computeUsage(`$
|
|
16311
|
-
output: computeUsage(`$
|
|
16641
|
+
prompt: computeUsage(`$3.50 / 1M tokens`),
|
|
16642
|
+
output: computeUsage(`$7.00 / 1M tokens`),
|
|
16312
16643
|
},
|
|
16313
16644
|
},
|
|
16314
16645
|
{
|
|
@@ -16403,7 +16734,7 @@ const _GoogleMetadataRegistration = $llmToolsMetadataRegister.register({
|
|
|
16403
16734
|
className: 'GoogleExecutionTools',
|
|
16404
16735
|
envVariables: ['GOOGLE_GENERATIVE_AI_API_KEY'],
|
|
16405
16736
|
trustLevel: 'CLOSED',
|
|
16406
|
-
order:
|
|
16737
|
+
order: MODEL_ORDERS.NORMAL,
|
|
16407
16738
|
getBoilerplateConfiguration() {
|
|
16408
16739
|
return {
|
|
16409
16740
|
title: 'Google Gemini',
|
|
@@ -16442,7 +16773,7 @@ const _GoogleMetadataRegistration = $llmToolsMetadataRegister.register({
|
|
|
16442
16773
|
/**
|
|
16443
16774
|
* List of available Google models with descriptions
|
|
16444
16775
|
*
|
|
16445
|
-
* Note: Done at 2025-
|
|
16776
|
+
* Note: Done at 2025-05-06
|
|
16446
16777
|
*
|
|
16447
16778
|
* @see https://ai.google.dev/models/gemini
|
|
16448
16779
|
* @public exported from `@promptbook/google`
|
|
@@ -16456,8 +16787,8 @@ const GOOGLE_MODELS = exportJson({
|
|
|
16456
16787
|
modelName: 'gemini-2.5-pro-preview-03-25',
|
|
16457
16788
|
modelDescription: 'Latest advanced multimodal model with exceptional reasoning, tool use, and instruction following. 1M token context window with improved vision capabilities for complex visual tasks.',
|
|
16458
16789
|
pricing: {
|
|
16459
|
-
prompt: computeUsage(`$
|
|
16460
|
-
output: computeUsage(`$
|
|
16790
|
+
prompt: computeUsage(`$8.00 / 1M tokens`),
|
|
16791
|
+
output: computeUsage(`$24.00 / 1M tokens`),
|
|
16461
16792
|
},
|
|
16462
16793
|
},
|
|
16463
16794
|
{
|
|
@@ -16496,8 +16827,8 @@ const GOOGLE_MODELS = exportJson({
|
|
|
16496
16827
|
modelName: 'gemini-1.5-flash',
|
|
16497
16828
|
modelDescription: 'Efficient model balancing speed and quality for general-purpose applications. 1M token context window with good multimodal capabilities and quick response times.',
|
|
16498
16829
|
pricing: {
|
|
16499
|
-
prompt: computeUsage(`$0.
|
|
16500
|
-
output: computeUsage(`$
|
|
16830
|
+
prompt: computeUsage(`$0.25 / 1M tokens`),
|
|
16831
|
+
output: computeUsage(`$0.75 / 1M tokens`),
|
|
16501
16832
|
},
|
|
16502
16833
|
},
|
|
16503
16834
|
{
|
|
@@ -16564,8 +16895,8 @@ const GOOGLE_MODELS = exportJson({
|
|
|
16564
16895
|
modelName: 'gemini-1.5-pro',
|
|
16565
16896
|
modelDescription: 'Flagship multimodal model with strong performance across text, code, vision, and audio tasks. 1M token context window with excellent reasoning capabilities.',
|
|
16566
16897
|
pricing: {
|
|
16567
|
-
prompt: computeUsage(`$
|
|
16568
|
-
output: computeUsage(`$
|
|
16898
|
+
prompt: computeUsage(`$6.00 / 1M tokens`),
|
|
16899
|
+
output: computeUsage(`$18.00 / 1M tokens`),
|
|
16569
16900
|
},
|
|
16570
16901
|
},
|
|
16571
16902
|
{
|
|
@@ -16668,7 +16999,7 @@ const _OpenAiMetadataRegistration = $llmToolsMetadataRegister.register({
|
|
|
16668
16999
|
className: 'OpenAiExecutionTools',
|
|
16669
17000
|
envVariables: ['OPENAI_API_KEY'],
|
|
16670
17001
|
trustLevel: 'CLOSED',
|
|
16671
|
-
order:
|
|
17002
|
+
order: MODEL_ORDERS.TOP_TIER,
|
|
16672
17003
|
getBoilerplateConfiguration() {
|
|
16673
17004
|
return {
|
|
16674
17005
|
title: 'Open AI',
|
|
@@ -16676,6 +17007,7 @@ const _OpenAiMetadataRegistration = $llmToolsMetadataRegister.register({
|
|
|
16676
17007
|
className: 'OpenAiExecutionTools',
|
|
16677
17008
|
options: {
|
|
16678
17009
|
apiKey: 'sk-',
|
|
17010
|
+
maxRequestsPerMinute: DEFAULT_MAX_REQUESTS_PER_MINUTE,
|
|
16679
17011
|
},
|
|
16680
17012
|
};
|
|
16681
17013
|
},
|
|
@@ -16695,9 +17027,9 @@ const _OpenAiMetadataRegistration = $llmToolsMetadataRegister.register({
|
|
|
16695
17027
|
},
|
|
16696
17028
|
});
|
|
16697
17029
|
/**
|
|
16698
|
-
*
|
|
17030
|
+
* Registration of the OpenAI Assistant metadata
|
|
16699
17031
|
*
|
|
16700
|
-
* Note: [🏐] Configurations registrations are done in
|
|
17032
|
+
* Note: [🏐] Configurations registrations are done in the metadata registration section, but the constructor registration is handled separately.
|
|
16701
17033
|
*
|
|
16702
17034
|
* @public exported from `@promptbook/core`
|
|
16703
17035
|
* @public exported from `@promptbook/wizzard`
|
|
@@ -16710,7 +17042,7 @@ const _OpenAiAssistantMetadataRegistration = $llmToolsMetadataRegister.register(
|
|
|
16710
17042
|
envVariables: null,
|
|
16711
17043
|
// <- TODO: ['OPENAI_API_KEY', 'OPENAI_ASSISTANT_ID']
|
|
16712
17044
|
trustLevel: 'CLOSED',
|
|
16713
|
-
order:
|
|
17045
|
+
order: MODEL_ORDERS.NORMAL,
|
|
16714
17046
|
getBoilerplateConfiguration() {
|
|
16715
17047
|
return {
|
|
16716
17048
|
title: 'Open AI Assistant',
|
|
@@ -16719,6 +17051,7 @@ const _OpenAiAssistantMetadataRegistration = $llmToolsMetadataRegister.register(
|
|
|
16719
17051
|
options: {
|
|
16720
17052
|
apiKey: 'sk-',
|
|
16721
17053
|
assistantId: 'asst_',
|
|
17054
|
+
maxRequestsPerMinute: DEFAULT_MAX_REQUESTS_PER_MINUTE,
|
|
16722
17055
|
},
|
|
16723
17056
|
};
|
|
16724
17057
|
},
|
|
@@ -16765,13 +17098,21 @@ resultContent, rawResponse) {
|
|
|
16765
17098
|
}
|
|
16766
17099
|
const inputTokens = rawResponse.usage.prompt_tokens;
|
|
16767
17100
|
const outputTokens = ((_b = rawResponse.usage) === null || _b === void 0 ? void 0 : _b.completion_tokens) || 0;
|
|
16768
|
-
|
|
17101
|
+
let isUncertain = false;
|
|
17102
|
+
let modelInfo = OPENAI_MODELS.find((model) => model.modelName === rawResponse.model);
|
|
17103
|
+
if (modelInfo === undefined) {
|
|
17104
|
+
// Note: Model is not in the list of known models, fallback to the family of the models and mark price as uncertain
|
|
17105
|
+
modelInfo = OPENAI_MODELS.find((model) => (rawResponse.model || SALT_NONCE).startsWith(model.modelName));
|
|
17106
|
+
if (modelInfo !== undefined) {
|
|
17107
|
+
isUncertain = true;
|
|
17108
|
+
}
|
|
17109
|
+
}
|
|
16769
17110
|
let price;
|
|
16770
17111
|
if (modelInfo === undefined || modelInfo.pricing === undefined) {
|
|
16771
17112
|
price = uncertainNumber();
|
|
16772
17113
|
}
|
|
16773
17114
|
else {
|
|
16774
|
-
price = uncertainNumber(inputTokens * modelInfo.pricing.prompt + outputTokens * modelInfo.pricing.output);
|
|
17115
|
+
price = uncertainNumber(inputTokens * modelInfo.pricing.prompt + outputTokens * modelInfo.pricing.output, isUncertain);
|
|
16775
17116
|
}
|
|
16776
17117
|
return {
|
|
16777
17118
|
price,
|
|
@@ -16789,9 +17130,6 @@ resultContent, rawResponse) {
|
|
|
16789
17130
|
* TODO: [🤝] DRY Maybe some common abstraction between `computeOpenAiUsage` and `computeAnthropicClaudeUsage`
|
|
16790
17131
|
*/
|
|
16791
17132
|
|
|
16792
|
-
// Default rate limits (requests per minute) - adjust as needed based on OpenAI tier
|
|
16793
|
-
const DEFAULT_RPM = 60;
|
|
16794
|
-
// <- TODO: !!! Put in some better place
|
|
16795
17133
|
/**
|
|
16796
17134
|
* Execution Tools for calling OpenAI API
|
|
16797
17135
|
*
|
|
@@ -16811,7 +17149,7 @@ class OpenAiExecutionTools {
|
|
|
16811
17149
|
this.client = null;
|
|
16812
17150
|
// TODO: Allow configuring rate limits via options
|
|
16813
17151
|
this.limiter = new Bottleneck({
|
|
16814
|
-
minTime: 60000 / (this.options.maxRequestsPerMinute ||
|
|
17152
|
+
minTime: 60000 / (this.options.maxRequestsPerMinute || DEFAULT_MAX_REQUESTS_PER_MINUTE),
|
|
16815
17153
|
});
|
|
16816
17154
|
}
|
|
16817
17155
|
get title() {
|
|
@@ -16912,7 +17250,6 @@ class OpenAiExecutionTools {
|
|
|
16912
17250
|
user: (_a = this.options.userId) === null || _a === void 0 ? void 0 : _a.toString(),
|
|
16913
17251
|
};
|
|
16914
17252
|
const start = $getCurrentDate();
|
|
16915
|
-
let complete;
|
|
16916
17253
|
if (this.options.isVerbose) {
|
|
16917
17254
|
console.info(colors.bgWhite('rawRequest'), JSON.stringify(rawRequest, null, 4));
|
|
16918
17255
|
}
|
|
@@ -16928,6 +17265,7 @@ class OpenAiExecutionTools {
|
|
|
16928
17265
|
if (this.options.isVerbose) {
|
|
16929
17266
|
console.info(colors.bgWhite('rawResponse'), JSON.stringify(rawResponse, null, 4));
|
|
16930
17267
|
}
|
|
17268
|
+
const complete = $getCurrentDate();
|
|
16931
17269
|
if (!rawResponse.choices[0]) {
|
|
16932
17270
|
throw new PipelineExecutionError('No choises from OpenAI');
|
|
16933
17271
|
}
|
|
@@ -16936,8 +17274,6 @@ class OpenAiExecutionTools {
|
|
|
16936
17274
|
throw new PipelineExecutionError('More than one choise from OpenAI');
|
|
16937
17275
|
}
|
|
16938
17276
|
const resultContent = rawResponse.choices[0].message.content;
|
|
16939
|
-
// eslint-disable-next-line prefer-const
|
|
16940
|
-
complete = $getCurrentDate();
|
|
16941
17277
|
const usage = computeOpenAiUsage(content || '', resultContent || '', rawResponse);
|
|
16942
17278
|
if (resultContent === null) {
|
|
16943
17279
|
throw new PipelineExecutionError('No response message from OpenAI');
|
|
@@ -16991,7 +17327,6 @@ class OpenAiExecutionTools {
|
|
|
16991
17327
|
user: (_a = this.options.userId) === null || _a === void 0 ? void 0 : _a.toString(),
|
|
16992
17328
|
};
|
|
16993
17329
|
const start = $getCurrentDate();
|
|
16994
|
-
let complete;
|
|
16995
17330
|
if (this.options.isVerbose) {
|
|
16996
17331
|
console.info(colors.bgWhite('rawRequest'), JSON.stringify(rawRequest, null, 4));
|
|
16997
17332
|
}
|
|
@@ -17007,6 +17342,7 @@ class OpenAiExecutionTools {
|
|
|
17007
17342
|
if (this.options.isVerbose) {
|
|
17008
17343
|
console.info(colors.bgWhite('rawResponse'), JSON.stringify(rawResponse, null, 4));
|
|
17009
17344
|
}
|
|
17345
|
+
const complete = $getCurrentDate();
|
|
17010
17346
|
if (!rawResponse.choices[0]) {
|
|
17011
17347
|
throw new PipelineExecutionError('No choises from OpenAI');
|
|
17012
17348
|
}
|
|
@@ -17015,8 +17351,6 @@ class OpenAiExecutionTools {
|
|
|
17015
17351
|
throw new PipelineExecutionError('More than one choise from OpenAI');
|
|
17016
17352
|
}
|
|
17017
17353
|
const resultContent = rawResponse.choices[0].text;
|
|
17018
|
-
// eslint-disable-next-line prefer-const
|
|
17019
|
-
complete = $getCurrentDate();
|
|
17020
17354
|
const usage = computeOpenAiUsage(content || '', resultContent || '', rawResponse);
|
|
17021
17355
|
return exportJson({
|
|
17022
17356
|
name: 'promptResult',
|
|
@@ -17057,7 +17391,6 @@ class OpenAiExecutionTools {
|
|
|
17057
17391
|
model: modelName,
|
|
17058
17392
|
};
|
|
17059
17393
|
const start = $getCurrentDate();
|
|
17060
|
-
let complete;
|
|
17061
17394
|
if (this.options.isVerbose) {
|
|
17062
17395
|
console.info(colors.bgWhite('rawRequest'), JSON.stringify(rawRequest, null, 4));
|
|
17063
17396
|
}
|
|
@@ -17073,12 +17406,11 @@ class OpenAiExecutionTools {
|
|
|
17073
17406
|
if (this.options.isVerbose) {
|
|
17074
17407
|
console.info(colors.bgWhite('rawResponse'), JSON.stringify(rawResponse, null, 4));
|
|
17075
17408
|
}
|
|
17409
|
+
const complete = $getCurrentDate();
|
|
17076
17410
|
if (rawResponse.data.length !== 1) {
|
|
17077
17411
|
throw new PipelineExecutionError(`Expected exactly 1 data item in response, got ${rawResponse.data.length}`);
|
|
17078
17412
|
}
|
|
17079
17413
|
const resultContent = rawResponse.data[0].embedding;
|
|
17080
|
-
// eslint-disable-next-line prefer-const
|
|
17081
|
-
complete = $getCurrentDate();
|
|
17082
17414
|
const usage = computeOpenAiUsage(content || '', '',
|
|
17083
17415
|
// <- Note: Embedding does not have result content
|
|
17084
17416
|
rawResponse);
|
|
@@ -17106,7 +17438,8 @@ class OpenAiExecutionTools {
|
|
|
17106
17438
|
* Get the model that should be used as default
|
|
17107
17439
|
*/
|
|
17108
17440
|
getDefaultModel(defaultModelName) {
|
|
17109
|
-
|
|
17441
|
+
// Note: Match exact or prefix for model families
|
|
17442
|
+
const model = OPENAI_MODELS.find(({ modelName }) => modelName === defaultModelName || modelName.startsWith(defaultModelName));
|
|
17110
17443
|
if (model === undefined) {
|
|
17111
17444
|
throw new UnexpectedError(spaceTrim((block) => `
|
|
17112
17445
|
Cannot find model in OpenAI models with name "${defaultModelName}" which should be used as default.
|
|
@@ -17148,7 +17481,7 @@ class OpenAiExecutionTools {
|
|
|
17148
17481
|
/**
|
|
17149
17482
|
* Execution Tools for calling OpenAI API Assistants
|
|
17150
17483
|
*
|
|
17151
|
-
* This is
|
|
17484
|
+
* This is useful for calling OpenAI API with a single assistant, for more wide usage use `OpenAiExecutionTools`.
|
|
17152
17485
|
*
|
|
17153
17486
|
* @public exported from `@promptbook/openai`
|
|
17154
17487
|
*/
|
|
@@ -17161,6 +17494,7 @@ class OpenAiAssistantExecutionTools extends OpenAiExecutionTools {
|
|
|
17161
17494
|
constructor(options) {
|
|
17162
17495
|
super(options);
|
|
17163
17496
|
this.assistantId = options.assistantId;
|
|
17497
|
+
// TODO: [👱] Make limiter same as in `OpenAiExecutionTools`
|
|
17164
17498
|
}
|
|
17165
17499
|
get title() {
|
|
17166
17500
|
return 'OpenAI Assistant';
|
|
@@ -17362,9 +17696,9 @@ const createOpenAiExecutionTools = Object.assign((options) => {
|
|
|
17362
17696
|
*/
|
|
17363
17697
|
const _OpenAiRegistration = $llmToolsRegister.register(createOpenAiExecutionTools);
|
|
17364
17698
|
/**
|
|
17365
|
-
*
|
|
17699
|
+
* Registration of the OpenAI Assistant provider
|
|
17366
17700
|
*
|
|
17367
|
-
* Note: [🏐] Configurations registrations are done in
|
|
17701
|
+
* Note: [🏐] Configurations registrations are done in register-constructor.ts BUT constructor register-constructor.ts
|
|
17368
17702
|
*
|
|
17369
17703
|
* @public exported from `@promptbook/openai`
|
|
17370
17704
|
* @public exported from `@promptbook/wizzard`
|
|
@@ -17377,9 +17711,8 @@ const _OpenAiAssistantRegistration = $llmToolsRegister.register(createOpenAiAssi
|
|
|
17377
17711
|
*/
|
|
17378
17712
|
|
|
17379
17713
|
/**
|
|
17380
|
-
*
|
|
17381
|
-
*
|
|
17382
|
-
* Note: It also checks if directory exists and creates it if not
|
|
17714
|
+
* Retrieves an intermediate source for a scraper based on the knowledge source.
|
|
17715
|
+
* Manages the caching and retrieval of intermediate scraper results for optimized performance.
|
|
17383
17716
|
*
|
|
17384
17717
|
* @private as internal utility for scrapers
|
|
17385
17718
|
*/
|
|
@@ -17522,7 +17855,7 @@ class MarkdownScraper {
|
|
|
17522
17855
|
const knowledge = await Promise.all(
|
|
17523
17856
|
// TODO: [🪂] Do not send all at once but in chunks
|
|
17524
17857
|
knowledgeTextPieces.map(async (knowledgeTextPiece, i) => {
|
|
17525
|
-
// Note:
|
|
17858
|
+
// Note: These are just default values, they will be overwritten by the actual values:
|
|
17526
17859
|
let name = `piece-${i}`;
|
|
17527
17860
|
let title = spaceTrim(knowledgeTextPiece.substring(0, 100));
|
|
17528
17861
|
const knowledgePieceContent = spaceTrim(knowledgeTextPiece);
|
|
@@ -17606,14 +17939,14 @@ const boilerplateScraperMetadata = $deepFreeze({
|
|
|
17606
17939
|
packageName: '@promptbook/boilerplate',
|
|
17607
17940
|
className: 'BoilerplateScraper',
|
|
17608
17941
|
mimeTypes: [
|
|
17609
|
-
'
|
|
17610
|
-
// <- TODO:
|
|
17942
|
+
'@@/@@',
|
|
17943
|
+
// <- TODO: @@ Add compatible mime types with Boilerplate scraper
|
|
17611
17944
|
],
|
|
17612
|
-
documentationUrl: 'https://github.com/webgptorg/promptbook/discussions
|
|
17945
|
+
documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
|
|
17613
17946
|
isAvilableInBrowser: false,
|
|
17614
17947
|
// <- Note: [🌏] Only `MarkdownScraper` makes sense to be available in the browser, for scraping non-markdown sources in the browser use a remote server
|
|
17615
17948
|
requiredExecutables: [
|
|
17616
|
-
/*
|
|
17949
|
+
/* @@ 'Pandoc' */
|
|
17617
17950
|
],
|
|
17618
17951
|
}); /* <- Note: [🤛] */
|
|
17619
17952
|
/**
|
|
@@ -17631,7 +17964,7 @@ const _BoilerplateScraperMetadataRegistration = $scrapersMetadataRegister.regist
|
|
|
17631
17964
|
*/
|
|
17632
17965
|
|
|
17633
17966
|
/**
|
|
17634
|
-
* Scraper of
|
|
17967
|
+
* Scraper of @@ files
|
|
17635
17968
|
*
|
|
17636
17969
|
* @see `documentationUrl` for more details
|
|
17637
17970
|
* @public exported from `@promptbook/boilerplate`
|
|
@@ -17649,30 +17982,30 @@ class BoilerplateScraper {
|
|
|
17649
17982
|
this.markdownScraper = new MarkdownScraper(tools, options);
|
|
17650
17983
|
}
|
|
17651
17984
|
/**
|
|
17652
|
-
* Convert the
|
|
17985
|
+
* Convert the `.@@` to `.md` file and returns intermediate source
|
|
17653
17986
|
*
|
|
17654
17987
|
* Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object
|
|
17655
17988
|
*/
|
|
17656
17989
|
async $convert(source) {
|
|
17657
17990
|
var _a;
|
|
17658
17991
|
const { rootDirname = process.cwd(), cacheDirname = DEFAULT_SCRAPE_CACHE_DIRNAME, intermediateFilesStrategy = DEFAULT_INTERMEDIATE_FILES_STRATEGY, isVerbose = DEFAULT_IS_VERBOSE, } = this.options;
|
|
17659
|
-
// TODO:
|
|
17992
|
+
// TODO: @@ Preserve or delete
|
|
17660
17993
|
if (!$isRunningInNode()) {
|
|
17661
17994
|
throw new KnowledgeScrapeError('BoilerplateScraper is only supported in Node environment');
|
|
17662
17995
|
}
|
|
17663
|
-
// TODO:
|
|
17996
|
+
// TODO: @@ Preserve or delete
|
|
17664
17997
|
if (this.tools.fs === undefined) {
|
|
17665
17998
|
throw new EnvironmentMismatchError('Can not scrape boilerplates without filesystem tools');
|
|
17666
17999
|
// <- TODO: [🧠] What is the best error type here`
|
|
17667
18000
|
}
|
|
17668
|
-
// TODO:
|
|
18001
|
+
// TODO: @@ Preserve, delete or modify
|
|
17669
18002
|
if (((_a = this.tools.executables) === null || _a === void 0 ? void 0 : _a.pandocPath) === undefined) {
|
|
17670
18003
|
throw new MissingToolsError('Pandoc is required for scraping .docx files');
|
|
17671
18004
|
}
|
|
17672
|
-
// TODO:
|
|
18005
|
+
// TODO: @@ Preserve, delete or modify
|
|
17673
18006
|
if (source.filename === null) {
|
|
17674
18007
|
// TODO: [🧠] Maybe save file as temporary
|
|
17675
|
-
throw new KnowledgeScrapeError('When parsing
|
|
18008
|
+
throw new KnowledgeScrapeError('When parsing .@@ file, it must be real file in the file system');
|
|
17676
18009
|
}
|
|
17677
18010
|
const extension = getFileExtension(source.filename);
|
|
17678
18011
|
const cacheFilehandler = await getScraperIntermediateSource(source, {
|
|
@@ -17682,7 +18015,7 @@ class BoilerplateScraper {
|
|
|
17682
18015
|
extension: 'md',
|
|
17683
18016
|
isVerbose,
|
|
17684
18017
|
});
|
|
17685
|
-
// TODO:
|
|
18018
|
+
// TODO: @@ Preserve, delete or modify
|
|
17686
18019
|
// Note: Running Pandoc ONLY if the file in the cache does not exist
|
|
17687
18020
|
if (!(await isFileExisting(cacheFilehandler.filename, this.tools.fs))) {
|
|
17688
18021
|
const command = `"${this.tools.executables.pandocPath}" -f ${extension} -t markdown "${source.filename}" -o "${cacheFilehandler.filename}"`;
|
|
@@ -17708,7 +18041,7 @@ class BoilerplateScraper {
|
|
|
17708
18041
|
*/
|
|
17709
18042
|
async scrape(source) {
|
|
17710
18043
|
const cacheFilehandler = await this.$convert(source);
|
|
17711
|
-
// TODO:
|
|
18044
|
+
// TODO: @@ Preserve, delete or modify
|
|
17712
18045
|
const markdownSource = {
|
|
17713
18046
|
source: source.source,
|
|
17714
18047
|
filename: cacheFilehandler.filename,
|
|
@@ -17739,7 +18072,7 @@ class BoilerplateScraper {
|
|
|
17739
18072
|
* TODO: [👣] Converted documents can act as cached items - there is no need to run conversion each time
|
|
17740
18073
|
* TODO: [🪂] Do it in parallel
|
|
17741
18074
|
* Note: No need to aggregate usage here, it is done by intercepting the llmTools
|
|
17742
|
-
*
|
|
18075
|
+
* @@ Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
|
|
17743
18076
|
*/
|
|
17744
18077
|
|
|
17745
18078
|
/**
|
|
@@ -18073,7 +18406,8 @@ class LegacyDocumentScraper {
|
|
|
18073
18406
|
*/
|
|
18074
18407
|
|
|
18075
18408
|
/**
|
|
18076
|
-
*
|
|
18409
|
+
* Creates a scraper for legacy document formats (.doc, .rtf, etc).
|
|
18410
|
+
* Uses LibreOffice for conversion to extract content from older document formats.
|
|
18077
18411
|
*
|
|
18078
18412
|
* @public exported from `@promptbook/legacy-documents`
|
|
18079
18413
|
*/
|
|
@@ -18100,7 +18434,7 @@ const _LegacyDocumentScraperRegistration = $scrapersRegister.register(createLega
|
|
|
18100
18434
|
*/
|
|
18101
18435
|
|
|
18102
18436
|
/**
|
|
18103
|
-
*
|
|
18437
|
+
* Creates a scraper for document content.
|
|
18104
18438
|
*
|
|
18105
18439
|
* @public exported from `@promptbook/documents`
|
|
18106
18440
|
*/
|
|
@@ -18127,7 +18461,7 @@ const _DocumentScraperRegistration = $scrapersRegister.register(createDocumentSc
|
|
|
18127
18461
|
*/
|
|
18128
18462
|
|
|
18129
18463
|
/**
|
|
18130
|
-
*
|
|
18464
|
+
* Creates a scraper for markdown content.
|
|
18131
18465
|
*
|
|
18132
18466
|
* @public exported from `@promptbook/markdown-utils`
|
|
18133
18467
|
*/
|
|
@@ -18233,8 +18567,8 @@ class MarkitdownScraper {
|
|
|
18233
18567
|
extension: 'md',
|
|
18234
18568
|
isVerbose,
|
|
18235
18569
|
});
|
|
18236
|
-
// TODO:
|
|
18237
|
-
// Note: Running
|
|
18570
|
+
// TODO: Determine if Markitdown conversion should run only if the cache file doesn't exist, or always.
|
|
18571
|
+
// Note: Running Markitdown conversion ONLY if the file in the cache does not exist
|
|
18238
18572
|
if (!(await isFileExisting(cacheFilehandler.filename, this.tools.fs))) {
|
|
18239
18573
|
const src = source.filename || source.url || null;
|
|
18240
18574
|
// console.log('!!', { src, source, cacheFilehandler });
|
|
@@ -18256,11 +18590,11 @@ class MarkitdownScraper {
|
|
|
18256
18590
|
return cacheFilehandler;
|
|
18257
18591
|
}
|
|
18258
18592
|
/**
|
|
18259
|
-
* Scrapes the
|
|
18593
|
+
* Scrapes the source document (PDF, DOCX, etc.) and returns the knowledge pieces or `null` if it can't scrape it.
|
|
18260
18594
|
*/
|
|
18261
18595
|
async scrape(source) {
|
|
18262
18596
|
const cacheFilehandler = await this.$convert(source);
|
|
18263
|
-
// TODO:
|
|
18597
|
+
// TODO: Ensure this correctly creates the source object for the internal MarkdownScraper using the converted file.
|
|
18264
18598
|
const markdownSource = {
|
|
18265
18599
|
source: source.source,
|
|
18266
18600
|
filename: cacheFilehandler.filename,
|
|
@@ -18404,7 +18738,8 @@ class PdfScraper {
|
|
|
18404
18738
|
*/
|
|
18405
18739
|
|
|
18406
18740
|
/**
|
|
18407
|
-
*
|
|
18741
|
+
* Factory function to create an instance of PdfScraper.
|
|
18742
|
+
* It bundles the scraper class with its metadata.
|
|
18408
18743
|
*
|
|
18409
18744
|
* @public exported from `@promptbook/pdf`
|
|
18410
18745
|
*/
|
|
@@ -18580,7 +18915,8 @@ class WebsiteScraper {
|
|
|
18580
18915
|
*/
|
|
18581
18916
|
|
|
18582
18917
|
/**
|
|
18583
|
-
*
|
|
18918
|
+
* Factory function to create an instance of WebsiteScraper.
|
|
18919
|
+
* It bundles the scraper class with its metadata.
|
|
18584
18920
|
*
|
|
18585
18921
|
* @public exported from `@promptbook/website-crawler`
|
|
18586
18922
|
*/
|