@promptbook/cli 0.92.0-8 → 0.92.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -6
- package/esm/index.es.js +1031 -670
- package/esm/index.es.js.map +1 -1
- package/esm/typings/src/_packages/browser.index.d.ts +2 -0
- package/esm/typings/src/_packages/core.index.d.ts +26 -14
- package/esm/typings/src/_packages/types.index.d.ts +6 -2
- package/esm/typings/src/collection/PipelineCollection.d.ts +0 -2
- package/esm/typings/src/collection/SimplePipelineCollection.d.ts +1 -1
- package/esm/typings/src/commands/FOREACH/ForeachJson.d.ts +6 -6
- package/esm/typings/src/commands/FOREACH/foreachCommandParser.d.ts +0 -2
- package/esm/typings/src/commands/FORMFACTOR/formfactorCommandParser.d.ts +1 -1
- package/esm/typings/src/commands/_BOILERPLATE/boilerplateCommandParser.d.ts +1 -1
- package/esm/typings/src/commands/_common/types/CommandParser.d.ts +36 -28
- package/esm/typings/src/config.d.ts +41 -11
- package/esm/typings/src/constants.d.ts +43 -2
- package/esm/typings/src/conversion/parsePipeline.d.ts +2 -2
- package/esm/typings/src/errors/0-BoilerplateError.d.ts +2 -2
- package/esm/typings/src/errors/CollectionError.d.ts +1 -1
- package/esm/typings/src/executables/$provideExecutablesForNode.d.ts +1 -1
- package/esm/typings/src/executables/apps/locateLibreoffice.d.ts +2 -1
- package/esm/typings/src/executables/apps/locatePandoc.d.ts +2 -1
- package/esm/typings/src/executables/locateApp.d.ts +2 -2
- package/esm/typings/src/executables/platforms/locateAppOnLinux.d.ts +2 -1
- package/esm/typings/src/executables/platforms/locateAppOnMacOs.d.ts +2 -1
- package/esm/typings/src/executables/platforms/locateAppOnWindows.d.ts +2 -1
- package/esm/typings/src/execution/AbstractTaskResult.d.ts +1 -1
- package/esm/typings/src/execution/CommonToolsOptions.d.ts +5 -1
- package/esm/typings/src/execution/ExecutionTask.d.ts +19 -1
- package/esm/typings/src/execution/LlmExecutionToolsConstructor.d.ts +2 -1
- package/esm/typings/src/execution/PipelineExecutorResult.d.ts +4 -2
- package/esm/typings/src/execution/PromptbookFetch.d.ts +1 -1
- package/esm/typings/src/execution/ScriptExecutionTools.d.ts +1 -1
- package/esm/typings/src/execution/createPipelineExecutor/$OngoingTaskResult.d.ts +12 -9
- package/esm/typings/src/execution/createPipelineExecutor/10-executePipeline.d.ts +13 -10
- package/esm/typings/src/execution/createPipelineExecutor/20-executeTask.d.ts +12 -9
- package/esm/typings/src/execution/createPipelineExecutor/30-executeFormatSubvalues.d.ts +15 -3
- package/esm/typings/src/execution/createPipelineExecutor/40-executeAttempts.d.ts +21 -15
- package/esm/typings/src/execution/createPipelineExecutor/computeCosineSimilarity.d.ts +13 -0
- package/esm/typings/src/execution/createPipelineExecutor/filterJustOutputParameters.d.ts +7 -6
- package/esm/typings/src/execution/createPipelineExecutor/getContextForTask.d.ts +5 -1
- package/esm/typings/src/execution/createPipelineExecutor/getExamplesForTask.d.ts +1 -1
- package/esm/typings/src/execution/createPipelineExecutor/getKnowledgeForTask.d.ts +12 -9
- package/esm/typings/src/execution/createPipelineExecutor/getReservedParametersForTask.d.ts +18 -5
- package/esm/typings/src/execution/createPipelineExecutor/knowledgePiecesToString.d.ts +9 -0
- package/esm/typings/src/execution/execution-report/ExecutionReportJson.d.ts +1 -1
- package/esm/typings/src/execution/execution-report/ExecutionReportString.d.ts +1 -1
- package/esm/typings/src/execution/translation/automatic-translate/automatic-translators/LindatAutomaticTranslator.d.ts +4 -4
- package/esm/typings/src/execution/utils/checkExpectations.d.ts +3 -3
- package/esm/typings/src/execution/utils/uncertainNumber.d.ts +3 -2
- package/esm/typings/src/execution/utils/usageToWorktime.d.ts +1 -1
- package/esm/typings/src/formats/_common/{FormatDefinition.d.ts → FormatParser.d.ts} +8 -6
- package/esm/typings/src/formats/_common/FormatSubvalueParser.d.ts +66 -0
- package/esm/typings/src/formats/csv/CsvFormatParser.d.ts +17 -0
- package/esm/typings/src/formats/csv/CsvSettings.d.ts +2 -2
- package/esm/typings/src/formats/csv/utils/csvParse.d.ts +12 -0
- package/esm/typings/src/formats/csv/utils/isValidCsvString.d.ts +1 -1
- package/esm/typings/src/formats/index.d.ts +2 -2
- package/esm/typings/src/formats/json/JsonFormatParser.d.ts +19 -0
- package/esm/typings/src/formats/json/utils/isValidJsonString.d.ts +1 -1
- package/esm/typings/src/formats/json/utils/jsonParse.d.ts +0 -3
- package/esm/typings/src/formats/text/{TextFormatDefinition.d.ts → TextFormatParser.d.ts} +7 -7
- package/esm/typings/src/formats/xml/XmlFormatParser.d.ts +19 -0
- package/esm/typings/src/formats/xml/utils/isValidXmlString.d.ts +1 -1
- package/esm/typings/src/formfactors/_boilerplate/BoilerplateFormfactorDefinition.d.ts +3 -2
- package/esm/typings/src/formfactors/_common/AbstractFormfactorDefinition.d.ts +16 -7
- package/esm/typings/src/formfactors/_common/FormfactorDefinition.d.ts +3 -1
- package/esm/typings/src/formfactors/_common/string_formfactor_name.d.ts +2 -1
- package/esm/typings/src/formfactors/chatbot/ChatbotFormfactorDefinition.d.ts +2 -2
- package/esm/typings/src/formfactors/completion/CompletionFormfactorDefinition.d.ts +29 -0
- package/esm/typings/src/formfactors/generator/GeneratorFormfactorDefinition.d.ts +2 -1
- package/esm/typings/src/formfactors/generic/GenericFormfactorDefinition.d.ts +2 -2
- package/esm/typings/src/formfactors/index.d.ts +33 -8
- package/esm/typings/src/formfactors/matcher/MatcherFormfactorDefinition.d.ts +4 -2
- package/esm/typings/src/formfactors/sheets/SheetsFormfactorDefinition.d.ts +3 -2
- package/esm/typings/src/formfactors/translator/TranslatorFormfactorDefinition.d.ts +3 -2
- package/esm/typings/src/high-level-abstractions/index.d.ts +2 -2
- package/esm/typings/src/llm-providers/_common/register/$llmToolsMetadataRegister.d.ts +3 -3
- package/esm/typings/src/llm-providers/_common/register/$llmToolsRegister.d.ts +3 -3
- package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsConfigurationFromEnv.d.ts +4 -4
- package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground.d.ts +4 -3
- package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsFromEnv.d.ts +18 -5
- package/esm/typings/src/llm-providers/_common/register/LlmToolsConfiguration.d.ts +11 -4
- package/esm/typings/src/llm-providers/_common/register/LlmToolsMetadata.d.ts +21 -42
- package/esm/typings/src/llm-providers/_common/register/LlmToolsOptions.d.ts +9 -2
- package/esm/typings/src/llm-providers/_common/register/createLlmToolsFromConfiguration.d.ts +13 -4
- package/esm/typings/src/llm-providers/_common/utils/cache/CacheItem.d.ts +10 -5
- package/esm/typings/src/llm-providers/_common/utils/cache/CacheLlmToolsOptions.d.ts +11 -3
- package/esm/typings/src/llm-providers/_common/utils/cache/cacheLlmTools.d.ts +3 -3
- package/esm/typings/src/llm-providers/_common/utils/count-total-usage/limitTotalUsage.d.ts +5 -5
- package/esm/typings/src/llm-providers/anthropic-claude/AnthropicClaudeExecutionTools.d.ts +6 -0
- package/esm/typings/src/llm-providers/anthropic-claude/anthropic-claude-models.d.ts +1 -1
- package/esm/typings/src/llm-providers/azure-openai/AzureOpenAiExecutionTools.d.ts +5 -1
- package/esm/typings/src/llm-providers/azure-openai/AzureOpenAiExecutionToolsOptions.d.ts +4 -4
- package/esm/typings/src/llm-providers/deepseek/deepseek-models.d.ts +1 -1
- package/esm/typings/src/llm-providers/google/google-models.d.ts +1 -1
- package/esm/typings/src/llm-providers/openai/OpenAiAssistantExecutionTools.d.ts +1 -1
- package/esm/typings/src/llm-providers/openai/OpenAiExecutionTools.d.ts +4 -0
- package/esm/typings/src/llm-providers/openai/openai-models.d.ts +1 -1
- package/esm/typings/src/llm-providers/openai/register-configuration.d.ts +2 -2
- package/esm/typings/src/llm-providers/openai/register-constructor.d.ts +2 -2
- package/esm/typings/src/migrations/migratePipeline.d.ts +9 -0
- package/esm/typings/src/other/templates/getBookTemplates.d.ts +2 -2
- package/esm/typings/src/pipeline/PipelineInterface/PipelineInterface.d.ts +3 -3
- package/esm/typings/src/pipeline/PipelineInterface/constants.d.ts +1 -1
- package/esm/typings/src/pipeline/PipelineInterface/getPipelineInterface.d.ts +1 -1
- package/esm/typings/src/pipeline/PipelineInterface/isPipelineImplementingInterface.d.ts +5 -4
- package/esm/typings/src/pipeline/PipelineInterface/isPipelineInterfacesEqual.d.ts +1 -1
- package/esm/typings/src/pipeline/PipelineJson/CommonTaskJson.d.ts +10 -7
- package/esm/typings/src/pipeline/PipelineJson/ParameterJson.d.ts +1 -1
- package/esm/typings/src/pipeline/PipelineJson/PipelineJson.d.ts +3 -2
- package/esm/typings/src/pipeline/PipelineString.d.ts +3 -1
- package/esm/typings/src/pipeline/book-notation.d.ts +2 -2
- package/esm/typings/src/postprocessing/utils/extractBlock.d.ts +1 -1
- package/esm/typings/src/postprocessing/utils/extractJsonBlock.d.ts +2 -2
- package/esm/typings/src/prepare/prepareTasks.d.ts +8 -5
- package/esm/typings/src/remote-server/openapi.d.ts +1 -1
- package/esm/typings/src/remote-server/socket-types/listModels/PromptbookServer_ListModels_Response.d.ts +1 -1
- package/esm/typings/src/remote-server/startRemoteServer.d.ts +1 -1
- package/esm/typings/src/remote-server/types/RemoteServerOptions.d.ts +7 -6
- package/esm/typings/src/scrapers/_boilerplate/BoilerplateScraper.d.ts +3 -3
- package/esm/typings/src/scrapers/_boilerplate/createBoilerplateScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/_boilerplate/register-metadata.d.ts +1 -1
- package/esm/typings/src/scrapers/_common/Converter.d.ts +3 -1
- package/esm/typings/src/scrapers/_common/Scraper.d.ts +4 -3
- package/esm/typings/src/scrapers/_common/ScraperIntermediateSource.d.ts +4 -2
- package/esm/typings/src/scrapers/_common/prepareKnowledgePieces.d.ts +2 -2
- package/esm/typings/src/scrapers/_common/register/$provideFilesystemForNode.d.ts +2 -1
- package/esm/typings/src/scrapers/_common/register/$provideScrapersForBrowser.d.ts +6 -3
- package/esm/typings/src/scrapers/_common/register/$provideScrapersForNode.d.ts +3 -5
- package/esm/typings/src/scrapers/_common/register/$scrapersMetadataRegister.d.ts +3 -3
- package/esm/typings/src/scrapers/_common/register/$scrapersRegister.d.ts +3 -2
- package/esm/typings/src/scrapers/_common/register/ScraperAndConverterMetadata.d.ts +8 -5
- package/esm/typings/src/scrapers/_common/register/ScraperConstructor.d.ts +2 -1
- package/esm/typings/src/scrapers/_common/utils/getScraperIntermediateSource.d.ts +6 -5
- package/esm/typings/src/scrapers/_common/utils/makeKnowledgeSourceHandler.d.ts +3 -1
- package/esm/typings/src/scrapers/document/createDocumentScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/document-legacy/createLegacyDocumentScraper.d.ts +2 -1
- package/esm/typings/src/scrapers/markdown/createMarkdownScraper.d.ts +4 -1
- package/esm/typings/src/scrapers/markitdown/MarkitdownScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/pdf/createPdfScraper.d.ts +2 -1
- package/esm/typings/src/scrapers/website/createWebsiteScraper.d.ts +3 -4
- package/esm/typings/src/scripting/javascript/JavascriptExecutionTools.d.ts +1 -1
- package/esm/typings/src/scripting/javascript/postprocessing-functions.d.ts +5 -1
- package/esm/typings/src/scripting/javascript/utils/preserve.d.ts +1 -1
- package/esm/typings/src/storage/_common/PromptbookStorage.d.ts +1 -1
- package/esm/typings/src/storage/file-cache-storage/FileCacheStorage.d.ts +12 -5
- package/esm/typings/src/storage/file-cache-storage/FileCacheStorageOptions.d.ts +4 -2
- package/esm/typings/src/storage/file-cache-storage/utils/nameToSubfolderPath.d.ts +2 -1
- package/esm/typings/src/storage/local-storage/getIndexedDbStorage.d.ts +11 -0
- package/esm/typings/src/storage/local-storage/utils/IndexedDbStorageOptions.d.ts +14 -0
- package/esm/typings/src/storage/local-storage/utils/makePromptbookStorageFromIndexedDb.d.ts +8 -0
- package/esm/typings/src/storage/local-storage/utils/makePromptbookStorageFromWebStorage.d.ts +2 -1
- package/esm/typings/src/types/IntermediateFilesStrategy.d.ts +2 -1
- package/esm/typings/src/types/ModelRequirements.d.ts +2 -2
- package/esm/typings/src/types/ModelVariant.d.ts +5 -5
- package/esm/typings/src/types/typeAliases.d.ts +22 -19
- package/esm/typings/src/utils/$Register.d.ts +8 -7
- package/esm/typings/src/utils/editable/edit-pipeline-string/addPipelineCommand.d.ts +2 -2
- package/esm/typings/src/utils/editable/edit-pipeline-string/deflatePipeline.d.ts +4 -1
- package/esm/typings/src/utils/editable/utils/isFlatPipeline.d.ts +2 -1
- package/esm/typings/src/utils/editable/utils/stringifyPipelineJson.d.ts +1 -1
- package/esm/typings/src/utils/environment/$getGlobalScope.d.ts +2 -1
- package/esm/typings/src/utils/expectation-counters/index.d.ts +1 -1
- package/esm/typings/src/utils/markdown/extractAllBlocksFromMarkdown.d.ts +2 -2
- package/esm/typings/src/utils/markdown/extractAllListItemsFromMarkdown.d.ts +1 -1
- package/esm/typings/src/utils/markdown/extractOneBlockFromMarkdown.d.ts +2 -2
- package/esm/typings/src/utils/normalization/nameToUriPart.d.ts +4 -4
- package/esm/typings/src/utils/normalization/nameToUriParts.d.ts +4 -4
- package/esm/typings/src/utils/normalization/normalize-to-kebab-case.d.ts +3 -3
- package/esm/typings/src/utils/normalization/normalizeTo_SCREAMING_CASE.d.ts +3 -3
- package/esm/typings/src/utils/normalization/normalizeTo_camelCase.d.ts +4 -4
- package/esm/typings/src/utils/normalization/normalizeTo_snake_case.d.ts +3 -3
- package/esm/typings/src/utils/normalization/removeDiacritics.d.ts +3 -3
- package/esm/typings/src/utils/normalization/searchKeywords.d.ts +4 -1
- package/esm/typings/src/utils/normalization/titleToName.d.ts +4 -4
- package/esm/typings/src/utils/organization/TODO_USE.d.ts +1 -1
- package/esm/typings/src/utils/organization/empty_object.d.ts +2 -2
- package/esm/typings/src/utils/organization/just.d.ts +1 -1
- package/esm/typings/src/utils/organization/just_empty_object.d.ts +4 -4
- package/esm/typings/src/utils/organization/keepUnused.d.ts +1 -1
- package/esm/typings/src/utils/parameters/mapAvailableToExpectedParameters.d.ts +7 -7
- package/esm/typings/src/utils/removeQuotes.d.ts +2 -2
- package/esm/typings/src/utils/serialization/clonePipeline.d.ts +4 -3
- package/esm/typings/src/utils/serialization/deepClone.d.ts +5 -1
- package/esm/typings/src/utils/trimCodeBlock.d.ts +1 -1
- package/esm/typings/src/utils/trimEndOfCodeBlock.d.ts +1 -1
- package/esm/typings/src/utils/unwrapResult.d.ts +2 -2
- package/esm/typings/src/utils/validators/javascriptName/isValidJavascriptName.d.ts +3 -3
- package/esm/typings/src/utils/validators/parameterName/validateParameterName.d.ts +5 -4
- package/esm/typings/src/utils/validators/semanticVersion/isValidPromptbookVersion.d.ts +1 -1
- package/esm/typings/src/utils/validators/semanticVersion/isValidSemanticVersion.d.ts +1 -1
- package/esm/typings/src/utils/validators/url/isHostnameOnPrivateNetwork.d.ts +1 -1
- package/esm/typings/src/utils/validators/url/isUrlOnPrivateNetwork.d.ts +1 -1
- package/esm/typings/src/utils/validators/url/isValidPipelineUrl.d.ts +1 -1
- package/esm/typings/src/utils/validators/url/isValidUrl.d.ts +1 -1
- package/esm/typings/src/version.d.ts +2 -1
- package/esm/typings/src/wizzard/wizzard.d.ts +1 -1
- package/package.json +14 -1
- package/umd/index.umd.js +1040 -679
- package/umd/index.umd.js.map +1 -1
- package/esm/typings/src/formats/_common/FormatSubvalueDefinition.d.ts +0 -31
- package/esm/typings/src/formats/csv/CsvFormatDefinition.d.ts +0 -17
- package/esm/typings/src/formats/json/JsonFormatDefinition.d.ts +0 -19
- package/esm/typings/src/formats/xml/XmlFormatDefinition.d.ts +0 -19
- /package/esm/typings/src/llm-providers/{multiple → _multiple}/MultipleLlmExecutionTools.d.ts +0 -0
- /package/esm/typings/src/llm-providers/{multiple → _multiple}/joinLlmExecutionTools.d.ts +0 -0
- /package/esm/typings/src/llm-providers/{multiple → _multiple}/playground/playground.d.ts +0 -0
package/umd/index.umd.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
(function (global, factory) {
|
|
2
|
-
typeof exports === 'object' && typeof module !== 'undefined' ? factory(exports, require('colors'), require('commander'), require('spacetrim'), require('waitasecond'), require('prompts'), require('path'), require('fs/promises'), require('dotenv'), require('crypto-js/enc-hex'), require('crypto-js/sha256'), require('crypto'), require('socket.io-client'), require('rxjs'), require('child_process'), require('jszip'), require('prettier'), require('prettier/parser-html'), require('papaparse'), require('crypto-js'), require('mime-types'), require('glob-promise'), require('moment'), require('express'), require('
|
|
3
|
-
typeof define === 'function' && define.amd ? define(['exports', 'colors', 'commander', 'spacetrim', 'waitasecond', 'prompts', 'path', 'fs/promises', 'dotenv', 'crypto-js/enc-hex', 'crypto-js/sha256', 'crypto', 'socket.io-client', 'rxjs', 'child_process', 'jszip', 'prettier', 'prettier/parser-html', 'papaparse', 'crypto-js', 'mime-types', 'glob-promise', 'moment', 'express', 'http', 'socket.io', '
|
|
4
|
-
(global = typeof globalThis !== 'undefined' ? globalThis : global || self, factory(global["promptbook-cli"] = {}, global.colors, global.commander, global.spaceTrim, global.waitasecond, global.prompts, global.path, global.promises, global.dotenv, global.hexEncoder, global.sha256, global.crypto, global.socket_ioClient, global.rxjs, global.child_process, global.JSZip, global.prettier, global.parserHtml, global.papaparse, global.cryptoJs, global.mimeTypes, global.glob, global.moment, global.express, global.
|
|
5
|
-
})(this, (function (exports, colors, commander, spaceTrim, waitasecond, prompts, path, promises, dotenv, hexEncoder, sha256, crypto, socket_ioClient, rxjs, child_process, JSZip, prettier, parserHtml, papaparse, cryptoJs, mimeTypes, glob, moment, express, http, socket_io,
|
|
2
|
+
typeof exports === 'object' && typeof module !== 'undefined' ? factory(exports, require('colors'), require('commander'), require('spacetrim'), require('waitasecond'), require('prompts'), require('path'), require('fs/promises'), require('dotenv'), require('crypto-js/enc-hex'), require('crypto-js/sha256'), require('crypto'), require('socket.io-client'), require('rxjs'), require('child_process'), require('jszip'), require('prettier'), require('prettier/parser-html'), require('papaparse'), require('crypto-js'), require('mime-types'), require('glob-promise'), require('moment'), require('express'), require('express-openapi-validator'), require('http'), require('socket.io'), require('swagger-ui-express'), require('@anthropic-ai/sdk'), require('bottleneck'), require('@azure/openai'), require('openai'), require('@mozilla/readability'), require('jsdom'), require('showdown')) :
|
|
3
|
+
typeof define === 'function' && define.amd ? define(['exports', 'colors', 'commander', 'spacetrim', 'waitasecond', 'prompts', 'path', 'fs/promises', 'dotenv', 'crypto-js/enc-hex', 'crypto-js/sha256', 'crypto', 'socket.io-client', 'rxjs', 'child_process', 'jszip', 'prettier', 'prettier/parser-html', 'papaparse', 'crypto-js', 'mime-types', 'glob-promise', 'moment', 'express', 'express-openapi-validator', 'http', 'socket.io', 'swagger-ui-express', '@anthropic-ai/sdk', 'bottleneck', '@azure/openai', 'openai', '@mozilla/readability', 'jsdom', 'showdown'], factory) :
|
|
4
|
+
(global = typeof globalThis !== 'undefined' ? globalThis : global || self, factory(global["promptbook-cli"] = {}, global.colors, global.commander, global.spaceTrim, global.waitasecond, global.prompts, global.path, global.promises, global.dotenv, global.hexEncoder, global.sha256, global.crypto, global.socket_ioClient, global.rxjs, global.child_process, global.JSZip, global.prettier, global.parserHtml, global.papaparse, global.cryptoJs, global.mimeTypes, global.glob, global.moment, global.express, global.OpenApiValidator, global.http, global.socket_io, global.swaggerUi, global.Anthropic, global.Bottleneck, global.openai, global.OpenAI, global.readability, global.jsdom, global.showdown));
|
|
5
|
+
})(this, (function (exports, colors, commander, spaceTrim, waitasecond, prompts, path, promises, dotenv, hexEncoder, sha256, crypto, socket_ioClient, rxjs, child_process, JSZip, prettier, parserHtml, papaparse, cryptoJs, mimeTypes, glob, moment, express, OpenApiValidator, http, socket_io, swaggerUi, Anthropic, Bottleneck, openai, OpenAI, readability, jsdom, showdown) { 'use strict';
|
|
6
6
|
|
|
7
7
|
function _interopDefaultLegacy (e) { return e && typeof e === 'object' && 'default' in e ? e : { 'default': e }; }
|
|
8
8
|
|
|
@@ -36,10 +36,11 @@
|
|
|
36
36
|
var glob__default = /*#__PURE__*/_interopDefaultLegacy(glob);
|
|
37
37
|
var moment__default = /*#__PURE__*/_interopDefaultLegacy(moment);
|
|
38
38
|
var express__default = /*#__PURE__*/_interopDefaultLegacy(express);
|
|
39
|
-
var http__default = /*#__PURE__*/_interopDefaultLegacy(http);
|
|
40
39
|
var OpenApiValidator__namespace = /*#__PURE__*/_interopNamespace(OpenApiValidator);
|
|
40
|
+
var http__default = /*#__PURE__*/_interopDefaultLegacy(http);
|
|
41
41
|
var swaggerUi__default = /*#__PURE__*/_interopDefaultLegacy(swaggerUi);
|
|
42
42
|
var Anthropic__default = /*#__PURE__*/_interopDefaultLegacy(Anthropic);
|
|
43
|
+
var Bottleneck__default = /*#__PURE__*/_interopDefaultLegacy(Bottleneck);
|
|
43
44
|
var OpenAI__default = /*#__PURE__*/_interopDefaultLegacy(OpenAI);
|
|
44
45
|
|
|
45
46
|
// ⚠️ WARNING: This code has been generated so that any manual changes will be overwritten
|
|
@@ -56,7 +57,7 @@
|
|
|
56
57
|
* @generated
|
|
57
58
|
* @see https://github.com/webgptorg/promptbook
|
|
58
59
|
*/
|
|
59
|
-
const PROMPTBOOK_ENGINE_VERSION = '0.92.0
|
|
60
|
+
const PROMPTBOOK_ENGINE_VERSION = '0.92.0';
|
|
60
61
|
/**
|
|
61
62
|
* TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
|
|
62
63
|
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
@@ -97,7 +98,7 @@
|
|
|
97
98
|
* Returns the same value that is passed as argument.
|
|
98
99
|
* No side effects.
|
|
99
100
|
*
|
|
100
|
-
* Note: It can be
|
|
101
|
+
* Note: It can be useful for:
|
|
101
102
|
*
|
|
102
103
|
* 1) Leveling indentation
|
|
103
104
|
* 2) Putting always-true or always-false conditions without getting eslint errors
|
|
@@ -172,6 +173,21 @@
|
|
|
172
173
|
* @public exported from `@promptbook/core`
|
|
173
174
|
*/
|
|
174
175
|
const DEFAULT_MAX_FILE_SIZE = 100 * 1024 * 1024; // 100MB
|
|
176
|
+
/**
|
|
177
|
+
* Threshold value that determines when a dataset is considered "big"
|
|
178
|
+
* and may require special handling or optimizations
|
|
179
|
+
*
|
|
180
|
+
* For example, when error occurs in one item of the big dataset, it will not fail the whole pipeline
|
|
181
|
+
*
|
|
182
|
+
* @public exported from `@promptbook/core`
|
|
183
|
+
*/
|
|
184
|
+
const BIG_DATASET_TRESHOLD = 50;
|
|
185
|
+
/**
|
|
186
|
+
* Placeholder text used to represent a placeholder value of failed operation
|
|
187
|
+
*
|
|
188
|
+
* @public exported from `@promptbook/core`
|
|
189
|
+
*/
|
|
190
|
+
const FAILED_VALUE_PLACEHOLDER = '!?';
|
|
175
191
|
/**
|
|
176
192
|
* Warning message for the generated sections and files files
|
|
177
193
|
*
|
|
@@ -256,6 +272,7 @@
|
|
|
256
272
|
* @public exported from `@promptbook/core`
|
|
257
273
|
*/
|
|
258
274
|
const DEFAULT_MAX_EXECUTION_ATTEMPTS = 10; // <- TODO: [🤹♂️]
|
|
275
|
+
// <- TODO: [🐝]
|
|
259
276
|
/**
|
|
260
277
|
* Where to store your books
|
|
261
278
|
* This is kind of a "src" for your books
|
|
@@ -327,7 +344,7 @@
|
|
|
327
344
|
const DEFAULT_REMOTE_SERVER_URL = REMOTE_SERVER_URLS[0].urls[0];
|
|
328
345
|
// <- TODO: [🧜♂️]
|
|
329
346
|
/**
|
|
330
|
-
*
|
|
347
|
+
* Default settings for parsing and generating CSV files in Promptbook.
|
|
331
348
|
*
|
|
332
349
|
* @public exported from `@promptbook/core`
|
|
333
350
|
*/
|
|
@@ -338,13 +355,13 @@
|
|
|
338
355
|
skipEmptyLines: true,
|
|
339
356
|
});
|
|
340
357
|
/**
|
|
341
|
-
*
|
|
358
|
+
* Controls whether verbose logging is enabled by default throughout the application.
|
|
342
359
|
*
|
|
343
360
|
* @public exported from `@promptbook/core`
|
|
344
361
|
*/
|
|
345
362
|
let DEFAULT_IS_VERBOSE = false;
|
|
346
363
|
/**
|
|
347
|
-
*
|
|
364
|
+
* Controls whether auto-installation of dependencies is enabled by default.
|
|
348
365
|
*
|
|
349
366
|
* @public exported from `@promptbook/core`
|
|
350
367
|
*/
|
|
@@ -356,7 +373,15 @@
|
|
|
356
373
|
*/
|
|
357
374
|
const DEFAULT_GET_PIPELINE_COLLECTION_FUNCTION_NAME = `getPipelineCollection`;
|
|
358
375
|
/**
|
|
359
|
-
*
|
|
376
|
+
* Default rate limits (requests per minute)
|
|
377
|
+
*
|
|
378
|
+
* Note: Adjust based on the provider tier you are have
|
|
379
|
+
*
|
|
380
|
+
* @public exported from `@promptbook/core`
|
|
381
|
+
*/
|
|
382
|
+
const DEFAULT_MAX_REQUESTS_PER_MINUTE = 60;
|
|
383
|
+
/**
|
|
384
|
+
* Indicates whether pipeline logic validation is enabled. When true, the pipeline logic is checked for consistency.
|
|
360
385
|
*
|
|
361
386
|
* @private within the repository
|
|
362
387
|
*/
|
|
@@ -513,7 +538,7 @@
|
|
|
513
538
|
${block(message)}
|
|
514
539
|
|
|
515
540
|
Note: This error should not happen.
|
|
516
|
-
It's
|
|
541
|
+
It's probably a bug in the pipeline collection
|
|
517
542
|
|
|
518
543
|
Please report issue:
|
|
519
544
|
${block(getErrorReportUrl(new Error(message)).href)}
|
|
@@ -696,7 +721,8 @@
|
|
|
696
721
|
}
|
|
697
722
|
|
|
698
723
|
/**
|
|
699
|
-
*
|
|
724
|
+
* Safely retrieves the global scope object (window in browser, global in Node.js)
|
|
725
|
+
* regardless of the JavaScript environment in which the code is running
|
|
700
726
|
*
|
|
701
727
|
* Note: `$` is used to indicate that this function is not a pure function - it access global scope
|
|
702
728
|
*
|
|
@@ -707,10 +733,10 @@
|
|
|
707
733
|
}
|
|
708
734
|
|
|
709
735
|
/**
|
|
710
|
-
*
|
|
736
|
+
* Normalizes a text string to SCREAMING_CASE (all uppercase with underscores).
|
|
711
737
|
*
|
|
712
|
-
* @param text
|
|
713
|
-
* @returns
|
|
738
|
+
* @param text The text string to be converted to SCREAMING_CASE format.
|
|
739
|
+
* @returns The normalized text in SCREAMING_CASE format.
|
|
714
740
|
* @example 'HELLO_WORLD'
|
|
715
741
|
* @example 'I_LOVE_PROMPTBOOK'
|
|
716
742
|
* @public exported from `@promptbook/utils`
|
|
@@ -762,10 +788,10 @@
|
|
|
762
788
|
*/
|
|
763
789
|
|
|
764
790
|
/**
|
|
765
|
-
*
|
|
791
|
+
* Normalizes a text string to snake_case format.
|
|
766
792
|
*
|
|
767
|
-
* @param text
|
|
768
|
-
* @returns
|
|
793
|
+
* @param text The text string to be converted to snake_case format.
|
|
794
|
+
* @returns The normalized text in snake_case format.
|
|
769
795
|
* @example 'hello_world'
|
|
770
796
|
* @example 'i_love_promptbook'
|
|
771
797
|
* @public exported from `@promptbook/utils`
|
|
@@ -775,11 +801,11 @@
|
|
|
775
801
|
}
|
|
776
802
|
|
|
777
803
|
/**
|
|
778
|
-
*
|
|
804
|
+
* Global registry for storing and managing registered entities of a given type.
|
|
779
805
|
*
|
|
780
806
|
* Note: `$` is used to indicate that this function is not a pure function - it accesses and adds variables in global scope.
|
|
781
807
|
*
|
|
782
|
-
* @private internal utility, exported are only
|
|
808
|
+
* @private internal utility, exported are only singleton instances of this class
|
|
783
809
|
*/
|
|
784
810
|
class $Register {
|
|
785
811
|
constructor(registerName) {
|
|
@@ -823,10 +849,10 @@
|
|
|
823
849
|
}
|
|
824
850
|
|
|
825
851
|
/**
|
|
826
|
-
*
|
|
852
|
+
* Register for LLM tools metadata.
|
|
827
853
|
*
|
|
828
854
|
* Note: `$` is used to indicate that this interacts with the global scope
|
|
829
|
-
* @singleton Only one instance of each register is created per build, but
|
|
855
|
+
* @singleton Only one instance of each register is created per build, but there can be more instances across different builds or environments.
|
|
830
856
|
* @public exported from `@promptbook/core`
|
|
831
857
|
*/
|
|
832
858
|
const $llmToolsMetadataRegister = new $Register('llm_tools_metadata');
|
|
@@ -835,10 +861,10 @@
|
|
|
835
861
|
*/
|
|
836
862
|
|
|
837
863
|
/**
|
|
838
|
-
*
|
|
864
|
+
* Register for LLM tools.
|
|
839
865
|
*
|
|
840
866
|
* Note: `$` is used to indicate that this interacts with the global scope
|
|
841
|
-
* @singleton Only one instance of each register is created per build, but
|
|
867
|
+
* @singleton Only one instance of each register is created per build, but there can be more instances across different builds or environments.
|
|
842
868
|
* @public exported from `@promptbook/core`
|
|
843
869
|
*/
|
|
844
870
|
const $llmToolsRegister = new $Register('llm_execution_tools_constructors');
|
|
@@ -990,7 +1016,7 @@
|
|
|
990
1016
|
* Just says that the variable is not used but should be kept
|
|
991
1017
|
* No side effects.
|
|
992
1018
|
*
|
|
993
|
-
* Note: It can be
|
|
1019
|
+
* Note: It can be useful for:
|
|
994
1020
|
*
|
|
995
1021
|
* 1) Suppressing eager optimization of unused imports
|
|
996
1022
|
* 2) Suppressing eslint errors of unused variables in the tests
|
|
@@ -1043,15 +1069,12 @@
|
|
|
1043
1069
|
}
|
|
1044
1070
|
throw new Error(spaceTrim__default["default"]((block) => `
|
|
1045
1071
|
${block(error.message)}
|
|
1046
|
-
|
|
1072
|
+
|
|
1047
1073
|
The JSON text:
|
|
1048
1074
|
${block(value)}
|
|
1049
1075
|
`));
|
|
1050
1076
|
}
|
|
1051
1077
|
}
|
|
1052
|
-
/**
|
|
1053
|
-
* TODO: !!!! Use in Promptbook.studio
|
|
1054
|
-
*/
|
|
1055
1078
|
|
|
1056
1079
|
/**
|
|
1057
1080
|
* Convert identification to Promptbook token
|
|
@@ -1091,7 +1114,7 @@
|
|
|
1091
1114
|
* Just marks a place of place where should be something implemented
|
|
1092
1115
|
* No side effects.
|
|
1093
1116
|
*
|
|
1094
|
-
* Note: It can be
|
|
1117
|
+
* Note: It can be useful suppressing eslint errors of unused variables
|
|
1095
1118
|
*
|
|
1096
1119
|
* @param value any values
|
|
1097
1120
|
* @returns void
|
|
@@ -1101,7 +1124,8 @@
|
|
|
1101
1124
|
}
|
|
1102
1125
|
|
|
1103
1126
|
/**
|
|
1104
|
-
*
|
|
1127
|
+
* Provides filesystem access (for example for Node.js-based scrapers)
|
|
1128
|
+
* Creates a standardized filesystem interface that scrapers can use for file operations.
|
|
1105
1129
|
*
|
|
1106
1130
|
* @public exported from `@promptbook/node`
|
|
1107
1131
|
*/
|
|
@@ -1476,8 +1500,12 @@
|
|
|
1476
1500
|
*/
|
|
1477
1501
|
|
|
1478
1502
|
/**
|
|
1479
|
-
*
|
|
1503
|
+
* Creates a deep clone of the given object
|
|
1504
|
+
*
|
|
1505
|
+
* Note: This method only works for objects that are fully serializable to JSON and do not contain functions, Dates, or special types.
|
|
1480
1506
|
*
|
|
1507
|
+
* @param objectValue The object to clone.
|
|
1508
|
+
* @returns A deep, writable clone of the input object.
|
|
1481
1509
|
* @public exported from `@promptbook/utils`
|
|
1482
1510
|
*/
|
|
1483
1511
|
function deepClone(objectValue) {
|
|
@@ -1532,6 +1560,26 @@
|
|
|
1532
1560
|
* TODO: [🧠] Is there a way how to meaningfully test this utility
|
|
1533
1561
|
*/
|
|
1534
1562
|
|
|
1563
|
+
// <- TODO: Maybe do better levels of trust
|
|
1564
|
+
/**
|
|
1565
|
+
* How is the model provider important?
|
|
1566
|
+
*
|
|
1567
|
+
* @public exported from `@promptbook/core`
|
|
1568
|
+
*/
|
|
1569
|
+
const MODEL_ORDERS = {
|
|
1570
|
+
/**
|
|
1571
|
+
* Top-tier models, e.g. OpenAI, Anthropic,...
|
|
1572
|
+
*/
|
|
1573
|
+
TOP_TIER: 333,
|
|
1574
|
+
/**
|
|
1575
|
+
* Mid-tier models, e.g. Llama, Mistral, etc.
|
|
1576
|
+
*/
|
|
1577
|
+
NORMAL: 100,
|
|
1578
|
+
/**
|
|
1579
|
+
* Low-tier models, e.g. Phi, Tiny, etc.
|
|
1580
|
+
*/
|
|
1581
|
+
LOW_TIER: 0,
|
|
1582
|
+
};
|
|
1535
1583
|
/**
|
|
1536
1584
|
* Order of keys in the pipeline JSON
|
|
1537
1585
|
*
|
|
@@ -1559,13 +1607,19 @@
|
|
|
1559
1607
|
*/
|
|
1560
1608
|
const REPLACING_NONCE = 'ptbkauk42kV2dzao34faw7FudQUHYPtW';
|
|
1561
1609
|
/**
|
|
1562
|
-
*
|
|
1610
|
+
* Nonce which is used as string which is not occurring in normal text
|
|
1611
|
+
*
|
|
1612
|
+
* @private within the repository
|
|
1613
|
+
*/
|
|
1614
|
+
const SALT_NONCE = 'ptbkghhewbvruets21t54et5';
|
|
1615
|
+
/**
|
|
1616
|
+
* Placeholder value indicating a parameter is missing its value.
|
|
1563
1617
|
*
|
|
1564
1618
|
* @private within the repository
|
|
1565
1619
|
*/
|
|
1566
1620
|
const RESERVED_PARAMETER_MISSING_VALUE = 'MISSING-' + REPLACING_NONCE;
|
|
1567
1621
|
/**
|
|
1568
|
-
*
|
|
1622
|
+
* Placeholder value indicating a parameter is restricted and cannot be used directly.
|
|
1569
1623
|
*
|
|
1570
1624
|
* @private within the repository
|
|
1571
1625
|
*/
|
|
@@ -1655,7 +1709,7 @@
|
|
|
1655
1709
|
return pipelineJsonStringified;
|
|
1656
1710
|
}
|
|
1657
1711
|
/**
|
|
1658
|
-
* TODO: [🐝] Not Working
|
|
1712
|
+
* TODO: [🐝] Not Working properly @see https://promptbook.studio/examples/mixed-knowledge.book
|
|
1659
1713
|
* TODO: [🧠][0] Maybe rename to `stringifyPipelineJson`, `stringifyIndexedJson`,...
|
|
1660
1714
|
* TODO: [🧠] Maybe more elegant solution than replacing via regex
|
|
1661
1715
|
* TODO: [🍙] Make some standard order of json properties
|
|
@@ -1730,7 +1784,7 @@
|
|
|
1730
1784
|
* Tests if given string is valid URL.
|
|
1731
1785
|
*
|
|
1732
1786
|
* Note: Dataurl are considered perfectly valid.
|
|
1733
|
-
* Note: There are two
|
|
1787
|
+
* Note: There are two similar functions:
|
|
1734
1788
|
* - `isValidUrl` which tests any URL
|
|
1735
1789
|
* - `isValidPipelineUrl` *(this one)* which tests just promptbook URL
|
|
1736
1790
|
*
|
|
@@ -2000,10 +2054,10 @@
|
|
|
2000
2054
|
*/
|
|
2001
2055
|
|
|
2002
2056
|
/**
|
|
2003
|
-
*
|
|
2057
|
+
* Removes diacritic marks (accents) from characters in a string.
|
|
2004
2058
|
*
|
|
2005
|
-
* @param input
|
|
2006
|
-
* @returns
|
|
2059
|
+
* @param input The string containing diacritics to be normalized.
|
|
2060
|
+
* @returns The string with diacritics removed or normalized.
|
|
2007
2061
|
* @public exported from `@promptbook/utils`
|
|
2008
2062
|
*/
|
|
2009
2063
|
function removeDiacritics(input) {
|
|
@@ -2017,10 +2071,10 @@
|
|
|
2017
2071
|
*/
|
|
2018
2072
|
|
|
2019
2073
|
/**
|
|
2020
|
-
*
|
|
2074
|
+
* Converts a given text to kebab-case format.
|
|
2021
2075
|
*
|
|
2022
|
-
* @param text
|
|
2023
|
-
* @returns
|
|
2076
|
+
* @param text The text to be converted.
|
|
2077
|
+
* @returns The kebab-case formatted string.
|
|
2024
2078
|
* @example 'hello-world'
|
|
2025
2079
|
* @example 'i-love-promptbook'
|
|
2026
2080
|
* @public exported from `@promptbook/utils`
|
|
@@ -2068,11 +2122,11 @@
|
|
|
2068
2122
|
*/
|
|
2069
2123
|
|
|
2070
2124
|
/**
|
|
2071
|
-
*
|
|
2125
|
+
* Converts a title string into a normalized name.
|
|
2072
2126
|
*
|
|
2073
|
-
* @param value
|
|
2074
|
-
* @returns
|
|
2075
|
-
* @example
|
|
2127
|
+
* @param value The title string to be converted to a name.
|
|
2128
|
+
* @returns A normalized name derived from the input title.
|
|
2129
|
+
* @example 'Hello World!' -> 'hello-world'
|
|
2076
2130
|
* @public exported from `@promptbook/utils`
|
|
2077
2131
|
*/
|
|
2078
2132
|
function titleToName(value) {
|
|
@@ -2092,7 +2146,8 @@
|
|
|
2092
2146
|
}
|
|
2093
2147
|
|
|
2094
2148
|
/**
|
|
2095
|
-
*
|
|
2149
|
+
* Converts a name to a properly formatted subfolder path for cache storage.
|
|
2150
|
+
* Handles normalization and path formatting to create consistent cache directory structures.
|
|
2096
2151
|
*
|
|
2097
2152
|
* @private for `FileCacheStorage`
|
|
2098
2153
|
*/
|
|
@@ -2101,7 +2156,10 @@
|
|
|
2101
2156
|
}
|
|
2102
2157
|
|
|
2103
2158
|
/**
|
|
2104
|
-
*
|
|
2159
|
+
* A storage implementation that caches data in files organized in a directory structure.
|
|
2160
|
+
* Provides methods for retrieving, storing, and managing cached data on the filesystem.
|
|
2161
|
+
*
|
|
2162
|
+
* This class implements the PromptbookStorage interface for filesystem-based caching.
|
|
2105
2163
|
*
|
|
2106
2164
|
* @public exported from `@promptbook/node`
|
|
2107
2165
|
*/
|
|
@@ -2114,7 +2172,8 @@
|
|
|
2114
2172
|
}
|
|
2115
2173
|
}
|
|
2116
2174
|
/**
|
|
2117
|
-
*
|
|
2175
|
+
* Converts a storage key to a filesystem path where the data should be stored.
|
|
2176
|
+
* Creates a consistent, deterministic file path based on the key string.
|
|
2118
2177
|
*/
|
|
2119
2178
|
getFilenameForKey(key) {
|
|
2120
2179
|
// TODO: [👬] DRY
|
|
@@ -2126,7 +2185,8 @@
|
|
|
2126
2185
|
...nameToSubfolderPath(hash /* <- TODO: [🎎] Maybe add some SHA256 prefix */), `${name.substring(0, MAX_FILENAME_LENGTH)}.json`);
|
|
2127
2186
|
}
|
|
2128
2187
|
/**
|
|
2129
|
-
*
|
|
2188
|
+
* Returns the current value associated with the given key, or null if the given key does not exist.
|
|
2189
|
+
* Retrieves the cached data from the file system storage.
|
|
2130
2190
|
*/
|
|
2131
2191
|
async getItem(key) {
|
|
2132
2192
|
const filename = this.getFilenameForKey(key);
|
|
@@ -2139,7 +2199,8 @@
|
|
|
2139
2199
|
return value;
|
|
2140
2200
|
}
|
|
2141
2201
|
/**
|
|
2142
|
-
*
|
|
2202
|
+
* Sets the value of the pair identified by key to value, creating a new key/value pair if none existed for key previously.
|
|
2203
|
+
* Persists data to the file system, creating necessary directory structure if it doesn't exist.
|
|
2143
2204
|
*/
|
|
2144
2205
|
async setItem(key, value) {
|
|
2145
2206
|
const filename = this.getFilenameForKey(key);
|
|
@@ -2151,7 +2212,8 @@
|
|
|
2151
2212
|
await promises.writeFile(filename, fileContent, 'utf-8');
|
|
2152
2213
|
}
|
|
2153
2214
|
/**
|
|
2154
|
-
*
|
|
2215
|
+
* Removes the key/value pair with the given key from the storage, if a key/value pair with the given key exists.
|
|
2216
|
+
* Deletes the corresponding file from the filesystem.
|
|
2155
2217
|
*/
|
|
2156
2218
|
async removeItem(key) {
|
|
2157
2219
|
const filename = this.getFilenameForKey(key);
|
|
@@ -2210,7 +2272,7 @@
|
|
|
2210
2272
|
}
|
|
2211
2273
|
|
|
2212
2274
|
/**
|
|
2213
|
-
* This error indicates that the pipeline collection cannot be
|
|
2275
|
+
* This error indicates that the pipeline collection cannot be properly loaded
|
|
2214
2276
|
*
|
|
2215
2277
|
* @public exported from `@promptbook/core`
|
|
2216
2278
|
*/
|
|
@@ -2274,7 +2336,7 @@
|
|
|
2274
2336
|
super(spaceTrim.spaceTrim((block) => `
|
|
2275
2337
|
${block(message)}
|
|
2276
2338
|
|
|
2277
|
-
Note: You have
|
|
2339
|
+
Note: You have probably forgot to provide some tools for pipeline execution or preparation
|
|
2278
2340
|
|
|
2279
2341
|
`));
|
|
2280
2342
|
this.name = 'MissingToolsError';
|
|
@@ -2337,7 +2399,7 @@
|
|
|
2337
2399
|
super(message);
|
|
2338
2400
|
this.name = 'PipelineExecutionError';
|
|
2339
2401
|
// TODO: [🐙] DRY - Maybe $randomId
|
|
2340
|
-
this.id = `error-${$randomToken(8 /* <- TODO: To global config + Use Base58 to avoid
|
|
2402
|
+
this.id = `error-${$randomToken(8 /* <- TODO: To global config + Use Base58 to avoid similar char conflicts */)}`;
|
|
2341
2403
|
Object.setPrototypeOf(this, PipelineExecutionError.prototype);
|
|
2342
2404
|
}
|
|
2343
2405
|
}
|
|
@@ -2534,10 +2596,10 @@
|
|
|
2534
2596
|
}
|
|
2535
2597
|
get title() {
|
|
2536
2598
|
// TODO: [🧠] Maybe fetch title+description from the remote server (as well as if model methods are defined)
|
|
2537
|
-
return '
|
|
2599
|
+
return 'Promptbook remote server';
|
|
2538
2600
|
}
|
|
2539
2601
|
get description() {
|
|
2540
|
-
return
|
|
2602
|
+
return `Models from Promptbook remote server ${this.options.remoteServerUrl}`;
|
|
2541
2603
|
}
|
|
2542
2604
|
/**
|
|
2543
2605
|
* Check the configuration of all execution tools
|
|
@@ -2552,7 +2614,7 @@
|
|
|
2552
2614
|
* List all available models that can be used
|
|
2553
2615
|
*/
|
|
2554
2616
|
async listModels() {
|
|
2555
|
-
// TODO: [👒] Listing models (and checking configuration)
|
|
2617
|
+
// TODO: [👒] Listing models (and checking configuration) probably should go through REST API not Socket.io
|
|
2556
2618
|
const socket = await createRemoteClient(this.options);
|
|
2557
2619
|
socket.emit('listModels-request', {
|
|
2558
2620
|
identification: this.options.identification,
|
|
@@ -2689,6 +2751,23 @@
|
|
|
2689
2751
|
return new Date().toISOString();
|
|
2690
2752
|
}
|
|
2691
2753
|
|
|
2754
|
+
/**
|
|
2755
|
+
* Parses the task and returns the list of all parameter names
|
|
2756
|
+
*
|
|
2757
|
+
* @param template the string template with parameters in {curly} braces
|
|
2758
|
+
* @returns the list of parameter names
|
|
2759
|
+
* @public exported from `@promptbook/utils`
|
|
2760
|
+
*/
|
|
2761
|
+
function extractParameterNames(template) {
|
|
2762
|
+
const matches = template.matchAll(/{\w+}/g);
|
|
2763
|
+
const parameterNames = new Set();
|
|
2764
|
+
for (const match of matches) {
|
|
2765
|
+
const parameterName = match[0].slice(1, -1);
|
|
2766
|
+
parameterNames.add(parameterName);
|
|
2767
|
+
}
|
|
2768
|
+
return parameterNames;
|
|
2769
|
+
}
|
|
2770
|
+
|
|
2692
2771
|
/**
|
|
2693
2772
|
* Intercepts LLM tools and counts total usage of the tools
|
|
2694
2773
|
*
|
|
@@ -2699,17 +2778,19 @@
|
|
|
2699
2778
|
* @public exported from `@promptbook/core`
|
|
2700
2779
|
*/
|
|
2701
2780
|
function cacheLlmTools(llmTools, options = {}) {
|
|
2702
|
-
const { storage = new MemoryStorage(), isCacheReloaded = false } = options;
|
|
2781
|
+
const { storage = new MemoryStorage(), isCacheReloaded = false, isVerbose = DEFAULT_IS_VERBOSE } = options;
|
|
2703
2782
|
const proxyTools = {
|
|
2704
2783
|
...llmTools,
|
|
2705
2784
|
// <- Note: [🥫]
|
|
2706
2785
|
get title() {
|
|
2707
|
-
|
|
2708
|
-
|
|
2786
|
+
return `${llmTools.title} (cached)`;
|
|
2787
|
+
// <- TODO: [🧈] Maybe standartize the suffix when wrapping `LlmExecutionTools` up
|
|
2788
|
+
// <- TODO: [🧈][🧠] Does it make sence to suffix "(cached)"?
|
|
2709
2789
|
},
|
|
2710
2790
|
get description() {
|
|
2711
|
-
|
|
2712
|
-
|
|
2791
|
+
return `${llmTools.description} (cached)`;
|
|
2792
|
+
// <- TODO: [🧈] Maybe standartize the suffix when wrapping `LlmExecutionTools` up
|
|
2793
|
+
// <- TODO: [🧈][🧠] Does it make sence to suffix "(cached)"?
|
|
2713
2794
|
},
|
|
2714
2795
|
listModels() {
|
|
2715
2796
|
// TODO: [🧠] Should be model listing also cached?
|
|
@@ -2719,13 +2800,37 @@
|
|
|
2719
2800
|
const callCommonModel = async (prompt) => {
|
|
2720
2801
|
const { parameters, content, modelRequirements } = prompt;
|
|
2721
2802
|
// <- Note: These are relevant things from the prompt that the cache key should depend on.
|
|
2803
|
+
// TODO: Maybe some standalone function for normalization of content for cache
|
|
2804
|
+
let normalizedContent = content;
|
|
2805
|
+
normalizedContent = normalizedContent.replace(/\s+/g, ' ');
|
|
2806
|
+
normalizedContent = normalizedContent.split('\r\n').join('\n');
|
|
2807
|
+
normalizedContent = spaceTrim__default["default"](normalizedContent);
|
|
2808
|
+
// Note: Do not need to save everything in the cache, just the relevant parameters
|
|
2809
|
+
const relevantParameterNames = extractParameterNames(content);
|
|
2810
|
+
const relevantParameters = Object.fromEntries(Object.entries(parameters).filter(([key]) => relevantParameterNames.has(key)));
|
|
2811
|
+
const keyHashBase = { relevantParameters, normalizedContent, modelRequirements };
|
|
2722
2812
|
const key = titleToName(prompt.title.substring(0, MAX_FILENAME_LENGTH - 10) +
|
|
2723
2813
|
'-' +
|
|
2724
|
-
sha256__default["default"](hexEncoder__default["default"].parse(JSON.stringify(
|
|
2814
|
+
sha256__default["default"](hexEncoder__default["default"].parse(JSON.stringify(keyHashBase)))
|
|
2815
|
+
.toString( /* hex */)
|
|
2816
|
+
.substring(0, 10 - 1));
|
|
2725
2817
|
const cacheItem = !isCacheReloaded ? await storage.getItem(key) : null;
|
|
2726
2818
|
if (cacheItem) {
|
|
2727
2819
|
return cacheItem.promptResult;
|
|
2728
2820
|
}
|
|
2821
|
+
if (isVerbose) {
|
|
2822
|
+
console.info('Cache miss for key:', key, {
|
|
2823
|
+
prompt,
|
|
2824
|
+
'prompt.title': prompt.title,
|
|
2825
|
+
MAX_FILENAME_LENGTH,
|
|
2826
|
+
keyHashBase,
|
|
2827
|
+
parameters,
|
|
2828
|
+
relevantParameters,
|
|
2829
|
+
content,
|
|
2830
|
+
normalizedContent,
|
|
2831
|
+
modelRequirements,
|
|
2832
|
+
});
|
|
2833
|
+
}
|
|
2729
2834
|
let promptResult;
|
|
2730
2835
|
variant: switch (prompt.modelRequirements.modelVariant) {
|
|
2731
2836
|
case 'CHAT':
|
|
@@ -2746,7 +2851,16 @@
|
|
|
2746
2851
|
await storage.setItem(key, {
|
|
2747
2852
|
date: $getCurrentDate(),
|
|
2748
2853
|
promptbookVersion: PROMPTBOOK_ENGINE_VERSION,
|
|
2749
|
-
|
|
2854
|
+
bookVersion: BOOK_LANGUAGE_VERSION,
|
|
2855
|
+
prompt: {
|
|
2856
|
+
...prompt,
|
|
2857
|
+
parameters: Object.entries(parameters).length === Object.entries(relevantParameters).length
|
|
2858
|
+
? parameters
|
|
2859
|
+
: {
|
|
2860
|
+
...relevantParameters,
|
|
2861
|
+
note: `<- Note: Only relevant parameters are stored in the cache`,
|
|
2862
|
+
},
|
|
2863
|
+
},
|
|
2750
2864
|
promptResult,
|
|
2751
2865
|
});
|
|
2752
2866
|
return promptResult;
|
|
@@ -2772,9 +2886,9 @@
|
|
|
2772
2886
|
/**
|
|
2773
2887
|
* TODO: [🧠][💸] Maybe make some common abstraction `interceptLlmTools` and use here (or use javascript Proxy?)
|
|
2774
2888
|
* TODO: [🧠] Is there some meaningfull way how to test this util
|
|
2775
|
-
* TODO: [👷♂️]
|
|
2776
|
-
*
|
|
2777
|
-
*
|
|
2889
|
+
* TODO: [👷♂️] Comprehensive manual about construction of llmTools
|
|
2890
|
+
* Detailed explanation about caching strategies and appropriate storage selection for different use cases
|
|
2891
|
+
* Examples of how to combine multiple interceptors for advanced caching, logging, and usage tracking
|
|
2778
2892
|
*/
|
|
2779
2893
|
|
|
2780
2894
|
/**
|
|
@@ -2904,12 +3018,14 @@
|
|
|
2904
3018
|
const spending = new rxjs.Subject();
|
|
2905
3019
|
const proxyTools = {
|
|
2906
3020
|
get title() {
|
|
2907
|
-
|
|
2908
|
-
|
|
3021
|
+
return `${llmTools.title} (+usage)`;
|
|
3022
|
+
// <- TODO: [🧈] Maybe standartize the suffix when wrapping `LlmExecutionTools` up
|
|
3023
|
+
// <- TODO: [🧈][🧠] Does it make sence to suffix "(+usage)"?
|
|
2909
3024
|
},
|
|
2910
3025
|
get description() {
|
|
2911
|
-
|
|
2912
|
-
|
|
3026
|
+
return `${llmTools.description} (+usage)`;
|
|
3027
|
+
// <- TODO: [🧈] Maybe standartize the suffix when wrapping `LlmExecutionTools` up
|
|
3028
|
+
// <- TODO: [🧈][🧠] Does it make sence to suffix "(+usage)"?
|
|
2913
3029
|
},
|
|
2914
3030
|
checkConfiguration() {
|
|
2915
3031
|
return /* not await */ llmTools.checkConfiguration();
|
|
@@ -2964,9 +3080,8 @@
|
|
|
2964
3080
|
*/
|
|
2965
3081
|
|
|
2966
3082
|
/**
|
|
2967
|
-
*
|
|
3083
|
+
* Provides LLM tools configuration by reading environment variables.
|
|
2968
3084
|
*
|
|
2969
|
-
* @@@ .env
|
|
2970
3085
|
* Note: `$` is used to indicate that this function is not a pure function - it uses filesystem to access `.env` file
|
|
2971
3086
|
*
|
|
2972
3087
|
* It looks for environment variables:
|
|
@@ -2974,7 +3089,8 @@
|
|
|
2974
3089
|
* - `process.env.ANTHROPIC_CLAUDE_API_KEY`
|
|
2975
3090
|
* - ...
|
|
2976
3091
|
*
|
|
2977
|
-
* @
|
|
3092
|
+
* @see Environment variables documentation or .env file for required variables.
|
|
3093
|
+
* @returns A promise that resolves to the LLM tools configuration, or null if configuration is incomplete or missing.
|
|
2978
3094
|
* @public exported from `@promptbook/node`
|
|
2979
3095
|
*/
|
|
2980
3096
|
async function $provideLlmToolsConfigurationFromEnv() {
|
|
@@ -3012,29 +3128,40 @@
|
|
|
3012
3128
|
return 'Multiple LLM Providers';
|
|
3013
3129
|
}
|
|
3014
3130
|
get description() {
|
|
3015
|
-
|
|
3131
|
+
const innerModelsTitlesAndDescriptions = this.llmExecutionTools
|
|
3132
|
+
.map(({ title, description }, index) => {
|
|
3133
|
+
const headLine = `${index + 1}) \`${title}\``;
|
|
3134
|
+
if (description === undefined) {
|
|
3135
|
+
return headLine;
|
|
3136
|
+
}
|
|
3137
|
+
return spaceTrim__default["default"]((block) => `
|
|
3138
|
+
${headLine}
|
|
3139
|
+
|
|
3140
|
+
${ /* <- Note: Indenting the description: */block(description)}
|
|
3141
|
+
`);
|
|
3142
|
+
})
|
|
3143
|
+
.join('\n\n');
|
|
3144
|
+
return spaceTrim__default["default"]((block) => `
|
|
3145
|
+
Multiple LLM Providers:
|
|
3146
|
+
|
|
3147
|
+
${block(innerModelsTitlesAndDescriptions)}
|
|
3148
|
+
`);
|
|
3016
3149
|
}
|
|
3017
3150
|
/**
|
|
3018
3151
|
* Check the configuration of all execution tools
|
|
3019
3152
|
*/
|
|
3020
3153
|
async checkConfiguration() {
|
|
3021
|
-
//
|
|
3022
|
-
|
|
3023
|
-
await llmExecutionTools.checkConfiguration();
|
|
3024
|
-
}
|
|
3154
|
+
// Note: Run checks in parallel
|
|
3155
|
+
await Promise.all(this.llmExecutionTools.map((tools) => tools.checkConfiguration()));
|
|
3025
3156
|
}
|
|
3026
3157
|
/**
|
|
3027
3158
|
* List all available models that can be used
|
|
3028
3159
|
* This lists is a combination of all available models from all execution tools
|
|
3029
3160
|
*/
|
|
3030
3161
|
async listModels() {
|
|
3031
|
-
|
|
3032
|
-
|
|
3033
|
-
|
|
3034
|
-
const models = await llmExecutionTools.listModels();
|
|
3035
|
-
availableModels.push(...models);
|
|
3036
|
-
}
|
|
3037
|
-
return availableModels;
|
|
3162
|
+
// Obtain all models in parallel and flatten
|
|
3163
|
+
const modelArrays = await Promise.all(this.llmExecutionTools.map((tools) => tools.listModels()));
|
|
3164
|
+
return modelArrays.flat();
|
|
3038
3165
|
}
|
|
3039
3166
|
/**
|
|
3040
3167
|
* Calls the best available chat model
|
|
@@ -3188,11 +3315,16 @@
|
|
|
3188
3315
|
*/
|
|
3189
3316
|
|
|
3190
3317
|
/**
|
|
3191
|
-
*
|
|
3318
|
+
* Creates LLM execution tools from provided configuration objects
|
|
3319
|
+
*
|
|
3320
|
+
* Instantiates and configures LLM tool instances for each configuration entry,
|
|
3321
|
+
* combining them into a unified interface via MultipleLlmExecutionTools.
|
|
3192
3322
|
*
|
|
3193
3323
|
* Note: This function is not cached, every call creates new instance of `MultipleLlmExecutionTools`
|
|
3194
3324
|
*
|
|
3195
|
-
* @
|
|
3325
|
+
* @param configuration Array of LLM tool configurations to instantiate
|
|
3326
|
+
* @param options Additional options for configuring the LLM tools
|
|
3327
|
+
* @returns A unified interface combining all successfully instantiated LLM tools
|
|
3196
3328
|
* @public exported from `@promptbook/core`
|
|
3197
3329
|
*/
|
|
3198
3330
|
function createLlmToolsFromConfiguration(configuration, options = {}) {
|
|
@@ -3231,7 +3363,11 @@
|
|
|
3231
3363
|
/**
|
|
3232
3364
|
* TODO: [🎌] Together with `createLlmToolsFromConfiguration` + 'EXECUTION_TOOLS_CLASSES' gets to `@promptbook/core` ALL model providers, make this more efficient
|
|
3233
3365
|
* TODO: [🧠][🎌] Dynamically install required providers
|
|
3234
|
-
* TODO:
|
|
3366
|
+
* TODO: We should implement an interactive configuration wizard that would:
|
|
3367
|
+
* 1. Detect which LLM providers are available in the environment
|
|
3368
|
+
* 2. Guide users through required configuration settings for each provider
|
|
3369
|
+
* 3. Allow testing connections before completing setup
|
|
3370
|
+
* 4. Generate appropriate configuration code for application integration
|
|
3235
3371
|
* TODO: [🧠][🍛] Which name is better `createLlmToolsFromConfig` or `createLlmToolsFromConfiguration`?
|
|
3236
3372
|
* TODO: [🧠] Is there some meaningfull way how to test this util
|
|
3237
3373
|
* TODO: This should be maybe not under `_common` but under `utils`
|
|
@@ -3239,11 +3375,14 @@
|
|
|
3239
3375
|
*/
|
|
3240
3376
|
|
|
3241
3377
|
/**
|
|
3242
|
-
*
|
|
3378
|
+
* Automatically configures LLM tools from environment variables in Node.js
|
|
3379
|
+
*
|
|
3380
|
+
* This utility function detects available LLM providers based on environment variables
|
|
3381
|
+
* and creates properly configured LLM execution tools for each detected provider.
|
|
3243
3382
|
*
|
|
3244
3383
|
* Note: This function is not cached, every call creates new instance of `MultipleLlmExecutionTools`
|
|
3245
3384
|
*
|
|
3246
|
-
*
|
|
3385
|
+
* Supports environment variables from .env files when dotenv is configured
|
|
3247
3386
|
* Note: `$` is used to indicate that this function is not a pure function - it uses filesystem to access `.env` file
|
|
3248
3387
|
*
|
|
3249
3388
|
* It looks for environment variables:
|
|
@@ -3251,7 +3390,8 @@
|
|
|
3251
3390
|
* - `process.env.ANTHROPIC_CLAUDE_API_KEY`
|
|
3252
3391
|
* - ...
|
|
3253
3392
|
*
|
|
3254
|
-
* @
|
|
3393
|
+
* @param options Configuration options for the LLM tools
|
|
3394
|
+
* @returns A unified interface containing all detected and configured LLM tools
|
|
3255
3395
|
* @public exported from `@promptbook/node`
|
|
3256
3396
|
*/
|
|
3257
3397
|
async function $provideLlmToolsFromEnv(options = {}) {
|
|
@@ -3277,7 +3417,16 @@
|
|
|
3277
3417
|
return createLlmToolsFromConfiguration(configuration, options);
|
|
3278
3418
|
}
|
|
3279
3419
|
/**
|
|
3280
|
-
* TODO:
|
|
3420
|
+
* TODO: The architecture for LLM tools configuration consists of three key functions:
|
|
3421
|
+
* 1. `$provideLlmToolsFromEnv` - High-level function that detects available providers from env vars and returns ready-to-use LLM tools
|
|
3422
|
+
* 2. `$provideLlmToolsConfigurationFromEnv` - Middle layer that extracts configuration objects from environment variables
|
|
3423
|
+
* 3. `createLlmToolsFromConfiguration` - Low-level function that instantiates LLM tools from explicit configuration
|
|
3424
|
+
*
|
|
3425
|
+
* This layered approach allows flexibility in how tools are configured:
|
|
3426
|
+
* - Use $provideLlmToolsFromEnv for automatic detection and setup in Node.js environments
|
|
3427
|
+
* - Use $provideLlmToolsConfigurationFromEnv to extract config objects for modification before instantiation
|
|
3428
|
+
* - Use createLlmToolsFromConfiguration for explicit control over tool configurations
|
|
3429
|
+
*
|
|
3281
3430
|
* TODO: [🧠][🍛] Which name is better `$provideLlmToolsFromEnv` or `$provideLlmToolsFromEnvironment`?
|
|
3282
3431
|
* TODO: [🧠] Is there some meaningfull way how to test this util
|
|
3283
3432
|
* Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
|
|
@@ -3445,15 +3594,19 @@
|
|
|
3445
3594
|
type: 'text',
|
|
3446
3595
|
name: 'username',
|
|
3447
3596
|
message: 'Enter your email:',
|
|
3448
|
-
validate
|
|
3597
|
+
validate(value) {
|
|
3598
|
+
return isValidEmail(value) ? true : 'Valid email is required';
|
|
3599
|
+
},
|
|
3449
3600
|
},
|
|
3450
3601
|
{
|
|
3451
3602
|
type: 'password',
|
|
3452
3603
|
name: 'password',
|
|
3453
3604
|
message: 'Enter your password:',
|
|
3454
|
-
validate
|
|
3455
|
-
|
|
3456
|
-
|
|
3605
|
+
validate(value) {
|
|
3606
|
+
return value.length /* <- TODO: [🧠] Better password validation */ > 0
|
|
3607
|
+
? true
|
|
3608
|
+
: 'Password is required';
|
|
3609
|
+
},
|
|
3457
3610
|
},
|
|
3458
3611
|
]);
|
|
3459
3612
|
const loginUrl = `${remoteServerUrl}/login`;
|
|
@@ -3690,7 +3843,8 @@
|
|
|
3690
3843
|
*/
|
|
3691
3844
|
|
|
3692
3845
|
/**
|
|
3693
|
-
*
|
|
3846
|
+
* Attempts to locate the specified application on a Linux system using the 'which' command.
|
|
3847
|
+
* Returns the path to the executable if found, or null otherwise.
|
|
3694
3848
|
*
|
|
3695
3849
|
* @private within the repository
|
|
3696
3850
|
*/
|
|
@@ -3733,7 +3887,8 @@
|
|
|
3733
3887
|
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
|
3734
3888
|
const userhome = require('userhome');
|
|
3735
3889
|
/**
|
|
3736
|
-
*
|
|
3890
|
+
* Attempts to locate the specified application on a macOS system by checking standard application paths and using mdfind.
|
|
3891
|
+
* Returns the path to the executable if found, or null otherwise.
|
|
3737
3892
|
*
|
|
3738
3893
|
* @private within the repository
|
|
3739
3894
|
*/
|
|
@@ -3765,7 +3920,8 @@
|
|
|
3765
3920
|
*/
|
|
3766
3921
|
|
|
3767
3922
|
/**
|
|
3768
|
-
*
|
|
3923
|
+
* Attempts to locate the specified application on a Windows system by searching common installation directories.
|
|
3924
|
+
* Returns the path to the executable if found, or null otherwise.
|
|
3769
3925
|
*
|
|
3770
3926
|
* @private within the repository
|
|
3771
3927
|
*/
|
|
@@ -3836,7 +3992,8 @@
|
|
|
3836
3992
|
*/
|
|
3837
3993
|
|
|
3838
3994
|
/**
|
|
3839
|
-
*
|
|
3995
|
+
* Locates the LibreOffice executable on the current system by searching platform-specific paths.
|
|
3996
|
+
* Returns the path to the executable if found, or null otherwise.
|
|
3840
3997
|
*
|
|
3841
3998
|
* @private within the repository
|
|
3842
3999
|
*/
|
|
@@ -3854,7 +4011,8 @@
|
|
|
3854
4011
|
*/
|
|
3855
4012
|
|
|
3856
4013
|
/**
|
|
3857
|
-
*
|
|
4014
|
+
* Locates the Pandoc executable on the current system by searching platform-specific paths.
|
|
4015
|
+
* Returns the path to the executable if found, or null otherwise.
|
|
3858
4016
|
*
|
|
3859
4017
|
* @private within the repository
|
|
3860
4018
|
*/
|
|
@@ -3872,7 +4030,7 @@
|
|
|
3872
4030
|
*/
|
|
3873
4031
|
|
|
3874
4032
|
/**
|
|
3875
|
-
*
|
|
4033
|
+
* Provides paths to required executables (i.e. as Pandoc and LibreOffice) for Node.js environments.
|
|
3876
4034
|
*
|
|
3877
4035
|
* @public exported from `@promptbook/node`
|
|
3878
4036
|
*/
|
|
@@ -3892,10 +4050,11 @@
|
|
|
3892
4050
|
*/
|
|
3893
4051
|
|
|
3894
4052
|
/**
|
|
3895
|
-
*
|
|
4053
|
+
* Registry for all available scrapers in the system.
|
|
4054
|
+
* Central point for registering and accessing different types of content scrapers.
|
|
3896
4055
|
*
|
|
3897
4056
|
* Note: `$` is used to indicate that this interacts with the global scope
|
|
3898
|
-
* @singleton Only one instance of each register is created per build, but
|
|
4057
|
+
* @singleton Only one instance of each register is created per build, but there can be more than one in different build modules
|
|
3899
4058
|
* @public exported from `@promptbook/core`
|
|
3900
4059
|
*/
|
|
3901
4060
|
const $scrapersRegister = new $Register('scraper_constructors');
|
|
@@ -3904,11 +4063,9 @@
|
|
|
3904
4063
|
*/
|
|
3905
4064
|
|
|
3906
4065
|
/**
|
|
3907
|
-
*
|
|
3908
|
-
*
|
|
3909
|
-
*
|
|
3910
|
-
* 2) @@@
|
|
3911
|
-
*
|
|
4066
|
+
* Provides a collection of scrapers optimized for Node.js environment.
|
|
4067
|
+
* 1) `provideScrapersForNode` use as default
|
|
4068
|
+
* 2) `provideScrapersForBrowser` use in limited browser environment *
|
|
3912
4069
|
* @public exported from `@promptbook/node`
|
|
3913
4070
|
*/
|
|
3914
4071
|
async function $provideScrapersForNode(tools, options) {
|
|
@@ -3933,10 +4090,10 @@
|
|
|
3933
4090
|
*/
|
|
3934
4091
|
|
|
3935
4092
|
/**
|
|
3936
|
-
*
|
|
4093
|
+
* Global registry for storing metadata about all available scrapers and converters.
|
|
3937
4094
|
*
|
|
3938
|
-
* Note: `$` is used to indicate that this interacts with the global scope
|
|
3939
|
-
* @singleton Only one instance of each register is created per build, but
|
|
4095
|
+
* Note: `$` is used to indicate that this interacts with the global scope.
|
|
4096
|
+
* @singleton Only one instance of each register is created per build, but there can be more in different contexts (e.g., tests).
|
|
3940
4097
|
* @public exported from `@promptbook/core`
|
|
3941
4098
|
*/
|
|
3942
4099
|
const $scrapersMetadataRegister = new $Register('scrapers_metadata');
|
|
@@ -4119,7 +4276,7 @@
|
|
|
4119
4276
|
/**
|
|
4120
4277
|
* Tests if given string is valid semantic version
|
|
4121
4278
|
*
|
|
4122
|
-
* Note: There are two
|
|
4279
|
+
* Note: There are two similar functions:
|
|
4123
4280
|
* - `isValidSemanticVersion` which tests any semantic version
|
|
4124
4281
|
* - `isValidPromptbookVersion` *(this one)* which tests just Promptbook versions
|
|
4125
4282
|
*
|
|
@@ -4141,7 +4298,7 @@
|
|
|
4141
4298
|
*
|
|
4142
4299
|
* @see https://www.npmjs.com/package/promptbook?activeTab=versions
|
|
4143
4300
|
* Note: When you are using for example promptbook 2.0.0 and there already is promptbook 3.0.0 it don`t know about it.
|
|
4144
|
-
* Note: There are two
|
|
4301
|
+
* Note: There are two similar functions:
|
|
4145
4302
|
* - `isValidSemanticVersion` which tests any semantic version
|
|
4146
4303
|
* - `isValidPromptbookVersion` *(this one)* which tests just Promptbook versions
|
|
4147
4304
|
*
|
|
@@ -4161,7 +4318,7 @@
|
|
|
4161
4318
|
/**
|
|
4162
4319
|
* Tests if given string is valid pipeline URL URL.
|
|
4163
4320
|
*
|
|
4164
|
-
* Note: There are two
|
|
4321
|
+
* Note: There are two similar functions:
|
|
4165
4322
|
* - `isValidUrl` which tests any URL
|
|
4166
4323
|
* - `isValidPipelineUrl` *(this one)* which tests just pipeline URL
|
|
4167
4324
|
*
|
|
@@ -4258,7 +4415,7 @@
|
|
|
4258
4415
|
${block(pipelineIdentification)}
|
|
4259
4416
|
`));
|
|
4260
4417
|
}
|
|
4261
|
-
// TODO: [🧠] Maybe do here some
|
|
4418
|
+
// TODO: [🧠] Maybe do here some proper JSON-schema / ZOD checking
|
|
4262
4419
|
if (!Array.isArray(pipeline.parameters)) {
|
|
4263
4420
|
// TODO: [🧠] what is the correct error tp throw - maybe PromptbookSchemaError
|
|
4264
4421
|
throw new ParseError(spaceTrim.spaceTrim((block) => `
|
|
@@ -4269,7 +4426,7 @@
|
|
|
4269
4426
|
${block(pipelineIdentification)}
|
|
4270
4427
|
`));
|
|
4271
4428
|
}
|
|
4272
|
-
// TODO: [🧠] Maybe do here some
|
|
4429
|
+
// TODO: [🧠] Maybe do here some proper JSON-schema / ZOD checking
|
|
4273
4430
|
if (!Array.isArray(pipeline.tasks)) {
|
|
4274
4431
|
// TODO: [🧠] what is the correct error tp throw - maybe PromptbookSchemaError
|
|
4275
4432
|
throw new ParseError(spaceTrim.spaceTrim((block) => `
|
|
@@ -4516,7 +4673,7 @@
|
|
|
4516
4673
|
* Function isValidJsonString will tell you if the string is valid JSON or not
|
|
4517
4674
|
*
|
|
4518
4675
|
* @param value The string to check
|
|
4519
|
-
* @returns
|
|
4676
|
+
* @returns `true` if the string is a valid JSON string, false otherwise
|
|
4520
4677
|
*
|
|
4521
4678
|
* @public exported from `@promptbook/utils`
|
|
4522
4679
|
*/
|
|
@@ -4628,7 +4785,7 @@
|
|
|
4628
4785
|
if (bookVersion !== `undefined`) {
|
|
4629
4786
|
commands.push(`BOOK VERSION ${bookVersion}`);
|
|
4630
4787
|
}
|
|
4631
|
-
// TODO: [main] !!5 This increases size of the bundle and is
|
|
4788
|
+
// TODO: [main] !!5 This increases size of the bundle and is probably not necessary
|
|
4632
4789
|
pipelineString = prettifyMarkdown(pipelineString);
|
|
4633
4790
|
for (const parameter of parameters.filter(({ isInput }) => isInput)) {
|
|
4634
4791
|
commands.push(`INPUT PARAMETER ${taskParameterJsonToString(parameter)}`);
|
|
@@ -4746,23 +4903,6 @@
|
|
|
4746
4903
|
* TODO: [🧠] Should be in generated .book.md file GENERATOR_WARNING
|
|
4747
4904
|
*/
|
|
4748
4905
|
|
|
4749
|
-
/**
|
|
4750
|
-
* Parses the task and returns the list of all parameter names
|
|
4751
|
-
*
|
|
4752
|
-
* @param template the string template with parameters in {curly} braces
|
|
4753
|
-
* @returns the list of parameter names
|
|
4754
|
-
* @public exported from `@promptbook/utils`
|
|
4755
|
-
*/
|
|
4756
|
-
function extractParameterNames(template) {
|
|
4757
|
-
const matches = template.matchAll(/{\w+}/g);
|
|
4758
|
-
const parameterNames = new Set();
|
|
4759
|
-
for (const match of matches) {
|
|
4760
|
-
const parameterName = match[0].slice(1, -1);
|
|
4761
|
-
parameterNames.add(parameterName);
|
|
4762
|
-
}
|
|
4763
|
-
return parameterNames;
|
|
4764
|
-
}
|
|
4765
|
-
|
|
4766
4906
|
/**
|
|
4767
4907
|
* Unprepare just strips the preparation data of the pipeline
|
|
4768
4908
|
*
|
|
@@ -4812,7 +4952,7 @@
|
|
|
4812
4952
|
/**
|
|
4813
4953
|
* Constructs a pipeline collection from pipelines
|
|
4814
4954
|
*
|
|
4815
|
-
* @param pipelines
|
|
4955
|
+
* @param pipelines Array of pipeline JSON objects to include in the collection
|
|
4816
4956
|
*
|
|
4817
4957
|
* Note: During the construction logic of all pipelines are validated
|
|
4818
4958
|
* Note: It is not recommended to use this constructor directly, use `createCollectionFromJson` *(or other variant)* instead
|
|
@@ -4924,15 +5064,18 @@
|
|
|
4924
5064
|
* @public exported from `@promptbook/core`
|
|
4925
5065
|
*/
|
|
4926
5066
|
function isPipelinePrepared(pipeline) {
|
|
4927
|
-
// Note: Ignoring `pipeline.preparations`
|
|
4928
|
-
// Note: Ignoring `pipeline.knowledgePieces`
|
|
5067
|
+
// Note: Ignoring `pipeline.preparations`
|
|
5068
|
+
// Note: Ignoring `pipeline.knowledgePieces`
|
|
4929
5069
|
if (pipeline.title === undefined || pipeline.title === '' || pipeline.title === DEFAULT_BOOK_TITLE) {
|
|
5070
|
+
// console.log('Pipeline is not prepared because title is undefined or empty', pipeline);
|
|
4930
5071
|
return false;
|
|
4931
5072
|
}
|
|
4932
5073
|
if (!pipeline.personas.every((persona) => persona.modelsRequirements !== undefined)) {
|
|
5074
|
+
// console.log('Pipeline is not prepared because personas are not prepared', pipeline.personas);
|
|
4933
5075
|
return false;
|
|
4934
5076
|
}
|
|
4935
5077
|
if (!pipeline.knowledgeSources.every((knowledgeSource) => knowledgeSource.preparationIds !== undefined)) {
|
|
5078
|
+
//console.log('Pipeline is not prepared because knowledge sources are not prepared', pipeline.knowledgeSources);
|
|
4936
5079
|
return false;
|
|
4937
5080
|
}
|
|
4938
5081
|
/*
|
|
@@ -4953,6 +5096,35 @@
|
|
|
4953
5096
|
* - [♨] Are tasks prepared
|
|
4954
5097
|
*/
|
|
4955
5098
|
|
|
5099
|
+
/**
|
|
5100
|
+
* Serializes an error into a [🚉] JSON-serializable object
|
|
5101
|
+
*
|
|
5102
|
+
* @public exported from `@promptbook/utils`
|
|
5103
|
+
*/
|
|
5104
|
+
function serializeError(error) {
|
|
5105
|
+
const { name, message, stack } = error;
|
|
5106
|
+
const { id } = error;
|
|
5107
|
+
if (!Object.keys(ALL_ERRORS).includes(name)) {
|
|
5108
|
+
console.error(spaceTrim__default["default"]((block) => `
|
|
5109
|
+
|
|
5110
|
+
Cannot serialize error with name "${name}"
|
|
5111
|
+
|
|
5112
|
+
Authors of Promptbook probably forgot to add this error into the list of errors:
|
|
5113
|
+
https://github.com/webgptorg/promptbook/blob/main/src/errors/0-index.ts
|
|
5114
|
+
|
|
5115
|
+
|
|
5116
|
+
${block(stack || message)}
|
|
5117
|
+
|
|
5118
|
+
`));
|
|
5119
|
+
}
|
|
5120
|
+
return {
|
|
5121
|
+
name: name,
|
|
5122
|
+
message,
|
|
5123
|
+
stack,
|
|
5124
|
+
id, // Include id in the serialized object
|
|
5125
|
+
};
|
|
5126
|
+
}
|
|
5127
|
+
|
|
4956
5128
|
/**
|
|
4957
5129
|
* Recursively converts JSON strings to JSON objects
|
|
4958
5130
|
|
|
@@ -5031,8 +5203,9 @@
|
|
|
5031
5203
|
*/
|
|
5032
5204
|
function createTask(options) {
|
|
5033
5205
|
const { taskType, taskProcessCallback } = options;
|
|
5206
|
+
let { title } = options;
|
|
5034
5207
|
// TODO: [🐙] DRY
|
|
5035
|
-
const taskId = `${taskType.toLowerCase().substring(0, 4)}-${$randomToken(8 /* <- TODO: To global config + Use Base58 to avoid
|
|
5208
|
+
const taskId = `${taskType.toLowerCase().substring(0, 4)}-${$randomToken(8 /* <- TODO: To global config + Use Base58 to avoid similar char conflicts */)}`;
|
|
5036
5209
|
let status = 'RUNNING';
|
|
5037
5210
|
const createdAt = new Date();
|
|
5038
5211
|
let updatedAt = createdAt;
|
|
@@ -5042,6 +5215,10 @@
|
|
|
5042
5215
|
const partialResultSubject = new rxjs.Subject();
|
|
5043
5216
|
// <- Note: Not using `BehaviorSubject` because on error we can't access the last value
|
|
5044
5217
|
const finalResultPromise = /* not await */ taskProcessCallback((newOngoingResult) => {
|
|
5218
|
+
if (newOngoingResult.title) {
|
|
5219
|
+
title = newOngoingResult.title;
|
|
5220
|
+
}
|
|
5221
|
+
updatedAt = new Date();
|
|
5045
5222
|
Object.assign(currentValue, newOngoingResult);
|
|
5046
5223
|
// <- TODO: assign deep
|
|
5047
5224
|
partialResultSubject.next(newOngoingResult);
|
|
@@ -5087,17 +5264,24 @@
|
|
|
5087
5264
|
return {
|
|
5088
5265
|
taskType,
|
|
5089
5266
|
taskId,
|
|
5267
|
+
get promptbookVersion() {
|
|
5268
|
+
return PROMPTBOOK_ENGINE_VERSION;
|
|
5269
|
+
},
|
|
5270
|
+
get title() {
|
|
5271
|
+
return title;
|
|
5272
|
+
// <- Note: [1] These must be getters to allow changing the value in the future
|
|
5273
|
+
},
|
|
5090
5274
|
get status() {
|
|
5091
5275
|
return status;
|
|
5092
|
-
// <- Note: [1]
|
|
5276
|
+
// <- Note: [1] --||--
|
|
5093
5277
|
},
|
|
5094
5278
|
get createdAt() {
|
|
5095
5279
|
return createdAt;
|
|
5096
|
-
// <- Note: [1]
|
|
5280
|
+
// <- Note: [1] --||--
|
|
5097
5281
|
},
|
|
5098
5282
|
get updatedAt() {
|
|
5099
5283
|
return updatedAt;
|
|
5100
|
-
// <- Note: [1]
|
|
5284
|
+
// <- Note: [1] --||--
|
|
5101
5285
|
},
|
|
5102
5286
|
asPromise,
|
|
5103
5287
|
asObservable() {
|
|
@@ -5105,15 +5289,15 @@
|
|
|
5105
5289
|
},
|
|
5106
5290
|
get errors() {
|
|
5107
5291
|
return errors;
|
|
5108
|
-
// <- Note: [1]
|
|
5292
|
+
// <- Note: [1] --||--
|
|
5109
5293
|
},
|
|
5110
5294
|
get warnings() {
|
|
5111
5295
|
return warnings;
|
|
5112
|
-
// <- Note: [1]
|
|
5296
|
+
// <- Note: [1] --||--
|
|
5113
5297
|
},
|
|
5114
5298
|
get currentValue() {
|
|
5115
5299
|
return currentValue;
|
|
5116
|
-
// <- Note: [1]
|
|
5300
|
+
// <- Note: [1] --||--
|
|
5117
5301
|
},
|
|
5118
5302
|
};
|
|
5119
5303
|
}
|
|
@@ -5122,35 +5306,6 @@
|
|
|
5122
5306
|
* TODO: [🐚] Split into more files and make `PrepareTask` & `RemoteTask` + split the function
|
|
5123
5307
|
*/
|
|
5124
5308
|
|
|
5125
|
-
/**
|
|
5126
|
-
* Serializes an error into a [🚉] JSON-serializable object
|
|
5127
|
-
*
|
|
5128
|
-
* @public exported from `@promptbook/utils`
|
|
5129
|
-
*/
|
|
5130
|
-
function serializeError(error) {
|
|
5131
|
-
const { name, message, stack } = error;
|
|
5132
|
-
const { id } = error;
|
|
5133
|
-
if (!Object.keys(ALL_ERRORS).includes(name)) {
|
|
5134
|
-
console.error(spaceTrim__default["default"]((block) => `
|
|
5135
|
-
|
|
5136
|
-
Cannot serialize error with name "${name}"
|
|
5137
|
-
|
|
5138
|
-
Authors of Promptbook probably forgot to add this error into the list of errors:
|
|
5139
|
-
https://github.com/webgptorg/promptbook/blob/main/src/errors/0-index.ts
|
|
5140
|
-
|
|
5141
|
-
|
|
5142
|
-
${block(stack || message)}
|
|
5143
|
-
|
|
5144
|
-
`));
|
|
5145
|
-
}
|
|
5146
|
-
return {
|
|
5147
|
-
name: name,
|
|
5148
|
-
message,
|
|
5149
|
-
stack,
|
|
5150
|
-
id, // Include id in the serialized object
|
|
5151
|
-
};
|
|
5152
|
-
}
|
|
5153
|
-
|
|
5154
5309
|
/**
|
|
5155
5310
|
* Format either small or big number
|
|
5156
5311
|
*
|
|
@@ -5393,7 +5548,7 @@
|
|
|
5393
5548
|
}
|
|
5394
5549
|
|
|
5395
5550
|
/**
|
|
5396
|
-
*
|
|
5551
|
+
* Contains configuration options for parsing and generating CSV files, such as delimiters and quoting rules.
|
|
5397
5552
|
*
|
|
5398
5553
|
* @public exported from `@promptbook/core`
|
|
5399
5554
|
*/
|
|
@@ -5402,11 +5557,29 @@
|
|
|
5402
5557
|
// encoding: 'utf-8',
|
|
5403
5558
|
});
|
|
5404
5559
|
|
|
5560
|
+
/**
|
|
5561
|
+
* Converts a CSV string into an object
|
|
5562
|
+
*
|
|
5563
|
+
* Note: This is wrapper around `papaparse.parse()` with better autohealing
|
|
5564
|
+
*
|
|
5565
|
+
* @private - for now until `@promptbook/csv` is released
|
|
5566
|
+
*/
|
|
5567
|
+
function csvParse(value /* <- TODO: string_csv */, settings, schema /* <- TODO: Make CSV Schemas */) {
|
|
5568
|
+
settings = { ...settings, ...MANDATORY_CSV_SETTINGS };
|
|
5569
|
+
// Note: Autoheal invalid '\n' characters
|
|
5570
|
+
if (settings.newline && !settings.newline.includes('\r') && value.includes('\r')) {
|
|
5571
|
+
console.warn('CSV string contains carriage return characters, but in the CSV settings the `newline` setting does not include them. Autohealing the CSV string.');
|
|
5572
|
+
value = value.replace(/\r\n/g, '\n').replace(/\r/g, '\n');
|
|
5573
|
+
}
|
|
5574
|
+
const csv = papaparse.parse(value, settings);
|
|
5575
|
+
return csv;
|
|
5576
|
+
}
|
|
5577
|
+
|
|
5405
5578
|
/**
|
|
5406
5579
|
* Function to check if a string is valid CSV
|
|
5407
5580
|
*
|
|
5408
5581
|
* @param value The string to check
|
|
5409
|
-
* @returns
|
|
5582
|
+
* @returns `true` if the string is a valid CSV string, false otherwise
|
|
5410
5583
|
*
|
|
5411
5584
|
* @public exported from `@promptbook/utils`
|
|
5412
5585
|
*/
|
|
@@ -5430,7 +5603,7 @@
|
|
|
5430
5603
|
* @public exported from `@promptbook/core`
|
|
5431
5604
|
* <- TODO: [🏢] Export from package `@promptbook/csv`
|
|
5432
5605
|
*/
|
|
5433
|
-
const
|
|
5606
|
+
const CsvFormatParser = {
|
|
5434
5607
|
formatName: 'CSV',
|
|
5435
5608
|
aliases: ['SPREADSHEET', 'TABLE'],
|
|
5436
5609
|
isValid(value, settings, schema) {
|
|
@@ -5442,12 +5615,12 @@
|
|
|
5442
5615
|
heal(value, settings, schema) {
|
|
5443
5616
|
throw new Error('Not implemented');
|
|
5444
5617
|
},
|
|
5445
|
-
|
|
5618
|
+
subvalueParsers: [
|
|
5446
5619
|
{
|
|
5447
5620
|
subvalueName: 'ROW',
|
|
5448
|
-
async mapValues(
|
|
5449
|
-
|
|
5450
|
-
const csv =
|
|
5621
|
+
async mapValues(options) {
|
|
5622
|
+
const { value, outputParameterName, settings, mapCallback, onProgress } = options;
|
|
5623
|
+
const csv = csvParse(value, settings);
|
|
5451
5624
|
if (csv.errors.length !== 0) {
|
|
5452
5625
|
throw new CsvFormatError(spaceTrim__default["default"]((block) => `
|
|
5453
5626
|
CSV parsing error
|
|
@@ -5462,23 +5635,37 @@
|
|
|
5462
5635
|
${block(value)}
|
|
5463
5636
|
`));
|
|
5464
5637
|
}
|
|
5465
|
-
const mappedData =
|
|
5638
|
+
const mappedData = [];
|
|
5639
|
+
const length = csv.data.length;
|
|
5640
|
+
for (let index = 0; index < length; index++) {
|
|
5641
|
+
const row = csv.data[index];
|
|
5466
5642
|
if (row[outputParameterName]) {
|
|
5467
5643
|
throw new CsvFormatError(`Can not overwrite existing column "${outputParameterName}" in CSV row`);
|
|
5468
5644
|
}
|
|
5469
|
-
|
|
5645
|
+
const mappedRow = {
|
|
5470
5646
|
...row,
|
|
5471
|
-
[outputParameterName]: await mapCallback(row, index),
|
|
5647
|
+
[outputParameterName]: await mapCallback(row, index, length),
|
|
5472
5648
|
};
|
|
5473
|
-
|
|
5649
|
+
mappedData.push(mappedRow);
|
|
5650
|
+
if (onProgress) {
|
|
5651
|
+
// Note: Report the CSV with all rows mapped so far
|
|
5652
|
+
/*
|
|
5653
|
+
// TODO: [🛕] Report progress with all the rows including the pending ones
|
|
5654
|
+
const progressData = mappedData.map((row, i) =>
|
|
5655
|
+
i > index ? { ...row, [outputParameterName]: PENDING_VALUE_PLACEHOLDER } : row,
|
|
5656
|
+
);
|
|
5657
|
+
*/
|
|
5658
|
+
await onProgress(papaparse.unparse(mappedData, { ...settings, ...MANDATORY_CSV_SETTINGS }));
|
|
5659
|
+
}
|
|
5660
|
+
}
|
|
5474
5661
|
return papaparse.unparse(mappedData, { ...settings, ...MANDATORY_CSV_SETTINGS });
|
|
5475
5662
|
},
|
|
5476
5663
|
},
|
|
5477
5664
|
{
|
|
5478
5665
|
subvalueName: 'CELL',
|
|
5479
|
-
async mapValues(
|
|
5480
|
-
|
|
5481
|
-
const csv =
|
|
5666
|
+
async mapValues(options) {
|
|
5667
|
+
const { value, settings, mapCallback, onProgress } = options;
|
|
5668
|
+
const csv = csvParse(value, settings);
|
|
5482
5669
|
if (csv.errors.length !== 0) {
|
|
5483
5670
|
throw new CsvFormatError(spaceTrim__default["default"]((block) => `
|
|
5484
5671
|
CSV parsing error
|
|
@@ -5494,9 +5681,9 @@
|
|
|
5494
5681
|
`));
|
|
5495
5682
|
}
|
|
5496
5683
|
const mappedData = await Promise.all(csv.data.map(async (row, rowIndex) => {
|
|
5497
|
-
return /* not await */ Promise.all(Object.entries(row).map(async ([key, value], columnIndex) => {
|
|
5684
|
+
return /* not await */ Promise.all(Object.entries(row).map(async ([key, value], columnIndex, array) => {
|
|
5498
5685
|
const index = rowIndex * Object.keys(row).length + columnIndex;
|
|
5499
|
-
return /* not await */ mapCallback({ [key]: value }, index);
|
|
5686
|
+
return /* not await */ mapCallback({ [key]: value }, index, array.length);
|
|
5500
5687
|
}));
|
|
5501
5688
|
}));
|
|
5502
5689
|
return papaparse.unparse(mappedData, { ...settings, ...MANDATORY_CSV_SETTINGS });
|
|
@@ -5505,10 +5692,10 @@
|
|
|
5505
5692
|
],
|
|
5506
5693
|
};
|
|
5507
5694
|
/**
|
|
5508
|
-
* TODO: [🍓] In `
|
|
5509
|
-
* TODO: [🍓] In `
|
|
5510
|
-
* TODO: [🍓] In `
|
|
5511
|
-
* TODO: [🍓] In `
|
|
5695
|
+
* TODO: [🍓] In `CsvFormatParser` implement simple `isValid`
|
|
5696
|
+
* TODO: [🍓] In `CsvFormatParser` implement partial `canBeValid`
|
|
5697
|
+
* TODO: [🍓] In `CsvFormatParser` implement `heal
|
|
5698
|
+
* TODO: [🍓] In `CsvFormatParser` implement `subvalueParsers`
|
|
5512
5699
|
* TODO: [🏢] Allow to expect something inside CSV objects and other formats
|
|
5513
5700
|
*/
|
|
5514
5701
|
|
|
@@ -5517,7 +5704,7 @@
|
|
|
5517
5704
|
*
|
|
5518
5705
|
* @private still in development [🏢]
|
|
5519
5706
|
*/
|
|
5520
|
-
const
|
|
5707
|
+
const JsonFormatParser = {
|
|
5521
5708
|
formatName: 'JSON',
|
|
5522
5709
|
mimeType: 'application/json',
|
|
5523
5710
|
isValid(value, settings, schema) {
|
|
@@ -5529,28 +5716,28 @@
|
|
|
5529
5716
|
heal(value, settings, schema) {
|
|
5530
5717
|
throw new Error('Not implemented');
|
|
5531
5718
|
},
|
|
5532
|
-
|
|
5719
|
+
subvalueParsers: [],
|
|
5533
5720
|
};
|
|
5534
5721
|
/**
|
|
5535
|
-
* TODO: [🧠] Maybe
|
|
5722
|
+
* TODO: [🧠] Maybe proper instance of object
|
|
5536
5723
|
* TODO: [0] Make string_serialized_json
|
|
5537
5724
|
* TODO: [1] Make type for JSON Settings and Schema
|
|
5538
5725
|
* TODO: [🧠] What to use for validating JSONs - JSON Schema, ZoD, typescript types/interfaces,...?
|
|
5539
|
-
* TODO: [🍓] In `
|
|
5540
|
-
* TODO: [🍓] In `
|
|
5541
|
-
* TODO: [🍓] In `
|
|
5542
|
-
* TODO: [🍓] In `
|
|
5726
|
+
* TODO: [🍓] In `JsonFormatParser` implement simple `isValid`
|
|
5727
|
+
* TODO: [🍓] In `JsonFormatParser` implement partial `canBeValid`
|
|
5728
|
+
* TODO: [🍓] In `JsonFormatParser` implement `heal
|
|
5729
|
+
* TODO: [🍓] In `JsonFormatParser` implement `subvalueParsers`
|
|
5543
5730
|
* TODO: [🏢] Allow to expect something inside JSON objects and other formats
|
|
5544
5731
|
*/
|
|
5545
5732
|
|
|
5546
5733
|
/**
|
|
5547
5734
|
* Definition for any text - this will be always valid
|
|
5548
5735
|
*
|
|
5549
|
-
* Note: This is not useful for validation, but for splitting and mapping with `
|
|
5736
|
+
* Note: This is not useful for validation, but for splitting and mapping with `subvalueParsers`
|
|
5550
5737
|
*
|
|
5551
5738
|
* @public exported from `@promptbook/core`
|
|
5552
5739
|
*/
|
|
5553
|
-
const
|
|
5740
|
+
const TextFormatParser = {
|
|
5554
5741
|
formatName: 'TEXT',
|
|
5555
5742
|
isValid(value) {
|
|
5556
5743
|
return typeof value === 'string';
|
|
@@ -5559,19 +5746,20 @@
|
|
|
5559
5746
|
return typeof partialValue === 'string';
|
|
5560
5747
|
},
|
|
5561
5748
|
heal() {
|
|
5562
|
-
throw new UnexpectedError('It does not make sense to call `
|
|
5749
|
+
throw new UnexpectedError('It does not make sense to call `TextFormatParser.heal`');
|
|
5563
5750
|
},
|
|
5564
|
-
|
|
5751
|
+
subvalueParsers: [
|
|
5565
5752
|
{
|
|
5566
5753
|
subvalueName: 'LINE',
|
|
5567
|
-
async mapValues(
|
|
5754
|
+
async mapValues(options) {
|
|
5755
|
+
const { value, mapCallback, onProgress } = options;
|
|
5568
5756
|
const lines = value.split('\n');
|
|
5569
|
-
const mappedLines = await Promise.all(lines.map((lineContent, lineNumber) =>
|
|
5757
|
+
const mappedLines = await Promise.all(lines.map((lineContent, lineNumber, array) =>
|
|
5570
5758
|
// TODO: [🧠] Maybe option to skip empty line
|
|
5571
5759
|
/* not await */ mapCallback({
|
|
5572
5760
|
lineContent,
|
|
5573
5761
|
// TODO: [🧠] Maybe also put here `lineNumber`
|
|
5574
|
-
}, lineNumber)));
|
|
5762
|
+
}, lineNumber, array.length)));
|
|
5575
5763
|
return mappedLines.join('\n');
|
|
5576
5764
|
},
|
|
5577
5765
|
},
|
|
@@ -5581,10 +5769,10 @@
|
|
|
5581
5769
|
/**
|
|
5582
5770
|
* TODO: [1] Make type for XML Text and Schema
|
|
5583
5771
|
* TODO: [🧠][🤠] Here should be all words, characters, lines, paragraphs, pages available as subvalues
|
|
5584
|
-
* TODO: [🍓] In `
|
|
5585
|
-
* TODO: [🍓] In `
|
|
5586
|
-
* TODO: [🍓] In `
|
|
5587
|
-
* TODO: [🍓] In `
|
|
5772
|
+
* TODO: [🍓] In `TextFormatParser` implement simple `isValid`
|
|
5773
|
+
* TODO: [🍓] In `TextFormatParser` implement partial `canBeValid`
|
|
5774
|
+
* TODO: [🍓] In `TextFormatParser` implement `heal
|
|
5775
|
+
* TODO: [🍓] In `TextFormatParser` implement `subvalueParsers`
|
|
5588
5776
|
* TODO: [🏢] Allow to expect something inside each item of list and other formats
|
|
5589
5777
|
*/
|
|
5590
5778
|
|
|
@@ -5592,7 +5780,7 @@
|
|
|
5592
5780
|
* Function to check if a string is valid XML
|
|
5593
5781
|
*
|
|
5594
5782
|
* @param value
|
|
5595
|
-
* @returns
|
|
5783
|
+
* @returns `true` if the string is a valid XML string, false otherwise
|
|
5596
5784
|
*
|
|
5597
5785
|
* @public exported from `@promptbook/utils`
|
|
5598
5786
|
*/
|
|
@@ -5617,7 +5805,7 @@
|
|
|
5617
5805
|
*
|
|
5618
5806
|
* @private still in development [🏢]
|
|
5619
5807
|
*/
|
|
5620
|
-
const
|
|
5808
|
+
const XmlFormatParser = {
|
|
5621
5809
|
formatName: 'XML',
|
|
5622
5810
|
mimeType: 'application/xml',
|
|
5623
5811
|
isValid(value, settings, schema) {
|
|
@@ -5629,17 +5817,17 @@
|
|
|
5629
5817
|
heal(value, settings, schema) {
|
|
5630
5818
|
throw new Error('Not implemented');
|
|
5631
5819
|
},
|
|
5632
|
-
|
|
5820
|
+
subvalueParsers: [],
|
|
5633
5821
|
};
|
|
5634
5822
|
/**
|
|
5635
|
-
* TODO: [🧠] Maybe
|
|
5823
|
+
* TODO: [🧠] Maybe proper instance of object
|
|
5636
5824
|
* TODO: [0] Make string_serialized_xml
|
|
5637
5825
|
* TODO: [1] Make type for XML Settings and Schema
|
|
5638
5826
|
* TODO: [🧠] What to use for validating XMLs - XSD,...
|
|
5639
|
-
* TODO: [🍓] In `
|
|
5640
|
-
* TODO: [🍓] In `
|
|
5641
|
-
* TODO: [🍓] In `
|
|
5642
|
-
* TODO: [🍓] In `
|
|
5827
|
+
* TODO: [🍓] In `XmlFormatParser` implement simple `isValid`
|
|
5828
|
+
* TODO: [🍓] In `XmlFormatParser` implement partial `canBeValid`
|
|
5829
|
+
* TODO: [🍓] In `XmlFormatParser` implement `heal
|
|
5830
|
+
* TODO: [🍓] In `XmlFormatParser` implement `subvalueParsers`
|
|
5643
5831
|
* TODO: [🏢] Allow to expect something inside XML and other formats
|
|
5644
5832
|
*/
|
|
5645
5833
|
|
|
@@ -5648,24 +5836,19 @@
|
|
|
5648
5836
|
*
|
|
5649
5837
|
* @private internal index of `...` <- TODO [🏢]
|
|
5650
5838
|
*/
|
|
5651
|
-
const FORMAT_DEFINITIONS = [
|
|
5652
|
-
JsonFormatDefinition,
|
|
5653
|
-
XmlFormatDefinition,
|
|
5654
|
-
TextFormatDefinition,
|
|
5655
|
-
CsvFormatDefinition,
|
|
5656
|
-
];
|
|
5839
|
+
const FORMAT_DEFINITIONS = [JsonFormatParser, XmlFormatParser, TextFormatParser, CsvFormatParser];
|
|
5657
5840
|
/**
|
|
5658
5841
|
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
5659
5842
|
*/
|
|
5660
5843
|
|
|
5661
5844
|
/**
|
|
5662
|
-
* Maps available parameters to expected parameters
|
|
5845
|
+
* Maps available parameters to expected parameters for a pipeline task.
|
|
5663
5846
|
*
|
|
5664
5847
|
* The strategy is:
|
|
5665
|
-
* 1)
|
|
5666
|
-
* 2)
|
|
5848
|
+
* 1) First, match parameters by name where both available and expected.
|
|
5849
|
+
* 2) Then, if there are unmatched expected and available parameters, map them by order.
|
|
5667
5850
|
*
|
|
5668
|
-
* @throws {PipelineExecutionError}
|
|
5851
|
+
* @throws {PipelineExecutionError} If the number of unmatched expected and available parameters does not match, or mapping is ambiguous.
|
|
5669
5852
|
* @private within the repository used in `createPipelineExecutor`
|
|
5670
5853
|
*/
|
|
5671
5854
|
function mapAvailableToExpectedParameters(options) {
|
|
@@ -5688,7 +5871,7 @@
|
|
|
5688
5871
|
else if (!availableParametersNames.has(parameterName) && expectedParameterNames.has(parameterName)) ;
|
|
5689
5872
|
}
|
|
5690
5873
|
if (expectedParameterNames.size === 0) {
|
|
5691
|
-
// Note: [👨👨👧] Now we can freeze `mappedParameters` to prevent
|
|
5874
|
+
// Note: [👨👨👧] Now we can freeze `mappedParameters` to prevent accidental modifications after mapping
|
|
5692
5875
|
Object.freeze(mappedParameters);
|
|
5693
5876
|
return mappedParameters;
|
|
5694
5877
|
}
|
|
@@ -5719,7 +5902,7 @@
|
|
|
5719
5902
|
for (let i = 0; i < expectedParameterNames.size; i++) {
|
|
5720
5903
|
mappedParameters[expectedParameterNamesArray[i]] = availableParameters[availableParametersNamesArray[i]];
|
|
5721
5904
|
}
|
|
5722
|
-
// Note: [👨👨👧] Now we can freeze `mappedParameters` to prevent
|
|
5905
|
+
// Note: [👨👨👧] Now we can freeze `mappedParameters` to prevent accidental modifications after mapping
|
|
5723
5906
|
Object.freeze(mappedParameters);
|
|
5724
5907
|
return mappedParameters;
|
|
5725
5908
|
}
|
|
@@ -5727,8 +5910,8 @@
|
|
|
5727
5910
|
/**
|
|
5728
5911
|
* Extracts all code blocks from markdown.
|
|
5729
5912
|
*
|
|
5730
|
-
* Note: There are multiple
|
|
5731
|
-
* - `extractBlock` just extracts the content of the code block which is also used as
|
|
5913
|
+
* Note: There are multiple similar functions:
|
|
5914
|
+
* - `extractBlock` just extracts the content of the code block which is also used as built-in function for postprocessing
|
|
5732
5915
|
* - `extractJsonBlock` extracts exactly one valid JSON code block
|
|
5733
5916
|
* - `extractOneBlockFromMarkdown` extracts exactly one code block with language of the code block
|
|
5734
5917
|
* - `extractAllBlocksFromMarkdown` extracts all code blocks with language of the code block
|
|
@@ -5778,7 +5961,7 @@
|
|
|
5778
5961
|
if (currentCodeBlock.content !== '') {
|
|
5779
5962
|
currentCodeBlock.content += '\n';
|
|
5780
5963
|
}
|
|
5781
|
-
currentCodeBlock.content += line.split('\\`\\`\\`').join('```') /* <- TODO: Maybe make
|
|
5964
|
+
currentCodeBlock.content += line.split('\\`\\`\\`').join('```') /* <- TODO: Maybe make proper unescape */;
|
|
5782
5965
|
}
|
|
5783
5966
|
}
|
|
5784
5967
|
if (currentCodeBlock !== null) {
|
|
@@ -5798,7 +5981,7 @@
|
|
|
5798
5981
|
* - When there are multiple JSON code blocks the function throws a `ParseError`
|
|
5799
5982
|
*
|
|
5800
5983
|
* Note: It is not important if marked as ```json BUT if it is VALID JSON
|
|
5801
|
-
* Note: There are multiple
|
|
5984
|
+
* Note: There are multiple similar function:
|
|
5802
5985
|
* - `extractBlock` just extracts the content of the code block which is also used as build-in function for postprocessing
|
|
5803
5986
|
* - `extractJsonBlock` extracts exactly one valid JSON code block
|
|
5804
5987
|
* - `extractOneBlockFromMarkdown` extracts exactly one code block with language of the code block
|
|
@@ -5823,7 +6006,7 @@
|
|
|
5823
6006
|
}
|
|
5824
6007
|
/**
|
|
5825
6008
|
* TODO: Add some auto-healing logic + extract YAML, JSON5, TOML, etc.
|
|
5826
|
-
* TODO: [🏢] Make this logic part of `
|
|
6009
|
+
* TODO: [🏢] Make this logic part of `JsonFormatParser` or `isValidJsonString`
|
|
5827
6010
|
*/
|
|
5828
6011
|
|
|
5829
6012
|
/**
|
|
@@ -6044,14 +6227,14 @@
|
|
|
6044
6227
|
PAGES: countPages,
|
|
6045
6228
|
};
|
|
6046
6229
|
/**
|
|
6047
|
-
* TODO: [🧠][🤠] This should be
|
|
6230
|
+
* TODO: [🧠][🤠] This should be probably as part of `TextFormatParser`
|
|
6048
6231
|
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
6049
6232
|
*/
|
|
6050
6233
|
|
|
6051
6234
|
/**
|
|
6052
6235
|
* Function checkExpectations will check if the expectations on given value are met
|
|
6053
6236
|
*
|
|
6054
|
-
* Note: There are two
|
|
6237
|
+
* Note: There are two similar functions:
|
|
6055
6238
|
* - `checkExpectations` which throws an error if the expectations are not met
|
|
6056
6239
|
* - `isPassingExpectations` which returns a boolean
|
|
6057
6240
|
*
|
|
@@ -6072,13 +6255,17 @@
|
|
|
6072
6255
|
}
|
|
6073
6256
|
/**
|
|
6074
6257
|
* TODO: [💝] Unite object for expecting amount and format
|
|
6075
|
-
* TODO: [🧠][🤠] This should be part of `
|
|
6258
|
+
* TODO: [🧠][🤠] This should be part of `TextFormatParser`
|
|
6076
6259
|
* Note: [💝] and [🤠] are interconnected together
|
|
6077
6260
|
*/
|
|
6078
6261
|
|
|
6079
6262
|
/**
|
|
6080
|
-
*
|
|
6263
|
+
* Executes a pipeline task with multiple attempts, including joker and retry logic. Handles different task types
|
|
6264
|
+
* (prompt, script, dialog, etc.), applies postprocessing, checks expectations, and updates the execution report.
|
|
6265
|
+
* Throws errors if execution fails after all attempts.
|
|
6081
6266
|
*
|
|
6267
|
+
* @param options - The options for execution, including task, parameters, pipeline, and configuration.
|
|
6268
|
+
* @returns The result string of the executed task.
|
|
6082
6269
|
* @private internal utility of `createPipelineExecutor`
|
|
6083
6270
|
*/
|
|
6084
6271
|
async function executeAttempts(options) {
|
|
@@ -6300,7 +6487,7 @@
|
|
|
6300
6487
|
if (task.format) {
|
|
6301
6488
|
if (task.format === 'JSON') {
|
|
6302
6489
|
if (!isValidJsonString($ongoingTaskResult.$resultString || '')) {
|
|
6303
|
-
// TODO: [🏢] Do more universally via `
|
|
6490
|
+
// TODO: [🏢] Do more universally via `FormatParser`
|
|
6304
6491
|
try {
|
|
6305
6492
|
$ongoingTaskResult.$resultString = extractJsonBlock($ongoingTaskResult.$resultString || '');
|
|
6306
6493
|
}
|
|
@@ -6402,12 +6589,16 @@
|
|
|
6402
6589
|
*/
|
|
6403
6590
|
|
|
6404
6591
|
/**
|
|
6405
|
-
*
|
|
6592
|
+
* Executes a pipeline task that requires mapping or iterating over subvalues of a parameter (such as rows in a CSV).
|
|
6593
|
+
* Handles format and subformat resolution, error handling, and progress reporting.
|
|
6594
|
+
*
|
|
6595
|
+
* @param options - Options for execution, including task details and progress callback.
|
|
6596
|
+
* @returns The result of the subvalue mapping or execution attempts.
|
|
6406
6597
|
*
|
|
6407
6598
|
* @private internal utility of `createPipelineExecutor`
|
|
6408
6599
|
*/
|
|
6409
6600
|
async function executeFormatSubvalues(options) {
|
|
6410
|
-
const { task, jokerParameterNames, parameters, priority, csvSettings, pipelineIdentification } = options;
|
|
6601
|
+
const { task, jokerParameterNames, parameters, priority, csvSettings, onProgress, pipelineIdentification } = options;
|
|
6411
6602
|
if (task.foreach === undefined) {
|
|
6412
6603
|
return /* not await */ executeAttempts(options);
|
|
6413
6604
|
}
|
|
@@ -6438,16 +6629,16 @@
|
|
|
6438
6629
|
${block(pipelineIdentification)}
|
|
6439
6630
|
`));
|
|
6440
6631
|
}
|
|
6441
|
-
const
|
|
6442
|
-
if (
|
|
6632
|
+
const subvalueParser = formatDefinition.subvalueParsers.find((subvalueParser) => [subvalueParser.subvalueName, ...(subvalueParser.aliases || [])].includes(task.foreach.subformatName));
|
|
6633
|
+
if (subvalueParser === undefined) {
|
|
6443
6634
|
throw new UnexpectedError(
|
|
6444
6635
|
// <- TODO: [🧠][🧐] Should be formats fixed per promptbook version or behave as plugins (=> change UnexpectedError)
|
|
6445
6636
|
spaceTrim__default["default"]((block) => `
|
|
6446
6637
|
Unsupported subformat name "${task.foreach.subformatName}" for format "${task.foreach.formatName}"
|
|
6447
6638
|
|
|
6448
6639
|
Available subformat names for format "${formatDefinition.formatName}":
|
|
6449
|
-
${block(formatDefinition.
|
|
6450
|
-
.map((
|
|
6640
|
+
${block(formatDefinition.subvalueParsers
|
|
6641
|
+
.map((subvalueParser) => subvalueParser.subvalueName)
|
|
6451
6642
|
.map((subvalueName) => `- ${subvalueName}`)
|
|
6452
6643
|
.join('\n'))}
|
|
6453
6644
|
|
|
@@ -6459,55 +6650,85 @@
|
|
|
6459
6650
|
let formatSettings;
|
|
6460
6651
|
if (formatDefinition.formatName === 'CSV') {
|
|
6461
6652
|
formatSettings = csvSettings;
|
|
6462
|
-
// <- TODO: [🤹♂️] More universal, make
|
|
6463
|
-
}
|
|
6464
|
-
const resultString = await
|
|
6465
|
-
|
|
6466
|
-
|
|
6467
|
-
|
|
6468
|
-
|
|
6469
|
-
|
|
6470
|
-
|
|
6471
|
-
|
|
6472
|
-
|
|
6473
|
-
|
|
6474
|
-
|
|
6475
|
-
|
|
6476
|
-
|
|
6653
|
+
// <- TODO: [🤹♂️] More universal, make similar pattern for other formats for example \n vs \r\n in text
|
|
6654
|
+
}
|
|
6655
|
+
const resultString = await subvalueParser.mapValues({
|
|
6656
|
+
value: parameterValue,
|
|
6657
|
+
outputParameterName: task.foreach.outputSubparameterName,
|
|
6658
|
+
settings: formatSettings,
|
|
6659
|
+
onProgress(partialResultString) {
|
|
6660
|
+
return onProgress(Object.freeze({
|
|
6661
|
+
[task.resultingParameterName]: partialResultString,
|
|
6662
|
+
}));
|
|
6663
|
+
},
|
|
6664
|
+
async mapCallback(subparameters, index, length) {
|
|
6665
|
+
let mappedParameters;
|
|
6666
|
+
try {
|
|
6667
|
+
mappedParameters = mapAvailableToExpectedParameters({
|
|
6668
|
+
expectedParameters: Object.fromEntries(task.foreach.inputSubparameterNames.map((subparameterName) => [subparameterName, null])),
|
|
6669
|
+
availableParameters: subparameters,
|
|
6670
|
+
});
|
|
6477
6671
|
}
|
|
6478
|
-
|
|
6479
|
-
|
|
6672
|
+
catch (error) {
|
|
6673
|
+
if (!(error instanceof PipelineExecutionError)) {
|
|
6674
|
+
throw error;
|
|
6675
|
+
}
|
|
6676
|
+
const highLevelError = new PipelineExecutionError(spaceTrim__default["default"]((block) => `
|
|
6677
|
+
${error.message}
|
|
6480
6678
|
|
|
6481
|
-
|
|
6482
|
-
|
|
6679
|
+
This is error in FOREACH command when mapping ${formatDefinition.formatName} ${subvalueParser.subvalueName} data (${index + 1}/${length})
|
|
6680
|
+
You have probably passed wrong data to pipeline or wrong data was generated which are processed by FOREACH command
|
|
6483
6681
|
|
|
6484
|
-
|
|
6485
|
-
|
|
6486
|
-
|
|
6487
|
-
|
|
6488
|
-
|
|
6489
|
-
|
|
6490
|
-
|
|
6491
|
-
|
|
6492
|
-
|
|
6493
|
-
|
|
6494
|
-
|
|
6495
|
-
|
|
6496
|
-
|
|
6497
|
-
|
|
6498
|
-
|
|
6499
|
-
|
|
6500
|
-
|
|
6501
|
-
|
|
6502
|
-
|
|
6503
|
-
|
|
6682
|
+
${block(pipelineIdentification)}
|
|
6683
|
+
`));
|
|
6684
|
+
if (length > BIG_DATASET_TRESHOLD) {
|
|
6685
|
+
console.error(highLevelError);
|
|
6686
|
+
return FAILED_VALUE_PLACEHOLDER;
|
|
6687
|
+
}
|
|
6688
|
+
throw highLevelError;
|
|
6689
|
+
}
|
|
6690
|
+
const allSubparameters = {
|
|
6691
|
+
...parameters,
|
|
6692
|
+
...mappedParameters,
|
|
6693
|
+
};
|
|
6694
|
+
Object.freeze(allSubparameters);
|
|
6695
|
+
try {
|
|
6696
|
+
const subresultString = await executeAttempts({
|
|
6697
|
+
...options,
|
|
6698
|
+
priority: priority + index,
|
|
6699
|
+
parameters: allSubparameters,
|
|
6700
|
+
pipelineIdentification: spaceTrim__default["default"]((block) => `
|
|
6701
|
+
${block(pipelineIdentification)}
|
|
6702
|
+
Subparameter index: ${index}
|
|
6703
|
+
`),
|
|
6704
|
+
});
|
|
6705
|
+
return subresultString;
|
|
6706
|
+
}
|
|
6707
|
+
catch (error) {
|
|
6708
|
+
if (length > BIG_DATASET_TRESHOLD) {
|
|
6709
|
+
console.error(spaceTrim__default["default"]((block) => `
|
|
6710
|
+
${error.message}
|
|
6711
|
+
|
|
6712
|
+
This is error in FOREACH command when processing ${formatDefinition.formatName} ${subvalueParser.subvalueName} data (${index + 1}/${length})
|
|
6713
|
+
|
|
6714
|
+
${block(pipelineIdentification)}
|
|
6715
|
+
`));
|
|
6716
|
+
return FAILED_VALUE_PLACEHOLDER;
|
|
6717
|
+
}
|
|
6718
|
+
throw error;
|
|
6719
|
+
}
|
|
6720
|
+
},
|
|
6504
6721
|
});
|
|
6505
6722
|
return resultString;
|
|
6506
6723
|
}
|
|
6507
6724
|
|
|
6508
6725
|
/**
|
|
6509
|
-
*
|
|
6726
|
+
* Returns the context for a given task, typically used to provide additional information or variables
|
|
6727
|
+
* required for the execution of the task within a pipeline. The context is returned as a string value
|
|
6728
|
+
* that may include markdown formatting.
|
|
6510
6729
|
*
|
|
6730
|
+
* @param task - The task for which the context is being generated. This should be a deeply immutable TaskJson object.
|
|
6731
|
+
* @returns The context as a string, formatted as markdown and parameter value.
|
|
6511
6732
|
* @private internal utility of `createPipelineExecutor`
|
|
6512
6733
|
*/
|
|
6513
6734
|
async function getContextForTask(task) {
|
|
@@ -6515,7 +6736,7 @@
|
|
|
6515
6736
|
}
|
|
6516
6737
|
|
|
6517
6738
|
/**
|
|
6518
|
-
*
|
|
6739
|
+
* Retrieves example values or templates for a given task, used to guide or validate pipeline execution.
|
|
6519
6740
|
*
|
|
6520
6741
|
* @private internal utility of `createPipelineExecutor`
|
|
6521
6742
|
*/
|
|
@@ -6524,91 +6745,128 @@
|
|
|
6524
6745
|
}
|
|
6525
6746
|
|
|
6526
6747
|
/**
|
|
6527
|
-
*
|
|
6748
|
+
* Computes the cosine similarity between two embedding vectors
|
|
6749
|
+
*
|
|
6750
|
+
* Note: This is helping function for RAG (retrieval-augmented generation)
|
|
6751
|
+
*
|
|
6752
|
+
* @param embeddingVector1
|
|
6753
|
+
* @param embeddingVector2
|
|
6754
|
+
* @returns Cosine similarity between the two vectors
|
|
6755
|
+
*
|
|
6756
|
+
* @public exported from `@promptbook/core`
|
|
6757
|
+
*/
|
|
6758
|
+
function computeCosineSimilarity(embeddingVector1, embeddingVector2) {
|
|
6759
|
+
if (embeddingVector1.length !== embeddingVector2.length) {
|
|
6760
|
+
throw new TypeError('Embedding vectors must have the same length');
|
|
6761
|
+
}
|
|
6762
|
+
const dotProduct = embeddingVector1.reduce((sum, value, index) => sum + value * embeddingVector2[index], 0);
|
|
6763
|
+
const magnitude1 = Math.sqrt(embeddingVector1.reduce((sum, value) => sum + value * value, 0));
|
|
6764
|
+
const magnitude2 = Math.sqrt(embeddingVector2.reduce((sum, value) => sum + value * value, 0));
|
|
6765
|
+
return 1 - dotProduct / (magnitude1 * magnitude2);
|
|
6766
|
+
}
|
|
6767
|
+
|
|
6768
|
+
/**
|
|
6528
6769
|
*
|
|
6529
|
-
*
|
|
6770
|
+
* @param knowledgePieces
|
|
6771
|
+
* @returns
|
|
6772
|
+
*
|
|
6773
|
+
* @private internal utility of `createPipelineExecutor`
|
|
6774
|
+
*/
|
|
6775
|
+
function knowledgePiecesToString(knowledgePieces) {
|
|
6776
|
+
return knowledgePieces
|
|
6777
|
+
.map((knowledgePiece) => {
|
|
6778
|
+
const { content } = knowledgePiece;
|
|
6779
|
+
return `- ${content}`;
|
|
6780
|
+
})
|
|
6781
|
+
.join('\n');
|
|
6782
|
+
// <- TODO: [🧠] Some smarter aggregation of knowledge pieces, single-line vs multi-line vs mixed
|
|
6783
|
+
}
|
|
6784
|
+
|
|
6785
|
+
/**
|
|
6786
|
+
* Retrieves the most relevant knowledge pieces for a given task using embedding-based similarity search.
|
|
6787
|
+
* This is where retrieval-augmented generation (RAG) is performed to enhance the task with external knowledge.
|
|
6530
6788
|
*
|
|
6531
6789
|
* @private internal utility of `createPipelineExecutor`
|
|
6532
6790
|
*/
|
|
6533
6791
|
async function getKnowledgeForTask(options) {
|
|
6534
|
-
const { tools, preparedPipeline, task } = options;
|
|
6792
|
+
const { tools, preparedPipeline, task, parameters } = options;
|
|
6535
6793
|
const firstKnowlegePiece = preparedPipeline.knowledgePieces[0];
|
|
6536
6794
|
const firstKnowlegeIndex = firstKnowlegePiece === null || firstKnowlegePiece === void 0 ? void 0 : firstKnowlegePiece.index[0];
|
|
6537
6795
|
// <- TODO: Do not use just first knowledge piece and first index to determine embedding model, use also keyword search
|
|
6538
6796
|
if (firstKnowlegePiece === undefined || firstKnowlegeIndex === undefined) {
|
|
6539
|
-
return '
|
|
6797
|
+
return ''; // <- Note: Np knowledge present, return empty string
|
|
6540
6798
|
}
|
|
6541
|
-
|
|
6542
|
-
|
|
6543
|
-
|
|
6544
|
-
|
|
6545
|
-
|
|
6546
|
-
|
|
6547
|
-
|
|
6548
|
-
|
|
6549
|
-
|
|
6550
|
-
|
|
6551
|
-
|
|
6552
|
-
|
|
6553
|
-
}
|
|
6554
|
-
|
|
6555
|
-
|
|
6556
|
-
|
|
6557
|
-
|
|
6558
|
-
|
|
6559
|
-
|
|
6560
|
-
|
|
6799
|
+
try {
|
|
6800
|
+
// TODO: [🚐] Make arrayable LLMs -> single LLM DRY
|
|
6801
|
+
const _llms = arrayableToArray(tools.llm);
|
|
6802
|
+
const llmTools = _llms.length === 1 ? _llms[0] : joinLlmExecutionTools(..._llms);
|
|
6803
|
+
const taskEmbeddingPrompt = {
|
|
6804
|
+
title: 'Knowledge Search',
|
|
6805
|
+
modelRequirements: {
|
|
6806
|
+
modelVariant: 'EMBEDDING',
|
|
6807
|
+
modelName: firstKnowlegeIndex.modelName,
|
|
6808
|
+
},
|
|
6809
|
+
content: task.content,
|
|
6810
|
+
parameters,
|
|
6811
|
+
};
|
|
6812
|
+
const taskEmbeddingResult = await llmTools.callEmbeddingModel(taskEmbeddingPrompt);
|
|
6813
|
+
const knowledgePiecesWithRelevance = preparedPipeline.knowledgePieces.map((knowledgePiece) => {
|
|
6814
|
+
const { index } = knowledgePiece;
|
|
6815
|
+
const knowledgePieceIndex = index.find((i) => i.modelName === firstKnowlegeIndex.modelName);
|
|
6816
|
+
// <- TODO: Do not use just first knowledge piece and first index to determine embedding model
|
|
6817
|
+
if (knowledgePieceIndex === undefined) {
|
|
6818
|
+
return {
|
|
6819
|
+
content: knowledgePiece.content,
|
|
6820
|
+
relevance: 0,
|
|
6821
|
+
};
|
|
6822
|
+
}
|
|
6823
|
+
const relevance = computeCosineSimilarity(knowledgePieceIndex.position, taskEmbeddingResult.content);
|
|
6561
6824
|
return {
|
|
6562
6825
|
content: knowledgePiece.content,
|
|
6563
|
-
relevance
|
|
6826
|
+
relevance,
|
|
6564
6827
|
};
|
|
6565
|
-
}
|
|
6566
|
-
const
|
|
6567
|
-
|
|
6568
|
-
|
|
6569
|
-
|
|
6570
|
-
|
|
6571
|
-
|
|
6572
|
-
|
|
6573
|
-
|
|
6574
|
-
|
|
6575
|
-
|
|
6576
|
-
|
|
6577
|
-
|
|
6578
|
-
|
|
6579
|
-
|
|
6580
|
-
|
|
6581
|
-
|
|
6582
|
-
|
|
6583
|
-
|
|
6584
|
-
|
|
6585
|
-
|
|
6586
|
-
|
|
6587
|
-
// TODO: !!!!!! Annotate + to new file
|
|
6588
|
-
function computeCosineSimilarity(embeddingVector1, embeddingVector2) {
|
|
6589
|
-
if (embeddingVector1.length !== embeddingVector2.length) {
|
|
6590
|
-
throw new TypeError('Embedding vectors must have the same length');
|
|
6828
|
+
});
|
|
6829
|
+
const knowledgePiecesSorted = knowledgePiecesWithRelevance.sort((a, b) => a.relevance - b.relevance);
|
|
6830
|
+
const knowledgePiecesLimited = knowledgePiecesSorted.slice(0, 5);
|
|
6831
|
+
/*
|
|
6832
|
+
console.log('`getKnowledgeForTask` Embedding', {
|
|
6833
|
+
task,
|
|
6834
|
+
taskEmbeddingPrompt,
|
|
6835
|
+
taskEmbeddingResult,
|
|
6836
|
+
firstKnowlegePiece,
|
|
6837
|
+
firstKnowlegeIndex,
|
|
6838
|
+
knowledgePiecesWithRelevance,
|
|
6839
|
+
knowledgePiecesSorted,
|
|
6840
|
+
knowledgePiecesLimited,
|
|
6841
|
+
});
|
|
6842
|
+
*/
|
|
6843
|
+
return knowledgePiecesToString(knowledgePiecesLimited);
|
|
6844
|
+
}
|
|
6845
|
+
catch (error) {
|
|
6846
|
+
assertsError(error);
|
|
6847
|
+
console.error('Error in `getKnowledgeForTask`', error);
|
|
6848
|
+
// Note: If the LLM fails, just return all knowledge pieces
|
|
6849
|
+
return knowledgePiecesToString(preparedPipeline.knowledgePieces);
|
|
6591
6850
|
}
|
|
6592
|
-
const dotProduct = embeddingVector1.reduce((sum, value, index) => sum + value * embeddingVector2[index], 0);
|
|
6593
|
-
const magnitude1 = Math.sqrt(embeddingVector1.reduce((sum, value) => sum + value * value, 0));
|
|
6594
|
-
const magnitude2 = Math.sqrt(embeddingVector2.reduce((sum, value) => sum + value * value, 0));
|
|
6595
|
-
return 1 - dotProduct / (magnitude1 * magnitude2);
|
|
6596
6851
|
}
|
|
6597
6852
|
/**
|
|
6598
|
-
* TODO: !!!! Verify if this is working
|
|
6599
6853
|
* TODO: [♨] Implement Better - use keyword search
|
|
6600
6854
|
* TODO: [♨] Examples of values
|
|
6601
6855
|
*/
|
|
6602
6856
|
|
|
6603
6857
|
/**
|
|
6604
|
-
*
|
|
6858
|
+
* Retrieves all reserved parameters for a given pipeline task, including context, knowledge, examples, and metadata.
|
|
6859
|
+
* Ensures all reserved parameters are defined and throws if any are missing.
|
|
6860
|
+
*
|
|
6861
|
+
* @param options - Options including tools, pipeline, task, and context.
|
|
6862
|
+
* @returns An object containing all reserved parameters for the task.
|
|
6605
6863
|
*
|
|
6606
6864
|
* @private internal utility of `createPipelineExecutor`
|
|
6607
6865
|
*/
|
|
6608
6866
|
async function getReservedParametersForTask(options) {
|
|
6609
|
-
const { tools, preparedPipeline, task, pipelineIdentification } = options;
|
|
6867
|
+
const { tools, preparedPipeline, task, parameters, pipelineIdentification, isVerbose } = options;
|
|
6610
6868
|
const context = await getContextForTask(); // <- [🏍]
|
|
6611
|
-
const knowledge = await getKnowledgeForTask({ tools, preparedPipeline, task });
|
|
6869
|
+
const knowledge = await getKnowledgeForTask({ tools, preparedPipeline, task, parameters });
|
|
6612
6870
|
const examples = await getExamplesForTask();
|
|
6613
6871
|
const currentDate = new Date().toISOString(); // <- TODO: [🧠][💩] Better
|
|
6614
6872
|
const modelName = RESERVED_PARAMETER_MISSING_VALUE;
|
|
@@ -6620,6 +6878,9 @@
|
|
|
6620
6878
|
currentDate,
|
|
6621
6879
|
modelName,
|
|
6622
6880
|
};
|
|
6881
|
+
if (isVerbose) {
|
|
6882
|
+
console.info('Reserved parameters for task:', { options, reservedParameters });
|
|
6883
|
+
}
|
|
6623
6884
|
// Note: Doublecheck that ALL reserved parameters are defined:
|
|
6624
6885
|
for (const parameterName of RESERVED_PARAMETER_NAMES) {
|
|
6625
6886
|
if (reservedParameters[parameterName] === undefined) {
|
|
@@ -6634,23 +6895,21 @@
|
|
|
6634
6895
|
}
|
|
6635
6896
|
|
|
6636
6897
|
/**
|
|
6637
|
-
*
|
|
6898
|
+
* Executes a single task within a pipeline, handling parameter validation, error checking, and progress reporting.
|
|
6899
|
+
*
|
|
6900
|
+
* @param options - Options for execution, including the task, pipeline, parameters, and callbacks.
|
|
6901
|
+
* @returns The output parameters produced by the task.
|
|
6638
6902
|
*
|
|
6639
6903
|
* @private internal utility of `createPipelineExecutor`
|
|
6640
6904
|
*/
|
|
6641
6905
|
async function executeTask(options) {
|
|
6642
6906
|
const { currentTask, preparedPipeline, parametersToPass, tools, onProgress, $executionReport, pipelineIdentification, maxExecutionAttempts, maxParallelCount, csvSettings, isVerbose, rootDirname, cacheDirname, intermediateFilesStrategy, isAutoInstalled, isNotPreparedWarningSupressed, } = options;
|
|
6643
6907
|
const priority = preparedPipeline.tasks.length - preparedPipeline.tasks.indexOf(currentTask);
|
|
6644
|
-
await onProgress({
|
|
6645
|
-
outputParameters: {
|
|
6646
|
-
[currentTask.resultingParameterName]: '', // <- TODO: [🧠] What is the best value here?
|
|
6647
|
-
},
|
|
6648
|
-
});
|
|
6649
6908
|
// Note: Check consistency of used and dependent parameters which was also done in `validatePipeline`, but it’s good to doublecheck
|
|
6650
6909
|
const usedParameterNames = extractParameterNamesFromTask(currentTask);
|
|
6651
6910
|
const dependentParameterNames = new Set(currentTask.dependentParameterNames);
|
|
6652
6911
|
// TODO: [👩🏾🤝👩🏻] Use here `mapAvailableToExpectedParameters`
|
|
6653
|
-
if (union(difference(usedParameterNames, dependentParameterNames), difference(dependentParameterNames, usedParameterNames)).size !== 0) {
|
|
6912
|
+
if (difference(union(difference(usedParameterNames, dependentParameterNames), difference(dependentParameterNames, usedParameterNames)), new Set(RESERVED_PARAMETER_NAMES)).size !== 0) {
|
|
6654
6913
|
throw new UnexpectedError(spaceTrim.spaceTrim((block) => `
|
|
6655
6914
|
Dependent parameters are not consistent with used parameters:
|
|
6656
6915
|
|
|
@@ -6668,13 +6927,16 @@
|
|
|
6668
6927
|
|
|
6669
6928
|
`));
|
|
6670
6929
|
}
|
|
6930
|
+
const reservedParameters = await getReservedParametersForTask({
|
|
6931
|
+
tools,
|
|
6932
|
+
preparedPipeline,
|
|
6933
|
+
task: currentTask,
|
|
6934
|
+
pipelineIdentification,
|
|
6935
|
+
parameters: parametersToPass,
|
|
6936
|
+
isVerbose,
|
|
6937
|
+
});
|
|
6671
6938
|
const definedParameters = Object.freeze({
|
|
6672
|
-
...
|
|
6673
|
-
tools,
|
|
6674
|
-
preparedPipeline,
|
|
6675
|
-
task: currentTask,
|
|
6676
|
-
pipelineIdentification,
|
|
6677
|
-
})),
|
|
6939
|
+
...reservedParameters,
|
|
6678
6940
|
...parametersToPass,
|
|
6679
6941
|
});
|
|
6680
6942
|
const definedParameterNames = new Set(Object.keys(definedParameters));
|
|
@@ -6719,6 +6981,7 @@
|
|
|
6719
6981
|
preparedPipeline,
|
|
6720
6982
|
tools,
|
|
6721
6983
|
$executionReport,
|
|
6984
|
+
onProgress,
|
|
6722
6985
|
pipelineIdentification,
|
|
6723
6986
|
maxExecutionAttempts,
|
|
6724
6987
|
maxParallelCount,
|
|
@@ -6746,7 +7009,8 @@
|
|
|
6746
7009
|
*/
|
|
6747
7010
|
|
|
6748
7011
|
/**
|
|
6749
|
-
*
|
|
7012
|
+
* Filters and returns only the output parameters from the provided pipeline execution options.
|
|
7013
|
+
* Adds warnings for any expected output parameters that are missing.
|
|
6750
7014
|
*
|
|
6751
7015
|
* @private internal utility of `createPipelineExecutor`
|
|
6752
7016
|
*/
|
|
@@ -6771,9 +7035,12 @@
|
|
|
6771
7035
|
}
|
|
6772
7036
|
|
|
6773
7037
|
/**
|
|
6774
|
-
*
|
|
7038
|
+
* Executes an entire pipeline, resolving tasks in dependency order, handling errors, and reporting progress.
|
|
6775
7039
|
*
|
|
6776
|
-
* Note: This is not a `PipelineExecutor` (which is
|
|
7040
|
+
* Note: This is not a `PipelineExecutor` (which is bound to a single pipeline), but a utility function used by `createPipelineExecutor` to create a `PipelineExecutor`.
|
|
7041
|
+
*
|
|
7042
|
+
* @param options - Options for execution, including input parameters, pipeline, and callbacks.
|
|
7043
|
+
* @returns The result of the pipeline execution, including output parameters, errors, and usage statistics.
|
|
6777
7044
|
*
|
|
6778
7045
|
* @private internal utility of `createPipelineExecutor`
|
|
6779
7046
|
*/
|
|
@@ -7096,10 +7363,27 @@
|
|
|
7096
7363
|
cacheDirname,
|
|
7097
7364
|
intermediateFilesStrategy,
|
|
7098
7365
|
isAutoInstalled,
|
|
7366
|
+
}).catch((error) => {
|
|
7367
|
+
assertsError(error);
|
|
7368
|
+
return exportJson({
|
|
7369
|
+
name: 'pipelineExecutorResult',
|
|
7370
|
+
message: `Unuccessful PipelineExecutorResult, last catch`,
|
|
7371
|
+
order: [],
|
|
7372
|
+
value: {
|
|
7373
|
+
isSuccessful: false,
|
|
7374
|
+
errors: [serializeError(error)],
|
|
7375
|
+
warnings: [],
|
|
7376
|
+
usage: UNCERTAIN_USAGE,
|
|
7377
|
+
executionReport: null,
|
|
7378
|
+
outputParameters: {},
|
|
7379
|
+
preparedPipeline,
|
|
7380
|
+
},
|
|
7381
|
+
});
|
|
7099
7382
|
});
|
|
7100
7383
|
};
|
|
7101
7384
|
const pipelineExecutor = (inputParameters) => createTask({
|
|
7102
7385
|
taskType: 'EXECUTION',
|
|
7386
|
+
title: pipeline.title,
|
|
7103
7387
|
taskProcessCallback(updateOngoingResult) {
|
|
7104
7388
|
return pipelineExecutorWithCallback(inputParameters, async (newOngoingResult) => {
|
|
7105
7389
|
updateOngoingResult(newOngoingResult);
|
|
@@ -7259,7 +7543,9 @@
|
|
|
7259
7543
|
}
|
|
7260
7544
|
|
|
7261
7545
|
/**
|
|
7262
|
-
*
|
|
7546
|
+
* Factory function that creates a handler for processing knowledge sources.
|
|
7547
|
+
* Provides standardized processing of different types of knowledge sources
|
|
7548
|
+
* across various scraper implementations.
|
|
7263
7549
|
*
|
|
7264
7550
|
* @public exported from `@promptbook/core`
|
|
7265
7551
|
*/
|
|
@@ -7398,7 +7684,7 @@
|
|
|
7398
7684
|
}
|
|
7399
7685
|
|
|
7400
7686
|
/**
|
|
7401
|
-
* Prepares the
|
|
7687
|
+
* Prepares the knowledge pieces
|
|
7402
7688
|
*
|
|
7403
7689
|
* @see https://github.com/webgptorg/promptbook/discussions/41
|
|
7404
7690
|
* @public exported from `@promptbook/core`
|
|
@@ -7494,15 +7780,18 @@
|
|
|
7494
7780
|
* TODO: [🧊] In future one preparation can take data from previous preparation and save tokens and time
|
|
7495
7781
|
* Put `knowledgePieces` into `PrepareKnowledgeOptions`
|
|
7496
7782
|
* TODO: [🪂] More than max things can run in parallel by acident [1,[2a,2b,_],[3a,3b,_]]
|
|
7497
|
-
* TODO: [🧠][❎] Do here
|
|
7783
|
+
* TODO: [🧠][❎] Do here proper M:N mapping
|
|
7498
7784
|
* [x] One source can make multiple pieces
|
|
7499
7785
|
* [ ] One piece can have multiple sources
|
|
7500
7786
|
*/
|
|
7501
7787
|
|
|
7502
7788
|
/**
|
|
7503
|
-
*
|
|
7789
|
+
* Prepares tasks by adding knowledge to the prompt and ensuring all necessary parameters are included.
|
|
7504
7790
|
*
|
|
7505
|
-
* @
|
|
7791
|
+
* @param tasks Sequence of tasks that are chained together to form a pipeline
|
|
7792
|
+
* @returns A promise that resolves to the prepared tasks.
|
|
7793
|
+
*
|
|
7794
|
+
* @private internal utility of `preparePipeline`
|
|
7506
7795
|
*/
|
|
7507
7796
|
async function prepareTasks(pipeline, tools, options) {
|
|
7508
7797
|
const { maxParallelCount = DEFAULT_MAX_PARALLEL_COUNT } = options;
|
|
@@ -7540,7 +7829,7 @@
|
|
|
7540
7829
|
return { tasksPrepared };
|
|
7541
7830
|
}
|
|
7542
7831
|
/**
|
|
7543
|
-
* TODO: [😂] Adding knowledge should be convert to async high-level abstractions,
|
|
7832
|
+
* TODO: [😂] Adding knowledge should be convert to async high-level abstractions, similar thing with expectations to sync high-level abstractions
|
|
7544
7833
|
* TODO: [🧠] Add context to each task (if missing)
|
|
7545
7834
|
* TODO: [🧠] What is better name `prepareTask` or `prepareTaskAndParameters`
|
|
7546
7835
|
* TODO: [♨][main] !!3 Prepare index the examples and maybe tasks
|
|
@@ -7676,7 +7965,7 @@
|
|
|
7676
7965
|
order: ORDER_OF_PIPELINE_JSON,
|
|
7677
7966
|
value: {
|
|
7678
7967
|
...pipeline,
|
|
7679
|
-
// <- TODO:
|
|
7968
|
+
// <- TODO: Probably deeply clone the pipeline because `$exportJson` freezes the subobjects
|
|
7680
7969
|
title,
|
|
7681
7970
|
knowledgeSources: knowledgeSourcesPrepared,
|
|
7682
7971
|
knowledgePieces: knowledgePiecesPrepared,
|
|
@@ -7952,7 +8241,7 @@
|
|
|
7952
8241
|
throw new ParseError(`Task section and example section must end with return statement -> {parameterName}`);
|
|
7953
8242
|
};
|
|
7954
8243
|
if ($taskJson.content === undefined) {
|
|
7955
|
-
throw new UnexpectedError(`Content is missing in the taskJson -
|
|
8244
|
+
throw new UnexpectedError(`Content is missing in the taskJson - probably commands are applied in wrong order`);
|
|
7956
8245
|
}
|
|
7957
8246
|
if (command.taskType === 'EXAMPLE') {
|
|
7958
8247
|
expectResultingParameterName();
|
|
@@ -8020,7 +8309,7 @@
|
|
|
8020
8309
|
/**
|
|
8021
8310
|
* Parses the boilerplate command
|
|
8022
8311
|
*
|
|
8023
|
-
* Note:
|
|
8312
|
+
* Note: @@ This command is used as boilerplate for new commands - it should NOT be used in any `.book` file
|
|
8024
8313
|
*
|
|
8025
8314
|
* @see `documentationUrl` for more details
|
|
8026
8315
|
* @private within the commands folder
|
|
@@ -8408,11 +8697,11 @@
|
|
|
8408
8697
|
};
|
|
8409
8698
|
|
|
8410
8699
|
/**
|
|
8411
|
-
*
|
|
8700
|
+
* Normalizes a given text to camelCase format.
|
|
8412
8701
|
*
|
|
8413
|
-
* @param text
|
|
8414
|
-
* @param _isFirstLetterCapital
|
|
8415
|
-
* @returns
|
|
8702
|
+
* @param text The text to be normalized.
|
|
8703
|
+
* @param _isFirstLetterCapital Whether the first letter should be capitalized.
|
|
8704
|
+
* @returns The camelCase formatted string.
|
|
8416
8705
|
* @example 'helloWorld'
|
|
8417
8706
|
* @example 'iLovePromptbook'
|
|
8418
8707
|
* @public exported from `@promptbook/utils`
|
|
@@ -8462,9 +8751,9 @@
|
|
|
8462
8751
|
/**
|
|
8463
8752
|
* Removes quotes from a string
|
|
8464
8753
|
*
|
|
8465
|
-
* Tip: This is very
|
|
8754
|
+
* Tip: This is very useful for post-processing of the result of the LLM model
|
|
8466
8755
|
* Note: This function removes only the same quotes from the beginning and the end of the string
|
|
8467
|
-
* Note: There are two
|
|
8756
|
+
* Note: There are two similar functions:
|
|
8468
8757
|
* - `removeQuotes` which removes only bounding quotes
|
|
8469
8758
|
* - `unwrapResult` which removes whole introduce sentence
|
|
8470
8759
|
*
|
|
@@ -8476,18 +8765,19 @@
|
|
|
8476
8765
|
if (text.startsWith('"') && text.endsWith('"')) {
|
|
8477
8766
|
return text.slice(1, -1);
|
|
8478
8767
|
}
|
|
8479
|
-
if (text.startsWith('
|
|
8768
|
+
if (text.startsWith("'") && text.endsWith("'")) {
|
|
8480
8769
|
return text.slice(1, -1);
|
|
8481
8770
|
}
|
|
8482
8771
|
return text;
|
|
8483
8772
|
}
|
|
8484
8773
|
|
|
8485
8774
|
/**
|
|
8486
|
-
* Function `validateParameterName` will
|
|
8775
|
+
* Function `validateParameterName` will normalize and validate a parameter name for use in pipelines.
|
|
8776
|
+
* It removes diacritics, emojis, and quotes, normalizes to camelCase, and checks for reserved names and invalid characters.
|
|
8487
8777
|
*
|
|
8488
|
-
* @param parameterName
|
|
8489
|
-
* @returns
|
|
8490
|
-
* @throws {ParseError}
|
|
8778
|
+
* @param parameterName The parameter name to validate and normalize.
|
|
8779
|
+
* @returns The validated and normalized parameter name.
|
|
8780
|
+
* @throws {ParseError} If the parameter name is empty, reserved, or contains invalid characters.
|
|
8491
8781
|
* @private within the repository
|
|
8492
8782
|
*/
|
|
8493
8783
|
function validateParameterName(parameterName) {
|
|
@@ -8557,8 +8847,6 @@
|
|
|
8557
8847
|
/**
|
|
8558
8848
|
* Parses the foreach command
|
|
8559
8849
|
*
|
|
8560
|
-
* Note: @@@ This command is used as foreach for new commands - it should NOT be used in any `.book` file
|
|
8561
|
-
*
|
|
8562
8850
|
* @see `documentationUrl` for more details
|
|
8563
8851
|
* @public exported from `@promptbook/editable`
|
|
8564
8852
|
*/
|
|
@@ -8615,14 +8903,14 @@
|
|
|
8615
8903
|
`));
|
|
8616
8904
|
// <- TODO: [🏢] List all supported format names
|
|
8617
8905
|
}
|
|
8618
|
-
const
|
|
8619
|
-
if (
|
|
8906
|
+
const subvalueParser = formatDefinition.subvalueParsers.find((subvalueParser) => [subvalueParser.subvalueName, ...(subvalueParser.aliases || [])].includes(subformatName));
|
|
8907
|
+
if (subvalueParser === undefined) {
|
|
8620
8908
|
throw new ParseError(spaceTrim__default["default"]((block) => `
|
|
8621
8909
|
Unsupported subformat name "${subformatName}" for format "${formatName}"
|
|
8622
8910
|
|
|
8623
8911
|
Available subformat names for format "${formatDefinition.formatName}":
|
|
8624
|
-
${block(formatDefinition.
|
|
8625
|
-
.map((
|
|
8912
|
+
${block(formatDefinition.subvalueParsers
|
|
8913
|
+
.map((subvalueParser) => subvalueParser.subvalueName)
|
|
8626
8914
|
.map((subvalueName) => `- ${subvalueName}`)
|
|
8627
8915
|
.join('\n'))}
|
|
8628
8916
|
`));
|
|
@@ -8799,14 +9087,14 @@
|
|
|
8799
9087
|
};
|
|
8800
9088
|
|
|
8801
9089
|
/**
|
|
8802
|
-
*
|
|
9090
|
+
* Chatbot form factor definition for conversational interfaces that interact with users in a chat-like manner.
|
|
8803
9091
|
*
|
|
8804
9092
|
* @public exported from `@promptbook/core`
|
|
8805
9093
|
*/
|
|
8806
9094
|
const ChatbotFormfactorDefinition = {
|
|
8807
9095
|
name: 'CHATBOT',
|
|
8808
9096
|
aliasNames: ['CHAT'],
|
|
8809
|
-
description:
|
|
9097
|
+
description: `A chatbot form factor for conversational user interfaces.`,
|
|
8810
9098
|
documentationUrl: `https://github.com/webgptorg/promptbook/discussions/174`,
|
|
8811
9099
|
pipelineInterface: {
|
|
8812
9100
|
inputParameters: [
|
|
@@ -8833,7 +9121,45 @@
|
|
|
8833
9121
|
};
|
|
8834
9122
|
|
|
8835
9123
|
/**
|
|
8836
|
-
*
|
|
9124
|
+
* Completion is formfactor that emulates completion models
|
|
9125
|
+
*
|
|
9126
|
+
* @public exported from `@promptbook/core`
|
|
9127
|
+
*/
|
|
9128
|
+
const CompletionFormfactorDefinition = {
|
|
9129
|
+
name: 'COMPLETION',
|
|
9130
|
+
description: `Completion is formfactor that emulates completion models`,
|
|
9131
|
+
documentationUrl: `https://github.com/webgptorg/promptbook/discussions/@@`,
|
|
9132
|
+
// <- TODO: https://github.com/webgptorg/promptbook/discussions/new?category=concepts
|
|
9133
|
+
// "🔠 Completion Formfactor"
|
|
9134
|
+
pipelineInterface: {
|
|
9135
|
+
inputParameters: [
|
|
9136
|
+
{
|
|
9137
|
+
name: 'inputText',
|
|
9138
|
+
description: `Input text to be completed`,
|
|
9139
|
+
isInput: true,
|
|
9140
|
+
isOutput: false,
|
|
9141
|
+
},
|
|
9142
|
+
{
|
|
9143
|
+
name: 'instructions',
|
|
9144
|
+
description: `Additional instructions for the model, for example the required length, empty by default`,
|
|
9145
|
+
isInput: true,
|
|
9146
|
+
isOutput: false,
|
|
9147
|
+
},
|
|
9148
|
+
],
|
|
9149
|
+
outputParameters: [
|
|
9150
|
+
{
|
|
9151
|
+
name: 'followingText',
|
|
9152
|
+
description: `Text that follows the input text`,
|
|
9153
|
+
isInput: false,
|
|
9154
|
+
isOutput: true,
|
|
9155
|
+
},
|
|
9156
|
+
],
|
|
9157
|
+
},
|
|
9158
|
+
};
|
|
9159
|
+
|
|
9160
|
+
/**
|
|
9161
|
+
* Generator form factor represents an application that generates content or data based on user input or predefined rules.
|
|
9162
|
+
* This form factor is used for apps that produce outputs, such as text, images, or other media, based on provided input.
|
|
8837
9163
|
*
|
|
8838
9164
|
* @public exported from `@promptbook/core`
|
|
8839
9165
|
*/
|
|
@@ -8862,7 +9188,7 @@
|
|
|
8862
9188
|
};
|
|
8863
9189
|
|
|
8864
9190
|
/**
|
|
8865
|
-
*
|
|
9191
|
+
* Pipeline interface which is equivalent to `any`
|
|
8866
9192
|
*
|
|
8867
9193
|
* @see https://github.com/webgptorg/promptbook/discussions/171
|
|
8868
9194
|
*
|
|
@@ -8877,13 +9203,13 @@
|
|
|
8877
9203
|
*/
|
|
8878
9204
|
|
|
8879
9205
|
/**
|
|
8880
|
-
*
|
|
9206
|
+
* A generic pipeline
|
|
8881
9207
|
*
|
|
8882
9208
|
* @public exported from `@promptbook/core`
|
|
8883
9209
|
*/
|
|
8884
9210
|
const GenericFormfactorDefinition = {
|
|
8885
9211
|
name: 'GENERIC',
|
|
8886
|
-
description:
|
|
9212
|
+
description: `A generic pipeline`,
|
|
8887
9213
|
documentationUrl: `https://github.com/webgptorg/promptbook/discussions/173`,
|
|
8888
9214
|
pipelineInterface: GENERIC_PIPELINE_INTERFACE,
|
|
8889
9215
|
};
|
|
@@ -8918,17 +9244,20 @@
|
|
|
8918
9244
|
};
|
|
8919
9245
|
|
|
8920
9246
|
/**
|
|
8921
|
-
* Matcher is form of app that
|
|
9247
|
+
* Matcher is form of app that evaluates (spreadsheet) content against defined criteria or patterns,
|
|
9248
|
+
* determining if it matches or meets specific requirements. Used for classification,
|
|
9249
|
+
* validation, filtering, and quality assessment of inputs.
|
|
8922
9250
|
*
|
|
8923
9251
|
* @public exported from `@promptbook/core`
|
|
8924
9252
|
*/
|
|
8925
9253
|
const MatcherFormfactorDefinition = {
|
|
8926
9254
|
name: 'EXPERIMENTAL_MATCHER',
|
|
8927
|
-
description:
|
|
9255
|
+
description: `An evaluation system that determines whether content meets specific criteria or patterns.
|
|
9256
|
+
Used for content validation, quality assessment, and intelligent filtering tasks. Currently in experimental phase.`,
|
|
8928
9257
|
documentationUrl: `https://github.com/webgptorg/promptbook/discussions/177`,
|
|
8929
9258
|
pipelineInterface: {
|
|
8930
9259
|
inputParameters: [
|
|
8931
|
-
/*
|
|
9260
|
+
/* Input parameters for content to be matched and criteria to match against */
|
|
8932
9261
|
{
|
|
8933
9262
|
name: 'nonce',
|
|
8934
9263
|
description: 'Just to prevent EXPERIMENTAL_MATCHER to be set as implicit formfactor',
|
|
@@ -8937,20 +9266,21 @@
|
|
|
8937
9266
|
},
|
|
8938
9267
|
],
|
|
8939
9268
|
outputParameters: [
|
|
8940
|
-
/*
|
|
9269
|
+
/* Output parameters containing match results, confidence scores, and relevant metadata */
|
|
8941
9270
|
],
|
|
8942
9271
|
},
|
|
8943
9272
|
};
|
|
8944
9273
|
|
|
8945
9274
|
/**
|
|
8946
|
-
* Sheets is form of app that
|
|
9275
|
+
* Sheets is form of app that processes tabular data in CSV format, allowing transformation
|
|
9276
|
+
* and analysis of structured data through AI-powered operations
|
|
8947
9277
|
*
|
|
8948
9278
|
* @public exported from `@promptbook/core`
|
|
8949
9279
|
*/
|
|
8950
9280
|
const SheetsFormfactorDefinition = {
|
|
8951
9281
|
name: 'SHEETS',
|
|
8952
9282
|
aliasNames: ['SHEETS', 'SHEET'],
|
|
8953
|
-
description:
|
|
9283
|
+
description: `A formfactor for processing spreadsheet-like data in CSV format, enabling AI transformations on tabular data`,
|
|
8954
9284
|
documentationUrl: `https://github.com/webgptorg/promptbook/discussions/176`,
|
|
8955
9285
|
pipelineInterface: {
|
|
8956
9286
|
inputParameters: [
|
|
@@ -8973,13 +9303,16 @@
|
|
|
8973
9303
|
};
|
|
8974
9304
|
|
|
8975
9305
|
/**
|
|
8976
|
-
* Translator is form of app that
|
|
9306
|
+
* Translator is form of app that transforms input text from one form to another,
|
|
9307
|
+
* such as language translation, style conversion, tone modification, or other text transformations.
|
|
8977
9308
|
*
|
|
8978
9309
|
* @public exported from `@promptbook/core`
|
|
8979
9310
|
*/
|
|
8980
9311
|
const TranslatorFormfactorDefinition = {
|
|
8981
9312
|
name: 'TRANSLATOR',
|
|
8982
|
-
description:
|
|
9313
|
+
description: `A text transformation system that converts input content into different forms,
|
|
9314
|
+
including language translations, paraphrasing, style conversions, and tone adjustments.
|
|
9315
|
+
This form factor takes one input and produces one transformed output.`,
|
|
8983
9316
|
documentationUrl: `https://github.com/webgptorg/promptbook/discussions/175`,
|
|
8984
9317
|
pipelineInterface: {
|
|
8985
9318
|
inputParameters: [
|
|
@@ -9016,6 +9349,8 @@
|
|
|
9016
9349
|
MatcherFormfactorDefinition,
|
|
9017
9350
|
GeneratorFormfactorDefinition,
|
|
9018
9351
|
ImageGeneratorFormfactorDefinition,
|
|
9352
|
+
CompletionFormfactorDefinition,
|
|
9353
|
+
// <- [🛬] When making new formfactor, copy the _boilerplate and link it here
|
|
9019
9354
|
];
|
|
9020
9355
|
/**
|
|
9021
9356
|
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
@@ -9024,7 +9359,7 @@
|
|
|
9024
9359
|
/**
|
|
9025
9360
|
* Parses the formfactor command
|
|
9026
9361
|
*
|
|
9027
|
-
* Note:
|
|
9362
|
+
* Note: This command is used as a formfactor for new commands and defines the app type format - it should NOT be used in any `.book` file
|
|
9028
9363
|
*
|
|
9029
9364
|
* @see `documentationUrl` for more details
|
|
9030
9365
|
* @public exported from `@promptbook/editable`
|
|
@@ -9046,7 +9381,7 @@
|
|
|
9046
9381
|
/**
|
|
9047
9382
|
* Description of the FORMFACTOR command
|
|
9048
9383
|
*/
|
|
9049
|
-
description:
|
|
9384
|
+
description: `Specifies the application type and interface requirements that this promptbook should conform to`,
|
|
9050
9385
|
/**
|
|
9051
9386
|
* Link to documentation
|
|
9052
9387
|
*/
|
|
@@ -9189,8 +9524,7 @@
|
|
|
9189
9524
|
};
|
|
9190
9525
|
|
|
9191
9526
|
/**
|
|
9192
|
-
*
|
|
9193
|
-
*
|
|
9527
|
+
* @see {@link ModelVariant}
|
|
9194
9528
|
* @public exported from `@promptbook/core`
|
|
9195
9529
|
*/
|
|
9196
9530
|
const MODEL_VARIANTS = ['COMPLETION', 'CHAT', 'EMBEDDING' /* <- TODO [🏳] */ /* <- [🤖] */];
|
|
@@ -9622,10 +9956,10 @@
|
|
|
9622
9956
|
}
|
|
9623
9957
|
|
|
9624
9958
|
/**
|
|
9625
|
-
*
|
|
9959
|
+
* Checks if the given value is a valid JavaScript identifier name.
|
|
9626
9960
|
*
|
|
9627
|
-
* @param javascriptName
|
|
9628
|
-
* @returns
|
|
9961
|
+
* @param javascriptName The value to check for JavaScript identifier validity.
|
|
9962
|
+
* @returns `true` if the value is a valid JavaScript name, false otherwise.
|
|
9629
9963
|
* @public exported from `@promptbook/utils`
|
|
9630
9964
|
*/
|
|
9631
9965
|
function isValidJavascriptName(javascriptName) {
|
|
@@ -10086,7 +10420,7 @@
|
|
|
10086
10420
|
// Arg1 Arg2 Arg3 | FOO
|
|
10087
10421
|
{
|
|
10088
10422
|
const commandNameRaw = items.slice(-1).join('_');
|
|
10089
|
-
const args = items.slice(0, -1); // <- Note: This is
|
|
10423
|
+
const args = items.slice(0, -1); // <- Note: This is probably not correct
|
|
10090
10424
|
const rawArgs = raw
|
|
10091
10425
|
.substring(0, raw.length - commandNameRaw.length)
|
|
10092
10426
|
.trim();
|
|
@@ -10105,7 +10439,10 @@
|
|
|
10105
10439
|
`));
|
|
10106
10440
|
}
|
|
10107
10441
|
/**
|
|
10108
|
-
*
|
|
10442
|
+
* Generates a markdown-formatted message listing all supported commands
|
|
10443
|
+
* with their descriptions and documentation links
|
|
10444
|
+
*
|
|
10445
|
+
* @returns A formatted markdown string containing all available commands and their details
|
|
10109
10446
|
*/
|
|
10110
10447
|
function getSupportedCommandsMessage() {
|
|
10111
10448
|
return COMMANDS.flatMap(({ name, aliasNames, description, documentationUrl }) =>
|
|
@@ -10116,7 +10453,10 @@
|
|
|
10116
10453
|
]).join('\n');
|
|
10117
10454
|
}
|
|
10118
10455
|
/**
|
|
10119
|
-
*
|
|
10456
|
+
* Attempts to parse a command variant using the provided input parameters
|
|
10457
|
+
*
|
|
10458
|
+
* @param input Object containing command parsing information including raw command text and normalized values
|
|
10459
|
+
* @returns A parsed Command object if successful, or null if the command cannot be parsed
|
|
10120
10460
|
*/
|
|
10121
10461
|
function parseCommandVariant(input) {
|
|
10122
10462
|
const { commandNameRaw, usagePlace, normalized, args, raw, rawArgs } = input;
|
|
@@ -10163,7 +10503,7 @@
|
|
|
10163
10503
|
}
|
|
10164
10504
|
|
|
10165
10505
|
/**
|
|
10166
|
-
*
|
|
10506
|
+
* Extracts the interface (input and output parameters) from a pipeline.
|
|
10167
10507
|
*
|
|
10168
10508
|
* @deprecated https://github.com/webgptorg/promptbook/pull/186
|
|
10169
10509
|
* @see https://github.com/webgptorg/promptbook/discussions/171
|
|
@@ -10196,7 +10536,7 @@
|
|
|
10196
10536
|
}
|
|
10197
10537
|
|
|
10198
10538
|
/**
|
|
10199
|
-
*
|
|
10539
|
+
* Checks if two pipeline interfaces are structurally identical.
|
|
10200
10540
|
*
|
|
10201
10541
|
* @deprecated https://github.com/webgptorg/promptbook/pull/186
|
|
10202
10542
|
* @see https://github.com/webgptorg/promptbook/discussions/171
|
|
@@ -10228,10 +10568,11 @@
|
|
|
10228
10568
|
}
|
|
10229
10569
|
|
|
10230
10570
|
/**
|
|
10231
|
-
*
|
|
10571
|
+
* Checks if a given pipeline satisfies the requirements of a specified pipeline interface.
|
|
10232
10572
|
*
|
|
10233
10573
|
* @deprecated https://github.com/webgptorg/promptbook/pull/186
|
|
10234
10574
|
* @see https://github.com/webgptorg/promptbook/discussions/171
|
|
10575
|
+
* @returns `true` if the pipeline implements the interface, `false` otherwise.
|
|
10235
10576
|
*
|
|
10236
10577
|
* @public exported from `@promptbook/core`
|
|
10237
10578
|
*/
|
|
@@ -10417,7 +10758,8 @@
|
|
|
10417
10758
|
}
|
|
10418
10759
|
|
|
10419
10760
|
/**
|
|
10420
|
-
*
|
|
10761
|
+
* Utility to determine if a pipeline string is in flat format.
|
|
10762
|
+
* A flat pipeline is a simple text without proper structure (headers, blocks, etc).
|
|
10421
10763
|
*
|
|
10422
10764
|
* @public exported from `@promptbook/editable`
|
|
10423
10765
|
*/
|
|
@@ -10438,7 +10780,10 @@
|
|
|
10438
10780
|
}
|
|
10439
10781
|
|
|
10440
10782
|
/**
|
|
10441
|
-
*
|
|
10783
|
+
* Converts a pipeline structure to its string representation.
|
|
10784
|
+
*
|
|
10785
|
+
* Transforms a flat, simple pipeline into a properly formatted pipeline string
|
|
10786
|
+
* with sections for title, prompt, and return statement.
|
|
10442
10787
|
*
|
|
10443
10788
|
* @public exported from `@promptbook/editable`
|
|
10444
10789
|
*/
|
|
@@ -10495,7 +10840,7 @@
|
|
|
10495
10840
|
* Note: It can not work with html syntax and comments
|
|
10496
10841
|
*
|
|
10497
10842
|
* @param markdown any valid markdown
|
|
10498
|
-
* @returns
|
|
10843
|
+
* @returns An array of strings, each representing an individual list item found in the markdown
|
|
10499
10844
|
* @public exported from `@promptbook/markdown-utils`
|
|
10500
10845
|
*/
|
|
10501
10846
|
function extractAllListItemsFromMarkdown(markdown) {
|
|
@@ -10520,8 +10865,8 @@
|
|
|
10520
10865
|
*
|
|
10521
10866
|
* - When there are multiple or no code blocks the function throws a `ParseError`
|
|
10522
10867
|
*
|
|
10523
|
-
* Note: There are multiple
|
|
10524
|
-
* - `extractBlock` just extracts the content of the code block which is also used as
|
|
10868
|
+
* Note: There are multiple similar functions:
|
|
10869
|
+
* - `extractBlock` just extracts the content of the code block which is also used as built-in function for postprocessing
|
|
10525
10870
|
* - `extractJsonBlock` extracts exactly one valid JSON code block
|
|
10526
10871
|
* - `extractOneBlockFromMarkdown` extracts exactly one code block with language of the code block
|
|
10527
10872
|
* - `extractAllBlocksFromMarkdown` extracts all code blocks with language of the code block
|
|
@@ -10676,7 +11021,7 @@
|
|
|
10676
11021
|
* Compile pipeline from string (markdown) format to JSON format synchronously
|
|
10677
11022
|
*
|
|
10678
11023
|
* Note: There are 3 similar functions:
|
|
10679
|
-
* - `compilePipeline` **(preferred)** - which
|
|
11024
|
+
* - `compilePipeline` **(preferred)** - which properly compiles the promptbook and uses embedding for external knowledge
|
|
10680
11025
|
* - `parsePipeline` - use only if you need to compile promptbook synchronously and it contains NO external knowledge
|
|
10681
11026
|
* - `preparePipeline` - just one step in the compilation process
|
|
10682
11027
|
*
|
|
@@ -11113,7 +11458,7 @@
|
|
|
11113
11458
|
* TODO: Use spaceTrim more effectively
|
|
11114
11459
|
* TODO: [🧠] Parameter flags - isInput, isOutput, isInternal
|
|
11115
11460
|
* TODO: [🥞] Not optimal parsing because `splitMarkdownIntoSections` is executed twice with same string, once through `flattenMarkdown` and second directly here
|
|
11116
|
-
* TODO: [♈]
|
|
11461
|
+
* TODO: [♈] Probably move expectations from tasks to parameters
|
|
11117
11462
|
* TODO: [🛠] Actions, instruments (and maybe knowledge) => Functions and tools
|
|
11118
11463
|
* TODO: [🍙] Make some standard order of json properties
|
|
11119
11464
|
*/
|
|
@@ -11158,21 +11503,63 @@
|
|
|
11158
11503
|
*/
|
|
11159
11504
|
function renderPromptbookMermaid(pipelineJson, options) {
|
|
11160
11505
|
const { linkTask = () => null } = options || {};
|
|
11506
|
+
const MERMAID_PREFIX = 'pipeline_';
|
|
11507
|
+
const MERMAID_KNOWLEDGE_NAME = MERMAID_PREFIX + 'knowledge';
|
|
11508
|
+
const MERMAID_RESERVED_NAME = MERMAID_PREFIX + 'reserved';
|
|
11509
|
+
const MERMAID_INPUT_NAME = MERMAID_PREFIX + 'input';
|
|
11510
|
+
const MERMAID_OUTPUT_NAME = MERMAID_PREFIX + 'output';
|
|
11161
11511
|
const parameterNameToTaskName = (parameterName) => {
|
|
11512
|
+
if (parameterName === 'knowledge') {
|
|
11513
|
+
return MERMAID_KNOWLEDGE_NAME;
|
|
11514
|
+
}
|
|
11515
|
+
else if (RESERVED_PARAMETER_NAMES.includes(parameterName)) {
|
|
11516
|
+
return MERMAID_RESERVED_NAME;
|
|
11517
|
+
}
|
|
11162
11518
|
const parameter = pipelineJson.parameters.find((parameter) => parameter.name === parameterName);
|
|
11163
11519
|
if (!parameter) {
|
|
11164
11520
|
throw new UnexpectedError(`Could not find {${parameterName}}`);
|
|
11165
|
-
// <- TODO:
|
|
11521
|
+
// <- TODO: This causes problems when {knowledge} and other reserved parameters are used
|
|
11166
11522
|
}
|
|
11167
11523
|
if (parameter.isInput) {
|
|
11168
|
-
return
|
|
11524
|
+
return MERMAID_INPUT_NAME;
|
|
11169
11525
|
}
|
|
11170
11526
|
const task = pipelineJson.tasks.find((task) => task.resultingParameterName === parameterName);
|
|
11171
11527
|
if (!task) {
|
|
11172
11528
|
throw new Error(`Could not find task for {${parameterName}}`);
|
|
11173
11529
|
}
|
|
11174
|
-
return task.name || normalizeTo_camelCase('task-' + titleToName(task.title));
|
|
11530
|
+
return MERMAID_PREFIX + (task.name || normalizeTo_camelCase('task-' + titleToName(task.title)));
|
|
11175
11531
|
};
|
|
11532
|
+
const inputAndIntermediateParametersMermaid = pipelineJson.tasks
|
|
11533
|
+
.flatMap(({ title, dependentParameterNames, resultingParameterName }) => [
|
|
11534
|
+
`${parameterNameToTaskName(resultingParameterName)}("${title}")`,
|
|
11535
|
+
...dependentParameterNames.map((dependentParameterName) => `${parameterNameToTaskName(dependentParameterName)}--"{${dependentParameterName}}"-->${parameterNameToTaskName(resultingParameterName)}`),
|
|
11536
|
+
])
|
|
11537
|
+
.join('\n');
|
|
11538
|
+
const outputParametersMermaid = pipelineJson.parameters
|
|
11539
|
+
.filter(({ isOutput }) => isOutput)
|
|
11540
|
+
.map(({ name }) => `${parameterNameToTaskName(name)}--"{${name}}"-->${MERMAID_OUTPUT_NAME}`)
|
|
11541
|
+
.join('\n');
|
|
11542
|
+
const linksMermaid = pipelineJson.tasks
|
|
11543
|
+
.map((task) => {
|
|
11544
|
+
const link = linkTask(task);
|
|
11545
|
+
if (link === null) {
|
|
11546
|
+
return '';
|
|
11547
|
+
}
|
|
11548
|
+
const { href, title } = link;
|
|
11549
|
+
const taskName = parameterNameToTaskName(task.resultingParameterName);
|
|
11550
|
+
return `click ${taskName} href "${href}" "${title}";`;
|
|
11551
|
+
})
|
|
11552
|
+
.filter((line) => line !== '')
|
|
11553
|
+
.join('\n');
|
|
11554
|
+
const interactionPointsMermaid = Object.entries({
|
|
11555
|
+
[MERMAID_INPUT_NAME]: 'Input',
|
|
11556
|
+
[MERMAID_OUTPUT_NAME]: 'Output',
|
|
11557
|
+
[MERMAID_RESERVED_NAME]: 'Other',
|
|
11558
|
+
[MERMAID_KNOWLEDGE_NAME]: 'Knowledge',
|
|
11559
|
+
})
|
|
11560
|
+
.filter(([MERMAID_NAME]) => (inputAndIntermediateParametersMermaid + outputParametersMermaid).includes(MERMAID_NAME))
|
|
11561
|
+
.map(([MERMAID_NAME, title]) => `${MERMAID_NAME}((${title})):::${MERMAID_NAME}`)
|
|
11562
|
+
.join('\n');
|
|
11176
11563
|
const promptbookMermaid = spaceTrim.spaceTrim((block) => `
|
|
11177
11564
|
|
|
11178
11565
|
%% 🔮 Tip: Open this on GitHub or in the VSCode website to see the Mermaid graph visually
|
|
@@ -11180,37 +11567,27 @@
|
|
|
11180
11567
|
flowchart LR
|
|
11181
11568
|
subgraph "${pipelineJson.title}"
|
|
11182
11569
|
|
|
11570
|
+
%% Basic configuration
|
|
11183
11571
|
direction TB
|
|
11184
11572
|
|
|
11185
|
-
|
|
11186
|
-
${block(
|
|
11187
|
-
.flatMap(({ title, dependentParameterNames, resultingParameterName }) => [
|
|
11188
|
-
`${parameterNameToTaskName(resultingParameterName)}("${title}")`,
|
|
11189
|
-
...dependentParameterNames.map((dependentParameterName) => `${parameterNameToTaskName(dependentParameterName)}--"{${dependentParameterName}}"-->${parameterNameToTaskName(resultingParameterName)}`),
|
|
11190
|
-
])
|
|
11191
|
-
.join('\n'))}
|
|
11573
|
+
%% Interaction points from pipeline to outside
|
|
11574
|
+
${block(interactionPointsMermaid)}
|
|
11192
11575
|
|
|
11193
|
-
|
|
11194
|
-
|
|
11195
|
-
.map(({ name }) => `${parameterNameToTaskName(name)}--"{${name}}"-->output`)
|
|
11196
|
-
.join('\n'))}
|
|
11197
|
-
output((Output)):::output
|
|
11576
|
+
%% Input and intermediate parameters
|
|
11577
|
+
${block(inputAndIntermediateParametersMermaid)}
|
|
11198
11578
|
|
|
11199
|
-
${block(pipelineJson.tasks
|
|
11200
|
-
.map((task) => {
|
|
11201
|
-
const link = linkTask(task);
|
|
11202
|
-
if (link === null) {
|
|
11203
|
-
return '';
|
|
11204
|
-
}
|
|
11205
|
-
const { href, title } = link;
|
|
11206
|
-
const taskName = parameterNameToTaskName(task.resultingParameterName);
|
|
11207
|
-
return `click ${taskName} href "${href}" "${title}";`;
|
|
11208
|
-
})
|
|
11209
|
-
.filter((line) => line !== '')
|
|
11210
|
-
.join('\n'))}
|
|
11211
11579
|
|
|
11212
|
-
|
|
11213
|
-
|
|
11580
|
+
%% Output parameters
|
|
11581
|
+
${block(outputParametersMermaid)}
|
|
11582
|
+
|
|
11583
|
+
%% Links
|
|
11584
|
+
${block(linksMermaid)}
|
|
11585
|
+
|
|
11586
|
+
%% Styles
|
|
11587
|
+
classDef ${MERMAID_INPUT_NAME} color: grey;
|
|
11588
|
+
classDef ${MERMAID_OUTPUT_NAME} color: grey;
|
|
11589
|
+
classDef ${MERMAID_RESERVED_NAME} color: grey;
|
|
11590
|
+
classDef ${MERMAID_KNOWLEDGE_NAME} color: grey;
|
|
11214
11591
|
|
|
11215
11592
|
end;
|
|
11216
11593
|
|
|
@@ -11250,11 +11627,11 @@
|
|
|
11250
11627
|
}
|
|
11251
11628
|
|
|
11252
11629
|
/**
|
|
11253
|
-
*
|
|
11630
|
+
* Converts a name string into a URI-compatible format.
|
|
11254
11631
|
*
|
|
11255
|
-
* @param name
|
|
11256
|
-
* @returns
|
|
11257
|
-
* @example
|
|
11632
|
+
* @param name The string to be converted to a URI-compatible format.
|
|
11633
|
+
* @returns A URI-compatible string derived from the input name.
|
|
11634
|
+
* @example 'Hello World' -> 'hello-world'
|
|
11258
11635
|
* @public exported from `@promptbook/utils`
|
|
11259
11636
|
*/
|
|
11260
11637
|
function nameToUriPart(name) {
|
|
@@ -11268,11 +11645,11 @@
|
|
|
11268
11645
|
}
|
|
11269
11646
|
|
|
11270
11647
|
/**
|
|
11271
|
-
*
|
|
11648
|
+
* Converts a given name into URI-compatible parts.
|
|
11272
11649
|
*
|
|
11273
|
-
* @param name
|
|
11274
|
-
* @returns
|
|
11275
|
-
* @example
|
|
11650
|
+
* @param name The name to be converted into URI parts.
|
|
11651
|
+
* @returns An array of URI-compatible parts derived from the name.
|
|
11652
|
+
* @example 'Example Name' -> ['example', 'name']
|
|
11276
11653
|
* @public exported from `@promptbook/utils`
|
|
11277
11654
|
*/
|
|
11278
11655
|
function nameToUriParts(name) {
|
|
@@ -11316,7 +11693,7 @@
|
|
|
11316
11693
|
/**
|
|
11317
11694
|
* Function trimCodeBlock will trim starting and ending code block from the string if it is present.
|
|
11318
11695
|
*
|
|
11319
|
-
* Note: This is
|
|
11696
|
+
* Note: This is useful for post-processing of the result of the chat LLM model
|
|
11320
11697
|
* when the model wraps the result in the (markdown) code block.
|
|
11321
11698
|
*
|
|
11322
11699
|
* @public exported from `@promptbook/utils`
|
|
@@ -11335,7 +11712,7 @@
|
|
|
11335
11712
|
/**
|
|
11336
11713
|
* Function trimEndOfCodeBlock will remove ending code block from the string if it is present.
|
|
11337
11714
|
*
|
|
11338
|
-
* Note: This is
|
|
11715
|
+
* Note: This is useful for post-processing of the result of the completion LLM model
|
|
11339
11716
|
* if you want to start code block in the prompt but you don't want to end it in the result.
|
|
11340
11717
|
*
|
|
11341
11718
|
* @public exported from `@promptbook/utils`
|
|
@@ -11350,9 +11727,9 @@
|
|
|
11350
11727
|
/**
|
|
11351
11728
|
* Removes quotes and optional introduce text from a string
|
|
11352
11729
|
*
|
|
11353
|
-
* Tip: This is very
|
|
11730
|
+
* Tip: This is very useful for post-processing of the result of the LLM model
|
|
11354
11731
|
* Note: This function trims the text and removes whole introduce sentence if it is present
|
|
11355
|
-
* Note: There are two
|
|
11732
|
+
* Note: There are two similar functions:
|
|
11356
11733
|
* - `removeQuotes` which removes only bounding quotes
|
|
11357
11734
|
* - `unwrapResult` which removes whole introduce sentence
|
|
11358
11735
|
*
|
|
@@ -11423,7 +11800,7 @@
|
|
|
11423
11800
|
*
|
|
11424
11801
|
* - When there are multiple or no code blocks the function throws a `ParseError`
|
|
11425
11802
|
*
|
|
11426
|
-
* Note: There are multiple
|
|
11803
|
+
* Note: There are multiple similar function:
|
|
11427
11804
|
* - `extractBlock` just extracts the content of the code block which is also used as build-in function for postprocessing
|
|
11428
11805
|
* - `extractJsonBlock` extracts exactly one valid JSON code block
|
|
11429
11806
|
* - `extractOneBlockFromMarkdown` extracts exactly one code block with language of the code block
|
|
@@ -11460,7 +11837,7 @@
|
|
|
11460
11837
|
})();
|
|
11461
11838
|
}
|
|
11462
11839
|
/**
|
|
11463
|
-
* TODO:
|
|
11840
|
+
* TODO: Probably remove in favour of `keepImported`
|
|
11464
11841
|
* TODO: [1] This maybe does memory leak
|
|
11465
11842
|
*/
|
|
11466
11843
|
|
|
@@ -11664,7 +12041,7 @@
|
|
|
11664
12041
|
*/
|
|
11665
12042
|
|
|
11666
12043
|
/**
|
|
11667
|
-
* Placeholder for better implementation of JavascriptExecutionTools - some
|
|
12044
|
+
* Placeholder for better implementation of JavascriptExecutionTools - some proper sandboxing
|
|
11668
12045
|
*
|
|
11669
12046
|
* @alias JavascriptExecutionTools
|
|
11670
12047
|
* @public exported from `@promptbook/javascript`
|
|
@@ -12038,7 +12415,7 @@
|
|
|
12038
12415
|
/**
|
|
12039
12416
|
* Function usageToWorktime will take usage and estimate saved worktime in hours of reading / writing
|
|
12040
12417
|
*
|
|
12041
|
-
* Note: This is an estimate based of
|
|
12418
|
+
* Note: This is an estimate based of these sources:
|
|
12042
12419
|
* - https://jecas.cz/doba-cteni
|
|
12043
12420
|
* - https://www.originalnitonery.cz/blog/psani-vsemi-deseti-se-muzete-naucit-i-sami-doma
|
|
12044
12421
|
*
|
|
@@ -13123,7 +13500,9 @@
|
|
|
13123
13500
|
type: 'text',
|
|
13124
13501
|
name: 'pipelineSource',
|
|
13125
13502
|
message: '',
|
|
13126
|
-
validate
|
|
13503
|
+
validate(value) {
|
|
13504
|
+
return value.length > 0 ? true : 'Pipeline source is required';
|
|
13505
|
+
},
|
|
13127
13506
|
});
|
|
13128
13507
|
if (!response.pipelineSource) {
|
|
13129
13508
|
console.error(colors__default["default"].red('Pipeline source is required'));
|
|
@@ -13248,10 +13627,10 @@
|
|
|
13248
13627
|
console.info(colors__default["default"].gray('--- Detailed Result ---'));
|
|
13249
13628
|
console.info({ isSuccessful, errors, warnings, outputParameters, executionReport });
|
|
13250
13629
|
}
|
|
13251
|
-
if (saveReport && saveReport.endsWith('.json')) {
|
|
13630
|
+
if (executionReport !== null && saveReport && saveReport.endsWith('.json')) {
|
|
13252
13631
|
await promises.writeFile(saveReport, JSON.stringify(executionReport, null, 4) + '\n', 'utf-8');
|
|
13253
13632
|
}
|
|
13254
|
-
else if (saveReport && saveReport.endsWith('.md')) {
|
|
13633
|
+
else if (executionReport !== null && saveReport && saveReport.endsWith('.md')) {
|
|
13255
13634
|
const executionReportString = executionReportJsonToString(executionReport);
|
|
13256
13635
|
await promises.writeFile(saveReport, executionReportString, 'utf-8');
|
|
13257
13636
|
}
|
|
@@ -13294,15 +13673,15 @@
|
|
|
13294
13673
|
* TODO: [🖇] What about symlinks? Maybe flag --follow-symlinks
|
|
13295
13674
|
*/
|
|
13296
13675
|
|
|
13297
|
-
// TODO:
|
|
13298
|
-
// TODO:
|
|
13676
|
+
// TODO: [🥺] List running services from REMOTE_SERVER_URLS
|
|
13677
|
+
// TODO: [🥺] Import directly from YML
|
|
13299
13678
|
/**
|
|
13300
|
-
* @private
|
|
13679
|
+
* @private [🥺] Decide how to expose this
|
|
13301
13680
|
*/
|
|
13302
13681
|
const openapiJson = {
|
|
13303
13682
|
openapi: '3.0.0',
|
|
13304
13683
|
info: {
|
|
13305
|
-
title: 'Promptbook Remote Server API (
|
|
13684
|
+
title: 'Promptbook Remote Server API ([🥺] From YML)',
|
|
13306
13685
|
version: '1.0.0',
|
|
13307
13686
|
description: 'API documentation for the Promptbook Remote Server',
|
|
13308
13687
|
},
|
|
@@ -13925,7 +14304,7 @@
|
|
|
13925
14304
|
// TODO: [main] !!4 Validate here userId (pass validator as dependency)
|
|
13926
14305
|
let llm;
|
|
13927
14306
|
if (isAnonymous === true) {
|
|
13928
|
-
// Note:
|
|
14307
|
+
// Note: Anonymous mode
|
|
13929
14308
|
// TODO: Maybe check that configuration is not empty
|
|
13930
14309
|
const { llmToolsConfiguration } = identification;
|
|
13931
14310
|
llm = createLlmToolsFromConfiguration(llmToolsConfiguration, { isVerbose });
|
|
@@ -13935,7 +14314,7 @@
|
|
|
13935
14314
|
llm = await createLlmExecutionTools(identification);
|
|
13936
14315
|
}
|
|
13937
14316
|
else {
|
|
13938
|
-
throw new PipelineExecutionError(`You must provide either llmToolsConfiguration or non-anonymous mode must be
|
|
14317
|
+
throw new PipelineExecutionError(`You must provide either llmToolsConfiguration or non-anonymous mode must be properly configured`);
|
|
13939
14318
|
}
|
|
13940
14319
|
const customExecutionTools = createExecutionTools ? await createExecutionTools(identification) : {};
|
|
13941
14320
|
const fs = customExecutionTools.fs || $provideFilesystemForNode();
|
|
@@ -13960,7 +14339,7 @@
|
|
|
13960
14339
|
response.setHeader('X-Powered-By', 'Promptbook engine');
|
|
13961
14340
|
next();
|
|
13962
14341
|
});
|
|
13963
|
-
// TODO:
|
|
14342
|
+
// TODO: [🥺] Expose openapiJson to consumer and also allow to add new routes
|
|
13964
14343
|
app.use(OpenApiValidator__namespace.middleware({
|
|
13965
14344
|
apiSpec: openapiJson,
|
|
13966
14345
|
ignorePaths(path) {
|
|
@@ -14000,7 +14379,7 @@
|
|
|
14000
14379
|
|
|
14001
14380
|
**Server port:** ${port}
|
|
14002
14381
|
**Startup date:** ${startupDate.toISOString()}
|
|
14003
|
-
**
|
|
14382
|
+
**Anonymous mode:** ${isAnonymousModeAllowed ? 'enabled' : 'disabled'}
|
|
14004
14383
|
**Application mode:** ${isApplicationModeAllowed ? 'enabled' : 'disabled'}
|
|
14005
14384
|
${block(!isApplicationModeAllowed || collection === null
|
|
14006
14385
|
? ''
|
|
@@ -14030,7 +14409,7 @@
|
|
|
14030
14409
|
To connect to this server use:
|
|
14031
14410
|
|
|
14032
14411
|
1) The client https://www.npmjs.com/package/@promptbook/remote-client
|
|
14033
|
-
2) OpenAI compatible client *(Not
|
|
14412
|
+
2) OpenAI compatible client *(Not working yet)*
|
|
14034
14413
|
3) REST API
|
|
14035
14414
|
|
|
14036
14415
|
For more information look at:
|
|
@@ -14114,12 +14493,13 @@
|
|
|
14114
14493
|
});
|
|
14115
14494
|
function exportExecutionTask(executionTask, isFull) {
|
|
14116
14495
|
// <- TODO: [🧠] This should be maybe method of `ExecutionTask` itself
|
|
14117
|
-
const { taskType, taskId, status, errors, warnings, createdAt, updatedAt, currentValue } = executionTask;
|
|
14496
|
+
const { taskType, promptbookVersion, taskId, title, status, errors, warnings, createdAt, updatedAt, currentValue, } = executionTask;
|
|
14118
14497
|
if (isFull) {
|
|
14119
14498
|
return {
|
|
14120
|
-
nonce: '✨',
|
|
14121
14499
|
taskId,
|
|
14500
|
+
title,
|
|
14122
14501
|
taskType,
|
|
14502
|
+
promptbookVersion,
|
|
14123
14503
|
status,
|
|
14124
14504
|
errors: errors.map(serializeError),
|
|
14125
14505
|
warnings: warnings.map(serializeError),
|
|
@@ -14130,9 +14510,10 @@
|
|
|
14130
14510
|
}
|
|
14131
14511
|
else {
|
|
14132
14512
|
return {
|
|
14133
|
-
nonce: '✨',
|
|
14134
14513
|
taskId,
|
|
14514
|
+
title,
|
|
14135
14515
|
taskType,
|
|
14516
|
+
promptbookVersion,
|
|
14136
14517
|
status,
|
|
14137
14518
|
createdAt,
|
|
14138
14519
|
updatedAt,
|
|
@@ -14282,7 +14663,7 @@
|
|
|
14282
14663
|
}
|
|
14283
14664
|
});
|
|
14284
14665
|
// -----------
|
|
14285
|
-
// TODO: [👒] Listing models (and checking configuration)
|
|
14666
|
+
// TODO: [👒] Listing models (and checking configuration) probably should go through REST API not Socket.io
|
|
14286
14667
|
socket.on('listModels-request', async (request) => {
|
|
14287
14668
|
const { identification } = request;
|
|
14288
14669
|
if (isVerbose) {
|
|
@@ -14304,7 +14685,7 @@
|
|
|
14304
14685
|
}
|
|
14305
14686
|
});
|
|
14306
14687
|
// -----------
|
|
14307
|
-
// TODO: [👒] Listing models (and checking configuration)
|
|
14688
|
+
// TODO: [👒] Listing models (and checking configuration) probably should go through REST API not Socket.io
|
|
14308
14689
|
socket.on('preparePipeline-request', async (request) => {
|
|
14309
14690
|
const { identification, pipeline } = request;
|
|
14310
14691
|
if (isVerbose) {
|
|
@@ -14364,7 +14745,7 @@
|
|
|
14364
14745
|
};
|
|
14365
14746
|
}
|
|
14366
14747
|
/**
|
|
14367
|
-
* TODO: [🌡] Add CORS and security -
|
|
14748
|
+
* TODO: [🌡] Add CORS and security - probably via `helmet`
|
|
14368
14749
|
* TODO: Split this file into multiple functions - handler for each request
|
|
14369
14750
|
* TODO: Maybe use `$exportJson`
|
|
14370
14751
|
* TODO: [🧠][🛍] Maybe not `isAnonymous: boolean` BUT `mode: 'ANONYMOUS'|'COLLECTION'`
|
|
@@ -14600,7 +14981,7 @@
|
|
|
14600
14981
|
program.alias('ptbk');
|
|
14601
14982
|
program.version(PROMPTBOOK_ENGINE_VERSION);
|
|
14602
14983
|
program.description(CLAIM);
|
|
14603
|
-
// Note:
|
|
14984
|
+
// Note: These options are valid for all commands
|
|
14604
14985
|
$initializeAboutCommand(program);
|
|
14605
14986
|
$initializeRunCommand(program);
|
|
14606
14987
|
$initializeLoginCommand(program);
|
|
@@ -14640,37 +15021,6 @@
|
|
|
14640
15021
|
* Note: [🟡] Code in this file should never be published outside of `@promptbook/cli`
|
|
14641
15022
|
*/
|
|
14642
15023
|
|
|
14643
|
-
/**
|
|
14644
|
-
* How is the model provider trusted?
|
|
14645
|
-
*
|
|
14646
|
-
* @public exported from `@promptbook/core`
|
|
14647
|
-
*/
|
|
14648
|
-
// <- TODO: Maybe do better levels of trust
|
|
14649
|
-
/**
|
|
14650
|
-
* How is the model provider important?
|
|
14651
|
-
*
|
|
14652
|
-
* @public exported from `@promptbook/core`
|
|
14653
|
-
*/
|
|
14654
|
-
const MODEL_ORDER = {
|
|
14655
|
-
/**
|
|
14656
|
-
* Top-tier models, e.g. OpenAI, Anthropic,...
|
|
14657
|
-
*/
|
|
14658
|
-
TOP_TIER: 333,
|
|
14659
|
-
/**
|
|
14660
|
-
* Mid-tier models, e.g. Llama, Mistral, etc.
|
|
14661
|
-
*/
|
|
14662
|
-
NORMAL: 100,
|
|
14663
|
-
/**
|
|
14664
|
-
* Low-tier models, e.g. Phi, Tiny, etc.
|
|
14665
|
-
*/
|
|
14666
|
-
LOW_TIER: 0,
|
|
14667
|
-
};
|
|
14668
|
-
/**
|
|
14669
|
-
* TODO: Add configuration schema and maybe some documentation link
|
|
14670
|
-
* TODO: Maybe constrain LlmToolsConfiguration[number] by generic to ensure that `createConfigurationFromEnv` and `getBoilerplateConfiguration` always create same `packageName` and `className`
|
|
14671
|
-
* TODO: [®] DRY Register logic
|
|
14672
|
-
*/
|
|
14673
|
-
|
|
14674
15024
|
/**
|
|
14675
15025
|
* Registration of LLM provider metadata
|
|
14676
15026
|
*
|
|
@@ -14686,7 +15036,7 @@
|
|
|
14686
15036
|
className: 'AnthropicClaudeExecutionTools',
|
|
14687
15037
|
envVariables: ['ANTHROPIC_CLAUDE_API_KEY'],
|
|
14688
15038
|
trustLevel: 'CLOSED',
|
|
14689
|
-
order:
|
|
15039
|
+
order: MODEL_ORDERS.TOP_TIER,
|
|
14690
15040
|
getBoilerplateConfiguration() {
|
|
14691
15041
|
return {
|
|
14692
15042
|
title: 'Anthropic Claude',
|
|
@@ -14696,6 +15046,7 @@
|
|
|
14696
15046
|
apiKey: 'sk-ant-api03-',
|
|
14697
15047
|
isProxied: true,
|
|
14698
15048
|
remoteServerUrl: DEFAULT_REMOTE_SERVER_URL,
|
|
15049
|
+
maxRequestsPerMinute: DEFAULT_MAX_REQUESTS_PER_MINUTE,
|
|
14699
15050
|
},
|
|
14700
15051
|
};
|
|
14701
15052
|
},
|
|
@@ -14731,7 +15082,7 @@
|
|
|
14731
15082
|
/**
|
|
14732
15083
|
* List of available Anthropic Claude models with pricing
|
|
14733
15084
|
*
|
|
14734
|
-
* Note: Done at
|
|
15085
|
+
* Note: Done at 2025-05-06
|
|
14735
15086
|
*
|
|
14736
15087
|
* @see https://docs.anthropic.com/en/docs/models-overview
|
|
14737
15088
|
* @public exported from `@promptbook/anthropic-claude`
|
|
@@ -14745,8 +15096,8 @@
|
|
|
14745
15096
|
modelName: 'claude-3-5-sonnet-20240620',
|
|
14746
15097
|
modelDescription: 'Latest Claude model with great reasoning, coding, and language understanding capabilities. 200K context window. Optimized balance of intelligence and speed.',
|
|
14747
15098
|
pricing: {
|
|
14748
|
-
prompt: computeUsage(`$
|
|
14749
|
-
output: computeUsage(`$
|
|
15099
|
+
prompt: computeUsage(`$2.50 / 1M tokens`),
|
|
15100
|
+
output: computeUsage(`$12.50 / 1M tokens`),
|
|
14750
15101
|
},
|
|
14751
15102
|
},
|
|
14752
15103
|
{
|
|
@@ -14755,8 +15106,8 @@
|
|
|
14755
15106
|
modelName: 'claude-3-opus-20240229',
|
|
14756
15107
|
modelDescription: 'Most capable Claude model excelling at complex reasoning, coding, and detailed instruction following. 200K context window. Best for sophisticated tasks requiring nuanced understanding.',
|
|
14757
15108
|
pricing: {
|
|
14758
|
-
prompt: computeUsage(`$
|
|
14759
|
-
output: computeUsage(`$
|
|
15109
|
+
prompt: computeUsage(`$12.00 / 1M tokens`),
|
|
15110
|
+
output: computeUsage(`$60.00 / 1M tokens`),
|
|
14760
15111
|
},
|
|
14761
15112
|
},
|
|
14762
15113
|
{
|
|
@@ -14815,8 +15166,8 @@
|
|
|
14815
15166
|
modelName: 'claude-3-7-sonnet-20250219',
|
|
14816
15167
|
modelDescription: 'Latest generation Claude model with advanced reasoning and language understanding. Enhanced capabilities over 3.5 with improved domain knowledge. 200K context window.',
|
|
14817
15168
|
pricing: {
|
|
14818
|
-
prompt: computeUsage(`$
|
|
14819
|
-
output: computeUsage(`$
|
|
15169
|
+
prompt: computeUsage(`$2.50 / 1M tokens`),
|
|
15170
|
+
output: computeUsage(`$12.50 / 1M tokens`),
|
|
14820
15171
|
},
|
|
14821
15172
|
},
|
|
14822
15173
|
{
|
|
@@ -14863,14 +15214,18 @@
|
|
|
14863
15214
|
/**
|
|
14864
15215
|
* Make UncertainNumber
|
|
14865
15216
|
*
|
|
14866
|
-
* @param value
|
|
15217
|
+
* @param value value of the uncertain number, if `NaN` or `undefined`, it will be set to 0 and `isUncertain=true`
|
|
15218
|
+
* @param isUncertain if `true`, the value is uncertain, otherwise depends on the value
|
|
14867
15219
|
*
|
|
14868
15220
|
* @private utility for initializating UncertainNumber
|
|
14869
15221
|
*/
|
|
14870
|
-
function uncertainNumber(value) {
|
|
15222
|
+
function uncertainNumber(value, isUncertain) {
|
|
14871
15223
|
if (value === null || value === undefined || Number.isNaN(value)) {
|
|
14872
15224
|
return UNCERTAIN_ZERO_VALUE;
|
|
14873
15225
|
}
|
|
15226
|
+
if (isUncertain === true) {
|
|
15227
|
+
return { value, isUncertain };
|
|
15228
|
+
}
|
|
14874
15229
|
return { value };
|
|
14875
15230
|
}
|
|
14876
15231
|
|
|
@@ -14936,6 +15291,8 @@
|
|
|
14936
15291
|
* Anthropic Claude API client.
|
|
14937
15292
|
*/
|
|
14938
15293
|
this.client = null;
|
|
15294
|
+
const rate = this.options.maxRequestsPerMinute || DEFAULT_MAX_REQUESTS_PER_MINUTE;
|
|
15295
|
+
this.limiter = new Bottleneck__default["default"]({ minTime: 60000 / rate });
|
|
14939
15296
|
}
|
|
14940
15297
|
get title() {
|
|
14941
15298
|
return 'Anthropic Claude';
|
|
@@ -14987,8 +15344,6 @@
|
|
|
14987
15344
|
// <- TODO: [🌾] Make some global max cap for maxTokens
|
|
14988
15345
|
temperature: modelRequirements.temperature,
|
|
14989
15346
|
system: modelRequirements.systemMessage,
|
|
14990
|
-
// <- TODO: [🈁] Use `seed` here AND/OR use is `isDeterministic` for entire execution tools
|
|
14991
|
-
// <- Note: [🧆]
|
|
14992
15347
|
messages: [
|
|
14993
15348
|
{
|
|
14994
15349
|
role: 'user',
|
|
@@ -14997,14 +15352,14 @@
|
|
|
14997
15352
|
// @see https://docs.anthropic.com/en/docs/test-and-evaluate/strengthen-guardrails/increase-consistency#specify-the-desired-output-format
|
|
14998
15353
|
},
|
|
14999
15354
|
],
|
|
15000
|
-
// TODO: Is here some equivalent of user identification?> user: this.options.user,
|
|
15001
15355
|
};
|
|
15002
15356
|
const start = $getCurrentDate();
|
|
15003
|
-
let complete;
|
|
15004
15357
|
if (this.options.isVerbose) {
|
|
15005
15358
|
console.info(colors__default["default"].bgWhite('rawRequest'), JSON.stringify(rawRequest, null, 4));
|
|
15006
15359
|
}
|
|
15007
|
-
const rawResponse = await
|
|
15360
|
+
const rawResponse = await this.limiter
|
|
15361
|
+
.schedule(() => client.messages.create(rawRequest))
|
|
15362
|
+
.catch((error) => {
|
|
15008
15363
|
if (this.options.isVerbose) {
|
|
15009
15364
|
console.info(colors__default["default"].bgRed('error'), error);
|
|
15010
15365
|
}
|
|
@@ -15024,12 +15379,11 @@
|
|
|
15024
15379
|
throw new PipelineExecutionError(`Returned content is not "text" type but "${contentBlock.type}"`);
|
|
15025
15380
|
}
|
|
15026
15381
|
const resultContent = contentBlock.text;
|
|
15027
|
-
|
|
15028
|
-
complete = $getCurrentDate();
|
|
15382
|
+
const complete = $getCurrentDate();
|
|
15029
15383
|
const usage = computeAnthropicClaudeUsage(rawPromptContent || '', resultContent || '', rawResponse);
|
|
15030
15384
|
return exportJson({
|
|
15031
15385
|
name: 'promptResult',
|
|
15032
|
-
message: `Result of \`
|
|
15386
|
+
message: `Result of \`AnthropicClaudeExecutionTools.callChatModel\``,
|
|
15033
15387
|
order: [],
|
|
15034
15388
|
value: {
|
|
15035
15389
|
content: resultContent,
|
|
@@ -15046,83 +15400,59 @@
|
|
|
15046
15400
|
},
|
|
15047
15401
|
});
|
|
15048
15402
|
}
|
|
15049
|
-
|
|
15050
|
-
|
|
15051
|
-
|
|
15052
|
-
|
|
15053
|
-
): Promise<CompletionPromptResult> {
|
|
15054
|
-
|
|
15403
|
+
/**
|
|
15404
|
+
* Calls Anthropic Claude API to use a completion model.
|
|
15405
|
+
*/
|
|
15406
|
+
async callCompletionModel(prompt) {
|
|
15055
15407
|
if (this.options.isVerbose) {
|
|
15056
15408
|
console.info('🖋 Anthropic Claude callCompletionModel call');
|
|
15057
15409
|
}
|
|
15058
|
-
|
|
15059
15410
|
const { content, parameters, modelRequirements } = prompt;
|
|
15060
|
-
|
|
15061
|
-
// TODO: [☂] Use here more modelRequirements
|
|
15062
15411
|
if (modelRequirements.modelVariant !== 'COMPLETION') {
|
|
15063
15412
|
throw new PipelineExecutionError('Use callCompletionModel only for COMPLETION variant');
|
|
15064
15413
|
}
|
|
15065
|
-
|
|
15414
|
+
const client = await this.getClient();
|
|
15066
15415
|
const modelName = modelRequirements.modelName || this.getDefaultChatModel().modelName;
|
|
15067
|
-
const
|
|
15416
|
+
const rawPromptContent = templateParameters(content, { ...parameters, modelName });
|
|
15417
|
+
const rawRequest = {
|
|
15068
15418
|
model: modelName,
|
|
15069
|
-
|
|
15070
|
-
|
|
15071
|
-
// <- TODO: Use here `systemMessage`, `temperature` and `seed`
|
|
15072
|
-
};
|
|
15073
|
-
|
|
15074
|
-
const rawRequest: xxxx.Completions.CompletionCreateParamsNonStreaming = {
|
|
15075
|
-
...modelSettings,
|
|
15419
|
+
max_tokens_to_sample: modelRequirements.maxTokens || 2000,
|
|
15420
|
+
temperature: modelRequirements.temperature,
|
|
15076
15421
|
prompt: rawPromptContent,
|
|
15077
|
-
user: this.options.user,
|
|
15078
15422
|
};
|
|
15079
|
-
const start
|
|
15080
|
-
|
|
15081
|
-
|
|
15082
|
-
|
|
15083
|
-
|
|
15084
|
-
|
|
15085
|
-
|
|
15086
|
-
|
|
15087
|
-
|
|
15088
|
-
}
|
|
15089
|
-
throw error;
|
|
15090
|
-
});
|
|
15091
|
-
|
|
15092
|
-
|
|
15423
|
+
const start = $getCurrentDate();
|
|
15424
|
+
const rawResponse = await this.limiter
|
|
15425
|
+
.schedule(() => client.completions.create(rawRequest))
|
|
15426
|
+
.catch((error) => {
|
|
15427
|
+
if (this.options.isVerbose) {
|
|
15428
|
+
console.info(colors__default["default"].bgRed('error'), error);
|
|
15429
|
+
}
|
|
15430
|
+
throw error;
|
|
15431
|
+
});
|
|
15093
15432
|
if (this.options.isVerbose) {
|
|
15094
|
-
console.info(
|
|
15095
|
-
}
|
|
15096
|
-
|
|
15097
|
-
if (!rawResponse.choices[0]) {
|
|
15098
|
-
throw new PipelineExecutionError('No choises from Anthropic Claude');
|
|
15433
|
+
console.info(colors__default["default"].bgWhite('rawResponse'), JSON.stringify(rawResponse, null, 4));
|
|
15099
15434
|
}
|
|
15100
|
-
|
|
15101
|
-
|
|
15102
|
-
// TODO: This should be maybe only warning
|
|
15103
|
-
throw new PipelineExecutionError('More than one choise from Anthropic Claude');
|
|
15435
|
+
if (!rawResponse.completion) {
|
|
15436
|
+
throw new PipelineExecutionError('No completion from Anthropic Claude');
|
|
15104
15437
|
}
|
|
15105
|
-
|
|
15106
|
-
const
|
|
15107
|
-
|
|
15108
|
-
|
|
15109
|
-
|
|
15110
|
-
|
|
15111
|
-
|
|
15112
|
-
|
|
15113
|
-
|
|
15114
|
-
|
|
15115
|
-
|
|
15116
|
-
|
|
15117
|
-
|
|
15118
|
-
|
|
15438
|
+
const resultContent = rawResponse.completion;
|
|
15439
|
+
const complete = $getCurrentDate();
|
|
15440
|
+
const usage = computeAnthropicClaudeUsage(rawPromptContent, resultContent, rawResponse);
|
|
15441
|
+
return exportJson({
|
|
15442
|
+
name: 'promptResult',
|
|
15443
|
+
message: `Result of \`AnthropicClaudeExecutionTools.callCompletionModel\``,
|
|
15444
|
+
order: [],
|
|
15445
|
+
value: {
|
|
15446
|
+
content: resultContent,
|
|
15447
|
+
modelName: rawResponse.model || modelName,
|
|
15448
|
+
timing: { start, complete },
|
|
15449
|
+
usage,
|
|
15450
|
+
rawPromptContent,
|
|
15451
|
+
rawRequest,
|
|
15452
|
+
rawResponse,
|
|
15119
15453
|
},
|
|
15120
|
-
usage,
|
|
15121
|
-
rawResponse,
|
|
15122
|
-
// <- [🗯]
|
|
15123
15454
|
});
|
|
15124
15455
|
}
|
|
15125
|
-
*/
|
|
15126
15456
|
// <- Note: [🤖] callXxxModel
|
|
15127
15457
|
/**
|
|
15128
15458
|
* Get the model that should be used as default
|
|
@@ -15131,7 +15461,7 @@
|
|
|
15131
15461
|
const model = ANTHROPIC_CLAUDE_MODELS.find(({ modelName }) => modelName.startsWith(defaultModelName));
|
|
15132
15462
|
if (model === undefined) {
|
|
15133
15463
|
throw new UnexpectedError(spaceTrim__default["default"]((block) => `
|
|
15134
|
-
Cannot find model in
|
|
15464
|
+
Cannot find model in Anthropic Claude models with name "${defaultModelName}" which should be used as default.
|
|
15135
15465
|
|
|
15136
15466
|
Available models:
|
|
15137
15467
|
${block(ANTHROPIC_CLAUDE_MODELS.map(({ modelName }) => `- "${modelName}"`).join('\n'))}
|
|
@@ -15224,9 +15554,9 @@
|
|
|
15224
15554
|
title: 'Azure Open AI',
|
|
15225
15555
|
packageName: '@promptbook/azure-openai',
|
|
15226
15556
|
className: 'AzureOpenAiExecutionTools',
|
|
15227
|
-
envVariables: ['
|
|
15557
|
+
envVariables: ['AZUREOPENAI_API_KEY', 'AZUREOPENAI_RESOURCE_NAME', 'AZUREOPENAI_DEPLOYMENT_NAME'],
|
|
15228
15558
|
trustLevel: 'CLOSED_BUSINESS',
|
|
15229
|
-
order:
|
|
15559
|
+
order: MODEL_ORDERS.NORMAL,
|
|
15230
15560
|
getBoilerplateConfiguration() {
|
|
15231
15561
|
return {
|
|
15232
15562
|
title: 'Azure Open AI',
|
|
@@ -15234,6 +15564,9 @@
|
|
|
15234
15564
|
className: 'AzureOpenAiExecutionTools',
|
|
15235
15565
|
options: {
|
|
15236
15566
|
apiKey: 'sk-',
|
|
15567
|
+
resourceName: 'my-resource-name',
|
|
15568
|
+
deploymentName: 'my-deployment-name',
|
|
15569
|
+
maxRequestsPerMinute: DEFAULT_MAX_REQUESTS_PER_MINUTE,
|
|
15237
15570
|
},
|
|
15238
15571
|
};
|
|
15239
15572
|
},
|
|
@@ -15247,15 +15580,15 @@
|
|
|
15247
15580
|
packageName: '@promptbook/azure-openai',
|
|
15248
15581
|
className: 'AzureOpenAiExecutionTools',
|
|
15249
15582
|
options: {
|
|
15583
|
+
apiKey: env.AZUREOPENAI_API_KEY,
|
|
15250
15584
|
resourceName: env.AZUREOPENAI_RESOURCE_NAME,
|
|
15251
15585
|
deploymentName: env.AZUREOPENAI_DEPLOYMENT_NAME,
|
|
15252
|
-
apiKey: env.AZUREOPENAI_API_KEY,
|
|
15253
15586
|
},
|
|
15254
15587
|
};
|
|
15255
15588
|
}
|
|
15256
|
-
else if (typeof env.
|
|
15257
|
-
typeof env.
|
|
15258
|
-
typeof env.
|
|
15589
|
+
else if (typeof env.AZUREOPENAI_API_KEY === 'string' ||
|
|
15590
|
+
typeof env.AZUREOPENAI_RESOURCE_NAME === 'string' ||
|
|
15591
|
+
typeof env.AZUREOPENAI_DEPLOYMENT_NAME === 'string') {
|
|
15259
15592
|
return null;
|
|
15260
15593
|
/*
|
|
15261
15594
|
Note: [🗨] Partial configuration is handled more gracefully elsewhere
|
|
@@ -15284,7 +15617,7 @@
|
|
|
15284
15617
|
/**
|
|
15285
15618
|
* List of available OpenAI models with pricing
|
|
15286
15619
|
*
|
|
15287
|
-
* Note: Done at
|
|
15620
|
+
* Note: Done at 2025-05-06
|
|
15288
15621
|
*
|
|
15289
15622
|
* @see https://platform.openai.com/docs/models/
|
|
15290
15623
|
* @see https://openai.com/api/pricing/
|
|
@@ -15545,7 +15878,7 @@
|
|
|
15545
15878
|
modelName: 'text-embedding-3-large',
|
|
15546
15879
|
modelDescription: "OpenAI's most capable text embedding model designed for high-quality embeddings for complex similarity tasks and information retrieval.",
|
|
15547
15880
|
pricing: {
|
|
15548
|
-
prompt: computeUsage(`$0.13
|
|
15881
|
+
prompt: computeUsage(`$0.13 / 1M tokens`),
|
|
15549
15882
|
// TODO: [🏏] Leverage the batch API @see https://platform.openai.com/docs/guides/batch
|
|
15550
15883
|
output: 0, // <- Note: [🆖] In Embedding models you dont pay for output
|
|
15551
15884
|
},
|
|
@@ -15638,8 +15971,8 @@
|
|
|
15638
15971
|
modelName: 'gpt-4o-mini',
|
|
15639
15972
|
modelDescription: 'Smaller, more cost-effective version of GPT-4o with good performance across text, vision, and audio tasks at reduced complexity.',
|
|
15640
15973
|
pricing: {
|
|
15641
|
-
prompt: computeUsage(`$
|
|
15642
|
-
output: computeUsage(`$
|
|
15974
|
+
prompt: computeUsage(`$0.15 / 1M tokens`),
|
|
15975
|
+
output: computeUsage(`$0.60 / 1M tokens`),
|
|
15643
15976
|
},
|
|
15644
15977
|
},
|
|
15645
15978
|
/**/
|
|
@@ -15661,7 +15994,7 @@
|
|
|
15661
15994
|
modelTitle: 'o1-preview-2024-09-12',
|
|
15662
15995
|
modelName: 'o1-preview-2024-09-12',
|
|
15663
15996
|
modelDescription: 'September 2024 version of O1 preview with specialized reasoning capabilities for complex tasks requiring precise analytical thinking.',
|
|
15664
|
-
// <- TODO: [💩] Some better system to organize
|
|
15997
|
+
// <- TODO: [💩] Some better system to organize these date suffixes and versions
|
|
15665
15998
|
pricing: {
|
|
15666
15999
|
prompt: computeUsage(`$15.00 / 1M tokens`),
|
|
15667
16000
|
output: computeUsage(`$60.00 / 1M tokens`),
|
|
@@ -15764,6 +16097,10 @@
|
|
|
15764
16097
|
* OpenAI Azure API client.
|
|
15765
16098
|
*/
|
|
15766
16099
|
this.client = null;
|
|
16100
|
+
// TODO: Allow configuring rate limits via options
|
|
16101
|
+
this.limiter = new Bottleneck__default["default"]({
|
|
16102
|
+
minTime: 60000 / (this.options.maxRequestsPerMinute || DEFAULT_MAX_REQUESTS_PER_MINUTE),
|
|
16103
|
+
});
|
|
15767
16104
|
}
|
|
15768
16105
|
get title() {
|
|
15769
16106
|
return 'Azure OpenAI';
|
|
@@ -15841,7 +16178,9 @@
|
|
|
15841
16178
|
console.info(colors__default["default"].bgWhite('messages'), JSON.stringify(messages, null, 4));
|
|
15842
16179
|
}
|
|
15843
16180
|
const rawRequest = [modelName, messages, modelSettings];
|
|
15844
|
-
const rawResponse = await this.
|
|
16181
|
+
const rawResponse = await this.limiter
|
|
16182
|
+
.schedule(() => this.withTimeout(client.getChatCompletions(...rawRequest)))
|
|
16183
|
+
.catch((error) => {
|
|
15845
16184
|
if (this.options.isVerbose) {
|
|
15846
16185
|
console.info(colors__default["default"].bgRed('error'), error);
|
|
15847
16186
|
}
|
|
@@ -15937,7 +16276,9 @@
|
|
|
15937
16276
|
[rawPromptContent],
|
|
15938
16277
|
modelSettings,
|
|
15939
16278
|
];
|
|
15940
|
-
const rawResponse = await this.
|
|
16279
|
+
const rawResponse = await this.limiter
|
|
16280
|
+
.schedule(() => this.withTimeout(client.getCompletions(...rawRequest)))
|
|
16281
|
+
.catch((error) => {
|
|
15941
16282
|
if (this.options.isVerbose) {
|
|
15942
16283
|
console.info(colors__default["default"].bgRed('error'), error);
|
|
15943
16284
|
}
|
|
@@ -16012,7 +16353,7 @@
|
|
|
16012
16353
|
});
|
|
16013
16354
|
}
|
|
16014
16355
|
/**
|
|
16015
|
-
* Changes Azure error (which is not
|
|
16356
|
+
* Changes Azure error (which is not proper Error but object) to proper Error
|
|
16016
16357
|
*/
|
|
16017
16358
|
transformAzureError(azureError) {
|
|
16018
16359
|
if (azureError instanceof UnexpectedError) {
|
|
@@ -16078,7 +16419,7 @@
|
|
|
16078
16419
|
className: 'DeepseekExecutionTools',
|
|
16079
16420
|
envVariables: ['DEEPSEEK_GENERATIVE_AI_API_KEY'],
|
|
16080
16421
|
trustLevel: 'UNTRUSTED',
|
|
16081
|
-
order:
|
|
16422
|
+
order: MODEL_ORDERS.NORMAL,
|
|
16082
16423
|
getBoilerplateConfiguration() {
|
|
16083
16424
|
return {
|
|
16084
16425
|
title: 'Deepseek',
|
|
@@ -16158,12 +16499,14 @@
|
|
|
16158
16499
|
const { vercelProvider, availableModels, userId, additionalChatSettings = {} } = options;
|
|
16159
16500
|
if (!/Vercel/i.test(title)) {
|
|
16160
16501
|
title = `${title} (through Vercel)`;
|
|
16502
|
+
// <- TODO: [🧈] Maybe standartize the suffix
|
|
16161
16503
|
} /* not else */
|
|
16162
16504
|
if (description === undefined) {
|
|
16163
16505
|
description = `Implementation of ${title} through Vercel`;
|
|
16164
16506
|
} /* not else */
|
|
16165
16507
|
if (!/Vercel/i.test(description)) {
|
|
16166
16508
|
description = `${description} (through Vercel)`;
|
|
16509
|
+
// <- TODO: [🧈] Maybe standartize the suffix
|
|
16167
16510
|
} /* not else */
|
|
16168
16511
|
return {
|
|
16169
16512
|
title,
|
|
@@ -16281,7 +16624,7 @@
|
|
|
16281
16624
|
/**
|
|
16282
16625
|
* List of available Deepseek models with descriptions
|
|
16283
16626
|
*
|
|
16284
|
-
* Note: Done at 2025-
|
|
16627
|
+
* Note: Done at 2025-05-06
|
|
16285
16628
|
*
|
|
16286
16629
|
* @see https://www.deepseek.com/models
|
|
16287
16630
|
* @public exported from `@promptbook/deepseek`
|
|
@@ -16295,8 +16638,8 @@
|
|
|
16295
16638
|
modelName: 'deepseek-chat',
|
|
16296
16639
|
modelDescription: 'General-purpose language model with strong performance across conversation, reasoning, and content generation. 128K context window with excellent instruction following capabilities.',
|
|
16297
16640
|
pricing: {
|
|
16298
|
-
prompt: computeUsage(`$
|
|
16299
|
-
output: computeUsage(`$
|
|
16641
|
+
prompt: computeUsage(`$0.80 / 1M tokens`),
|
|
16642
|
+
output: computeUsage(`$1.60 / 1M tokens`),
|
|
16300
16643
|
},
|
|
16301
16644
|
},
|
|
16302
16645
|
{
|
|
@@ -16305,8 +16648,8 @@
|
|
|
16305
16648
|
modelName: 'deepseek-reasoner',
|
|
16306
16649
|
modelDescription: 'Specialized model focused on complex reasoning tasks like mathematical problem-solving and logical analysis. Enhanced step-by-step reasoning with explicit chain-of-thought processes. 128K context window.',
|
|
16307
16650
|
pricing: {
|
|
16308
|
-
prompt: computeUsage(`$
|
|
16309
|
-
output: computeUsage(`$
|
|
16651
|
+
prompt: computeUsage(`$3.50 / 1M tokens`),
|
|
16652
|
+
output: computeUsage(`$7.00 / 1M tokens`),
|
|
16310
16653
|
},
|
|
16311
16654
|
},
|
|
16312
16655
|
{
|
|
@@ -16401,7 +16744,7 @@
|
|
|
16401
16744
|
className: 'GoogleExecutionTools',
|
|
16402
16745
|
envVariables: ['GOOGLE_GENERATIVE_AI_API_KEY'],
|
|
16403
16746
|
trustLevel: 'CLOSED',
|
|
16404
|
-
order:
|
|
16747
|
+
order: MODEL_ORDERS.NORMAL,
|
|
16405
16748
|
getBoilerplateConfiguration() {
|
|
16406
16749
|
return {
|
|
16407
16750
|
title: 'Google Gemini',
|
|
@@ -16440,7 +16783,7 @@
|
|
|
16440
16783
|
/**
|
|
16441
16784
|
* List of available Google models with descriptions
|
|
16442
16785
|
*
|
|
16443
|
-
* Note: Done at 2025-
|
|
16786
|
+
* Note: Done at 2025-05-06
|
|
16444
16787
|
*
|
|
16445
16788
|
* @see https://ai.google.dev/models/gemini
|
|
16446
16789
|
* @public exported from `@promptbook/google`
|
|
@@ -16454,8 +16797,8 @@
|
|
|
16454
16797
|
modelName: 'gemini-2.5-pro-preview-03-25',
|
|
16455
16798
|
modelDescription: 'Latest advanced multimodal model with exceptional reasoning, tool use, and instruction following. 1M token context window with improved vision capabilities for complex visual tasks.',
|
|
16456
16799
|
pricing: {
|
|
16457
|
-
prompt: computeUsage(`$
|
|
16458
|
-
output: computeUsage(`$
|
|
16800
|
+
prompt: computeUsage(`$8.00 / 1M tokens`),
|
|
16801
|
+
output: computeUsage(`$24.00 / 1M tokens`),
|
|
16459
16802
|
},
|
|
16460
16803
|
},
|
|
16461
16804
|
{
|
|
@@ -16494,8 +16837,8 @@
|
|
|
16494
16837
|
modelName: 'gemini-1.5-flash',
|
|
16495
16838
|
modelDescription: 'Efficient model balancing speed and quality for general-purpose applications. 1M token context window with good multimodal capabilities and quick response times.',
|
|
16496
16839
|
pricing: {
|
|
16497
|
-
prompt: computeUsage(`$0.
|
|
16498
|
-
output: computeUsage(`$
|
|
16840
|
+
prompt: computeUsage(`$0.25 / 1M tokens`),
|
|
16841
|
+
output: computeUsage(`$0.75 / 1M tokens`),
|
|
16499
16842
|
},
|
|
16500
16843
|
},
|
|
16501
16844
|
{
|
|
@@ -16562,8 +16905,8 @@
|
|
|
16562
16905
|
modelName: 'gemini-1.5-pro',
|
|
16563
16906
|
modelDescription: 'Flagship multimodal model with strong performance across text, code, vision, and audio tasks. 1M token context window with excellent reasoning capabilities.',
|
|
16564
16907
|
pricing: {
|
|
16565
|
-
prompt: computeUsage(`$
|
|
16566
|
-
output: computeUsage(`$
|
|
16908
|
+
prompt: computeUsage(`$6.00 / 1M tokens`),
|
|
16909
|
+
output: computeUsage(`$18.00 / 1M tokens`),
|
|
16567
16910
|
},
|
|
16568
16911
|
},
|
|
16569
16912
|
{
|
|
@@ -16666,7 +17009,7 @@
|
|
|
16666
17009
|
className: 'OpenAiExecutionTools',
|
|
16667
17010
|
envVariables: ['OPENAI_API_KEY'],
|
|
16668
17011
|
trustLevel: 'CLOSED',
|
|
16669
|
-
order:
|
|
17012
|
+
order: MODEL_ORDERS.TOP_TIER,
|
|
16670
17013
|
getBoilerplateConfiguration() {
|
|
16671
17014
|
return {
|
|
16672
17015
|
title: 'Open AI',
|
|
@@ -16674,6 +17017,7 @@
|
|
|
16674
17017
|
className: 'OpenAiExecutionTools',
|
|
16675
17018
|
options: {
|
|
16676
17019
|
apiKey: 'sk-',
|
|
17020
|
+
maxRequestsPerMinute: DEFAULT_MAX_REQUESTS_PER_MINUTE,
|
|
16677
17021
|
},
|
|
16678
17022
|
};
|
|
16679
17023
|
},
|
|
@@ -16693,9 +17037,9 @@
|
|
|
16693
17037
|
},
|
|
16694
17038
|
});
|
|
16695
17039
|
/**
|
|
16696
|
-
*
|
|
17040
|
+
* Registration of the OpenAI Assistant metadata
|
|
16697
17041
|
*
|
|
16698
|
-
* Note: [🏐] Configurations registrations are done in
|
|
17042
|
+
* Note: [🏐] Configurations registrations are done in the metadata registration section, but the constructor registration is handled separately.
|
|
16699
17043
|
*
|
|
16700
17044
|
* @public exported from `@promptbook/core`
|
|
16701
17045
|
* @public exported from `@promptbook/wizzard`
|
|
@@ -16708,7 +17052,7 @@
|
|
|
16708
17052
|
envVariables: null,
|
|
16709
17053
|
// <- TODO: ['OPENAI_API_KEY', 'OPENAI_ASSISTANT_ID']
|
|
16710
17054
|
trustLevel: 'CLOSED',
|
|
16711
|
-
order:
|
|
17055
|
+
order: MODEL_ORDERS.NORMAL,
|
|
16712
17056
|
getBoilerplateConfiguration() {
|
|
16713
17057
|
return {
|
|
16714
17058
|
title: 'Open AI Assistant',
|
|
@@ -16717,6 +17061,7 @@
|
|
|
16717
17061
|
options: {
|
|
16718
17062
|
apiKey: 'sk-',
|
|
16719
17063
|
assistantId: 'asst_',
|
|
17064
|
+
maxRequestsPerMinute: DEFAULT_MAX_REQUESTS_PER_MINUTE,
|
|
16720
17065
|
},
|
|
16721
17066
|
};
|
|
16722
17067
|
},
|
|
@@ -16763,13 +17108,21 @@
|
|
|
16763
17108
|
}
|
|
16764
17109
|
const inputTokens = rawResponse.usage.prompt_tokens;
|
|
16765
17110
|
const outputTokens = ((_b = rawResponse.usage) === null || _b === void 0 ? void 0 : _b.completion_tokens) || 0;
|
|
16766
|
-
|
|
17111
|
+
let isUncertain = false;
|
|
17112
|
+
let modelInfo = OPENAI_MODELS.find((model) => model.modelName === rawResponse.model);
|
|
17113
|
+
if (modelInfo === undefined) {
|
|
17114
|
+
// Note: Model is not in the list of known models, fallback to the family of the models and mark price as uncertain
|
|
17115
|
+
modelInfo = OPENAI_MODELS.find((model) => (rawResponse.model || SALT_NONCE).startsWith(model.modelName));
|
|
17116
|
+
if (modelInfo !== undefined) {
|
|
17117
|
+
isUncertain = true;
|
|
17118
|
+
}
|
|
17119
|
+
}
|
|
16767
17120
|
let price;
|
|
16768
17121
|
if (modelInfo === undefined || modelInfo.pricing === undefined) {
|
|
16769
17122
|
price = uncertainNumber();
|
|
16770
17123
|
}
|
|
16771
17124
|
else {
|
|
16772
|
-
price = uncertainNumber(inputTokens * modelInfo.pricing.prompt + outputTokens * modelInfo.pricing.output);
|
|
17125
|
+
price = uncertainNumber(inputTokens * modelInfo.pricing.prompt + outputTokens * modelInfo.pricing.output, isUncertain);
|
|
16773
17126
|
}
|
|
16774
17127
|
return {
|
|
16775
17128
|
price,
|
|
@@ -16804,6 +17157,10 @@
|
|
|
16804
17157
|
* OpenAI API client.
|
|
16805
17158
|
*/
|
|
16806
17159
|
this.client = null;
|
|
17160
|
+
// TODO: Allow configuring rate limits via options
|
|
17161
|
+
this.limiter = new Bottleneck__default["default"]({
|
|
17162
|
+
minTime: 60000 / (this.options.maxRequestsPerMinute || DEFAULT_MAX_REQUESTS_PER_MINUTE),
|
|
17163
|
+
});
|
|
16807
17164
|
}
|
|
16808
17165
|
get title() {
|
|
16809
17166
|
return 'OpenAI';
|
|
@@ -16903,11 +17260,12 @@
|
|
|
16903
17260
|
user: (_a = this.options.userId) === null || _a === void 0 ? void 0 : _a.toString(),
|
|
16904
17261
|
};
|
|
16905
17262
|
const start = $getCurrentDate();
|
|
16906
|
-
let complete;
|
|
16907
17263
|
if (this.options.isVerbose) {
|
|
16908
17264
|
console.info(colors__default["default"].bgWhite('rawRequest'), JSON.stringify(rawRequest, null, 4));
|
|
16909
17265
|
}
|
|
16910
|
-
const rawResponse = await
|
|
17266
|
+
const rawResponse = await this.limiter
|
|
17267
|
+
.schedule(() => client.chat.completions.create(rawRequest))
|
|
17268
|
+
.catch((error) => {
|
|
16911
17269
|
assertsError(error);
|
|
16912
17270
|
if (this.options.isVerbose) {
|
|
16913
17271
|
console.info(colors__default["default"].bgRed('error'), error);
|
|
@@ -16917,6 +17275,7 @@
|
|
|
16917
17275
|
if (this.options.isVerbose) {
|
|
16918
17276
|
console.info(colors__default["default"].bgWhite('rawResponse'), JSON.stringify(rawResponse, null, 4));
|
|
16919
17277
|
}
|
|
17278
|
+
const complete = $getCurrentDate();
|
|
16920
17279
|
if (!rawResponse.choices[0]) {
|
|
16921
17280
|
throw new PipelineExecutionError('No choises from OpenAI');
|
|
16922
17281
|
}
|
|
@@ -16925,8 +17284,6 @@
|
|
|
16925
17284
|
throw new PipelineExecutionError('More than one choise from OpenAI');
|
|
16926
17285
|
}
|
|
16927
17286
|
const resultContent = rawResponse.choices[0].message.content;
|
|
16928
|
-
// eslint-disable-next-line prefer-const
|
|
16929
|
-
complete = $getCurrentDate();
|
|
16930
17287
|
const usage = computeOpenAiUsage(content || '', resultContent || '', rawResponse);
|
|
16931
17288
|
if (resultContent === null) {
|
|
16932
17289
|
throw new PipelineExecutionError('No response message from OpenAI');
|
|
@@ -16980,11 +17337,12 @@
|
|
|
16980
17337
|
user: (_a = this.options.userId) === null || _a === void 0 ? void 0 : _a.toString(),
|
|
16981
17338
|
};
|
|
16982
17339
|
const start = $getCurrentDate();
|
|
16983
|
-
let complete;
|
|
16984
17340
|
if (this.options.isVerbose) {
|
|
16985
17341
|
console.info(colors__default["default"].bgWhite('rawRequest'), JSON.stringify(rawRequest, null, 4));
|
|
16986
17342
|
}
|
|
16987
|
-
const rawResponse = await
|
|
17343
|
+
const rawResponse = await this.limiter
|
|
17344
|
+
.schedule(() => client.completions.create(rawRequest))
|
|
17345
|
+
.catch((error) => {
|
|
16988
17346
|
assertsError(error);
|
|
16989
17347
|
if (this.options.isVerbose) {
|
|
16990
17348
|
console.info(colors__default["default"].bgRed('error'), error);
|
|
@@ -16994,6 +17352,7 @@
|
|
|
16994
17352
|
if (this.options.isVerbose) {
|
|
16995
17353
|
console.info(colors__default["default"].bgWhite('rawResponse'), JSON.stringify(rawResponse, null, 4));
|
|
16996
17354
|
}
|
|
17355
|
+
const complete = $getCurrentDate();
|
|
16997
17356
|
if (!rawResponse.choices[0]) {
|
|
16998
17357
|
throw new PipelineExecutionError('No choises from OpenAI');
|
|
16999
17358
|
}
|
|
@@ -17002,8 +17361,6 @@
|
|
|
17002
17361
|
throw new PipelineExecutionError('More than one choise from OpenAI');
|
|
17003
17362
|
}
|
|
17004
17363
|
const resultContent = rawResponse.choices[0].text;
|
|
17005
|
-
// eslint-disable-next-line prefer-const
|
|
17006
|
-
complete = $getCurrentDate();
|
|
17007
17364
|
const usage = computeOpenAiUsage(content || '', resultContent || '', rawResponse);
|
|
17008
17365
|
return exportJson({
|
|
17009
17366
|
name: 'promptResult',
|
|
@@ -17044,11 +17401,12 @@
|
|
|
17044
17401
|
model: modelName,
|
|
17045
17402
|
};
|
|
17046
17403
|
const start = $getCurrentDate();
|
|
17047
|
-
let complete;
|
|
17048
17404
|
if (this.options.isVerbose) {
|
|
17049
17405
|
console.info(colors__default["default"].bgWhite('rawRequest'), JSON.stringify(rawRequest, null, 4));
|
|
17050
17406
|
}
|
|
17051
|
-
const rawResponse = await
|
|
17407
|
+
const rawResponse = await this.limiter
|
|
17408
|
+
.schedule(() => client.embeddings.create(rawRequest))
|
|
17409
|
+
.catch((error) => {
|
|
17052
17410
|
assertsError(error);
|
|
17053
17411
|
if (this.options.isVerbose) {
|
|
17054
17412
|
console.info(colors__default["default"].bgRed('error'), error);
|
|
@@ -17058,12 +17416,11 @@
|
|
|
17058
17416
|
if (this.options.isVerbose) {
|
|
17059
17417
|
console.info(colors__default["default"].bgWhite('rawResponse'), JSON.stringify(rawResponse, null, 4));
|
|
17060
17418
|
}
|
|
17419
|
+
const complete = $getCurrentDate();
|
|
17061
17420
|
if (rawResponse.data.length !== 1) {
|
|
17062
17421
|
throw new PipelineExecutionError(`Expected exactly 1 data item in response, got ${rawResponse.data.length}`);
|
|
17063
17422
|
}
|
|
17064
17423
|
const resultContent = rawResponse.data[0].embedding;
|
|
17065
|
-
// eslint-disable-next-line prefer-const
|
|
17066
|
-
complete = $getCurrentDate();
|
|
17067
17424
|
const usage = computeOpenAiUsage(content || '', '',
|
|
17068
17425
|
// <- Note: Embedding does not have result content
|
|
17069
17426
|
rawResponse);
|
|
@@ -17091,7 +17448,8 @@
|
|
|
17091
17448
|
* Get the model that should be used as default
|
|
17092
17449
|
*/
|
|
17093
17450
|
getDefaultModel(defaultModelName) {
|
|
17094
|
-
|
|
17451
|
+
// Note: Match exact or prefix for model families
|
|
17452
|
+
const model = OPENAI_MODELS.find(({ modelName }) => modelName === defaultModelName || modelName.startsWith(defaultModelName));
|
|
17095
17453
|
if (model === undefined) {
|
|
17096
17454
|
throw new UnexpectedError(spaceTrim__default["default"]((block) => `
|
|
17097
17455
|
Cannot find model in OpenAI models with name "${defaultModelName}" which should be used as default.
|
|
@@ -17133,7 +17491,7 @@
|
|
|
17133
17491
|
/**
|
|
17134
17492
|
* Execution Tools for calling OpenAI API Assistants
|
|
17135
17493
|
*
|
|
17136
|
-
* This is
|
|
17494
|
+
* This is useful for calling OpenAI API with a single assistant, for more wide usage use `OpenAiExecutionTools`.
|
|
17137
17495
|
*
|
|
17138
17496
|
* @public exported from `@promptbook/openai`
|
|
17139
17497
|
*/
|
|
@@ -17146,6 +17504,7 @@
|
|
|
17146
17504
|
constructor(options) {
|
|
17147
17505
|
super(options);
|
|
17148
17506
|
this.assistantId = options.assistantId;
|
|
17507
|
+
// TODO: [👱] Make limiter same as in `OpenAiExecutionTools`
|
|
17149
17508
|
}
|
|
17150
17509
|
get title() {
|
|
17151
17510
|
return 'OpenAI Assistant';
|
|
@@ -17347,9 +17706,9 @@
|
|
|
17347
17706
|
*/
|
|
17348
17707
|
const _OpenAiRegistration = $llmToolsRegister.register(createOpenAiExecutionTools);
|
|
17349
17708
|
/**
|
|
17350
|
-
*
|
|
17709
|
+
* Registration of the OpenAI Assistant provider
|
|
17351
17710
|
*
|
|
17352
|
-
* Note: [🏐] Configurations registrations are done in
|
|
17711
|
+
* Note: [🏐] Configurations registrations are done in register-constructor.ts BUT constructor register-constructor.ts
|
|
17353
17712
|
*
|
|
17354
17713
|
* @public exported from `@promptbook/openai`
|
|
17355
17714
|
* @public exported from `@promptbook/wizzard`
|
|
@@ -17362,9 +17721,8 @@
|
|
|
17362
17721
|
*/
|
|
17363
17722
|
|
|
17364
17723
|
/**
|
|
17365
|
-
*
|
|
17366
|
-
*
|
|
17367
|
-
* Note: It also checks if directory exists and creates it if not
|
|
17724
|
+
* Retrieves an intermediate source for a scraper based on the knowledge source.
|
|
17725
|
+
* Manages the caching and retrieval of intermediate scraper results for optimized performance.
|
|
17368
17726
|
*
|
|
17369
17727
|
* @private as internal utility for scrapers
|
|
17370
17728
|
*/
|
|
@@ -17507,7 +17865,7 @@
|
|
|
17507
17865
|
const knowledge = await Promise.all(
|
|
17508
17866
|
// TODO: [🪂] Do not send all at once but in chunks
|
|
17509
17867
|
knowledgeTextPieces.map(async (knowledgeTextPiece, i) => {
|
|
17510
|
-
// Note:
|
|
17868
|
+
// Note: These are just default values, they will be overwritten by the actual values:
|
|
17511
17869
|
let name = `piece-${i}`;
|
|
17512
17870
|
let title = spaceTrim__default["default"](knowledgeTextPiece.substring(0, 100));
|
|
17513
17871
|
const knowledgePieceContent = spaceTrim__default["default"](knowledgeTextPiece);
|
|
@@ -17591,14 +17949,14 @@
|
|
|
17591
17949
|
packageName: '@promptbook/boilerplate',
|
|
17592
17950
|
className: 'BoilerplateScraper',
|
|
17593
17951
|
mimeTypes: [
|
|
17594
|
-
'
|
|
17595
|
-
// <- TODO:
|
|
17952
|
+
'@@/@@',
|
|
17953
|
+
// <- TODO: @@ Add compatible mime types with Boilerplate scraper
|
|
17596
17954
|
],
|
|
17597
|
-
documentationUrl: 'https://github.com/webgptorg/promptbook/discussions
|
|
17955
|
+
documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
|
|
17598
17956
|
isAvilableInBrowser: false,
|
|
17599
17957
|
// <- Note: [🌏] Only `MarkdownScraper` makes sense to be available in the browser, for scraping non-markdown sources in the browser use a remote server
|
|
17600
17958
|
requiredExecutables: [
|
|
17601
|
-
/*
|
|
17959
|
+
/* @@ 'Pandoc' */
|
|
17602
17960
|
],
|
|
17603
17961
|
}); /* <- Note: [🤛] */
|
|
17604
17962
|
/**
|
|
@@ -17616,7 +17974,7 @@
|
|
|
17616
17974
|
*/
|
|
17617
17975
|
|
|
17618
17976
|
/**
|
|
17619
|
-
* Scraper of
|
|
17977
|
+
* Scraper of @@ files
|
|
17620
17978
|
*
|
|
17621
17979
|
* @see `documentationUrl` for more details
|
|
17622
17980
|
* @public exported from `@promptbook/boilerplate`
|
|
@@ -17634,30 +17992,30 @@
|
|
|
17634
17992
|
this.markdownScraper = new MarkdownScraper(tools, options);
|
|
17635
17993
|
}
|
|
17636
17994
|
/**
|
|
17637
|
-
* Convert the
|
|
17995
|
+
* Convert the `.@@` to `.md` file and returns intermediate source
|
|
17638
17996
|
*
|
|
17639
17997
|
* Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object
|
|
17640
17998
|
*/
|
|
17641
17999
|
async $convert(source) {
|
|
17642
18000
|
var _a;
|
|
17643
18001
|
const { rootDirname = process.cwd(), cacheDirname = DEFAULT_SCRAPE_CACHE_DIRNAME, intermediateFilesStrategy = DEFAULT_INTERMEDIATE_FILES_STRATEGY, isVerbose = DEFAULT_IS_VERBOSE, } = this.options;
|
|
17644
|
-
// TODO:
|
|
18002
|
+
// TODO: @@ Preserve or delete
|
|
17645
18003
|
if (!$isRunningInNode()) {
|
|
17646
18004
|
throw new KnowledgeScrapeError('BoilerplateScraper is only supported in Node environment');
|
|
17647
18005
|
}
|
|
17648
|
-
// TODO:
|
|
18006
|
+
// TODO: @@ Preserve or delete
|
|
17649
18007
|
if (this.tools.fs === undefined) {
|
|
17650
18008
|
throw new EnvironmentMismatchError('Can not scrape boilerplates without filesystem tools');
|
|
17651
18009
|
// <- TODO: [🧠] What is the best error type here`
|
|
17652
18010
|
}
|
|
17653
|
-
// TODO:
|
|
18011
|
+
// TODO: @@ Preserve, delete or modify
|
|
17654
18012
|
if (((_a = this.tools.executables) === null || _a === void 0 ? void 0 : _a.pandocPath) === undefined) {
|
|
17655
18013
|
throw new MissingToolsError('Pandoc is required for scraping .docx files');
|
|
17656
18014
|
}
|
|
17657
|
-
// TODO:
|
|
18015
|
+
// TODO: @@ Preserve, delete or modify
|
|
17658
18016
|
if (source.filename === null) {
|
|
17659
18017
|
// TODO: [🧠] Maybe save file as temporary
|
|
17660
|
-
throw new KnowledgeScrapeError('When parsing
|
|
18018
|
+
throw new KnowledgeScrapeError('When parsing .@@ file, it must be real file in the file system');
|
|
17661
18019
|
}
|
|
17662
18020
|
const extension = getFileExtension(source.filename);
|
|
17663
18021
|
const cacheFilehandler = await getScraperIntermediateSource(source, {
|
|
@@ -17667,7 +18025,7 @@
|
|
|
17667
18025
|
extension: 'md',
|
|
17668
18026
|
isVerbose,
|
|
17669
18027
|
});
|
|
17670
|
-
// TODO:
|
|
18028
|
+
// TODO: @@ Preserve, delete or modify
|
|
17671
18029
|
// Note: Running Pandoc ONLY if the file in the cache does not exist
|
|
17672
18030
|
if (!(await isFileExisting(cacheFilehandler.filename, this.tools.fs))) {
|
|
17673
18031
|
const command = `"${this.tools.executables.pandocPath}" -f ${extension} -t markdown "${source.filename}" -o "${cacheFilehandler.filename}"`;
|
|
@@ -17693,7 +18051,7 @@
|
|
|
17693
18051
|
*/
|
|
17694
18052
|
async scrape(source) {
|
|
17695
18053
|
const cacheFilehandler = await this.$convert(source);
|
|
17696
|
-
// TODO:
|
|
18054
|
+
// TODO: @@ Preserve, delete or modify
|
|
17697
18055
|
const markdownSource = {
|
|
17698
18056
|
source: source.source,
|
|
17699
18057
|
filename: cacheFilehandler.filename,
|
|
@@ -17724,7 +18082,7 @@
|
|
|
17724
18082
|
* TODO: [👣] Converted documents can act as cached items - there is no need to run conversion each time
|
|
17725
18083
|
* TODO: [🪂] Do it in parallel
|
|
17726
18084
|
* Note: No need to aggregate usage here, it is done by intercepting the llmTools
|
|
17727
|
-
*
|
|
18085
|
+
* @@ Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
|
|
17728
18086
|
*/
|
|
17729
18087
|
|
|
17730
18088
|
/**
|
|
@@ -18058,7 +18416,8 @@
|
|
|
18058
18416
|
*/
|
|
18059
18417
|
|
|
18060
18418
|
/**
|
|
18061
|
-
*
|
|
18419
|
+
* Creates a scraper for legacy document formats (.doc, .rtf, etc).
|
|
18420
|
+
* Uses LibreOffice for conversion to extract content from older document formats.
|
|
18062
18421
|
*
|
|
18063
18422
|
* @public exported from `@promptbook/legacy-documents`
|
|
18064
18423
|
*/
|
|
@@ -18085,7 +18444,7 @@
|
|
|
18085
18444
|
*/
|
|
18086
18445
|
|
|
18087
18446
|
/**
|
|
18088
|
-
*
|
|
18447
|
+
* Creates a scraper for document content.
|
|
18089
18448
|
*
|
|
18090
18449
|
* @public exported from `@promptbook/documents`
|
|
18091
18450
|
*/
|
|
@@ -18112,7 +18471,7 @@
|
|
|
18112
18471
|
*/
|
|
18113
18472
|
|
|
18114
18473
|
/**
|
|
18115
|
-
*
|
|
18474
|
+
* Creates a scraper for markdown content.
|
|
18116
18475
|
*
|
|
18117
18476
|
* @public exported from `@promptbook/markdown-utils`
|
|
18118
18477
|
*/
|
|
@@ -18218,8 +18577,8 @@
|
|
|
18218
18577
|
extension: 'md',
|
|
18219
18578
|
isVerbose,
|
|
18220
18579
|
});
|
|
18221
|
-
// TODO:
|
|
18222
|
-
// Note: Running
|
|
18580
|
+
// TODO: Determine if Markitdown conversion should run only if the cache file doesn't exist, or always.
|
|
18581
|
+
// Note: Running Markitdown conversion ONLY if the file in the cache does not exist
|
|
18223
18582
|
if (!(await isFileExisting(cacheFilehandler.filename, this.tools.fs))) {
|
|
18224
18583
|
const src = source.filename || source.url || null;
|
|
18225
18584
|
// console.log('!!', { src, source, cacheFilehandler });
|
|
@@ -18241,11 +18600,11 @@
|
|
|
18241
18600
|
return cacheFilehandler;
|
|
18242
18601
|
}
|
|
18243
18602
|
/**
|
|
18244
|
-
* Scrapes the
|
|
18603
|
+
* Scrapes the source document (PDF, DOCX, etc.) and returns the knowledge pieces or `null` if it can't scrape it.
|
|
18245
18604
|
*/
|
|
18246
18605
|
async scrape(source) {
|
|
18247
18606
|
const cacheFilehandler = await this.$convert(source);
|
|
18248
|
-
// TODO:
|
|
18607
|
+
// TODO: Ensure this correctly creates the source object for the internal MarkdownScraper using the converted file.
|
|
18249
18608
|
const markdownSource = {
|
|
18250
18609
|
source: source.source,
|
|
18251
18610
|
filename: cacheFilehandler.filename,
|
|
@@ -18389,7 +18748,8 @@
|
|
|
18389
18748
|
*/
|
|
18390
18749
|
|
|
18391
18750
|
/**
|
|
18392
|
-
*
|
|
18751
|
+
* Factory function to create an instance of PdfScraper.
|
|
18752
|
+
* It bundles the scraper class with its metadata.
|
|
18393
18753
|
*
|
|
18394
18754
|
* @public exported from `@promptbook/pdf`
|
|
18395
18755
|
*/
|
|
@@ -18565,7 +18925,8 @@
|
|
|
18565
18925
|
*/
|
|
18566
18926
|
|
|
18567
18927
|
/**
|
|
18568
|
-
*
|
|
18928
|
+
* Factory function to create an instance of WebsiteScraper.
|
|
18929
|
+
* It bundles the scraper class with its metadata.
|
|
18569
18930
|
*
|
|
18570
18931
|
* @public exported from `@promptbook/website-crawler`
|
|
18571
18932
|
*/
|