@promptbook/core 0.72.0-8 → 0.72.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -8
- package/esm/index.es.js +1630 -2236
- package/esm/index.es.js.map +1 -1
- package/esm/typings/src/_packages/browser.index.d.ts +2 -0
- package/esm/typings/src/_packages/cli.index.d.ts +20 -0
- package/esm/typings/src/_packages/core.index.d.ts +42 -30
- package/esm/typings/src/_packages/documents.index.d.ts +8 -0
- package/esm/typings/src/_packages/legacy-documents.index.d.ts +8 -0
- package/esm/typings/src/_packages/markdown-utils.index.d.ts +6 -0
- package/esm/typings/src/_packages/node.index.d.ts +12 -4
- package/esm/typings/src/_packages/pdf.index.d.ts +8 -0
- package/esm/typings/src/_packages/types.index.d.ts +33 -17
- package/esm/typings/src/_packages/utils.index.d.ts +2 -2
- package/esm/typings/src/_packages/website-crawler.index.d.ts +8 -0
- package/esm/typings/src/collection/PipelineCollection.d.ts +1 -1
- package/esm/typings/src/collection/SimplePipelineCollection.d.ts +2 -2
- package/esm/typings/src/collection/collectionToJson.d.ts +1 -1
- package/esm/typings/src/collection/constructors/createCollectionFromDirectory.d.ts +5 -3
- package/esm/typings/src/collection/constructors/createCollectionFromJson.d.ts +1 -1
- package/esm/typings/src/collection/constructors/createCollectionFromPromise.d.ts +1 -1
- package/esm/typings/src/commands/_common/types/CommandParser.d.ts +5 -5
- package/esm/typings/src/config.d.ts +26 -13
- package/esm/typings/src/conversion/pipelineStringToJson.d.ts +3 -1
- package/esm/typings/src/dialogs/callback/CallbackInterfaceToolsOptions.d.ts +2 -2
- package/esm/typings/src/dialogs/simple-prompt/SimplePromptInterfaceTools.d.ts +3 -3
- package/esm/typings/src/executables/$provideExecutablesForNode.d.ts +12 -0
- package/esm/typings/src/executables/apps/locateLibreoffice.d.ts +11 -0
- package/esm/typings/src/executables/apps/locatePandoc.d.ts +11 -0
- package/esm/typings/src/executables/locateApp.d.ts +33 -0
- package/esm/typings/src/executables/platforms/locateAppOnLinux.d.ts +12 -0
- package/esm/typings/src/executables/platforms/locateAppOnMacOs.d.ts +12 -0
- package/esm/typings/src/executables/platforms/locateAppOnWindows.d.ts +12 -0
- package/esm/typings/src/execution/{CommonExecutionToolsOptions.d.ts → CommonToolsOptions.d.ts} +1 -1
- package/esm/typings/src/execution/EmbeddingVector.d.ts +1 -1
- package/esm/typings/src/execution/Executables.d.ts +18 -0
- package/esm/typings/src/execution/ExecutionTools.d.ts +32 -6
- package/esm/typings/src/execution/FilesystemTools.d.ts +9 -0
- package/esm/typings/src/execution/LlmExecutionTools.d.ts +2 -2
- package/esm/typings/src/execution/PipelineExecutorResult.d.ts +2 -2
- package/esm/typings/src/execution/assertsExecutionSuccessful.d.ts +3 -2
- package/esm/typings/src/execution/createPipelineExecutor/00-CreatePipelineExecutorOptions.d.ts +29 -6
- package/esm/typings/src/execution/createPipelineExecutor/10-executePipeline.d.ts +3 -12
- package/esm/typings/src/execution/createPipelineExecutor/20-executeTemplate.d.ts +4 -18
- package/esm/typings/src/execution/createPipelineExecutor/40-executeAttempts.d.ts +9 -19
- package/esm/typings/src/execution/translation/automatic-translate/automatic-translators/LindatAutomaticTranslator.d.ts +11 -3
- package/esm/typings/src/execution/translation/automatic-translate/translateMessages.d.ts +1 -0
- package/esm/typings/src/execution/utils/$provideExecutionToolsForNode.d.ts +13 -0
- package/esm/typings/src/execution/utils/addUsage.d.ts +1 -1
- package/esm/typings/src/execution/utils/forEachAsync.d.ts +1 -1
- package/esm/typings/src/formats/_common/FormatDefinition.d.ts +2 -2
- package/esm/typings/src/formats/_common/FormatSubvalueDefinition.d.ts +2 -2
- package/esm/typings/src/llm-providers/_common/{$llmToolsMetadataRegister.d.ts → register/$llmToolsMetadataRegister.d.ts} +4 -1
- package/esm/typings/src/llm-providers/_common/{$llmToolsRegister.d.ts → register/$llmToolsRegister.d.ts} +5 -2
- package/esm/typings/src/llm-providers/_common/{createLlmToolsFromConfigurationFromEnv.d.ts → register/$provideLlmToolsConfigurationFromEnv.d.ts} +3 -3
- package/esm/typings/src/llm-providers/_common/{getLlmToolsForCli.d.ts → register/$provideLlmToolsForCli.d.ts} +4 -11
- package/esm/typings/src/llm-providers/_common/{getLlmToolsForTestingAndScriptsAndPlayground.d.ts → register/$provideLlmToolsForTestingAndScriptsAndPlayground.d.ts} +4 -3
- package/esm/typings/src/llm-providers/_common/{createLlmToolsFromEnv.d.ts → register/$provideLlmToolsFromEnv.d.ts} +6 -5
- package/esm/typings/src/llm-providers/_common/{$registeredLlmToolsMessage.d.ts → register/$registeredLlmToolsMessage.d.ts} +5 -2
- package/esm/typings/src/llm-providers/_common/{LlmToolsConfiguration.d.ts → register/LlmToolsConfiguration.d.ts} +6 -5
- package/esm/typings/src/llm-providers/_common/{LlmToolsMetadata.d.ts → register/LlmToolsMetadata.d.ts} +5 -4
- package/esm/typings/src/llm-providers/_common/{LlmToolsOptions.d.ts → register/LlmToolsOptions.d.ts} +4 -1
- package/esm/typings/src/llm-providers/_common/{createLlmToolsFromConfiguration.d.ts → register/createLlmToolsFromConfiguration.d.ts} +12 -4
- package/esm/typings/src/llm-providers/_common/utils/cache/CacheLlmToolsOptions.d.ts +4 -1
- package/esm/typings/src/llm-providers/anthropic-claude/AnthropicClaudeExecutionTools.d.ts +1 -1
- package/esm/typings/src/llm-providers/anthropic-claude/AnthropicClaudeExecutionToolsOptions.d.ts +5 -5
- package/esm/typings/src/llm-providers/anthropic-claude/anthropic-claude-models.d.ts +1 -1
- package/esm/typings/src/llm-providers/anthropic-claude/register-configuration.d.ts +4 -3
- package/esm/typings/src/llm-providers/anthropic-claude/register-constructor.d.ts +4 -3
- package/esm/typings/src/llm-providers/azure-openai/AzureOpenAiExecutionTools.d.ts +1 -1
- package/esm/typings/src/llm-providers/azure-openai/AzureOpenAiExecutionToolsOptions.d.ts +4 -3
- package/esm/typings/src/llm-providers/azure-openai/register-configuration.d.ts +4 -3
- package/esm/typings/src/llm-providers/azure-openai/register-constructor.d.ts +4 -3
- package/esm/typings/src/llm-providers/langtail/LangtailExecutionTools.d.ts +1 -1
- package/esm/typings/src/llm-providers/mocked/$fakeTextToExpectations.d.ts +1 -1
- package/esm/typings/src/llm-providers/mocked/MockedEchoLlmExecutionTools.d.ts +4 -4
- package/esm/typings/src/llm-providers/mocked/MockedFackedLlmExecutionTools.d.ts +4 -4
- package/esm/typings/src/llm-providers/multiple/MultipleLlmExecutionTools.d.ts +4 -4
- package/esm/typings/src/llm-providers/multiple/joinLlmExecutionTools.d.ts +1 -1
- package/esm/typings/src/llm-providers/openai/OpenAiAssistantExecutionTools.d.ts +0 -1
- package/esm/typings/src/llm-providers/openai/OpenAiExecutionTools.d.ts +2 -2
- package/esm/typings/src/llm-providers/openai/OpenAiExecutionToolsOptions.d.ts +4 -4
- package/esm/typings/src/llm-providers/openai/openai-models.d.ts +1 -1
- package/esm/typings/src/llm-providers/openai/register-configuration.d.ts +5 -4
- package/esm/typings/src/llm-providers/openai/register-constructor.d.ts +5 -4
- package/esm/typings/src/llm-providers/remote/RemoteLlmExecutionTools.d.ts +4 -4
- package/esm/typings/src/llm-providers/remote/interfaces/PromptbookServer_ListModels_Request.d.ts +16 -7
- package/esm/typings/src/llm-providers/remote/interfaces/PromptbookServer_ListModels_Response.d.ts +4 -4
- package/esm/typings/src/llm-providers/remote/interfaces/PromptbookServer_Prompt_Request.d.ts +7 -13
- package/esm/typings/src/llm-providers/remote/interfaces/PromptbookServer_Prompt_Response.d.ts +3 -3
- package/esm/typings/src/llm-providers/remote/interfaces/RemoteLlmExecutionToolsOptions.d.ts +11 -16
- package/esm/typings/src/llm-providers/remote/interfaces/RemoteServerOptions.d.ts +24 -9
- package/esm/typings/src/llm-providers/remote/startRemoteServer.d.ts +1 -1
- package/esm/typings/src/personas/preparePersona.d.ts +2 -1
- package/esm/typings/src/prepare/PrepareAndScrapeOptions.d.ts +10 -28
- package/esm/typings/src/prepare/preparePipeline.d.ts +2 -1
- package/esm/typings/src/prepare/prepareTemplates.d.ts +3 -2
- package/esm/typings/src/scrapers/_common/Converter.d.ts +4 -10
- package/esm/typings/src/scrapers/_common/Scraper.d.ts +5 -17
- package/esm/typings/src/scrapers/_common/prepareKnowledgePieces.d.ts +2 -1
- package/esm/typings/src/scrapers/_common/register/$provideFilesystemForNode.d.ts +11 -0
- package/esm/typings/src/scrapers/_common/register/$provideScrapersForBrowser.d.ts +12 -0
- package/esm/typings/src/scrapers/_common/register/$provideScrapersForNode.d.ts +15 -0
- package/esm/typings/src/scrapers/_common/register/$registeredScrapersMessage.d.ts +12 -0
- package/esm/typings/src/scrapers/_common/register/$scrapersMetadataRegister.d.ts +13 -0
- package/esm/typings/src/scrapers/_common/register/$scrapersRegister.d.ts +13 -0
- package/esm/typings/src/scrapers/_common/register/ScraperAndConverterMetadata.d.ts +41 -0
- package/esm/typings/src/scrapers/_common/register/ScraperConstructor.d.ts +12 -0
- package/esm/typings/src/scrapers/_common/utils/getScraperIntermediateSource.d.ts +3 -2
- package/esm/typings/src/scrapers/_common/utils/makeKnowledgeSourceHandler.d.ts +3 -2
- package/esm/typings/src/scrapers/document/{documentScraper.d.ts → DocumentScraper.d.ts} +18 -12
- package/esm/typings/src/scrapers/document/createDocumentScraper.d.ts +20 -0
- package/esm/typings/src/scrapers/document/register-constructor.d.ts +13 -0
- package/esm/typings/src/scrapers/document/register-metadata.d.ts +24 -0
- package/esm/typings/src/scrapers/document-legacy/LegacyDocumentScraper.d.ts +43 -0
- package/esm/typings/src/scrapers/document-legacy/createLegacyDocumentScraper.d.ts +20 -0
- package/esm/typings/src/scrapers/document-legacy/register-constructor.d.ts +13 -0
- package/esm/typings/src/scrapers/document-legacy/register-metadata.d.ts +24 -0
- package/esm/typings/src/scrapers/markdown/MarkdownScraper.d.ts +29 -0
- package/esm/typings/src/scrapers/markdown/createMarkdownScraper.d.ts +20 -0
- package/esm/typings/src/scrapers/markdown/register-constructor.d.ts +13 -0
- package/esm/typings/src/scrapers/markdown/register-metadata.d.ts +24 -0
- package/esm/typings/src/scrapers/pdf/PdfScraper.d.ts +40 -0
- package/esm/typings/src/scrapers/pdf/createPdfScraper.d.ts +20 -0
- package/esm/typings/src/scrapers/pdf/register-constructor.d.ts +13 -0
- package/esm/typings/src/scrapers/pdf/register-metadata.d.ts +24 -0
- package/esm/typings/src/scrapers/website/WebsiteScraper.d.ts +50 -0
- package/esm/typings/src/scrapers/website/createWebsiteScraper.d.ts +20 -0
- package/esm/typings/src/scrapers/website/register-constructor.d.ts +13 -0
- package/esm/typings/src/scrapers/website/register-metadata.d.ts +24 -0
- package/esm/typings/src/scrapers/website/utils/createShowdownConverter.d.ts +7 -0
- package/esm/typings/src/scripting/javascript/JavascriptExecutionToolsOptions.d.ts +2 -2
- package/esm/typings/src/scripting/javascript/utils/preserve.d.ts +1 -1
- package/esm/typings/src/scripting/python/PythonExecutionTools.d.ts +3 -3
- package/esm/typings/src/scripting/typescript/TypescriptExecutionTools.d.ts +3 -3
- package/esm/typings/src/storage/file-cache-storage/FileCacheStorage.d.ts +5 -3
- package/esm/typings/src/storage/file-cache-storage/utils/nameToSubfolderPath.d.ts +1 -1
- package/esm/typings/src/storage/{utils → memory/utils}/PrefixStorage.d.ts +1 -1
- package/esm/typings/src/storage/{utils → memory/utils}/makePromptbookStorageFromWebStorage.d.ts +1 -1
- package/esm/typings/src/types/Arrayable.d.ts +1 -1
- package/esm/typings/src/types/IntermediateFilesStrategy.d.ts +7 -0
- package/esm/typings/src/types/PipelineJson/KnowledgePieceJson.d.ts +4 -4
- package/esm/typings/src/types/PipelineJson/KnowledgeSourceJson.d.ts +1 -1
- package/esm/typings/src/types/PipelineJson/PersonaJson.d.ts +1 -1
- package/esm/typings/src/types/PipelineJson/TemplateJsonCommon.d.ts +2 -2
- package/esm/typings/src/types/Prompt.d.ts +2 -1
- package/esm/typings/src/types/execution-report/ExecutionReportJson.d.ts +1 -1
- package/esm/typings/src/types/typeAliases.d.ts +18 -8
- package/esm/typings/src/utils/$Register.d.ts +20 -7
- package/esm/typings/src/utils/FromtoItems.d.ts +1 -1
- package/esm/typings/src/utils/arrayableToArray.d.ts +1 -1
- package/esm/typings/src/utils/emojis.d.ts +1 -1
- package/esm/typings/src/utils/execCommand/$execCommand.d.ts +5 -3
- package/esm/typings/src/utils/execCommand/$execCommandNormalizeOptions.d.ts +12 -0
- package/esm/typings/src/utils/execCommand/$execCommands.d.ts +2 -1
- package/esm/typings/src/utils/execCommand/ExecCommandOptions.d.ts +45 -0
- package/esm/typings/src/utils/expectation-counters/countSentences.d.ts +1 -1
- package/esm/typings/src/utils/files/isDirectoryExisting.d.ts +14 -0
- package/esm/typings/src/utils/files/isExecutable.d.ts +11 -0
- package/esm/typings/src/utils/files/isFileExisting.d.ts +13 -0
- package/esm/typings/src/utils/files/isFileExisting.test.d.ts +1 -0
- package/esm/typings/src/utils/files/{$listAllFiles.d.ts → listAllFiles.d.ts} +3 -4
- package/esm/typings/src/utils/files/listAllFiles.test.d.ts +1 -0
- package/esm/typings/src/utils/markdown/extractAllBlocksFromMarkdown.d.ts +1 -1
- package/esm/typings/src/utils/markdown/splitMarkdownIntoSections.d.ts +1 -1
- package/esm/typings/src/utils/normalization/IKeywords.d.ts +2 -2
- package/esm/typings/src/utils/normalization/parseKeywords.d.ts +2 -2
- package/esm/typings/src/utils/normalization/parseKeywordsFromString.d.ts +2 -2
- package/esm/typings/src/utils/normalization/searchKeywords.d.ts +2 -2
- package/esm/typings/src/utils/organization/TODO_USE.d.ts +1 -1
- package/esm/typings/src/utils/organization/keepUnused.d.ts +1 -1
- package/esm/typings/src/utils/random/$randomSeed.d.ts +1 -1
- package/esm/typings/src/utils/sets/intersection.d.ts +1 -1
- package/esm/typings/src/utils/sets/union.d.ts +1 -1
- package/esm/typings/src/utils/unwrapResult.d.ts +4 -4
- package/package.json +3 -6
- package/umd/index.umd.js +1650 -2245
- package/umd/index.umd.js.map +1 -1
- package/esm/typings/src/execution/createPipelineExecutor/00-CreatePipelineExecutorSettings.d.ts +0 -29
- package/esm/typings/src/scrapers/document-legacy/legacyDocumentScraper.d.ts +0 -37
- package/esm/typings/src/scrapers/index.d.ts +0 -8
- package/esm/typings/src/scrapers/markdown/markdownScraper.d.ts +0 -29
- package/esm/typings/src/scrapers/pdf/pdfScraper.d.ts +0 -35
- package/esm/typings/src/scrapers/website/utils/markdownConverter.d.ts +0 -12
- package/esm/typings/src/scrapers/website/websiteScraper.d.ts +0 -43
- package/esm/typings/src/utils/execCommand/IExecCommandOptions.d.ts +0 -23
- package/esm/typings/src/utils/execCommand/execCommandNormalizeOptions.d.ts +0 -10
- package/esm/typings/src/utils/files/$isDirectoryExisting.d.ts +0 -15
- package/esm/typings/src/utils/files/$isFileExisting.d.ts +0 -14
- /package/esm/typings/src/{scrapers/website/utils/markdownConverter.test.d.ts → executables/apps/locateLibreoffice.test.d.ts} +0 -0
- /package/esm/typings/src/{scrapers/website/websiteScraper.test.d.ts → executables/apps/locatePandoc.test.d.ts} +0 -0
- /package/esm/typings/src/{utils/files/$isDirectoryExisting.test.d.ts → executables/locateApp.test.d.ts} +0 -0
- /package/esm/typings/src/scrapers/document/{documentScraper.test.d.ts → DocumentScraper.test.d.ts} +0 -0
- /package/esm/typings/src/scrapers/document-legacy/{legacyDocumentScraper.test.d.ts → LegacyDocumentScraper.test.d.ts} +0 -0
- /package/esm/typings/src/scrapers/markdown/{markdownScraper.test.d.ts → MarkdownScraper.test.d.ts} +0 -0
- /package/esm/typings/src/{utils/files/$isFileExisting.test.d.ts → scrapers/website/utils/createShowdownConverter.test.d.ts} +0 -0
- /package/esm/typings/src/utils/files/{$listAllFiles.test.d.ts → isDirectoryExisting.test.d.ts} +0 -0
package/umd/index.umd.js
CHANGED
|
@@ -1,14 +1,13 @@
|
|
|
1
1
|
(function (global, factory) {
|
|
2
|
-
typeof exports === 'object' && typeof module !== 'undefined' ? factory(exports, require('spacetrim'), require('prettier'), require('prettier/parser-html'), require('
|
|
3
|
-
typeof define === 'function' && define.amd ? define(['exports', 'spacetrim', 'prettier', 'prettier/parser-html', '
|
|
4
|
-
(global = typeof globalThis !== 'undefined' ? globalThis : global || self, factory(global["promptbook-core"] = {}, global.spaceTrim, global.prettier, global.parserHtml, global.
|
|
5
|
-
})(this, (function (exports, spaceTrim, prettier, parserHtml,
|
|
2
|
+
typeof exports === 'object' && typeof module !== 'undefined' ? factory(exports, require('spacetrim'), require('prettier'), require('prettier/parser-html'), require('waitasecond'), require('papaparse'), require('path'), require('crypto-js'), require('crypto-js/enc-hex'), require('mime-types'), require('crypto-js/sha256'), require('moment')) :
|
|
3
|
+
typeof define === 'function' && define.amd ? define(['exports', 'spacetrim', 'prettier', 'prettier/parser-html', 'waitasecond', 'papaparse', 'path', 'crypto-js', 'crypto-js/enc-hex', 'mime-types', 'crypto-js/sha256', 'moment'], factory) :
|
|
4
|
+
(global = typeof globalThis !== 'undefined' ? globalThis : global || self, factory(global["promptbook-core"] = {}, global.spaceTrim, global.prettier, global.parserHtml, global.waitasecond, global.papaparse, global.path, global.cryptoJs, global.hexEncoder, global.mimeTypes, global.sha256, global.moment));
|
|
5
|
+
})(this, (function (exports, spaceTrim, prettier, parserHtml, waitasecond, papaparse, path, cryptoJs, hexEncoder, mimeTypes, sha256, moment) { 'use strict';
|
|
6
6
|
|
|
7
7
|
function _interopDefaultLegacy (e) { return e && typeof e === 'object' && 'default' in e ? e : { 'default': e }; }
|
|
8
8
|
|
|
9
9
|
var spaceTrim__default = /*#__PURE__*/_interopDefaultLegacy(spaceTrim);
|
|
10
10
|
var parserHtml__default = /*#__PURE__*/_interopDefaultLegacy(parserHtml);
|
|
11
|
-
var colors__default = /*#__PURE__*/_interopDefaultLegacy(colors);
|
|
12
11
|
var hexEncoder__default = /*#__PURE__*/_interopDefaultLegacy(hexEncoder);
|
|
13
12
|
var sha256__default = /*#__PURE__*/_interopDefaultLegacy(sha256);
|
|
14
13
|
var moment__default = /*#__PURE__*/_interopDefaultLegacy(moment);
|
|
@@ -17,7 +16,7 @@
|
|
|
17
16
|
/**
|
|
18
17
|
* The version of the Promptbook library
|
|
19
18
|
*/
|
|
20
|
-
var PROMPTBOOK_VERSION = '0.72.0-
|
|
19
|
+
var PROMPTBOOK_VERSION = '0.72.0-34';
|
|
21
20
|
// TODO: [main] !!!! List here all the versions and annotate + put into script
|
|
22
21
|
|
|
23
22
|
/*! *****************************************************************************
|
|
@@ -224,7 +223,6 @@
|
|
|
224
223
|
pipelineString += '\n\n';
|
|
225
224
|
pipelineString += description;
|
|
226
225
|
}
|
|
227
|
-
// TODO:> const commands: Array<Command>
|
|
228
226
|
var commands = [];
|
|
229
227
|
if (pipelineUrl) {
|
|
230
228
|
commands.push("PIPELINE URL ".concat(pipelineUrl));
|
|
@@ -280,7 +278,6 @@
|
|
|
280
278
|
pipelineString += '\n\n';
|
|
281
279
|
pipelineString += description_1;
|
|
282
280
|
}
|
|
283
|
-
// TODO:> const commands: Array<Command>
|
|
284
281
|
var commands_1 = [];
|
|
285
282
|
var contentLanguage = 'text';
|
|
286
283
|
if (templateType === 'PROMPT_TEMPLATE') {
|
|
@@ -640,7 +637,7 @@
|
|
|
640
637
|
*
|
|
641
638
|
* @public exported from `@promptbook/core`
|
|
642
639
|
*/
|
|
643
|
-
var CLAIM = "
|
|
640
|
+
var CLAIM = "Build responsible, controlled and transparent applications on top of LLM models!";
|
|
644
641
|
// <- TODO: [🧠] Better system for generator warnings - not always "code" and "by `@promptbook/cli`"
|
|
645
642
|
/**
|
|
646
643
|
* The maximum number of iterations for a loops
|
|
@@ -655,37 +652,44 @@
|
|
|
655
652
|
*/
|
|
656
653
|
var IMMEDIATE_TIME = 10;
|
|
657
654
|
/**
|
|
658
|
-
* The maximum
|
|
655
|
+
* The maximum length of the (generated) filename
|
|
659
656
|
*
|
|
660
657
|
* @public exported from `@promptbook/core`
|
|
661
658
|
*/
|
|
662
|
-
var
|
|
659
|
+
var MAX_FILENAME_LENGTH = 30;
|
|
663
660
|
/**
|
|
664
|
-
*
|
|
661
|
+
* Strategy for caching the intermediate results for knowledge sources
|
|
665
662
|
*
|
|
666
663
|
* @public exported from `@promptbook/core`
|
|
667
664
|
*/
|
|
668
|
-
var
|
|
665
|
+
var DEFAULT_INTERMEDIATE_FILES_STRATEGY = 'HIDE_AND_KEEP';
|
|
666
|
+
// <- TODO: [😡] Change to 'VISIBLE'
|
|
669
667
|
/**
|
|
670
|
-
* The maximum
|
|
668
|
+
* The maximum number of (LLM) tasks running in parallel
|
|
671
669
|
*
|
|
672
670
|
* @public exported from `@promptbook/core`
|
|
673
671
|
*/
|
|
674
|
-
var
|
|
672
|
+
var DEFAULT_MAX_PARALLEL_COUNT = 5; // <- TODO: [🤹♂️]
|
|
673
|
+
/**
|
|
674
|
+
* The maximum number of attempts to execute LLM task before giving up
|
|
675
|
+
*
|
|
676
|
+
* @public exported from `@promptbook/core`
|
|
677
|
+
*/
|
|
678
|
+
var DEFAULT_MAX_EXECUTION_ATTEMPTS = 3; // <- TODO: [🤹♂️]
|
|
675
679
|
/**
|
|
676
680
|
* @@@
|
|
677
681
|
* TODO: [🐝][main] !!! Use
|
|
678
682
|
*
|
|
679
683
|
* @public exported from `@promptbook/core`
|
|
680
684
|
*/
|
|
681
|
-
var
|
|
685
|
+
var DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_DEPTH = 3;
|
|
682
686
|
/**
|
|
683
687
|
* @@@
|
|
684
688
|
* TODO: [🐝][main] !!! Use
|
|
685
689
|
*
|
|
686
690
|
* @public exported from `@promptbook/core`
|
|
687
691
|
*/
|
|
688
|
-
var
|
|
692
|
+
var DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_TOTAL = 200;
|
|
689
693
|
/**
|
|
690
694
|
* Where to store the cache of executions for promptbook CLI
|
|
691
695
|
*
|
|
@@ -693,7 +697,7 @@
|
|
|
693
697
|
*
|
|
694
698
|
* @public exported from `@promptbook/core`
|
|
695
699
|
*/
|
|
696
|
-
var
|
|
700
|
+
var DEFAULT_EXECUTIONS_CACHE_DIRNAME = '/.promptbook/executions-cache';
|
|
697
701
|
/**
|
|
698
702
|
* Where to store the scrape cache
|
|
699
703
|
*
|
|
@@ -701,13 +705,13 @@
|
|
|
701
705
|
*
|
|
702
706
|
* @public exported from `@promptbook/core`
|
|
703
707
|
*/
|
|
704
|
-
var
|
|
708
|
+
var DEFAULT_SCRAPE_CACHE_DIRNAME = '/.promptbook/scrape-cache';
|
|
705
709
|
/**
|
|
706
710
|
* The name of the builded pipeline collection made by CLI `ptbk make` and for lookup in `createCollectionFromDirectory`
|
|
707
711
|
*
|
|
708
712
|
* @public exported from `@promptbook/core`
|
|
709
713
|
*/
|
|
710
|
-
var
|
|
714
|
+
var DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME = "index";
|
|
711
715
|
/**
|
|
712
716
|
* Nonce which is used for replacing things in strings
|
|
713
717
|
*
|
|
@@ -780,7 +784,13 @@
|
|
|
780
784
|
*
|
|
781
785
|
* @public exported from `@promptbook/core`
|
|
782
786
|
*/
|
|
783
|
-
var
|
|
787
|
+
var DEFAULT_IS_VERBOSE = false;
|
|
788
|
+
/**
|
|
789
|
+
* @@@
|
|
790
|
+
*
|
|
791
|
+
* @public exported from `@promptbook/core`
|
|
792
|
+
*/
|
|
793
|
+
var DEFAULT_IS_AUTO_INSTALLED = false;
|
|
784
794
|
/**
|
|
785
795
|
* @@@
|
|
786
796
|
*
|
|
@@ -1529,7 +1539,7 @@
|
|
|
1529
1539
|
return __generator(this, function (_d) {
|
|
1530
1540
|
switch (_d.label) {
|
|
1531
1541
|
case 0:
|
|
1532
|
-
_a = options || {}, _b = _a.isVerbose, isVerbose = _b === void 0 ?
|
|
1542
|
+
_a = options || {}, _b = _a.isVerbose, isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b, _c = _a.isLazyLoaded, isLazyLoaded = _c === void 0 ? false : _c;
|
|
1533
1543
|
collection = createCollectionFromPromise(function () { return __awaiter(_this, void 0, void 0, function () {
|
|
1534
1544
|
return __generator(this, function (_a) {
|
|
1535
1545
|
if (isVerbose) {
|
|
@@ -1651,19 +1661,19 @@
|
|
|
1651
1661
|
];
|
|
1652
1662
|
|
|
1653
1663
|
/**
|
|
1654
|
-
* This error indicates that
|
|
1664
|
+
* This error type indicates that some tools are missing for pipeline execution or preparation
|
|
1655
1665
|
*
|
|
1656
1666
|
* @public exported from `@promptbook/core`
|
|
1657
1667
|
*/
|
|
1658
|
-
var
|
|
1659
|
-
__extends(
|
|
1660
|
-
function
|
|
1661
|
-
var _this = _super.call(this, message) || this;
|
|
1662
|
-
_this.name = '
|
|
1663
|
-
Object.setPrototypeOf(_this,
|
|
1668
|
+
var MissingToolsError = /** @class */ (function (_super) {
|
|
1669
|
+
__extends(MissingToolsError, _super);
|
|
1670
|
+
function MissingToolsError(message) {
|
|
1671
|
+
var _this = _super.call(this, spaceTrim.spaceTrim(function (block) { return "\n ".concat(block(message), "\n\n Note: You have probbably forgot to provide some tools for pipeline execution or preparation\n\n "); })) || this;
|
|
1672
|
+
_this.name = 'MissingToolsError';
|
|
1673
|
+
Object.setPrototypeOf(_this, MissingToolsError.prototype);
|
|
1664
1674
|
return _this;
|
|
1665
1675
|
}
|
|
1666
|
-
return
|
|
1676
|
+
return MissingToolsError;
|
|
1667
1677
|
}(Error));
|
|
1668
1678
|
|
|
1669
1679
|
/**
|
|
@@ -1743,1042 +1753,187 @@
|
|
|
1743
1753
|
}
|
|
1744
1754
|
|
|
1745
1755
|
/**
|
|
1746
|
-
*
|
|
1756
|
+
* Represents the usage with no resources consumed
|
|
1747
1757
|
*
|
|
1748
1758
|
* @public exported from `@promptbook/core`
|
|
1749
1759
|
*/
|
|
1750
|
-
var
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
|
|
1758
|
-
|
|
1759
|
-
|
|
1760
|
-
|
|
1760
|
+
var ZERO_USAGE = $deepFreeze({
|
|
1761
|
+
price: { value: 0 },
|
|
1762
|
+
input: {
|
|
1763
|
+
tokensCount: { value: 0 },
|
|
1764
|
+
charactersCount: { value: 0 },
|
|
1765
|
+
wordsCount: { value: 0 },
|
|
1766
|
+
sentencesCount: { value: 0 },
|
|
1767
|
+
linesCount: { value: 0 },
|
|
1768
|
+
paragraphsCount: { value: 0 },
|
|
1769
|
+
pagesCount: { value: 0 },
|
|
1770
|
+
},
|
|
1771
|
+
output: {
|
|
1772
|
+
tokensCount: { value: 0 },
|
|
1773
|
+
charactersCount: { value: 0 },
|
|
1774
|
+
wordsCount: { value: 0 },
|
|
1775
|
+
sentencesCount: { value: 0 },
|
|
1776
|
+
linesCount: { value: 0 },
|
|
1777
|
+
paragraphsCount: { value: 0 },
|
|
1778
|
+
pagesCount: { value: 0 },
|
|
1779
|
+
},
|
|
1780
|
+
});
|
|
1761
1781
|
/**
|
|
1762
|
-
*
|
|
1763
|
-
*
|
|
1764
|
-
* Note: `$` is used to indicate that this function is not a pure function - it looks at the global object to determine the environment
|
|
1782
|
+
* Represents the usage with unknown resources consumed
|
|
1765
1783
|
*
|
|
1766
|
-
* @public exported from `@promptbook/
|
|
1784
|
+
* @public exported from `@promptbook/core`
|
|
1767
1785
|
*/
|
|
1768
|
-
var
|
|
1786
|
+
var UNCERTAIN_USAGE = $deepFreeze({
|
|
1787
|
+
price: { value: 0, isUncertain: true },
|
|
1788
|
+
input: {
|
|
1789
|
+
tokensCount: { value: 0, isUncertain: true },
|
|
1790
|
+
charactersCount: { value: 0, isUncertain: true },
|
|
1791
|
+
wordsCount: { value: 0, isUncertain: true },
|
|
1792
|
+
sentencesCount: { value: 0, isUncertain: true },
|
|
1793
|
+
linesCount: { value: 0, isUncertain: true },
|
|
1794
|
+
paragraphsCount: { value: 0, isUncertain: true },
|
|
1795
|
+
pagesCount: { value: 0, isUncertain: true },
|
|
1796
|
+
},
|
|
1797
|
+
output: {
|
|
1798
|
+
tokensCount: { value: 0, isUncertain: true },
|
|
1799
|
+
charactersCount: { value: 0, isUncertain: true },
|
|
1800
|
+
wordsCount: { value: 0, isUncertain: true },
|
|
1801
|
+
sentencesCount: { value: 0, isUncertain: true },
|
|
1802
|
+
linesCount: { value: 0, isUncertain: true },
|
|
1803
|
+
paragraphsCount: { value: 0, isUncertain: true },
|
|
1804
|
+
pagesCount: { value: 0, isUncertain: true },
|
|
1805
|
+
},
|
|
1806
|
+
});
|
|
1769
1807
|
|
|
1770
1808
|
/**
|
|
1771
|
-
* This error
|
|
1809
|
+
* This error indicates errors during the execution of the pipeline
|
|
1772
1810
|
*
|
|
1773
1811
|
* @public exported from `@promptbook/core`
|
|
1774
1812
|
*/
|
|
1775
|
-
var
|
|
1776
|
-
__extends(
|
|
1777
|
-
function
|
|
1813
|
+
var PipelineExecutionError = /** @class */ (function (_super) {
|
|
1814
|
+
__extends(PipelineExecutionError, _super);
|
|
1815
|
+
function PipelineExecutionError(message) {
|
|
1778
1816
|
var _this = _super.call(this, message) || this;
|
|
1779
|
-
_this.name = '
|
|
1780
|
-
Object.setPrototypeOf(_this,
|
|
1817
|
+
_this.name = 'PipelineExecutionError';
|
|
1818
|
+
Object.setPrototypeOf(_this, PipelineExecutionError.prototype);
|
|
1781
1819
|
return _this;
|
|
1782
1820
|
}
|
|
1783
|
-
return
|
|
1821
|
+
return PipelineExecutionError;
|
|
1784
1822
|
}(Error));
|
|
1785
1823
|
|
|
1786
1824
|
/**
|
|
1787
|
-
*
|
|
1825
|
+
* Multiple LLM Execution Tools is a proxy server that uses multiple execution tools internally and exposes the executor interface externally.
|
|
1788
1826
|
*
|
|
1789
|
-
*
|
|
1827
|
+
* Note: Internal utility of `joinLlmExecutionTools` but exposed type
|
|
1828
|
+
* @public exported from `@promptbook/core`
|
|
1790
1829
|
*/
|
|
1791
|
-
function
|
|
1792
|
-
|
|
1793
|
-
|
|
1794
|
-
|
|
1795
|
-
|
|
1796
|
-
|
|
1797
|
-
|
|
1798
|
-
|
|
1799
|
-
if (typeof options === 'string') {
|
|
1800
|
-
// TODO: [1] DRY default values
|
|
1801
|
-
command = options;
|
|
1802
|
-
cwd = process.cwd();
|
|
1803
|
-
crashOnError = true;
|
|
1804
|
-
timeout = Infinity;
|
|
1805
|
-
}
|
|
1806
|
-
else {
|
|
1807
|
-
/*
|
|
1808
|
-
TODO:
|
|
1809
|
-
if ((options as any).commands !== undefined) {
|
|
1810
|
-
commands = (options as any).commands;
|
|
1811
|
-
} else {
|
|
1812
|
-
commands = [(options as any).command];
|
|
1830
|
+
var MultipleLlmExecutionTools = /** @class */ (function () {
|
|
1831
|
+
/**
|
|
1832
|
+
* Gets array of execution tools in order of priority
|
|
1833
|
+
*/
|
|
1834
|
+
function MultipleLlmExecutionTools() {
|
|
1835
|
+
var llmExecutionTools = [];
|
|
1836
|
+
for (var _i = 0; _i < arguments.length; _i++) {
|
|
1837
|
+
llmExecutionTools[_i] = arguments[_i];
|
|
1813
1838
|
}
|
|
1814
|
-
|
|
1815
|
-
// TODO: [1] DRY default values
|
|
1816
|
-
command = options.command;
|
|
1817
|
-
cwd = (_b = options.cwd) !== null && _b !== void 0 ? _b : process.cwd();
|
|
1818
|
-
crashOnError = (_c = options.crashOnError) !== null && _c !== void 0 ? _c : true;
|
|
1819
|
-
timeout = (_d = options.timeout) !== null && _d !== void 0 ? _d : Infinity;
|
|
1820
|
-
}
|
|
1821
|
-
// TODO: /(-[a-zA-Z0-9-]+\s+[^\s]*)|[^\s]*/g
|
|
1822
|
-
var _ = Array.from(command.matchAll(/(".*")|([^\s]*)/g))
|
|
1823
|
-
.map(function (_a) {
|
|
1824
|
-
var _b = __read(_a, 1), match = _b[0];
|
|
1825
|
-
return match;
|
|
1826
|
-
})
|
|
1827
|
-
.filter(function (arg) { return arg !== ''; });
|
|
1828
|
-
if (_.length > 1) {
|
|
1829
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
1830
|
-
_a = __read(_), command = _a[0], args = _a.slice(1);
|
|
1831
|
-
}
|
|
1832
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
1833
|
-
if (options.args) {
|
|
1834
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
1835
|
-
args = __spreadArray(__spreadArray([], __read(args), false), __read(options.args), false);
|
|
1836
|
-
}
|
|
1837
|
-
var humanReadableCommand = !['npx', 'npm'].includes(command) ? command : args[0];
|
|
1838
|
-
if (['ts-node'].includes(humanReadableCommand)) {
|
|
1839
|
-
humanReadableCommand += " ".concat(args[1]);
|
|
1840
|
-
}
|
|
1841
|
-
return { command: command, humanReadableCommand: humanReadableCommand, args: args, cwd: cwd, crashOnError: crashOnError, timeout: timeout };
|
|
1842
|
-
}
|
|
1843
|
-
// TODO: This should show type error> execCommandNormalizeOptions({ command: '', commands: [''] });
|
|
1844
|
-
|
|
1845
|
-
/**
|
|
1846
|
-
* Run one command in a shell
|
|
1847
|
-
*
|
|
1848
|
-
* Note: There are 2 similar functions in the codebase:
|
|
1849
|
-
* - `$execCommand` which runs a single command
|
|
1850
|
-
* - `$execCommands` which runs multiple commands
|
|
1851
|
-
*
|
|
1852
|
-
* @public exported from `@promptbook/node`
|
|
1853
|
-
*/
|
|
1854
|
-
function $execCommand(options) {
|
|
1855
|
-
if (!$isRunningInNode()) {
|
|
1856
|
-
throw new EnvironmentMismatchError('Function `$execCommand` can run only in Node environment.js');
|
|
1839
|
+
this.llmExecutionTools = llmExecutionTools;
|
|
1857
1840
|
}
|
|
1858
|
-
|
|
1859
|
-
|
|
1860
|
-
|
|
1861
|
-
|
|
1862
|
-
|
|
1863
|
-
|
|
1864
|
-
|
|
1865
|
-
|
|
1866
|
-
|
|
1867
|
-
|
|
1868
|
-
|
|
1869
|
-
|
|
1870
|
-
|
|
1871
|
-
|
|
1872
|
-
|
|
1873
|
-
|
|
1874
|
-
|
|
1875
|
-
|
|
1876
|
-
|
|
1877
|
-
|
|
1878
|
-
|
|
1879
|
-
|
|
1880
|
-
|
|
1881
|
-
var
|
|
1882
|
-
|
|
1883
|
-
|
|
1884
|
-
|
|
1885
|
-
|
|
1886
|
-
|
|
1887
|
-
|
|
1888
|
-
|
|
1889
|
-
|
|
1890
|
-
|
|
1891
|
-
|
|
1892
|
-
|
|
1893
|
-
|
|
1894
|
-
|
|
1895
|
-
|
|
1896
|
-
|
|
1897
|
-
|
|
1898
|
-
|
|
1899
|
-
|
|
1900
|
-
|
|
1901
|
-
|
|
1902
|
-
|
|
1903
|
-
|
|
1904
|
-
|
|
1905
|
-
|
|
1906
|
-
|
|
1907
|
-
|
|
1908
|
-
|
|
1841
|
+
Object.defineProperty(MultipleLlmExecutionTools.prototype, "title", {
|
|
1842
|
+
get: function () {
|
|
1843
|
+
return 'Multiple LLM Providers';
|
|
1844
|
+
},
|
|
1845
|
+
enumerable: false,
|
|
1846
|
+
configurable: true
|
|
1847
|
+
});
|
|
1848
|
+
Object.defineProperty(MultipleLlmExecutionTools.prototype, "description", {
|
|
1849
|
+
get: function () {
|
|
1850
|
+
return this.llmExecutionTools.map(function (_a, index) {
|
|
1851
|
+
var title = _a.title;
|
|
1852
|
+
return "".concat(index + 1, ") `").concat(title, "`");
|
|
1853
|
+
}).join('\n');
|
|
1854
|
+
},
|
|
1855
|
+
enumerable: false,
|
|
1856
|
+
configurable: true
|
|
1857
|
+
});
|
|
1858
|
+
/**
|
|
1859
|
+
* Check the configuration of all execution tools
|
|
1860
|
+
*/
|
|
1861
|
+
MultipleLlmExecutionTools.prototype.checkConfiguration = function () {
|
|
1862
|
+
return __awaiter(this, void 0, void 0, function () {
|
|
1863
|
+
var _a, _b, llmExecutionTools, e_1_1;
|
|
1864
|
+
var e_1, _c;
|
|
1865
|
+
return __generator(this, function (_d) {
|
|
1866
|
+
switch (_d.label) {
|
|
1867
|
+
case 0:
|
|
1868
|
+
_d.trys.push([0, 5, 6, 7]);
|
|
1869
|
+
_a = __values(this.llmExecutionTools), _b = _a.next();
|
|
1870
|
+
_d.label = 1;
|
|
1871
|
+
case 1:
|
|
1872
|
+
if (!!_b.done) return [3 /*break*/, 4];
|
|
1873
|
+
llmExecutionTools = _b.value;
|
|
1874
|
+
return [4 /*yield*/, llmExecutionTools.checkConfiguration()];
|
|
1875
|
+
case 2:
|
|
1876
|
+
_d.sent();
|
|
1877
|
+
_d.label = 3;
|
|
1878
|
+
case 3:
|
|
1879
|
+
_b = _a.next();
|
|
1880
|
+
return [3 /*break*/, 1];
|
|
1881
|
+
case 4: return [3 /*break*/, 7];
|
|
1882
|
+
case 5:
|
|
1883
|
+
e_1_1 = _d.sent();
|
|
1884
|
+
e_1 = { error: e_1_1 };
|
|
1885
|
+
return [3 /*break*/, 7];
|
|
1886
|
+
case 6:
|
|
1887
|
+
try {
|
|
1888
|
+
if (_b && !_b.done && (_c = _a.return)) _c.call(_a);
|
|
1889
|
+
}
|
|
1890
|
+
finally { if (e_1) throw e_1.error; }
|
|
1891
|
+
return [7 /*endfinally*/];
|
|
1892
|
+
case 7: return [2 /*return*/];
|
|
1909
1893
|
}
|
|
1910
|
-
};
|
|
1911
|
-
commandProcess.on('close', finishWithCode);
|
|
1912
|
-
commandProcess.on('exit', finishWithCode);
|
|
1913
|
-
commandProcess.on('disconnect', function () {
|
|
1914
|
-
// Note: Unexpected disconnection should always result in rejection
|
|
1915
|
-
reject(new Error("Command \"".concat(humanReadableCommand, "\" disconnected")));
|
|
1916
1894
|
});
|
|
1917
|
-
|
|
1918
|
-
|
|
1919
|
-
|
|
1920
|
-
|
|
1921
|
-
|
|
1922
|
-
|
|
1923
|
-
|
|
1924
|
-
|
|
1925
|
-
|
|
1926
|
-
|
|
1927
|
-
|
|
1928
|
-
|
|
1929
|
-
|
|
1930
|
-
|
|
1931
|
-
|
|
1932
|
-
|
|
1933
|
-
|
|
1934
|
-
|
|
1935
|
-
|
|
1936
|
-
|
|
1937
|
-
|
|
1938
|
-
|
|
1939
|
-
|
|
1940
|
-
|
|
1941
|
-
|
|
1942
|
-
|
|
1943
|
-
|
|
1944
|
-
|
|
1945
|
-
|
|
1946
|
-
|
|
1947
|
-
|
|
1948
|
-
|
|
1949
|
-
|
|
1950
|
-
|
|
1951
|
-
|
|
1952
|
-
|
|
1953
|
-
|
|
1954
|
-
|
|
1955
|
-
|
|
1956
|
-
|
|
1957
|
-
|
|
1958
|
-
|
|
1959
|
-
return [2 /*return*/, false];
|
|
1960
|
-
}
|
|
1961
|
-
return [4 /*yield*/, promises.stat(filename)
|
|
1962
|
-
.then(function (fileStat) { return fileStat.isFile(); })
|
|
1963
|
-
.catch(function () { return false; })];
|
|
1964
|
-
case 2:
|
|
1965
|
-
isFile = _a.sent();
|
|
1966
|
-
return [2 /*return*/, isFile];
|
|
1967
|
-
}
|
|
1968
|
-
});
|
|
1969
|
-
});
|
|
1970
|
-
}
|
|
1971
|
-
/**
|
|
1972
|
-
* Note: [🟢 <- TODO: [🦖] !!!!!! Split scrapers into packages and enable] Code in this file should never be published outside of `@promptbook/node` and `@promptbook/cli`
|
|
1973
|
-
* TODO: [🐠] This can be a validator - with variants that return true/false and variants that throw errors with meaningless messages
|
|
1974
|
-
* TODO: [🖇] What about symlinks?
|
|
1975
|
-
*/
|
|
1976
|
-
|
|
1977
|
-
/**
|
|
1978
|
-
* Get the file extension from a file name
|
|
1979
|
-
*
|
|
1980
|
-
* @private within the repository
|
|
1981
|
-
*/
|
|
1982
|
-
function getFileExtension(value) {
|
|
1983
|
-
var match = value.match(/\.([0-9a-z]+)(?:[?#]|$)/i);
|
|
1984
|
-
return match ? match[1].toLowerCase() : null;
|
|
1985
|
-
}
|
|
1986
|
-
|
|
1987
|
-
var defaultDiacriticsRemovalMap = [
|
|
1988
|
-
{
|
|
1989
|
-
base: 'A',
|
|
1990
|
-
letters: '\u0041\u24B6\uFF21\u00C0\u00C1\u00C2\u1EA6\u1EA4\u1EAA\u1EA8\u00C3\u0100\u0102\u1EB0\u1EAE\u1EB4\u1EB2\u0226\u01E0\u00C4\u01DE\u1EA2\u00C5\u01FA\u01CD\u0200\u0202\u1EA0\u1EAC\u1EB6\u1E00\u0104\u023A\u2C6F',
|
|
1991
|
-
},
|
|
1992
|
-
{ base: 'AA', letters: '\uA732' },
|
|
1993
|
-
{ base: 'AE', letters: '\u00C6\u01FC\u01E2' },
|
|
1994
|
-
{ base: 'AO', letters: '\uA734' },
|
|
1995
|
-
{ base: 'AU', letters: '\uA736' },
|
|
1996
|
-
{ base: 'AV', letters: '\uA738\uA73A' },
|
|
1997
|
-
{ base: 'AY', letters: '\uA73C' },
|
|
1998
|
-
{
|
|
1999
|
-
base: 'B',
|
|
2000
|
-
letters: '\u0042\u24B7\uFF22\u1E02\u1E04\u1E06\u0243\u0182\u0181',
|
|
2001
|
-
},
|
|
2002
|
-
{
|
|
2003
|
-
base: 'C',
|
|
2004
|
-
letters: '\u0043\u24B8\uFF23\u0106\u0108\u010A\u010C\u00C7\u1E08\u0187\u023B\uA73E',
|
|
2005
|
-
},
|
|
2006
|
-
{
|
|
2007
|
-
base: 'D',
|
|
2008
|
-
letters: '\u0044\u24B9\uFF24\u1E0A\u010E\u1E0C\u1E10\u1E12\u1E0E\u0110\u018B\u018A\u0189\uA779\u00D0',
|
|
2009
|
-
},
|
|
2010
|
-
{ base: 'DZ', letters: '\u01F1\u01C4' },
|
|
2011
|
-
{ base: 'Dz', letters: '\u01F2\u01C5' },
|
|
2012
|
-
{
|
|
2013
|
-
base: 'E',
|
|
2014
|
-
letters: '\u0045\u24BA\uFF25\u00C8\u00C9\u00CA\u1EC0\u1EBE\u1EC4\u1EC2\u1EBC\u0112\u1E14\u1E16\u0114\u0116\u00CB\u1EBA\u011A\u0204\u0206\u1EB8\u1EC6\u0228\u1E1C\u0118\u1E18\u1E1A\u0190\u018E',
|
|
2015
|
-
},
|
|
2016
|
-
{ base: 'F', letters: '\u0046\u24BB\uFF26\u1E1E\u0191\uA77B' },
|
|
2017
|
-
{
|
|
2018
|
-
base: 'G',
|
|
2019
|
-
letters: '\u0047\u24BC\uFF27\u01F4\u011C\u1E20\u011E\u0120\u01E6\u0122\u01E4\u0193\uA7A0\uA77D\uA77E',
|
|
2020
|
-
},
|
|
2021
|
-
{
|
|
2022
|
-
base: 'H',
|
|
2023
|
-
letters: '\u0048\u24BD\uFF28\u0124\u1E22\u1E26\u021E\u1E24\u1E28\u1E2A\u0126\u2C67\u2C75\uA78D',
|
|
2024
|
-
},
|
|
2025
|
-
{
|
|
2026
|
-
base: 'I',
|
|
2027
|
-
letters: '\u0049\u24BE\uFF29\u00CC\u00CD\u00CE\u0128\u012A\u012C\u0130\u00CF\u1E2E\u1EC8\u01CF\u0208\u020A\u1ECA\u012E\u1E2C\u0197',
|
|
2028
|
-
},
|
|
2029
|
-
{ base: 'J', letters: '\u004A\u24BF\uFF2A\u0134\u0248' },
|
|
2030
|
-
{
|
|
2031
|
-
base: 'K',
|
|
2032
|
-
letters: '\u004B\u24C0\uFF2B\u1E30\u01E8\u1E32\u0136\u1E34\u0198\u2C69\uA740\uA742\uA744\uA7A2',
|
|
2033
|
-
},
|
|
2034
|
-
{
|
|
2035
|
-
base: 'L',
|
|
2036
|
-
letters: '\u004C\u24C1\uFF2C\u013F\u0139\u013D\u1E36\u1E38\u013B\u1E3C\u1E3A\u0141\u023D\u2C62\u2C60\uA748\uA746\uA780',
|
|
2037
|
-
},
|
|
2038
|
-
{ base: 'LJ', letters: '\u01C7' },
|
|
2039
|
-
{ base: 'Lj', letters: '\u01C8' },
|
|
2040
|
-
{ base: 'M', letters: '\u004D\u24C2\uFF2D\u1E3E\u1E40\u1E42\u2C6E\u019C' },
|
|
2041
|
-
{
|
|
2042
|
-
base: 'N',
|
|
2043
|
-
letters: '\u004E\u24C3\uFF2E\u01F8\u0143\u00D1\u1E44\u0147\u1E46\u0145\u1E4A\u1E48\u0220\u019D\uA790\uA7A4',
|
|
2044
|
-
},
|
|
2045
|
-
{ base: 'NJ', letters: '\u01CA' },
|
|
2046
|
-
{ base: 'Nj', letters: '\u01CB' },
|
|
2047
|
-
{
|
|
2048
|
-
base: 'O',
|
|
2049
|
-
letters: '\u004F\u24C4\uFF2F\u00D2\u00D3\u00D4\u1ED2\u1ED0\u1ED6\u1ED4\u00D5\u1E4C\u022C\u1E4E\u014C\u1E50\u1E52\u014E\u022E\u0230\u00D6\u022A\u1ECE\u0150\u01D1\u020C\u020E\u01A0\u1EDC\u1EDA\u1EE0\u1EDE\u1EE2\u1ECC\u1ED8\u01EA\u01EC\u00D8\u01FE\u0186\u019F\uA74A\uA74C',
|
|
2050
|
-
},
|
|
2051
|
-
{ base: 'OI', letters: '\u01A2' },
|
|
2052
|
-
{ base: 'OO', letters: '\uA74E' },
|
|
2053
|
-
{ base: 'OU', letters: '\u0222' },
|
|
2054
|
-
{ base: 'OE', letters: '\u008C\u0152' },
|
|
2055
|
-
{ base: 'oe', letters: '\u009C\u0153' },
|
|
2056
|
-
{
|
|
2057
|
-
base: 'P',
|
|
2058
|
-
letters: '\u0050\u24C5\uFF30\u1E54\u1E56\u01A4\u2C63\uA750\uA752\uA754',
|
|
2059
|
-
},
|
|
2060
|
-
{ base: 'Q', letters: '\u0051\u24C6\uFF31\uA756\uA758\u024A' },
|
|
2061
|
-
{
|
|
2062
|
-
base: 'R',
|
|
2063
|
-
letters: '\u0052\u24C7\uFF32\u0154\u1E58\u0158\u0210\u0212\u1E5A\u1E5C\u0156\u1E5E\u024C\u2C64\uA75A\uA7A6\uA782',
|
|
2064
|
-
},
|
|
2065
|
-
{
|
|
2066
|
-
base: 'S',
|
|
2067
|
-
letters: '\u0053\u24C8\uFF33\u1E9E\u015A\u1E64\u015C\u1E60\u0160\u1E66\u1E62\u1E68\u0218\u015E\u2C7E\uA7A8\uA784',
|
|
2068
|
-
},
|
|
2069
|
-
{
|
|
2070
|
-
base: 'T',
|
|
2071
|
-
letters: '\u0054\u24C9\uFF34\u1E6A\u0164\u1E6C\u021A\u0162\u1E70\u1E6E\u0166\u01AC\u01AE\u023E\uA786',
|
|
2072
|
-
},
|
|
2073
|
-
{ base: 'TZ', letters: '\uA728' },
|
|
2074
|
-
{
|
|
2075
|
-
base: 'U',
|
|
2076
|
-
letters: '\u0055\u24CA\uFF35\u00D9\u00DA\u00DB\u0168\u1E78\u016A\u1E7A\u016C\u00DC\u01DB\u01D7\u01D5\u01D9\u1EE6\u016E\u0170\u01D3\u0214\u0216\u01AF\u1EEA\u1EE8\u1EEE\u1EEC\u1EF0\u1EE4\u1E72\u0172\u1E76\u1E74\u0244',
|
|
2077
|
-
},
|
|
2078
|
-
{ base: 'V', letters: '\u0056\u24CB\uFF36\u1E7C\u1E7E\u01B2\uA75E\u0245' },
|
|
2079
|
-
{ base: 'VY', letters: '\uA760' },
|
|
2080
|
-
{
|
|
2081
|
-
base: 'W',
|
|
2082
|
-
letters: '\u0057\u24CC\uFF37\u1E80\u1E82\u0174\u1E86\u1E84\u1E88\u2C72',
|
|
2083
|
-
},
|
|
2084
|
-
{ base: 'X', letters: '\u0058\u24CD\uFF38\u1E8A\u1E8C' },
|
|
2085
|
-
{
|
|
2086
|
-
base: 'Y',
|
|
2087
|
-
letters: '\u0059\u24CE\uFF39\u1EF2\u00DD\u0176\u1EF8\u0232\u1E8E\u0178\u1EF6\u1EF4\u01B3\u024E\u1EFE',
|
|
2088
|
-
},
|
|
2089
|
-
{
|
|
2090
|
-
base: 'Z',
|
|
2091
|
-
letters: '\u005A\u24CF\uFF3A\u0179\u1E90\u017B\u017D\u1E92\u1E94\u01B5\u0224\u2C7F\u2C6B\uA762',
|
|
2092
|
-
},
|
|
2093
|
-
{
|
|
2094
|
-
base: 'a',
|
|
2095
|
-
letters: '\u0061\u24D0\uFF41\u1E9A\u00E0\u00E1\u00E2\u1EA7\u1EA5\u1EAB\u1EA9\u00E3\u0101\u0103\u1EB1\u1EAF\u1EB5\u1EB3\u0227\u01E1\u00E4\u01DF\u1EA3\u00E5\u01FB\u01CE\u0201\u0203\u1EA1\u1EAD\u1EB7\u1E01\u0105\u2C65\u0250',
|
|
2096
|
-
},
|
|
2097
|
-
{ base: 'aa', letters: '\uA733' },
|
|
2098
|
-
{ base: 'ae', letters: '\u00E6\u01FD\u01E3' },
|
|
2099
|
-
{ base: 'ao', letters: '\uA735' },
|
|
2100
|
-
{ base: 'au', letters: '\uA737' },
|
|
2101
|
-
{ base: 'av', letters: '\uA739\uA73B' },
|
|
2102
|
-
{ base: 'ay', letters: '\uA73D' },
|
|
2103
|
-
{
|
|
2104
|
-
base: 'b',
|
|
2105
|
-
letters: '\u0062\u24D1\uFF42\u1E03\u1E05\u1E07\u0180\u0183\u0253',
|
|
2106
|
-
},
|
|
2107
|
-
{
|
|
2108
|
-
base: 'c',
|
|
2109
|
-
letters: '\u0063\u24D2\uFF43\u0107\u0109\u010B\u010D\u00E7\u1E09\u0188\u023C\uA73F\u2184',
|
|
2110
|
-
},
|
|
2111
|
-
{
|
|
2112
|
-
base: 'd',
|
|
2113
|
-
letters: '\u0064\u24D3\uFF44\u1E0B\u010F\u1E0D\u1E11\u1E13\u1E0F\u0111\u018C\u0256\u0257\uA77A',
|
|
2114
|
-
},
|
|
2115
|
-
{ base: 'dz', letters: '\u01F3\u01C6' },
|
|
2116
|
-
{
|
|
2117
|
-
base: 'e',
|
|
2118
|
-
letters: '\u0065\u24D4\uFF45\u00E8\u00E9\u00EA\u1EC1\u1EBF\u1EC5\u1EC3\u1EBD\u0113\u1E15\u1E17\u0115\u0117\u00EB\u1EBB\u011B\u0205\u0207\u1EB9\u1EC7\u0229\u1E1D\u0119\u1E19\u1E1B\u0247\u025B\u01DD',
|
|
2119
|
-
},
|
|
2120
|
-
{ base: 'f', letters: '\u0066\u24D5\uFF46\u1E1F\u0192\uA77C' },
|
|
2121
|
-
{
|
|
2122
|
-
base: 'g',
|
|
2123
|
-
letters: '\u0067\u24D6\uFF47\u01F5\u011D\u1E21\u011F\u0121\u01E7\u0123\u01E5\u0260\uA7A1\u1D79\uA77F',
|
|
2124
|
-
},
|
|
2125
|
-
{
|
|
2126
|
-
base: 'h',
|
|
2127
|
-
letters: '\u0068\u24D7\uFF48\u0125\u1E23\u1E27\u021F\u1E25\u1E29\u1E2B\u1E96\u0127\u2C68\u2C76\u0265',
|
|
2128
|
-
},
|
|
2129
|
-
{ base: 'hv', letters: '\u0195' },
|
|
2130
|
-
{
|
|
2131
|
-
base: 'i',
|
|
2132
|
-
letters: '\u0069\u24D8\uFF49\u00EC\u00ED\u00EE\u0129\u012B\u012D\u00EF\u1E2F\u1EC9\u01D0\u0209\u020B\u1ECB\u012F\u1E2D\u0268\u0131',
|
|
2133
|
-
},
|
|
2134
|
-
{ base: 'j', letters: '\u006A\u24D9\uFF4A\u0135\u01F0\u0249' },
|
|
2135
|
-
{
|
|
2136
|
-
base: 'k',
|
|
2137
|
-
letters: '\u006B\u24DA\uFF4B\u1E31\u01E9\u1E33\u0137\u1E35\u0199\u2C6A\uA741\uA743\uA745\uA7A3',
|
|
2138
|
-
},
|
|
2139
|
-
{
|
|
2140
|
-
base: 'l',
|
|
2141
|
-
letters: '\u006C\u24DB\uFF4C\u0140\u013A\u013E\u1E37\u1E39\u013C\u1E3D\u1E3B\u017F\u0142\u019A\u026B\u2C61\uA749\uA781\uA747',
|
|
2142
|
-
},
|
|
2143
|
-
{ base: 'lj', letters: '\u01C9' },
|
|
2144
|
-
{ base: 'm', letters: '\u006D\u24DC\uFF4D\u1E3F\u1E41\u1E43\u0271\u026F' },
|
|
2145
|
-
{
|
|
2146
|
-
base: 'n',
|
|
2147
|
-
letters: '\u006E\u24DD\uFF4E\u01F9\u0144\u00F1\u1E45\u0148\u1E47\u0146\u1E4B\u1E49\u019E\u0272\u0149\uA791\uA7A5',
|
|
2148
|
-
},
|
|
2149
|
-
{ base: 'nj', letters: '\u01CC' },
|
|
2150
|
-
{
|
|
2151
|
-
base: 'o',
|
|
2152
|
-
letters: '\u006F\u24DE\uFF4F\u00F2\u00F3\u00F4\u1ED3\u1ED1\u1ED7\u1ED5\u00F5\u1E4D\u022D\u1E4F\u014D\u1E51\u1E53\u014F\u022F\u0231\u00F6\u022B\u1ECF\u0151\u01D2\u020D\u020F\u01A1\u1EDD\u1EDB\u1EE1\u1EDF\u1EE3\u1ECD\u1ED9\u01EB\u01ED\u00F8\u01FF\u0254\uA74B\uA74D\u0275',
|
|
2153
|
-
},
|
|
2154
|
-
{ base: 'oi', letters: '\u01A3' },
|
|
2155
|
-
{ base: 'ou', letters: '\u0223' },
|
|
2156
|
-
{ base: 'oo', letters: '\uA74F' },
|
|
2157
|
-
{
|
|
2158
|
-
base: 'p',
|
|
2159
|
-
letters: '\u0070\u24DF\uFF50\u1E55\u1E57\u01A5\u1D7D\uA751\uA753\uA755',
|
|
2160
|
-
},
|
|
2161
|
-
{ base: 'q', letters: '\u0071\u24E0\uFF51\u024B\uA757\uA759' },
|
|
2162
|
-
{
|
|
2163
|
-
base: 'r',
|
|
2164
|
-
letters: '\u0072\u24E1\uFF52\u0155\u1E59\u0159\u0211\u0213\u1E5B\u1E5D\u0157\u1E5F\u024D\u027D\uA75B\uA7A7\uA783',
|
|
2165
|
-
},
|
|
2166
|
-
{
|
|
2167
|
-
base: 's',
|
|
2168
|
-
letters: '\u0073\u24E2\uFF53\u00DF\u015B\u1E65\u015D\u1E61\u0161\u1E67\u1E63\u1E69\u0219\u015F\u023F\uA7A9\uA785\u1E9B',
|
|
2169
|
-
},
|
|
2170
|
-
{
|
|
2171
|
-
base: 't',
|
|
2172
|
-
letters: '\u0074\u24E3\uFF54\u1E6B\u1E97\u0165\u1E6D\u021B\u0163\u1E71\u1E6F\u0167\u01AD\u0288\u2C66\uA787',
|
|
2173
|
-
},
|
|
2174
|
-
{ base: 'tz', letters: '\uA729' },
|
|
2175
|
-
{
|
|
2176
|
-
base: 'u',
|
|
2177
|
-
letters: '\u0075\u24E4\uFF55\u00F9\u00FA\u00FB\u0169\u1E79\u016B\u1E7B\u016D\u00FC\u01DC\u01D8\u01D6\u01DA\u1EE7\u016F\u0171\u01D4\u0215\u0217\u01B0\u1EEB\u1EE9\u1EEF\u1EED\u1EF1\u1EE5\u1E73\u0173\u1E77\u1E75\u0289',
|
|
2178
|
-
},
|
|
2179
|
-
{ base: 'v', letters: '\u0076\u24E5\uFF56\u1E7D\u1E7F\u028B\uA75F\u028C' },
|
|
2180
|
-
{ base: 'vy', letters: '\uA761' },
|
|
2181
|
-
{
|
|
2182
|
-
base: 'w',
|
|
2183
|
-
letters: '\u0077\u24E6\uFF57\u1E81\u1E83\u0175\u1E87\u1E85\u1E98\u1E89\u2C73',
|
|
2184
|
-
},
|
|
2185
|
-
{ base: 'x', letters: '\u0078\u24E7\uFF58\u1E8B\u1E8D' },
|
|
2186
|
-
{
|
|
2187
|
-
base: 'y',
|
|
2188
|
-
letters: '\u0079\u24E8\uFF59\u1EF3\u00FD\u0177\u1EF9\u0233\u1E8F\u00FF\u1EF7\u1E99\u1EF5\u01B4\u024F\u1EFF',
|
|
2189
|
-
},
|
|
2190
|
-
{
|
|
2191
|
-
base: 'z',
|
|
2192
|
-
letters: '\u007A\u24E9\uFF5A\u017A\u1E91\u017C\u017E\u1E93\u1E95\u01B6\u0225\u0240\u2C6C\uA763',
|
|
2193
|
-
},
|
|
2194
|
-
];
|
|
2195
|
-
/**
|
|
2196
|
-
* Map of letters from diacritic variant to diacritless variant
|
|
2197
|
-
* Contains lowercase and uppercase separatelly
|
|
2198
|
-
*
|
|
2199
|
-
* > "á" => "a"
|
|
2200
|
-
* > "ě" => "e"
|
|
2201
|
-
* > "Ă" => "A"
|
|
2202
|
-
* > ...
|
|
2203
|
-
*
|
|
2204
|
-
* @public exported from `@promptbook/utils`
|
|
2205
|
-
*/
|
|
2206
|
-
var DIACRITIC_VARIANTS_LETTERS = {};
|
|
2207
|
-
// tslint:disable-next-line: prefer-for-of
|
|
2208
|
-
for (var i = 0; i < defaultDiacriticsRemovalMap.length; i++) {
|
|
2209
|
-
var letters = defaultDiacriticsRemovalMap[i].letters;
|
|
2210
|
-
// tslint:disable-next-line: prefer-for-of
|
|
2211
|
-
for (var j = 0; j < letters.length; j++) {
|
|
2212
|
-
DIACRITIC_VARIANTS_LETTERS[letters[j]] = defaultDiacriticsRemovalMap[i].base;
|
|
2213
|
-
}
|
|
2214
|
-
}
|
|
2215
|
-
// <- TODO: [🍓] Put to maker function to save execution time if not needed
|
|
2216
|
-
/*
|
|
2217
|
-
@see https://stackoverflow.com/questions/990904/remove-accents-diacritics-in-a-string-in-javascript
|
|
2218
|
-
Licensed under the Apache License, Version 2.0 (the "License");
|
|
2219
|
-
you may not use this file except in compliance with the License.
|
|
2220
|
-
You may obtain a copy of the License at
|
|
2221
|
-
|
|
2222
|
-
http://www.apache.org/licenses/LICENSE-2.0
|
|
2223
|
-
|
|
2224
|
-
Unless required by applicable law or agreed to in writing, software
|
|
2225
|
-
distributed under the License is distributed on an "AS IS" BASIS,
|
|
2226
|
-
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
2227
|
-
See the License for the specific language governing permissions and
|
|
2228
|
-
limitations under the License.
|
|
2229
|
-
*/
|
|
2230
|
-
|
|
2231
|
-
/**
|
|
2232
|
-
* @@@
|
|
2233
|
-
*
|
|
2234
|
-
* @param input @@@
|
|
2235
|
-
* @returns @@@
|
|
2236
|
-
* @public exported from `@promptbook/utils`
|
|
2237
|
-
*/
|
|
2238
|
-
function removeDiacritics(input) {
|
|
2239
|
-
/*eslint no-control-regex: "off"*/
|
|
2240
|
-
return input.replace(/[^\u0000-\u007E]/g, function (a) {
|
|
2241
|
-
return DIACRITIC_VARIANTS_LETTERS[a] || a;
|
|
2242
|
-
});
|
|
2243
|
-
}
|
|
2244
|
-
/**
|
|
2245
|
-
* TODO: [Ж] Variant for cyrillic (and in general non-latin) letters
|
|
2246
|
-
*/
|
|
2247
|
-
|
|
2248
|
-
/**
|
|
2249
|
-
* @@@
|
|
2250
|
-
*
|
|
2251
|
-
* @param text @@@
|
|
2252
|
-
* @returns @@@
|
|
2253
|
-
* @example 'hello-world'
|
|
2254
|
-
* @example 'i-love-promptbook'
|
|
2255
|
-
* @public exported from `@promptbook/utils`
|
|
2256
|
-
*/
|
|
2257
|
-
function normalizeToKebabCase(text) {
|
|
2258
|
-
var e_1, _a;
|
|
2259
|
-
text = removeDiacritics(text);
|
|
2260
|
-
var charType;
|
|
2261
|
-
var lastCharType = 'OTHER';
|
|
2262
|
-
var normalizedName = '';
|
|
2263
|
-
try {
|
|
2264
|
-
for (var text_1 = __values(text), text_1_1 = text_1.next(); !text_1_1.done; text_1_1 = text_1.next()) {
|
|
2265
|
-
var char = text_1_1.value;
|
|
2266
|
-
var normalizedChar = void 0;
|
|
2267
|
-
if (/^[a-z]$/.test(char)) {
|
|
2268
|
-
charType = 'LOWERCASE';
|
|
2269
|
-
normalizedChar = char;
|
|
2270
|
-
}
|
|
2271
|
-
else if (/^[A-Z]$/.test(char)) {
|
|
2272
|
-
charType = 'UPPERCASE';
|
|
2273
|
-
normalizedChar = char.toLowerCase();
|
|
2274
|
-
}
|
|
2275
|
-
else if (/^[0-9]$/.test(char)) {
|
|
2276
|
-
charType = 'NUMBER';
|
|
2277
|
-
normalizedChar = char;
|
|
2278
|
-
}
|
|
2279
|
-
else {
|
|
2280
|
-
charType = 'OTHER';
|
|
2281
|
-
normalizedChar = '-';
|
|
2282
|
-
}
|
|
2283
|
-
if (charType !== lastCharType &&
|
|
2284
|
-
!(lastCharType === 'UPPERCASE' && charType === 'LOWERCASE') &&
|
|
2285
|
-
!(lastCharType === 'NUMBER') &&
|
|
2286
|
-
!(charType === 'NUMBER')) {
|
|
2287
|
-
normalizedName += '-';
|
|
2288
|
-
}
|
|
2289
|
-
normalizedName += normalizedChar;
|
|
2290
|
-
lastCharType = charType;
|
|
2291
|
-
}
|
|
2292
|
-
}
|
|
2293
|
-
catch (e_1_1) { e_1 = { error: e_1_1 }; }
|
|
2294
|
-
finally {
|
|
2295
|
-
try {
|
|
2296
|
-
if (text_1_1 && !text_1_1.done && (_a = text_1.return)) _a.call(text_1);
|
|
2297
|
-
}
|
|
2298
|
-
finally { if (e_1) throw e_1.error; }
|
|
2299
|
-
}
|
|
2300
|
-
normalizedName = normalizedName.split(/-+/g).join('-');
|
|
2301
|
-
normalizedName = normalizedName.split(/-?\/-?/g).join('/');
|
|
2302
|
-
normalizedName = normalizedName.replace(/^-/, '');
|
|
2303
|
-
normalizedName = normalizedName.replace(/-$/, '');
|
|
2304
|
-
return normalizedName;
|
|
2305
|
-
}
|
|
2306
|
-
|
|
2307
|
-
/**
|
|
2308
|
-
* Removes emojis from a string and fix whitespaces
|
|
2309
|
-
*
|
|
2310
|
-
* @param text with emojis
|
|
2311
|
-
* @returns text without emojis
|
|
2312
|
-
* @public exported from `@promptbook/utils`
|
|
2313
|
-
*/
|
|
2314
|
-
function removeEmojis(text) {
|
|
2315
|
-
// Replace emojis (and also ZWJ sequence) with hyphens
|
|
2316
|
-
text = text.replace(/(\p{Extended_Pictographic})\p{Modifier_Symbol}/gu, '$1');
|
|
2317
|
-
text = text.replace(/(\p{Extended_Pictographic})[\u{FE00}-\u{FE0F}]/gu, '$1');
|
|
2318
|
-
text = text.replace(/(\p{Extended_Pictographic})(\u{200D}\p{Extended_Pictographic})*/gu, '$1');
|
|
2319
|
-
text = text.replace(/\p{Extended_Pictographic}/gu, '');
|
|
2320
|
-
return text;
|
|
2321
|
-
}
|
|
2322
|
-
|
|
2323
|
-
/**
|
|
2324
|
-
* Tests if given string is valid URL.
|
|
2325
|
-
*
|
|
2326
|
-
* Note: This does not check if the file exists only if the path is valid
|
|
2327
|
-
* @public exported from `@promptbook/utils`
|
|
2328
|
-
*/
|
|
2329
|
-
function isValidFilePath(filename) {
|
|
2330
|
-
if (typeof filename !== 'string') {
|
|
2331
|
-
return false;
|
|
2332
|
-
}
|
|
2333
|
-
var filenameSlashes = filename.split('\\').join('/');
|
|
2334
|
-
// Absolute Unix path: /hello.txt
|
|
2335
|
-
if (/^(\/)/i.test(filenameSlashes)) {
|
|
2336
|
-
return true;
|
|
2337
|
-
}
|
|
2338
|
-
// Absolute Windows path: /hello.txt
|
|
2339
|
-
if (/^([A-Z]{1,2}:\/?)\//i.test(filenameSlashes)) {
|
|
2340
|
-
return true;
|
|
2341
|
-
}
|
|
2342
|
-
// Relative path: ./hello.txt
|
|
2343
|
-
if (/^(\.\.?\/)+/i.test(filenameSlashes)) {
|
|
2344
|
-
return true;
|
|
2345
|
-
}
|
|
2346
|
-
return false;
|
|
2347
|
-
}
|
|
2348
|
-
|
|
2349
|
-
/**
|
|
2350
|
-
* @@@
|
|
2351
|
-
*
|
|
2352
|
-
* @param value @@@
|
|
2353
|
-
* @returns @@@
|
|
2354
|
-
* @example @@@
|
|
2355
|
-
* @public exported from `@promptbook/utils`
|
|
2356
|
-
*/
|
|
2357
|
-
function titleToName(value) {
|
|
2358
|
-
if (isValidUrl(value)) {
|
|
2359
|
-
value = value.replace(/^https?:\/\//, '');
|
|
2360
|
-
value = value.replace(/\.html$/, '');
|
|
2361
|
-
}
|
|
2362
|
-
else if (isValidFilePath(value)) {
|
|
2363
|
-
value = path.basename(value);
|
|
2364
|
-
// Note: Keeping extension in the name
|
|
2365
|
-
}
|
|
2366
|
-
value = value.split('/').join('-');
|
|
2367
|
-
value = removeEmojis(value);
|
|
2368
|
-
value = normalizeToKebabCase(value);
|
|
2369
|
-
// TODO: [🧠] Maybe warn or add some padding to short name which are not good identifiers
|
|
2370
|
-
return value;
|
|
2371
|
-
}
|
|
2372
|
-
|
|
2373
|
-
/**
|
|
2374
|
-
* @@@
|
|
2375
|
-
*
|
|
2376
|
-
* @private for `FileCacheStorage`
|
|
2377
|
-
*/
|
|
2378
|
-
function nameToSubfolderPath(name) {
|
|
2379
|
-
return [name.substr(0, 1).toLowerCase(), name.substr(1, 1).toLowerCase()];
|
|
2380
|
-
}
|
|
2381
|
-
|
|
2382
|
-
/**
|
|
2383
|
-
* Just marks a place of place where should be something implemented
|
|
2384
|
-
* No side effects.
|
|
2385
|
-
*
|
|
2386
|
-
* Note: It can be usefull suppressing eslint errors of unused variables
|
|
2387
|
-
*
|
|
2388
|
-
* @param value any values
|
|
2389
|
-
* @returns void
|
|
2390
|
-
* @private within the repository
|
|
2391
|
-
*/
|
|
2392
|
-
function TODO_USE() {
|
|
2393
|
-
var value = [];
|
|
2394
|
-
for (var _i = 0; _i < arguments.length; _i++) {
|
|
2395
|
-
value[_i] = arguments[_i];
|
|
2396
|
-
}
|
|
2397
|
-
}
|
|
2398
|
-
|
|
2399
|
-
/**
|
|
2400
|
-
* Create a filename for intermediate cache for scrapers
|
|
2401
|
-
*
|
|
2402
|
-
* Note: It also checks if directory exists and creates it if not
|
|
2403
|
-
*
|
|
2404
|
-
* @private as internal utility for scrapers
|
|
2405
|
-
*/
|
|
2406
|
-
function getScraperIntermediateSource(source, options) {
|
|
2407
|
-
return __awaiter(this, void 0, void 0, function () {
|
|
2408
|
-
var sourceFilename, url, rootDirname, cacheDirname, isCacheCleaned, extension, isVerbose, hash, semanticName, pieces, name, cacheFilename, isDestroyed, fileHandler;
|
|
2409
|
-
return __generator(this, function (_a) {
|
|
2410
|
-
switch (_a.label) {
|
|
2411
|
-
case 0:
|
|
2412
|
-
sourceFilename = source.filename, url = source.url;
|
|
2413
|
-
rootDirname = options.rootDirname, cacheDirname = options.cacheDirname, isCacheCleaned = options.isCacheCleaned, extension = options.extension, isVerbose = options.isVerbose;
|
|
2414
|
-
hash = cryptoJs.SHA256(
|
|
2415
|
-
// <- TODO: [🥬] Encapsulate sha256 to some private utility function
|
|
2416
|
-
hexEncoder__default["default"].parse(sourceFilename || url || 'untitled'))
|
|
2417
|
-
.toString( /* hex */)
|
|
2418
|
-
.substring(0, 20);
|
|
2419
|
-
semanticName = normalizeToKebabCase(titleToName((sourceFilename || url || '').split('intermediate').join(''))).substring(0, 20);
|
|
2420
|
-
pieces = ['intermediate', semanticName, hash].filter(function (piece) { return piece !== ''; });
|
|
2421
|
-
name = pieces.join('-').split('--').join('-');
|
|
2422
|
-
// <- TODO: Use MAX_FILENAME_LENGTH
|
|
2423
|
-
TODO_USE(rootDirname); // <- TODO: !!!!!!
|
|
2424
|
-
cacheFilename = path.join.apply(void 0, __spreadArray(__spreadArray([process.cwd(),
|
|
2425
|
-
cacheDirname], __read(nameToSubfolderPath(hash /* <- TODO: [🎎] Maybe add some SHA256 prefix */)), false), [name], false)).split('\\')
|
|
2426
|
-
.join('/') +
|
|
2427
|
-
'.' +
|
|
2428
|
-
extension;
|
|
2429
|
-
return [4 /*yield*/, promises.mkdir(path.dirname(cacheFilename), { recursive: true })];
|
|
2430
|
-
case 1:
|
|
2431
|
-
_a.sent();
|
|
2432
|
-
isDestroyed = true;
|
|
2433
|
-
fileHandler = {
|
|
2434
|
-
filename: cacheFilename,
|
|
2435
|
-
get isDestroyed() {
|
|
2436
|
-
return isDestroyed;
|
|
2437
|
-
},
|
|
2438
|
-
destroy: function () {
|
|
2439
|
-
return __awaiter(this, void 0, void 0, function () {
|
|
2440
|
-
return __generator(this, function (_a) {
|
|
2441
|
-
switch (_a.label) {
|
|
2442
|
-
case 0:
|
|
2443
|
-
if (!isCacheCleaned) return [3 /*break*/, 2];
|
|
2444
|
-
if (isVerbose) {
|
|
2445
|
-
console.info('legacyDocumentScraper: Clening cache');
|
|
2446
|
-
}
|
|
2447
|
-
return [4 /*yield*/, promises.rm(cacheFilename)];
|
|
2448
|
-
case 1:
|
|
2449
|
-
_a.sent();
|
|
2450
|
-
_a.label = 2;
|
|
2451
|
-
case 2:
|
|
2452
|
-
isDestroyed = true;
|
|
2453
|
-
return [2 /*return*/];
|
|
2454
|
-
}
|
|
2455
|
-
});
|
|
2456
|
-
});
|
|
2457
|
-
},
|
|
2458
|
-
};
|
|
2459
|
-
return [2 /*return*/, fileHandler];
|
|
2460
|
-
}
|
|
2461
|
-
});
|
|
2462
|
-
});
|
|
2463
|
-
}
|
|
2464
|
-
/**
|
|
2465
|
-
* Note: Not using `FileCacheStorage` for two reasons:
|
|
2466
|
-
* 1) Need to store more than serialized JSONs
|
|
2467
|
-
* 2) Need to switch between a `rootDirname` and `cacheDirname` <- TODO: !!!!
|
|
2468
|
-
* TODO: [🐱🐉][🧠] Make some smart crop
|
|
2469
|
-
*/
|
|
2470
|
-
|
|
2471
|
-
var PipelineCollection = [{title:"Prepare Knowledge from Markdown",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.ptbk.md",parameters:[{name:"knowledgeContent",description:"Markdown document content",isInput:true,isOutput:false},{name:"knowledgePieces",description:"The knowledge JSON object",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, extract the important knowledge from the document.\n\n# Rules\n\n- Make pieces of information concise, clear, and easy to understand\n- One piece of information should be approximately 1 paragraph\n- Divide the paragraphs by markdown horizontal lines ---\n- Omit irrelevant information\n- Group redundant information\n- Write just extracted information, nothing else\n\n# The document\n\nTake information from this document:\n\n> {knowledgeContent}",resultingParameterName:"knowledgePieces",dependentParameterNames:["knowledgeContent"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-knowledge-from-markdown.ptbk.md"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-keywords.ptbk.md",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"keywords",description:"Keywords separated by comma",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, detect the important keywords in the document.\n\n# Rules\n\n- Write just keywords separated by comma\n\n# The document\n\nTake information from this document:\n\n> {knowledgePieceContent}",resultingParameterName:"keywords",dependentParameterNames:["knowledgePieceContent"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-knowledge-keywords.ptbk.md"},{title:"Prepare Title",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-title.ptbk.md",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"title",description:"The title of the document",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",content:"You are experienced content creator, write best title for the document.\n\n# Rules\n\n- Write just title, nothing else\n- Title should be concise and clear\n- Write maximum 5 words for the title\n\n# The document\n\n> {knowledgePieceContent}",resultingParameterName:"title",expectations:{words:{min:1,max:8}},dependentParameterNames:["knowledgePieceContent"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-knowledge-title.ptbk.md"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-persona.ptbk.md",parameters:[{name:"availableModelNames",description:"List of available model names separated by comma (,)",isInput:true,isOutput:false},{name:"personaDescription",description:"Description of the persona",isInput:true,isOutput:false},{name:"modelRequirements",description:"Specific requirements for the model",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"make-model-requirements",title:"Make modelRequirements",content:"You are experienced AI engineer, you need to create virtual assistant.\nWrite\n\n## Sample\n\n```json\n{\n\"modelName\": \"gpt-4o\",\n\"systemMessage\": \"You are experienced AI engineer and helpfull assistant.\",\n\"temperature\": 0.7\n}\n```\n\n## Instructions\n\n- Your output format is JSON object\n- Write just the JSON object, no other text should be present\n- It contains the following keys:\n - `modelName`: The name of the model to use\n - `systemMessage`: The system message to provide context to the model\n - `temperature`: The sampling temperature to use\n\n### Key `modelName`\n\nPick from the following models:\n\n- {availableModelNames}\n\n### Key `systemMessage`\n\nThe system message is used to communicate instructions or provide context to the model at the beginning of a conversation. It is displayed in a different format compared to user messages, helping the model understand its role in the conversation. The system message typically guides the model's behavior, sets the tone, or specifies desired output from the model. By utilizing the system message effectively, users can steer the model towards generating more accurate and relevant responses.\n\nFor example:\n\n> You are an experienced AI engineer and helpful assistant.\n\n> You are a friendly and knowledgeable chatbot.\n\n### Key `temperature`\n\nThe sampling temperature, between 0 and 1. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. If set to 0, the model will use log probability to automatically increase the temperature until certain thresholds are hit.\n\nYou can pick a value between 0 and 2. For example:\n\n- `0.1`: Low temperature, extremely conservative and deterministic\n- `0.5`: Medium temperature, balanced between conservative and creative\n- `1.0`: High temperature, creative and bit random\n- `1.5`: Very high temperature, extremely creative and often chaotic and unpredictable\n- `2.0`: Maximum temperature, completely random and unpredictable, for some extreme creative use cases\n\n# The assistant\n\nTake this description of the persona:\n\n> {personaDescription}",resultingParameterName:"modelRequirements",format:"JSON",dependentParameterNames:["availableModelNames","personaDescription"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-persona.ptbk.md"}];
|
|
2472
|
-
|
|
2473
|
-
/**
|
|
2474
|
-
* This error indicates errors during the execution of the pipeline
|
|
2475
|
-
*
|
|
2476
|
-
* @public exported from `@promptbook/core`
|
|
2477
|
-
*/
|
|
2478
|
-
var PipelineExecutionError = /** @class */ (function (_super) {
|
|
2479
|
-
__extends(PipelineExecutionError, _super);
|
|
2480
|
-
function PipelineExecutionError(message) {
|
|
2481
|
-
var _this = _super.call(this, message) || this;
|
|
2482
|
-
_this.name = 'PipelineExecutionError';
|
|
2483
|
-
Object.setPrototypeOf(_this, PipelineExecutionError.prototype);
|
|
2484
|
-
return _this;
|
|
2485
|
-
}
|
|
2486
|
-
return PipelineExecutionError;
|
|
2487
|
-
}(Error));
|
|
2488
|
-
|
|
2489
|
-
/**
|
|
2490
|
-
* This error indicates that the pipeline collection cannot be propperly loaded
|
|
2491
|
-
*
|
|
2492
|
-
* @public exported from `@promptbook/core`
|
|
2493
|
-
*/
|
|
2494
|
-
var CollectionError = /** @class */ (function (_super) {
|
|
2495
|
-
__extends(CollectionError, _super);
|
|
2496
|
-
function CollectionError(message) {
|
|
2497
|
-
var _this = _super.call(this, message) || this;
|
|
2498
|
-
_this.name = 'CollectionError';
|
|
2499
|
-
Object.setPrototypeOf(_this, CollectionError.prototype);
|
|
2500
|
-
return _this;
|
|
2501
|
-
}
|
|
2502
|
-
return CollectionError;
|
|
2503
|
-
}(Error));
|
|
2504
|
-
|
|
2505
|
-
/**
|
|
2506
|
-
* This error occurs when some expectation is not met in the execution of the pipeline
|
|
2507
|
-
*
|
|
2508
|
-
* @public exported from `@promptbook/core`
|
|
2509
|
-
* Note: Do not throw this error, its reserved for `checkExpectations` and `createPipelineExecutor` and public ONLY to be serializable through remote server
|
|
2510
|
-
* Note: Always thrown in `checkExpectations` and catched in `createPipelineExecutor` and rethrown as `PipelineExecutionError`
|
|
2511
|
-
* Note: This is a kindof subtype of PipelineExecutionError
|
|
2512
|
-
*/
|
|
2513
|
-
var ExpectError = /** @class */ (function (_super) {
|
|
2514
|
-
__extends(ExpectError, _super);
|
|
2515
|
-
function ExpectError(message) {
|
|
2516
|
-
var _this = _super.call(this, message) || this;
|
|
2517
|
-
_this.name = 'ExpectError';
|
|
2518
|
-
Object.setPrototypeOf(_this, ExpectError.prototype);
|
|
2519
|
-
return _this;
|
|
2520
|
-
}
|
|
2521
|
-
return ExpectError;
|
|
2522
|
-
}(Error));
|
|
2523
|
-
|
|
2524
|
-
/**
|
|
2525
|
-
* This error type indicates that some limit was reached
|
|
2526
|
-
*
|
|
2527
|
-
* @public exported from `@promptbook/core`
|
|
2528
|
-
*/
|
|
2529
|
-
var LimitReachedError = /** @class */ (function (_super) {
|
|
2530
|
-
__extends(LimitReachedError, _super);
|
|
2531
|
-
function LimitReachedError(message) {
|
|
2532
|
-
var _this = _super.call(this, message) || this;
|
|
2533
|
-
_this.name = 'LimitReachedError';
|
|
2534
|
-
Object.setPrototypeOf(_this, LimitReachedError.prototype);
|
|
2535
|
-
return _this;
|
|
2536
|
-
}
|
|
2537
|
-
return LimitReachedError;
|
|
2538
|
-
}(Error));
|
|
2539
|
-
|
|
2540
|
-
/**
|
|
2541
|
-
* This error type indicates that some part of the code is not implemented yet
|
|
2542
|
-
*
|
|
2543
|
-
* @public exported from `@promptbook/core`
|
|
2544
|
-
*/
|
|
2545
|
-
var NotYetImplementedError = /** @class */ (function (_super) {
|
|
2546
|
-
__extends(NotYetImplementedError, _super);
|
|
2547
|
-
function NotYetImplementedError(message) {
|
|
2548
|
-
var _this = _super.call(this, spaceTrim.spaceTrim(function (block) { return "\n ".concat(block(message), "\n\n Note: This feature is not implemented yet but it will be soon.\n\n If you want speed up the implementation or just read more, look here:\n https://github.com/webgptorg/promptbook\n\n Or contact us on me@pavolhejny.com\n\n "); })) || this;
|
|
2549
|
-
_this.name = 'NotYetImplementedError';
|
|
2550
|
-
Object.setPrototypeOf(_this, NotYetImplementedError.prototype);
|
|
2551
|
-
return _this;
|
|
2552
|
-
}
|
|
2553
|
-
return NotYetImplementedError;
|
|
2554
|
-
}(Error));
|
|
2555
|
-
|
|
2556
|
-
/**
|
|
2557
|
-
* Index of all custom errors
|
|
2558
|
-
*
|
|
2559
|
-
* @public exported from `@promptbook/core`
|
|
2560
|
-
*/
|
|
2561
|
-
var ERRORS = {
|
|
2562
|
-
ExpectError: ExpectError,
|
|
2563
|
-
CollectionError: CollectionError,
|
|
2564
|
-
EnvironmentMismatchError: EnvironmentMismatchError,
|
|
2565
|
-
LimitReachedError: LimitReachedError,
|
|
2566
|
-
NotFoundError: NotFoundError,
|
|
2567
|
-
NotYetImplementedError: NotYetImplementedError,
|
|
2568
|
-
ParseError: ParseError,
|
|
2569
|
-
PipelineExecutionError: PipelineExecutionError,
|
|
2570
|
-
PipelineLogicError: PipelineLogicError,
|
|
2571
|
-
PipelineUrlError: PipelineUrlError,
|
|
2572
|
-
UnexpectedError: UnexpectedError,
|
|
2573
|
-
// TODO: [🪑]> VersionMismatchError,
|
|
2574
|
-
};
|
|
2575
|
-
|
|
2576
|
-
/**
|
|
2577
|
-
* Deserializes the error object
|
|
2578
|
-
*
|
|
2579
|
-
* @public exported from `@promptbook/utils`
|
|
2580
|
-
*/
|
|
2581
|
-
function deserializeError(error) {
|
|
2582
|
-
if (error.name === 'Error') {
|
|
2583
|
-
return new Error(error.message);
|
|
2584
|
-
}
|
|
2585
|
-
var CustomError = ERRORS[error.name];
|
|
2586
|
-
return new CustomError(error.message);
|
|
2587
|
-
}
|
|
2588
|
-
|
|
2589
|
-
/**
|
|
2590
|
-
* Asserts that the execution of a Promptbook is successful
|
|
2591
|
-
*
|
|
2592
|
-
* @param executionResult - The partial result of the Promptbook execution
|
|
2593
|
-
* @throws {PipelineExecutionError} If the execution is not successful or if multiple errors occurred
|
|
2594
|
-
* @public exported from `@promptbook/core`
|
|
2595
|
-
*/
|
|
2596
|
-
function assertsExecutionSuccessful(executionResult) {
|
|
2597
|
-
var isSuccessful = executionResult.isSuccessful, errors = executionResult.errors;
|
|
2598
|
-
if (isSuccessful === true) {
|
|
2599
|
-
return;
|
|
2600
|
-
}
|
|
2601
|
-
if (errors.length === 0) {
|
|
2602
|
-
throw new PipelineExecutionError("Promptbook Execution failed because of unknown reason");
|
|
2603
|
-
}
|
|
2604
|
-
else if (errors.length === 1) {
|
|
2605
|
-
throw deserializeError(errors[0]);
|
|
2606
|
-
}
|
|
2607
|
-
else {
|
|
2608
|
-
throw new PipelineExecutionError(spaceTrim.spaceTrim(function (block) { return "\n Multiple errors occurred during Promptbook execution\n\n ".concat(block(errors
|
|
2609
|
-
.map(function (_a, index) {
|
|
2610
|
-
var name = _a.name, stack = _a.stack, message = _a.message;
|
|
2611
|
-
return spaceTrim.spaceTrim(function (block) { return "\n ".concat(name, " ").concat(index + 1, ":\n ").concat(block(stack || message), "\n "); });
|
|
2612
|
-
})
|
|
2613
|
-
.join('\n')), "\n "); }));
|
|
2614
|
-
}
|
|
2615
|
-
}
|
|
2616
|
-
/**
|
|
2617
|
-
* TODO: [🧠] Can this return type be better typed than void
|
|
2618
|
-
*/
|
|
2619
|
-
|
|
2620
|
-
/**
|
|
2621
|
-
* Determine if the pipeline is fully prepared
|
|
2622
|
-
*
|
|
2623
|
-
* @public exported from `@promptbook/core`
|
|
2624
|
-
*/
|
|
2625
|
-
function isPipelinePrepared(pipeline) {
|
|
2626
|
-
// Note: Ignoring `pipeline.preparations` @@@
|
|
2627
|
-
// Note: Ignoring `pipeline.knowledgePieces` @@@
|
|
2628
|
-
if (!pipeline.personas.every(function (persona) { return persona.modelRequirements !== undefined; })) {
|
|
2629
|
-
return false;
|
|
2630
|
-
}
|
|
2631
|
-
if (!pipeline.knowledgeSources.every(function (knowledgeSource) { return knowledgeSource.preparationIds !== undefined; })) {
|
|
2632
|
-
return false;
|
|
2633
|
-
}
|
|
2634
|
-
/*
|
|
2635
|
-
TODO: [🧠][🍫] `templates` can not be determined if they are fully prepared SO ignoring them
|
|
2636
|
-
> if (!pipeline.templates.every(({ preparedContent }) => preparedContent === undefined)) {
|
|
2637
|
-
> return false;
|
|
2638
|
-
> }
|
|
2639
|
-
*/
|
|
2640
|
-
return true;
|
|
2641
|
-
}
|
|
2642
|
-
/**
|
|
2643
|
-
* TODO: [🔃][main] !!!!! If the pipeline was prepared with different version or different set of models, prepare it once again
|
|
2644
|
-
* TODO: [🐠] Maybe base this on `makeValidator`
|
|
2645
|
-
* TODO: [🧊] Pipeline can be partially prepared, this should return true ONLY if fully prepared
|
|
2646
|
-
* TODO: [🧿] Maybe do same process with same granularity and subfinctions as `preparePipeline`
|
|
2647
|
-
* - [🏍] ? Is context in each template
|
|
2648
|
-
* - [♨] Are samples prepared
|
|
2649
|
-
* - [♨] Are templates prepared
|
|
2650
|
-
*/
|
|
2651
|
-
|
|
2652
|
-
/**
|
|
2653
|
-
* Serializes an error into a [🚉] JSON-serializable object
|
|
2654
|
-
*
|
|
2655
|
-
* @public exported from `@promptbook/utils`
|
|
2656
|
-
*/
|
|
2657
|
-
function serializeError(error) {
|
|
2658
|
-
var name = error.name, message = error.message, stack = error.stack;
|
|
2659
|
-
if (!__spreadArray(['Error'], __read(Object.keys(ERRORS)), false).includes(name)) {
|
|
2660
|
-
throw new UnexpectedError(spaceTrim__default["default"](function (block) { return "\n \n Cannot serialize error with name \"".concat(name, "\"\n\n ").concat(block(stack || message), "\n \n "); }));
|
|
2661
|
-
}
|
|
2662
|
-
return {
|
|
2663
|
-
name: name,
|
|
2664
|
-
message: message,
|
|
2665
|
-
stack: stack,
|
|
2666
|
-
};
|
|
2667
|
-
}
|
|
2668
|
-
|
|
2669
|
-
/**
|
|
2670
|
-
* Multiple LLM Execution Tools is a proxy server that uses multiple execution tools internally and exposes the executor interface externally.
|
|
2671
|
-
*
|
|
2672
|
-
* Note: Internal utility of `joinLlmExecutionTools` but exposed type
|
|
2673
|
-
* @public exported from `@promptbook/types`
|
|
2674
|
-
*/
|
|
2675
|
-
var MultipleLlmExecutionTools = /** @class */ (function () {
|
|
2676
|
-
/**
|
|
2677
|
-
* Gets array of execution tools in order of priority
|
|
2678
|
-
*/
|
|
2679
|
-
function MultipleLlmExecutionTools() {
|
|
2680
|
-
var llmExecutionTools = [];
|
|
2681
|
-
for (var _i = 0; _i < arguments.length; _i++) {
|
|
2682
|
-
llmExecutionTools[_i] = arguments[_i];
|
|
2683
|
-
}
|
|
2684
|
-
this.llmExecutionTools = llmExecutionTools;
|
|
2685
|
-
}
|
|
2686
|
-
Object.defineProperty(MultipleLlmExecutionTools.prototype, "title", {
|
|
2687
|
-
get: function () {
|
|
2688
|
-
return 'Multiple LLM Providers';
|
|
2689
|
-
},
|
|
2690
|
-
enumerable: false,
|
|
2691
|
-
configurable: true
|
|
2692
|
-
});
|
|
2693
|
-
Object.defineProperty(MultipleLlmExecutionTools.prototype, "description", {
|
|
2694
|
-
get: function () {
|
|
2695
|
-
return this.llmExecutionTools.map(function (_a, index) {
|
|
2696
|
-
var title = _a.title;
|
|
2697
|
-
return "".concat(index + 1, ") `").concat(title, "`");
|
|
2698
|
-
}).join('\n');
|
|
2699
|
-
},
|
|
2700
|
-
enumerable: false,
|
|
2701
|
-
configurable: true
|
|
2702
|
-
});
|
|
2703
|
-
/**
|
|
2704
|
-
* Check the configuration of all execution tools
|
|
2705
|
-
*/
|
|
2706
|
-
MultipleLlmExecutionTools.prototype.checkConfiguration = function () {
|
|
2707
|
-
return __awaiter(this, void 0, void 0, function () {
|
|
2708
|
-
var _a, _b, llmExecutionTools, e_1_1;
|
|
2709
|
-
var e_1, _c;
|
|
2710
|
-
return __generator(this, function (_d) {
|
|
2711
|
-
switch (_d.label) {
|
|
2712
|
-
case 0:
|
|
2713
|
-
_d.trys.push([0, 5, 6, 7]);
|
|
2714
|
-
_a = __values(this.llmExecutionTools), _b = _a.next();
|
|
2715
|
-
_d.label = 1;
|
|
2716
|
-
case 1:
|
|
2717
|
-
if (!!_b.done) return [3 /*break*/, 4];
|
|
2718
|
-
llmExecutionTools = _b.value;
|
|
2719
|
-
return [4 /*yield*/, llmExecutionTools.checkConfiguration()];
|
|
2720
|
-
case 2:
|
|
2721
|
-
_d.sent();
|
|
2722
|
-
_d.label = 3;
|
|
2723
|
-
case 3:
|
|
2724
|
-
_b = _a.next();
|
|
2725
|
-
return [3 /*break*/, 1];
|
|
2726
|
-
case 4: return [3 /*break*/, 7];
|
|
2727
|
-
case 5:
|
|
2728
|
-
e_1_1 = _d.sent();
|
|
2729
|
-
e_1 = { error: e_1_1 };
|
|
2730
|
-
return [3 /*break*/, 7];
|
|
2731
|
-
case 6:
|
|
2732
|
-
try {
|
|
2733
|
-
if (_b && !_b.done && (_c = _a.return)) _c.call(_a);
|
|
2734
|
-
}
|
|
2735
|
-
finally { if (e_1) throw e_1.error; }
|
|
2736
|
-
return [7 /*endfinally*/];
|
|
2737
|
-
case 7: return [2 /*return*/];
|
|
2738
|
-
}
|
|
2739
|
-
});
|
|
2740
|
-
});
|
|
2741
|
-
};
|
|
2742
|
-
/**
|
|
2743
|
-
* List all available models that can be used
|
|
2744
|
-
* This lists is a combination of all available models from all execution tools
|
|
2745
|
-
*/
|
|
2746
|
-
MultipleLlmExecutionTools.prototype.listModels = function () {
|
|
2747
|
-
return __awaiter(this, void 0, void 0, function () {
|
|
2748
|
-
var availableModels, _a, _b, llmExecutionTools, models, e_2_1;
|
|
2749
|
-
var e_2, _c;
|
|
2750
|
-
return __generator(this, function (_d) {
|
|
2751
|
-
switch (_d.label) {
|
|
2752
|
-
case 0:
|
|
2753
|
-
availableModels = [];
|
|
2754
|
-
_d.label = 1;
|
|
2755
|
-
case 1:
|
|
2756
|
-
_d.trys.push([1, 6, 7, 8]);
|
|
2757
|
-
_a = __values(this.llmExecutionTools), _b = _a.next();
|
|
2758
|
-
_d.label = 2;
|
|
2759
|
-
case 2:
|
|
2760
|
-
if (!!_b.done) return [3 /*break*/, 5];
|
|
2761
|
-
llmExecutionTools = _b.value;
|
|
2762
|
-
return [4 /*yield*/, llmExecutionTools.listModels()];
|
|
2763
|
-
case 3:
|
|
2764
|
-
models = _d.sent();
|
|
2765
|
-
availableModels.push.apply(availableModels, __spreadArray([], __read(models), false));
|
|
2766
|
-
_d.label = 4;
|
|
2767
|
-
case 4:
|
|
2768
|
-
_b = _a.next();
|
|
2769
|
-
return [3 /*break*/, 2];
|
|
2770
|
-
case 5: return [3 /*break*/, 8];
|
|
2771
|
-
case 6:
|
|
2772
|
-
e_2_1 = _d.sent();
|
|
2773
|
-
e_2 = { error: e_2_1 };
|
|
2774
|
-
return [3 /*break*/, 8];
|
|
2775
|
-
case 7:
|
|
2776
|
-
try {
|
|
2777
|
-
if (_b && !_b.done && (_c = _a.return)) _c.call(_a);
|
|
2778
|
-
}
|
|
2779
|
-
finally { if (e_2) throw e_2.error; }
|
|
2780
|
-
return [7 /*endfinally*/];
|
|
2781
|
-
case 8: return [2 /*return*/, availableModels];
|
|
1895
|
+
});
|
|
1896
|
+
};
|
|
1897
|
+
/**
|
|
1898
|
+
* List all available models that can be used
|
|
1899
|
+
* This lists is a combination of all available models from all execution tools
|
|
1900
|
+
*/
|
|
1901
|
+
MultipleLlmExecutionTools.prototype.listModels = function () {
|
|
1902
|
+
return __awaiter(this, void 0, void 0, function () {
|
|
1903
|
+
var availableModels, _a, _b, llmExecutionTools, models, e_2_1;
|
|
1904
|
+
var e_2, _c;
|
|
1905
|
+
return __generator(this, function (_d) {
|
|
1906
|
+
switch (_d.label) {
|
|
1907
|
+
case 0:
|
|
1908
|
+
availableModels = [];
|
|
1909
|
+
_d.label = 1;
|
|
1910
|
+
case 1:
|
|
1911
|
+
_d.trys.push([1, 6, 7, 8]);
|
|
1912
|
+
_a = __values(this.llmExecutionTools), _b = _a.next();
|
|
1913
|
+
_d.label = 2;
|
|
1914
|
+
case 2:
|
|
1915
|
+
if (!!_b.done) return [3 /*break*/, 5];
|
|
1916
|
+
llmExecutionTools = _b.value;
|
|
1917
|
+
return [4 /*yield*/, llmExecutionTools.listModels()];
|
|
1918
|
+
case 3:
|
|
1919
|
+
models = _d.sent();
|
|
1920
|
+
availableModels.push.apply(availableModels, __spreadArray([], __read(models), false));
|
|
1921
|
+
_d.label = 4;
|
|
1922
|
+
case 4:
|
|
1923
|
+
_b = _a.next();
|
|
1924
|
+
return [3 /*break*/, 2];
|
|
1925
|
+
case 5: return [3 /*break*/, 8];
|
|
1926
|
+
case 6:
|
|
1927
|
+
e_2_1 = _d.sent();
|
|
1928
|
+
e_2 = { error: e_2_1 };
|
|
1929
|
+
return [3 /*break*/, 8];
|
|
1930
|
+
case 7:
|
|
1931
|
+
try {
|
|
1932
|
+
if (_b && !_b.done && (_c = _a.return)) _c.call(_a);
|
|
1933
|
+
}
|
|
1934
|
+
finally { if (e_2) throw e_2.error; }
|
|
1935
|
+
return [7 /*endfinally*/];
|
|
1936
|
+
case 8: return [2 /*return*/, availableModels];
|
|
2782
1937
|
}
|
|
2783
1938
|
});
|
|
2784
1939
|
});
|
|
@@ -2958,25 +2113,6 @@
|
|
|
2958
2113
|
* TODO: [👷♂️] @@@ Manual about construction of llmTools
|
|
2959
2114
|
*/
|
|
2960
2115
|
|
|
2961
|
-
/**
|
|
2962
|
-
* Takes an item or an array of items and returns an array of items
|
|
2963
|
-
*
|
|
2964
|
-
* 1) Any item except array and undefined returns array with that one item (also null)
|
|
2965
|
-
* 2) Undefined returns empty array
|
|
2966
|
-
* 3) Array returns itself
|
|
2967
|
-
*
|
|
2968
|
-
* @private internal utility
|
|
2969
|
-
*/
|
|
2970
|
-
function arrayableToArray(input) {
|
|
2971
|
-
if (input === undefined) {
|
|
2972
|
-
return [];
|
|
2973
|
-
}
|
|
2974
|
-
if (input instanceof Array) {
|
|
2975
|
-
return input;
|
|
2976
|
-
}
|
|
2977
|
-
return [input];
|
|
2978
|
-
}
|
|
2979
|
-
|
|
2980
2116
|
/**
|
|
2981
2117
|
* @@@
|
|
2982
2118
|
*
|
|
@@ -2992,66 +2128,13 @@
|
|
|
2992
2128
|
> if (value && typeof value === 'object') {
|
|
2993
2129
|
> deepClone(value);
|
|
2994
2130
|
> }
|
|
2995
|
-
> }
|
|
2996
|
-
> return Object.assign({}, objectValue);
|
|
2997
|
-
*/
|
|
2998
|
-
}
|
|
2999
|
-
/**
|
|
3000
|
-
* TODO: [🧠] Is there a way how to meaningfully test this utility
|
|
3001
|
-
*/
|
|
3002
|
-
|
|
3003
|
-
/**
|
|
3004
|
-
* Represents the usage with no resources consumed
|
|
3005
|
-
*
|
|
3006
|
-
* @public exported from `@promptbook/core`
|
|
3007
|
-
*/
|
|
3008
|
-
var ZERO_USAGE = $deepFreeze({
|
|
3009
|
-
price: { value: 0 },
|
|
3010
|
-
input: {
|
|
3011
|
-
tokensCount: { value: 0 },
|
|
3012
|
-
charactersCount: { value: 0 },
|
|
3013
|
-
wordsCount: { value: 0 },
|
|
3014
|
-
sentencesCount: { value: 0 },
|
|
3015
|
-
linesCount: { value: 0 },
|
|
3016
|
-
paragraphsCount: { value: 0 },
|
|
3017
|
-
pagesCount: { value: 0 },
|
|
3018
|
-
},
|
|
3019
|
-
output: {
|
|
3020
|
-
tokensCount: { value: 0 },
|
|
3021
|
-
charactersCount: { value: 0 },
|
|
3022
|
-
wordsCount: { value: 0 },
|
|
3023
|
-
sentencesCount: { value: 0 },
|
|
3024
|
-
linesCount: { value: 0 },
|
|
3025
|
-
paragraphsCount: { value: 0 },
|
|
3026
|
-
pagesCount: { value: 0 },
|
|
3027
|
-
},
|
|
3028
|
-
});
|
|
2131
|
+
> }
|
|
2132
|
+
> return Object.assign({}, objectValue);
|
|
2133
|
+
*/
|
|
2134
|
+
}
|
|
3029
2135
|
/**
|
|
3030
|
-
*
|
|
3031
|
-
*
|
|
3032
|
-
* @public exported from `@promptbook/core`
|
|
2136
|
+
* TODO: [🧠] Is there a way how to meaningfully test this utility
|
|
3033
2137
|
*/
|
|
3034
|
-
var UNCERTAIN_USAGE = $deepFreeze({
|
|
3035
|
-
price: { value: 0, isUncertain: true },
|
|
3036
|
-
input: {
|
|
3037
|
-
tokensCount: { value: 0, isUncertain: true },
|
|
3038
|
-
charactersCount: { value: 0, isUncertain: true },
|
|
3039
|
-
wordsCount: { value: 0, isUncertain: true },
|
|
3040
|
-
sentencesCount: { value: 0, isUncertain: true },
|
|
3041
|
-
linesCount: { value: 0, isUncertain: true },
|
|
3042
|
-
paragraphsCount: { value: 0, isUncertain: true },
|
|
3043
|
-
pagesCount: { value: 0, isUncertain: true },
|
|
3044
|
-
},
|
|
3045
|
-
output: {
|
|
3046
|
-
tokensCount: { value: 0, isUncertain: true },
|
|
3047
|
-
charactersCount: { value: 0, isUncertain: true },
|
|
3048
|
-
wordsCount: { value: 0, isUncertain: true },
|
|
3049
|
-
sentencesCount: { value: 0, isUncertain: true },
|
|
3050
|
-
linesCount: { value: 0, isUncertain: true },
|
|
3051
|
-
paragraphsCount: { value: 0, isUncertain: true },
|
|
3052
|
-
pagesCount: { value: 0, isUncertain: true },
|
|
3053
|
-
},
|
|
3054
|
-
});
|
|
3055
2138
|
|
|
3056
2139
|
/**
|
|
3057
2140
|
* Function `addUsage` will add multiple usages into one
|
|
@@ -3125,6 +2208,292 @@
|
|
|
3125
2208
|
}, deepClone(ZERO_USAGE));
|
|
3126
2209
|
}
|
|
3127
2210
|
|
|
2211
|
+
/**
|
|
2212
|
+
* Intercepts LLM tools and counts total usage of the tools
|
|
2213
|
+
*
|
|
2214
|
+
* @param llmTools LLM tools to be intercepted with usage counting
|
|
2215
|
+
* @returns LLM tools with same functionality with added total cost counting
|
|
2216
|
+
* @public exported from `@promptbook/core`
|
|
2217
|
+
*/
|
|
2218
|
+
function countTotalUsage(llmTools) {
|
|
2219
|
+
var _this = this;
|
|
2220
|
+
var totalUsage = ZERO_USAGE;
|
|
2221
|
+
var proxyTools = {
|
|
2222
|
+
get title() {
|
|
2223
|
+
// TODO: [🧠] Maybe put here some suffix
|
|
2224
|
+
return llmTools.title;
|
|
2225
|
+
},
|
|
2226
|
+
get description() {
|
|
2227
|
+
// TODO: [🧠] Maybe put here some suffix
|
|
2228
|
+
return llmTools.description;
|
|
2229
|
+
},
|
|
2230
|
+
checkConfiguration: function () {
|
|
2231
|
+
return __awaiter(this, void 0, void 0, function () {
|
|
2232
|
+
return __generator(this, function (_a) {
|
|
2233
|
+
return [2 /*return*/, /* not await */ llmTools.checkConfiguration()];
|
|
2234
|
+
});
|
|
2235
|
+
});
|
|
2236
|
+
},
|
|
2237
|
+
listModels: function () {
|
|
2238
|
+
return /* not await */ llmTools.listModels();
|
|
2239
|
+
},
|
|
2240
|
+
getTotalUsage: function () {
|
|
2241
|
+
// <- Note: [🥫] Not using getter `get totalUsage` but `getTotalUsage` to allow this object to be proxied
|
|
2242
|
+
return totalUsage;
|
|
2243
|
+
},
|
|
2244
|
+
};
|
|
2245
|
+
if (llmTools.callChatModel !== undefined) {
|
|
2246
|
+
proxyTools.callChatModel = function (prompt) { return __awaiter(_this, void 0, void 0, function () {
|
|
2247
|
+
var promptResult;
|
|
2248
|
+
return __generator(this, function (_a) {
|
|
2249
|
+
switch (_a.label) {
|
|
2250
|
+
case 0: return [4 /*yield*/, llmTools.callChatModel(prompt)];
|
|
2251
|
+
case 1:
|
|
2252
|
+
promptResult = _a.sent();
|
|
2253
|
+
totalUsage = addUsage(totalUsage, promptResult.usage);
|
|
2254
|
+
return [2 /*return*/, promptResult];
|
|
2255
|
+
}
|
|
2256
|
+
});
|
|
2257
|
+
}); };
|
|
2258
|
+
}
|
|
2259
|
+
if (llmTools.callCompletionModel !== undefined) {
|
|
2260
|
+
proxyTools.callCompletionModel = function (prompt) { return __awaiter(_this, void 0, void 0, function () {
|
|
2261
|
+
var promptResult;
|
|
2262
|
+
return __generator(this, function (_a) {
|
|
2263
|
+
switch (_a.label) {
|
|
2264
|
+
case 0: return [4 /*yield*/, llmTools.callCompletionModel(prompt)];
|
|
2265
|
+
case 1:
|
|
2266
|
+
promptResult = _a.sent();
|
|
2267
|
+
totalUsage = addUsage(totalUsage, promptResult.usage);
|
|
2268
|
+
return [2 /*return*/, promptResult];
|
|
2269
|
+
}
|
|
2270
|
+
});
|
|
2271
|
+
}); };
|
|
2272
|
+
}
|
|
2273
|
+
if (llmTools.callEmbeddingModel !== undefined) {
|
|
2274
|
+
proxyTools.callEmbeddingModel = function (prompt) { return __awaiter(_this, void 0, void 0, function () {
|
|
2275
|
+
var promptResult;
|
|
2276
|
+
return __generator(this, function (_a) {
|
|
2277
|
+
switch (_a.label) {
|
|
2278
|
+
case 0: return [4 /*yield*/, llmTools.callEmbeddingModel(prompt)];
|
|
2279
|
+
case 1:
|
|
2280
|
+
promptResult = _a.sent();
|
|
2281
|
+
totalUsage = addUsage(totalUsage, promptResult.usage);
|
|
2282
|
+
return [2 /*return*/, promptResult];
|
|
2283
|
+
}
|
|
2284
|
+
});
|
|
2285
|
+
}); };
|
|
2286
|
+
}
|
|
2287
|
+
// <- Note: [🤖]
|
|
2288
|
+
return proxyTools;
|
|
2289
|
+
}
|
|
2290
|
+
/**
|
|
2291
|
+
* TODO: [🧠][💸] Maybe make some common abstraction `interceptLlmTools` and use here (or use javascript Proxy?)
|
|
2292
|
+
* TODO: [🧠] Is there some meaningfull way how to test this util
|
|
2293
|
+
* TODO: [🧠][🌯] Maybe a way how to hide ability to `get totalUsage`
|
|
2294
|
+
* > const [llmToolsWithUsage,getUsage] = countTotalUsage(llmTools);
|
|
2295
|
+
* TODO: [👷♂️] @@@ Manual about construction of llmTools
|
|
2296
|
+
*/
|
|
2297
|
+
|
|
2298
|
+
var PipelineCollection = [{title:"Prepare Knowledge from Markdown",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.ptbk.md",parameters:[{name:"knowledgeContent",description:"Markdown document content",isInput:true,isOutput:false},{name:"knowledgePieces",description:"The knowledge JSON object",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, extract the important knowledge from the document.\n\n# Rules\n\n- Make pieces of information concise, clear, and easy to understand\n- One piece of information should be approximately 1 paragraph\n- Divide the paragraphs by markdown horizontal lines ---\n- Omit irrelevant information\n- Group redundant information\n- Write just extracted information, nothing else\n\n# The document\n\nTake information from this document:\n\n> {knowledgeContent}",resultingParameterName:"knowledgePieces",dependentParameterNames:["knowledgeContent"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-knowledge-from-markdown.ptbk.md"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-keywords.ptbk.md",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"keywords",description:"Keywords separated by comma",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, detect the important keywords in the document.\n\n# Rules\n\n- Write just keywords separated by comma\n\n# The document\n\nTake information from this document:\n\n> {knowledgePieceContent}",resultingParameterName:"keywords",dependentParameterNames:["knowledgePieceContent"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-knowledge-keywords.ptbk.md"},{title:"Prepare Title",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-title.ptbk.md",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"title",description:"The title of the document",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"knowledge",title:"Knowledge",content:"You are experienced content creator, write best title for the document.\n\n# Rules\n\n- Write just title, nothing else\n- Title should be concise and clear\n- Write maximum 5 words for the title\n\n# The document\n\n> {knowledgePieceContent}",resultingParameterName:"title",expectations:{words:{min:1,max:8}},dependentParameterNames:["knowledgePieceContent"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-knowledge-title.ptbk.md"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-persona.ptbk.md",parameters:[{name:"availableModelNames",description:"List of available model names separated by comma (,)",isInput:true,isOutput:false},{name:"personaDescription",description:"Description of the persona",isInput:true,isOutput:false},{name:"modelRequirements",description:"Specific requirements for the model",isInput:false,isOutput:true}],templates:[{templateType:"PROMPT_TEMPLATE",name:"make-model-requirements",title:"Make modelRequirements",content:"You are experienced AI engineer, you need to create virtual assistant.\nWrite\n\n## Sample\n\n```json\n{\n\"modelName\": \"gpt-4o\",\n\"systemMessage\": \"You are experienced AI engineer and helpfull assistant.\",\n\"temperature\": 0.7\n}\n```\n\n## Instructions\n\n- Your output format is JSON object\n- Write just the JSON object, no other text should be present\n- It contains the following keys:\n - `modelName`: The name of the model to use\n - `systemMessage`: The system message to provide context to the model\n - `temperature`: The sampling temperature to use\n\n### Key `modelName`\n\nPick from the following models:\n\n- {availableModelNames}\n\n### Key `systemMessage`\n\nThe system message is used to communicate instructions or provide context to the model at the beginning of a conversation. It is displayed in a different format compared to user messages, helping the model understand its role in the conversation. The system message typically guides the model's behavior, sets the tone, or specifies desired output from the model. By utilizing the system message effectively, users can steer the model towards generating more accurate and relevant responses.\n\nFor example:\n\n> You are an experienced AI engineer and helpful assistant.\n\n> You are a friendly and knowledgeable chatbot.\n\n### Key `temperature`\n\nThe sampling temperature, between 0 and 1. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. If set to 0, the model will use log probability to automatically increase the temperature until certain thresholds are hit.\n\nYou can pick a value between 0 and 2. For example:\n\n- `0.1`: Low temperature, extremely conservative and deterministic\n- `0.5`: Medium temperature, balanced between conservative and creative\n- `1.0`: High temperature, creative and bit random\n- `1.5`: Very high temperature, extremely creative and often chaotic and unpredictable\n- `2.0`: Maximum temperature, completely random and unpredictable, for some extreme creative use cases\n\n# The assistant\n\nTake this description of the persona:\n\n> {personaDescription}",resultingParameterName:"modelRequirements",format:"JSON",dependentParameterNames:["availableModelNames","personaDescription"]}],knowledgeSources:[],knowledgePieces:[],personas:[],preparations:[],sourceFile:"./promptbook-collection/prepare-persona.ptbk.md"}];
|
|
2299
|
+
|
|
2300
|
+
/**
|
|
2301
|
+
* This error indicates that the pipeline collection cannot be propperly loaded
|
|
2302
|
+
*
|
|
2303
|
+
* @public exported from `@promptbook/core`
|
|
2304
|
+
*/
|
|
2305
|
+
var CollectionError = /** @class */ (function (_super) {
|
|
2306
|
+
__extends(CollectionError, _super);
|
|
2307
|
+
function CollectionError(message) {
|
|
2308
|
+
var _this = _super.call(this, message) || this;
|
|
2309
|
+
_this.name = 'CollectionError';
|
|
2310
|
+
Object.setPrototypeOf(_this, CollectionError.prototype);
|
|
2311
|
+
return _this;
|
|
2312
|
+
}
|
|
2313
|
+
return CollectionError;
|
|
2314
|
+
}(Error));
|
|
2315
|
+
|
|
2316
|
+
/**
|
|
2317
|
+
* This error type indicates that you try to use a feature that is not available in the current environment
|
|
2318
|
+
*
|
|
2319
|
+
* @public exported from `@promptbook/core`
|
|
2320
|
+
*/
|
|
2321
|
+
var EnvironmentMismatchError = /** @class */ (function (_super) {
|
|
2322
|
+
__extends(EnvironmentMismatchError, _super);
|
|
2323
|
+
function EnvironmentMismatchError(message) {
|
|
2324
|
+
var _this = _super.call(this, message) || this;
|
|
2325
|
+
_this.name = 'EnvironmentMismatchError';
|
|
2326
|
+
Object.setPrototypeOf(_this, EnvironmentMismatchError.prototype);
|
|
2327
|
+
return _this;
|
|
2328
|
+
}
|
|
2329
|
+
return EnvironmentMismatchError;
|
|
2330
|
+
}(Error));
|
|
2331
|
+
|
|
2332
|
+
/**
|
|
2333
|
+
* This error occurs when some expectation is not met in the execution of the pipeline
|
|
2334
|
+
*
|
|
2335
|
+
* @public exported from `@promptbook/core`
|
|
2336
|
+
* Note: Do not throw this error, its reserved for `checkExpectations` and `createPipelineExecutor` and public ONLY to be serializable through remote server
|
|
2337
|
+
* Note: Always thrown in `checkExpectations` and catched in `createPipelineExecutor` and rethrown as `PipelineExecutionError`
|
|
2338
|
+
* Note: This is a kindof subtype of PipelineExecutionError
|
|
2339
|
+
*/
|
|
2340
|
+
var ExpectError = /** @class */ (function (_super) {
|
|
2341
|
+
__extends(ExpectError, _super);
|
|
2342
|
+
function ExpectError(message) {
|
|
2343
|
+
var _this = _super.call(this, message) || this;
|
|
2344
|
+
_this.name = 'ExpectError';
|
|
2345
|
+
Object.setPrototypeOf(_this, ExpectError.prototype);
|
|
2346
|
+
return _this;
|
|
2347
|
+
}
|
|
2348
|
+
return ExpectError;
|
|
2349
|
+
}(Error));
|
|
2350
|
+
|
|
2351
|
+
/**
|
|
2352
|
+
* This error type indicates that some limit was reached
|
|
2353
|
+
*
|
|
2354
|
+
* @public exported from `@promptbook/core`
|
|
2355
|
+
*/
|
|
2356
|
+
var LimitReachedError = /** @class */ (function (_super) {
|
|
2357
|
+
__extends(LimitReachedError, _super);
|
|
2358
|
+
function LimitReachedError(message) {
|
|
2359
|
+
var _this = _super.call(this, message) || this;
|
|
2360
|
+
_this.name = 'LimitReachedError';
|
|
2361
|
+
Object.setPrototypeOf(_this, LimitReachedError.prototype);
|
|
2362
|
+
return _this;
|
|
2363
|
+
}
|
|
2364
|
+
return LimitReachedError;
|
|
2365
|
+
}(Error));
|
|
2366
|
+
|
|
2367
|
+
/**
|
|
2368
|
+
* This error type indicates that some part of the code is not implemented yet
|
|
2369
|
+
*
|
|
2370
|
+
* @public exported from `@promptbook/core`
|
|
2371
|
+
*/
|
|
2372
|
+
var NotYetImplementedError = /** @class */ (function (_super) {
|
|
2373
|
+
__extends(NotYetImplementedError, _super);
|
|
2374
|
+
function NotYetImplementedError(message) {
|
|
2375
|
+
var _this = _super.call(this, spaceTrim.spaceTrim(function (block) { return "\n ".concat(block(message), "\n\n Note: This feature is not implemented yet but it will be soon.\n\n If you want speed up the implementation or just read more, look here:\n https://github.com/webgptorg/promptbook\n\n Or contact us on me@pavolhejny.com\n\n "); })) || this;
|
|
2376
|
+
_this.name = 'NotYetImplementedError';
|
|
2377
|
+
Object.setPrototypeOf(_this, NotYetImplementedError.prototype);
|
|
2378
|
+
return _this;
|
|
2379
|
+
}
|
|
2380
|
+
return NotYetImplementedError;
|
|
2381
|
+
}(Error));
|
|
2382
|
+
|
|
2383
|
+
/**
|
|
2384
|
+
* Index of all custom errors
|
|
2385
|
+
*
|
|
2386
|
+
* @public exported from `@promptbook/core`
|
|
2387
|
+
*/
|
|
2388
|
+
var ERRORS = {
|
|
2389
|
+
ExpectError: ExpectError,
|
|
2390
|
+
CollectionError: CollectionError,
|
|
2391
|
+
EnvironmentMismatchError: EnvironmentMismatchError,
|
|
2392
|
+
LimitReachedError: LimitReachedError,
|
|
2393
|
+
NotFoundError: NotFoundError,
|
|
2394
|
+
NotYetImplementedError: NotYetImplementedError,
|
|
2395
|
+
ParseError: ParseError,
|
|
2396
|
+
PipelineExecutionError: PipelineExecutionError,
|
|
2397
|
+
PipelineLogicError: PipelineLogicError,
|
|
2398
|
+
PipelineUrlError: PipelineUrlError,
|
|
2399
|
+
UnexpectedError: UnexpectedError,
|
|
2400
|
+
// TODO: [🪑]> VersionMismatchError,
|
|
2401
|
+
};
|
|
2402
|
+
|
|
2403
|
+
/**
|
|
2404
|
+
* Deserializes the error object
|
|
2405
|
+
*
|
|
2406
|
+
* @public exported from `@promptbook/utils`
|
|
2407
|
+
*/
|
|
2408
|
+
function deserializeError(error) {
|
|
2409
|
+
if (error.name === 'Error') {
|
|
2410
|
+
return new Error(error.message);
|
|
2411
|
+
}
|
|
2412
|
+
var CustomError = ERRORS[error.name];
|
|
2413
|
+
return new CustomError(error.message);
|
|
2414
|
+
}
|
|
2415
|
+
|
|
2416
|
+
/**
|
|
2417
|
+
* Asserts that the execution of a Promptbook is successful
|
|
2418
|
+
*
|
|
2419
|
+
* @param executionResult - The partial result of the Promptbook execution
|
|
2420
|
+
* @throws {PipelineExecutionError} If the execution is not successful or if multiple errors occurred
|
|
2421
|
+
* @public exported from `@promptbook/core`
|
|
2422
|
+
*/
|
|
2423
|
+
function assertsExecutionSuccessful(executionResult) {
|
|
2424
|
+
var isSuccessful = executionResult.isSuccessful, errors = executionResult.errors;
|
|
2425
|
+
if (isSuccessful === true) {
|
|
2426
|
+
return;
|
|
2427
|
+
}
|
|
2428
|
+
if (errors.length === 0) {
|
|
2429
|
+
throw new PipelineExecutionError("Promptbook Execution failed because of unknown reason");
|
|
2430
|
+
}
|
|
2431
|
+
else if (errors.length === 1) {
|
|
2432
|
+
throw deserializeError(errors[0]);
|
|
2433
|
+
}
|
|
2434
|
+
else {
|
|
2435
|
+
throw new PipelineExecutionError(spaceTrim.spaceTrim(function (block) { return "\n Multiple errors occurred during Promptbook execution\n\n ".concat(block(errors
|
|
2436
|
+
.map(function (_a, index) {
|
|
2437
|
+
var name = _a.name, stack = _a.stack, message = _a.message;
|
|
2438
|
+
return spaceTrim.spaceTrim(function (block) { return "\n ".concat(name, " ").concat(index + 1, ":\n ").concat(block(stack || message), "\n "); });
|
|
2439
|
+
})
|
|
2440
|
+
.join('\n')), "\n "); }));
|
|
2441
|
+
}
|
|
2442
|
+
}
|
|
2443
|
+
/**
|
|
2444
|
+
* TODO: [🐚] This function should be removed OR changed OR be completely rewritten
|
|
2445
|
+
* TODO: [🧠] Can this return type be better typed than void
|
|
2446
|
+
*/
|
|
2447
|
+
|
|
2448
|
+
/**
|
|
2449
|
+
* Determine if the pipeline is fully prepared
|
|
2450
|
+
*
|
|
2451
|
+
* @public exported from `@promptbook/core`
|
|
2452
|
+
*/
|
|
2453
|
+
function isPipelinePrepared(pipeline) {
|
|
2454
|
+
// Note: Ignoring `pipeline.preparations` @@@
|
|
2455
|
+
// Note: Ignoring `pipeline.knowledgePieces` @@@
|
|
2456
|
+
if (!pipeline.personas.every(function (persona) { return persona.modelRequirements !== undefined; })) {
|
|
2457
|
+
return false;
|
|
2458
|
+
}
|
|
2459
|
+
if (!pipeline.knowledgeSources.every(function (knowledgeSource) { return knowledgeSource.preparationIds !== undefined; })) {
|
|
2460
|
+
return false;
|
|
2461
|
+
}
|
|
2462
|
+
/*
|
|
2463
|
+
TODO: [🧠][🍫] `templates` can not be determined if they are fully prepared SO ignoring them
|
|
2464
|
+
> if (!pipeline.templates.every(({ preparedContent }) => preparedContent === undefined)) {
|
|
2465
|
+
> return false;
|
|
2466
|
+
> }
|
|
2467
|
+
*/
|
|
2468
|
+
return true;
|
|
2469
|
+
}
|
|
2470
|
+
/**
|
|
2471
|
+
* TODO: [🔃][main] !!!!! If the pipeline was prepared with different version or different set of models, prepare it once again
|
|
2472
|
+
* TODO: [🐠] Maybe base this on `makeValidator`
|
|
2473
|
+
* TODO: [🧊] Pipeline can be partially prepared, this should return true ONLY if fully prepared
|
|
2474
|
+
* TODO: [🧿] Maybe do same process with same granularity and subfinctions as `preparePipeline`
|
|
2475
|
+
* - [🏍] ? Is context in each template
|
|
2476
|
+
* - [♨] Are samples prepared
|
|
2477
|
+
* - [♨] Are templates prepared
|
|
2478
|
+
*/
|
|
2479
|
+
|
|
2480
|
+
/**
|
|
2481
|
+
* Serializes an error into a [🚉] JSON-serializable object
|
|
2482
|
+
*
|
|
2483
|
+
* @public exported from `@promptbook/utils`
|
|
2484
|
+
*/
|
|
2485
|
+
function serializeError(error) {
|
|
2486
|
+
var name = error.name, message = error.message, stack = error.stack;
|
|
2487
|
+
if (!__spreadArray(['Error'], __read(Object.keys(ERRORS)), false).includes(name)) {
|
|
2488
|
+
throw new UnexpectedError(spaceTrim__default["default"](function (block) { return "\n \n Cannot serialize error with name \"".concat(name, "\"\n\n ").concat(block(stack || message), "\n \n "); }));
|
|
2489
|
+
}
|
|
2490
|
+
return {
|
|
2491
|
+
name: name,
|
|
2492
|
+
message: message,
|
|
2493
|
+
stack: stack,
|
|
2494
|
+
};
|
|
2495
|
+
}
|
|
2496
|
+
|
|
3128
2497
|
/**
|
|
3129
2498
|
* Parses the given script and returns the list of all used variables that are not defined in the script
|
|
3130
2499
|
*
|
|
@@ -3330,6 +2699,23 @@
|
|
|
3330
2699
|
return union;
|
|
3331
2700
|
}
|
|
3332
2701
|
|
|
2702
|
+
/**
|
|
2703
|
+
* Just marks a place of place where should be something implemented
|
|
2704
|
+
* No side effects.
|
|
2705
|
+
*
|
|
2706
|
+
* Note: It can be usefull suppressing eslint errors of unused variables
|
|
2707
|
+
*
|
|
2708
|
+
* @param value any values
|
|
2709
|
+
* @returns void
|
|
2710
|
+
* @private within the repository
|
|
2711
|
+
*/
|
|
2712
|
+
function TODO_USE() {
|
|
2713
|
+
var value = [];
|
|
2714
|
+
for (var _i = 0; _i < arguments.length; _i++) {
|
|
2715
|
+
value[_i] = arguments[_i];
|
|
2716
|
+
}
|
|
2717
|
+
}
|
|
2718
|
+
|
|
3333
2719
|
/**
|
|
3334
2720
|
* This error indicates problems parsing the format value
|
|
3335
2721
|
*
|
|
@@ -3843,6 +3229,25 @@
|
|
|
3843
3229
|
* TODO: [🏢] Make this logic part of `JsonFormatDefinition` or `isValidJsonString`
|
|
3844
3230
|
*/
|
|
3845
3231
|
|
|
3232
|
+
/**
|
|
3233
|
+
* Takes an item or an array of items and returns an array of items
|
|
3234
|
+
*
|
|
3235
|
+
* 1) Any item except array and undefined returns array with that one item (also null)
|
|
3236
|
+
* 2) Undefined returns empty array
|
|
3237
|
+
* 3) Array returns itself
|
|
3238
|
+
*
|
|
3239
|
+
* @private internal utility
|
|
3240
|
+
*/
|
|
3241
|
+
function arrayableToArray(input) {
|
|
3242
|
+
if (input === undefined) {
|
|
3243
|
+
return [];
|
|
3244
|
+
}
|
|
3245
|
+
if (input instanceof Array) {
|
|
3246
|
+
return input;
|
|
3247
|
+
}
|
|
3248
|
+
return [input];
|
|
3249
|
+
}
|
|
3250
|
+
|
|
3846
3251
|
/**
|
|
3847
3252
|
* Just says that the variable is not used but should be kept
|
|
3848
3253
|
* No side effects.
|
|
@@ -3944,69 +3349,330 @@
|
|
|
3944
3349
|
}
|
|
3945
3350
|
|
|
3946
3351
|
/**
|
|
3947
|
-
* Counts number of characters in the text
|
|
3352
|
+
* Counts number of characters in the text
|
|
3353
|
+
*
|
|
3354
|
+
* @public exported from `@promptbook/utils`
|
|
3355
|
+
*/
|
|
3356
|
+
function countCharacters(text) {
|
|
3357
|
+
// Remove null characters
|
|
3358
|
+
text = text.replace(/\0/g, '');
|
|
3359
|
+
// Replace emojis (and also ZWJ sequence) with hyphens
|
|
3360
|
+
text = text.replace(/(\p{Extended_Pictographic})\p{Modifier_Symbol}/gu, '$1');
|
|
3361
|
+
text = text.replace(/(\p{Extended_Pictographic})[\u{FE00}-\u{FE0F}]/gu, '$1');
|
|
3362
|
+
text = text.replace(/\p{Extended_Pictographic}(\u{200D}\p{Extended_Pictographic})*/gu, '-');
|
|
3363
|
+
return text.length;
|
|
3364
|
+
}
|
|
3365
|
+
|
|
3366
|
+
/**
|
|
3367
|
+
* Counts number of lines in the text
|
|
3368
|
+
*
|
|
3369
|
+
* @public exported from `@promptbook/utils`
|
|
3370
|
+
*/
|
|
3371
|
+
function countLines(text) {
|
|
3372
|
+
if (text === '') {
|
|
3373
|
+
return 0;
|
|
3374
|
+
}
|
|
3375
|
+
return text.split('\n').length;
|
|
3376
|
+
}
|
|
3377
|
+
|
|
3378
|
+
/**
|
|
3379
|
+
* Counts number of pages in the text
|
|
3380
|
+
*
|
|
3381
|
+
* @public exported from `@promptbook/utils`
|
|
3382
|
+
*/
|
|
3383
|
+
function countPages(text) {
|
|
3384
|
+
var sentencesPerPage = 5; // Assuming each page has 5 sentences
|
|
3385
|
+
var sentences = text.split(/[.!?]+/).filter(function (sentence) { return sentence.trim() !== ''; });
|
|
3386
|
+
var pageCount = Math.ceil(sentences.length / sentencesPerPage);
|
|
3387
|
+
return pageCount;
|
|
3388
|
+
}
|
|
3389
|
+
|
|
3390
|
+
/**
|
|
3391
|
+
* Counts number of paragraphs in the text
|
|
3392
|
+
*
|
|
3393
|
+
* @public exported from `@promptbook/utils`
|
|
3394
|
+
*/
|
|
3395
|
+
function countParagraphs(text) {
|
|
3396
|
+
return text.split(/\n\s*\n/).filter(function (paragraph) { return paragraph.trim() !== ''; }).length;
|
|
3397
|
+
}
|
|
3398
|
+
|
|
3399
|
+
/**
|
|
3400
|
+
* Split text into sentences
|
|
3401
|
+
*
|
|
3402
|
+
* @public exported from `@promptbook/utils`
|
|
3403
|
+
*/
|
|
3404
|
+
function splitIntoSentences(text) {
|
|
3405
|
+
return text.split(/[.!?]+/).filter(function (sentence) { return sentence.trim() !== ''; });
|
|
3406
|
+
}
|
|
3407
|
+
/**
|
|
3408
|
+
* Counts number of sentences in the text
|
|
3409
|
+
*
|
|
3410
|
+
* @public exported from `@promptbook/utils`
|
|
3411
|
+
*/
|
|
3412
|
+
function countSentences(text) {
|
|
3413
|
+
return splitIntoSentences(text).length;
|
|
3414
|
+
}
|
|
3415
|
+
|
|
3416
|
+
var defaultDiacriticsRemovalMap = [
|
|
3417
|
+
{
|
|
3418
|
+
base: 'A',
|
|
3419
|
+
letters: '\u0041\u24B6\uFF21\u00C0\u00C1\u00C2\u1EA6\u1EA4\u1EAA\u1EA8\u00C3\u0100\u0102\u1EB0\u1EAE\u1EB4\u1EB2\u0226\u01E0\u00C4\u01DE\u1EA2\u00C5\u01FA\u01CD\u0200\u0202\u1EA0\u1EAC\u1EB6\u1E00\u0104\u023A\u2C6F',
|
|
3420
|
+
},
|
|
3421
|
+
{ base: 'AA', letters: '\uA732' },
|
|
3422
|
+
{ base: 'AE', letters: '\u00C6\u01FC\u01E2' },
|
|
3423
|
+
{ base: 'AO', letters: '\uA734' },
|
|
3424
|
+
{ base: 'AU', letters: '\uA736' },
|
|
3425
|
+
{ base: 'AV', letters: '\uA738\uA73A' },
|
|
3426
|
+
{ base: 'AY', letters: '\uA73C' },
|
|
3427
|
+
{
|
|
3428
|
+
base: 'B',
|
|
3429
|
+
letters: '\u0042\u24B7\uFF22\u1E02\u1E04\u1E06\u0243\u0182\u0181',
|
|
3430
|
+
},
|
|
3431
|
+
{
|
|
3432
|
+
base: 'C',
|
|
3433
|
+
letters: '\u0043\u24B8\uFF23\u0106\u0108\u010A\u010C\u00C7\u1E08\u0187\u023B\uA73E',
|
|
3434
|
+
},
|
|
3435
|
+
{
|
|
3436
|
+
base: 'D',
|
|
3437
|
+
letters: '\u0044\u24B9\uFF24\u1E0A\u010E\u1E0C\u1E10\u1E12\u1E0E\u0110\u018B\u018A\u0189\uA779\u00D0',
|
|
3438
|
+
},
|
|
3439
|
+
{ base: 'DZ', letters: '\u01F1\u01C4' },
|
|
3440
|
+
{ base: 'Dz', letters: '\u01F2\u01C5' },
|
|
3441
|
+
{
|
|
3442
|
+
base: 'E',
|
|
3443
|
+
letters: '\u0045\u24BA\uFF25\u00C8\u00C9\u00CA\u1EC0\u1EBE\u1EC4\u1EC2\u1EBC\u0112\u1E14\u1E16\u0114\u0116\u00CB\u1EBA\u011A\u0204\u0206\u1EB8\u1EC6\u0228\u1E1C\u0118\u1E18\u1E1A\u0190\u018E',
|
|
3444
|
+
},
|
|
3445
|
+
{ base: 'F', letters: '\u0046\u24BB\uFF26\u1E1E\u0191\uA77B' },
|
|
3446
|
+
{
|
|
3447
|
+
base: 'G',
|
|
3448
|
+
letters: '\u0047\u24BC\uFF27\u01F4\u011C\u1E20\u011E\u0120\u01E6\u0122\u01E4\u0193\uA7A0\uA77D\uA77E',
|
|
3449
|
+
},
|
|
3450
|
+
{
|
|
3451
|
+
base: 'H',
|
|
3452
|
+
letters: '\u0048\u24BD\uFF28\u0124\u1E22\u1E26\u021E\u1E24\u1E28\u1E2A\u0126\u2C67\u2C75\uA78D',
|
|
3453
|
+
},
|
|
3454
|
+
{
|
|
3455
|
+
base: 'I',
|
|
3456
|
+
letters: '\u0049\u24BE\uFF29\u00CC\u00CD\u00CE\u0128\u012A\u012C\u0130\u00CF\u1E2E\u1EC8\u01CF\u0208\u020A\u1ECA\u012E\u1E2C\u0197',
|
|
3457
|
+
},
|
|
3458
|
+
{ base: 'J', letters: '\u004A\u24BF\uFF2A\u0134\u0248' },
|
|
3459
|
+
{
|
|
3460
|
+
base: 'K',
|
|
3461
|
+
letters: '\u004B\u24C0\uFF2B\u1E30\u01E8\u1E32\u0136\u1E34\u0198\u2C69\uA740\uA742\uA744\uA7A2',
|
|
3462
|
+
},
|
|
3463
|
+
{
|
|
3464
|
+
base: 'L',
|
|
3465
|
+
letters: '\u004C\u24C1\uFF2C\u013F\u0139\u013D\u1E36\u1E38\u013B\u1E3C\u1E3A\u0141\u023D\u2C62\u2C60\uA748\uA746\uA780',
|
|
3466
|
+
},
|
|
3467
|
+
{ base: 'LJ', letters: '\u01C7' },
|
|
3468
|
+
{ base: 'Lj', letters: '\u01C8' },
|
|
3469
|
+
{ base: 'M', letters: '\u004D\u24C2\uFF2D\u1E3E\u1E40\u1E42\u2C6E\u019C' },
|
|
3470
|
+
{
|
|
3471
|
+
base: 'N',
|
|
3472
|
+
letters: '\u004E\u24C3\uFF2E\u01F8\u0143\u00D1\u1E44\u0147\u1E46\u0145\u1E4A\u1E48\u0220\u019D\uA790\uA7A4',
|
|
3473
|
+
},
|
|
3474
|
+
{ base: 'NJ', letters: '\u01CA' },
|
|
3475
|
+
{ base: 'Nj', letters: '\u01CB' },
|
|
3476
|
+
{
|
|
3477
|
+
base: 'O',
|
|
3478
|
+
letters: '\u004F\u24C4\uFF2F\u00D2\u00D3\u00D4\u1ED2\u1ED0\u1ED6\u1ED4\u00D5\u1E4C\u022C\u1E4E\u014C\u1E50\u1E52\u014E\u022E\u0230\u00D6\u022A\u1ECE\u0150\u01D1\u020C\u020E\u01A0\u1EDC\u1EDA\u1EE0\u1EDE\u1EE2\u1ECC\u1ED8\u01EA\u01EC\u00D8\u01FE\u0186\u019F\uA74A\uA74C',
|
|
3479
|
+
},
|
|
3480
|
+
{ base: 'OI', letters: '\u01A2' },
|
|
3481
|
+
{ base: 'OO', letters: '\uA74E' },
|
|
3482
|
+
{ base: 'OU', letters: '\u0222' },
|
|
3483
|
+
{ base: 'OE', letters: '\u008C\u0152' },
|
|
3484
|
+
{ base: 'oe', letters: '\u009C\u0153' },
|
|
3485
|
+
{
|
|
3486
|
+
base: 'P',
|
|
3487
|
+
letters: '\u0050\u24C5\uFF30\u1E54\u1E56\u01A4\u2C63\uA750\uA752\uA754',
|
|
3488
|
+
},
|
|
3489
|
+
{ base: 'Q', letters: '\u0051\u24C6\uFF31\uA756\uA758\u024A' },
|
|
3490
|
+
{
|
|
3491
|
+
base: 'R',
|
|
3492
|
+
letters: '\u0052\u24C7\uFF32\u0154\u1E58\u0158\u0210\u0212\u1E5A\u1E5C\u0156\u1E5E\u024C\u2C64\uA75A\uA7A6\uA782',
|
|
3493
|
+
},
|
|
3494
|
+
{
|
|
3495
|
+
base: 'S',
|
|
3496
|
+
letters: '\u0053\u24C8\uFF33\u1E9E\u015A\u1E64\u015C\u1E60\u0160\u1E66\u1E62\u1E68\u0218\u015E\u2C7E\uA7A8\uA784',
|
|
3497
|
+
},
|
|
3498
|
+
{
|
|
3499
|
+
base: 'T',
|
|
3500
|
+
letters: '\u0054\u24C9\uFF34\u1E6A\u0164\u1E6C\u021A\u0162\u1E70\u1E6E\u0166\u01AC\u01AE\u023E\uA786',
|
|
3501
|
+
},
|
|
3502
|
+
{ base: 'TZ', letters: '\uA728' },
|
|
3503
|
+
{
|
|
3504
|
+
base: 'U',
|
|
3505
|
+
letters: '\u0055\u24CA\uFF35\u00D9\u00DA\u00DB\u0168\u1E78\u016A\u1E7A\u016C\u00DC\u01DB\u01D7\u01D5\u01D9\u1EE6\u016E\u0170\u01D3\u0214\u0216\u01AF\u1EEA\u1EE8\u1EEE\u1EEC\u1EF0\u1EE4\u1E72\u0172\u1E76\u1E74\u0244',
|
|
3506
|
+
},
|
|
3507
|
+
{ base: 'V', letters: '\u0056\u24CB\uFF36\u1E7C\u1E7E\u01B2\uA75E\u0245' },
|
|
3508
|
+
{ base: 'VY', letters: '\uA760' },
|
|
3509
|
+
{
|
|
3510
|
+
base: 'W',
|
|
3511
|
+
letters: '\u0057\u24CC\uFF37\u1E80\u1E82\u0174\u1E86\u1E84\u1E88\u2C72',
|
|
3512
|
+
},
|
|
3513
|
+
{ base: 'X', letters: '\u0058\u24CD\uFF38\u1E8A\u1E8C' },
|
|
3514
|
+
{
|
|
3515
|
+
base: 'Y',
|
|
3516
|
+
letters: '\u0059\u24CE\uFF39\u1EF2\u00DD\u0176\u1EF8\u0232\u1E8E\u0178\u1EF6\u1EF4\u01B3\u024E\u1EFE',
|
|
3517
|
+
},
|
|
3518
|
+
{
|
|
3519
|
+
base: 'Z',
|
|
3520
|
+
letters: '\u005A\u24CF\uFF3A\u0179\u1E90\u017B\u017D\u1E92\u1E94\u01B5\u0224\u2C7F\u2C6B\uA762',
|
|
3521
|
+
},
|
|
3522
|
+
{
|
|
3523
|
+
base: 'a',
|
|
3524
|
+
letters: '\u0061\u24D0\uFF41\u1E9A\u00E0\u00E1\u00E2\u1EA7\u1EA5\u1EAB\u1EA9\u00E3\u0101\u0103\u1EB1\u1EAF\u1EB5\u1EB3\u0227\u01E1\u00E4\u01DF\u1EA3\u00E5\u01FB\u01CE\u0201\u0203\u1EA1\u1EAD\u1EB7\u1E01\u0105\u2C65\u0250',
|
|
3525
|
+
},
|
|
3526
|
+
{ base: 'aa', letters: '\uA733' },
|
|
3527
|
+
{ base: 'ae', letters: '\u00E6\u01FD\u01E3' },
|
|
3528
|
+
{ base: 'ao', letters: '\uA735' },
|
|
3529
|
+
{ base: 'au', letters: '\uA737' },
|
|
3530
|
+
{ base: 'av', letters: '\uA739\uA73B' },
|
|
3531
|
+
{ base: 'ay', letters: '\uA73D' },
|
|
3532
|
+
{
|
|
3533
|
+
base: 'b',
|
|
3534
|
+
letters: '\u0062\u24D1\uFF42\u1E03\u1E05\u1E07\u0180\u0183\u0253',
|
|
3535
|
+
},
|
|
3536
|
+
{
|
|
3537
|
+
base: 'c',
|
|
3538
|
+
letters: '\u0063\u24D2\uFF43\u0107\u0109\u010B\u010D\u00E7\u1E09\u0188\u023C\uA73F\u2184',
|
|
3539
|
+
},
|
|
3540
|
+
{
|
|
3541
|
+
base: 'd',
|
|
3542
|
+
letters: '\u0064\u24D3\uFF44\u1E0B\u010F\u1E0D\u1E11\u1E13\u1E0F\u0111\u018C\u0256\u0257\uA77A',
|
|
3543
|
+
},
|
|
3544
|
+
{ base: 'dz', letters: '\u01F3\u01C6' },
|
|
3545
|
+
{
|
|
3546
|
+
base: 'e',
|
|
3547
|
+
letters: '\u0065\u24D4\uFF45\u00E8\u00E9\u00EA\u1EC1\u1EBF\u1EC5\u1EC3\u1EBD\u0113\u1E15\u1E17\u0115\u0117\u00EB\u1EBB\u011B\u0205\u0207\u1EB9\u1EC7\u0229\u1E1D\u0119\u1E19\u1E1B\u0247\u025B\u01DD',
|
|
3548
|
+
},
|
|
3549
|
+
{ base: 'f', letters: '\u0066\u24D5\uFF46\u1E1F\u0192\uA77C' },
|
|
3550
|
+
{
|
|
3551
|
+
base: 'g',
|
|
3552
|
+
letters: '\u0067\u24D6\uFF47\u01F5\u011D\u1E21\u011F\u0121\u01E7\u0123\u01E5\u0260\uA7A1\u1D79\uA77F',
|
|
3553
|
+
},
|
|
3554
|
+
{
|
|
3555
|
+
base: 'h',
|
|
3556
|
+
letters: '\u0068\u24D7\uFF48\u0125\u1E23\u1E27\u021F\u1E25\u1E29\u1E2B\u1E96\u0127\u2C68\u2C76\u0265',
|
|
3557
|
+
},
|
|
3558
|
+
{ base: 'hv', letters: '\u0195' },
|
|
3559
|
+
{
|
|
3560
|
+
base: 'i',
|
|
3561
|
+
letters: '\u0069\u24D8\uFF49\u00EC\u00ED\u00EE\u0129\u012B\u012D\u00EF\u1E2F\u1EC9\u01D0\u0209\u020B\u1ECB\u012F\u1E2D\u0268\u0131',
|
|
3562
|
+
},
|
|
3563
|
+
{ base: 'j', letters: '\u006A\u24D9\uFF4A\u0135\u01F0\u0249' },
|
|
3564
|
+
{
|
|
3565
|
+
base: 'k',
|
|
3566
|
+
letters: '\u006B\u24DA\uFF4B\u1E31\u01E9\u1E33\u0137\u1E35\u0199\u2C6A\uA741\uA743\uA745\uA7A3',
|
|
3567
|
+
},
|
|
3568
|
+
{
|
|
3569
|
+
base: 'l',
|
|
3570
|
+
letters: '\u006C\u24DB\uFF4C\u0140\u013A\u013E\u1E37\u1E39\u013C\u1E3D\u1E3B\u017F\u0142\u019A\u026B\u2C61\uA749\uA781\uA747',
|
|
3571
|
+
},
|
|
3572
|
+
{ base: 'lj', letters: '\u01C9' },
|
|
3573
|
+
{ base: 'm', letters: '\u006D\u24DC\uFF4D\u1E3F\u1E41\u1E43\u0271\u026F' },
|
|
3574
|
+
{
|
|
3575
|
+
base: 'n',
|
|
3576
|
+
letters: '\u006E\u24DD\uFF4E\u01F9\u0144\u00F1\u1E45\u0148\u1E47\u0146\u1E4B\u1E49\u019E\u0272\u0149\uA791\uA7A5',
|
|
3577
|
+
},
|
|
3578
|
+
{ base: 'nj', letters: '\u01CC' },
|
|
3579
|
+
{
|
|
3580
|
+
base: 'o',
|
|
3581
|
+
letters: '\u006F\u24DE\uFF4F\u00F2\u00F3\u00F4\u1ED3\u1ED1\u1ED7\u1ED5\u00F5\u1E4D\u022D\u1E4F\u014D\u1E51\u1E53\u014F\u022F\u0231\u00F6\u022B\u1ECF\u0151\u01D2\u020D\u020F\u01A1\u1EDD\u1EDB\u1EE1\u1EDF\u1EE3\u1ECD\u1ED9\u01EB\u01ED\u00F8\u01FF\u0254\uA74B\uA74D\u0275',
|
|
3582
|
+
},
|
|
3583
|
+
{ base: 'oi', letters: '\u01A3' },
|
|
3584
|
+
{ base: 'ou', letters: '\u0223' },
|
|
3585
|
+
{ base: 'oo', letters: '\uA74F' },
|
|
3586
|
+
{
|
|
3587
|
+
base: 'p',
|
|
3588
|
+
letters: '\u0070\u24DF\uFF50\u1E55\u1E57\u01A5\u1D7D\uA751\uA753\uA755',
|
|
3589
|
+
},
|
|
3590
|
+
{ base: 'q', letters: '\u0071\u24E0\uFF51\u024B\uA757\uA759' },
|
|
3591
|
+
{
|
|
3592
|
+
base: 'r',
|
|
3593
|
+
letters: '\u0072\u24E1\uFF52\u0155\u1E59\u0159\u0211\u0213\u1E5B\u1E5D\u0157\u1E5F\u024D\u027D\uA75B\uA7A7\uA783',
|
|
3594
|
+
},
|
|
3595
|
+
{
|
|
3596
|
+
base: 's',
|
|
3597
|
+
letters: '\u0073\u24E2\uFF53\u00DF\u015B\u1E65\u015D\u1E61\u0161\u1E67\u1E63\u1E69\u0219\u015F\u023F\uA7A9\uA785\u1E9B',
|
|
3598
|
+
},
|
|
3599
|
+
{
|
|
3600
|
+
base: 't',
|
|
3601
|
+
letters: '\u0074\u24E3\uFF54\u1E6B\u1E97\u0165\u1E6D\u021B\u0163\u1E71\u1E6F\u0167\u01AD\u0288\u2C66\uA787',
|
|
3602
|
+
},
|
|
3603
|
+
{ base: 'tz', letters: '\uA729' },
|
|
3604
|
+
{
|
|
3605
|
+
base: 'u',
|
|
3606
|
+
letters: '\u0075\u24E4\uFF55\u00F9\u00FA\u00FB\u0169\u1E79\u016B\u1E7B\u016D\u00FC\u01DC\u01D8\u01D6\u01DA\u1EE7\u016F\u0171\u01D4\u0215\u0217\u01B0\u1EEB\u1EE9\u1EEF\u1EED\u1EF1\u1EE5\u1E73\u0173\u1E77\u1E75\u0289',
|
|
3607
|
+
},
|
|
3608
|
+
{ base: 'v', letters: '\u0076\u24E5\uFF56\u1E7D\u1E7F\u028B\uA75F\u028C' },
|
|
3609
|
+
{ base: 'vy', letters: '\uA761' },
|
|
3610
|
+
{
|
|
3611
|
+
base: 'w',
|
|
3612
|
+
letters: '\u0077\u24E6\uFF57\u1E81\u1E83\u0175\u1E87\u1E85\u1E98\u1E89\u2C73',
|
|
3613
|
+
},
|
|
3614
|
+
{ base: 'x', letters: '\u0078\u24E7\uFF58\u1E8B\u1E8D' },
|
|
3615
|
+
{
|
|
3616
|
+
base: 'y',
|
|
3617
|
+
letters: '\u0079\u24E8\uFF59\u1EF3\u00FD\u0177\u1EF9\u0233\u1E8F\u00FF\u1EF7\u1E99\u1EF5\u01B4\u024F\u1EFF',
|
|
3618
|
+
},
|
|
3619
|
+
{
|
|
3620
|
+
base: 'z',
|
|
3621
|
+
letters: '\u007A\u24E9\uFF5A\u017A\u1E91\u017C\u017E\u1E93\u1E95\u01B6\u0225\u0240\u2C6C\uA763',
|
|
3622
|
+
},
|
|
3623
|
+
];
|
|
3624
|
+
/**
|
|
3625
|
+
* Map of letters from diacritic variant to diacritless variant
|
|
3626
|
+
* Contains lowercase and uppercase separatelly
|
|
3948
3627
|
*
|
|
3949
|
-
*
|
|
3950
|
-
|
|
3951
|
-
|
|
3952
|
-
|
|
3953
|
-
text = text.replace(/\0/g, '');
|
|
3954
|
-
// Replace emojis (and also ZWJ sequence) with hyphens
|
|
3955
|
-
text = text.replace(/(\p{Extended_Pictographic})\p{Modifier_Symbol}/gu, '$1');
|
|
3956
|
-
text = text.replace(/(\p{Extended_Pictographic})[\u{FE00}-\u{FE0F}]/gu, '$1');
|
|
3957
|
-
text = text.replace(/\p{Extended_Pictographic}(\u{200D}\p{Extended_Pictographic})*/gu, '-');
|
|
3958
|
-
return text.length;
|
|
3959
|
-
}
|
|
3960
|
-
|
|
3961
|
-
/**
|
|
3962
|
-
* Counts number of lines in the text
|
|
3628
|
+
* > "á" => "a"
|
|
3629
|
+
* > "ě" => "e"
|
|
3630
|
+
* > "Ă" => "A"
|
|
3631
|
+
* > ...
|
|
3963
3632
|
*
|
|
3964
3633
|
* @public exported from `@promptbook/utils`
|
|
3965
3634
|
*/
|
|
3966
|
-
|
|
3967
|
-
|
|
3968
|
-
|
|
3635
|
+
var DIACRITIC_VARIANTS_LETTERS = {};
|
|
3636
|
+
// tslint:disable-next-line: prefer-for-of
|
|
3637
|
+
for (var i = 0; i < defaultDiacriticsRemovalMap.length; i++) {
|
|
3638
|
+
var letters = defaultDiacriticsRemovalMap[i].letters;
|
|
3639
|
+
// tslint:disable-next-line: prefer-for-of
|
|
3640
|
+
for (var j = 0; j < letters.length; j++) {
|
|
3641
|
+
DIACRITIC_VARIANTS_LETTERS[letters[j]] = defaultDiacriticsRemovalMap[i].base;
|
|
3969
3642
|
}
|
|
3970
|
-
return text.split('\n').length;
|
|
3971
3643
|
}
|
|
3644
|
+
// <- TODO: [🍓] Put to maker function to save execution time if not needed
|
|
3645
|
+
/*
|
|
3646
|
+
@see https://stackoverflow.com/questions/990904/remove-accents-diacritics-in-a-string-in-javascript
|
|
3647
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
3648
|
+
you may not use this file except in compliance with the License.
|
|
3649
|
+
You may obtain a copy of the License at
|
|
3972
3650
|
|
|
3973
|
-
|
|
3974
|
-
* Counts number of pages in the text
|
|
3975
|
-
*
|
|
3976
|
-
* @public exported from `@promptbook/utils`
|
|
3977
|
-
*/
|
|
3978
|
-
function countPages(text) {
|
|
3979
|
-
var sentencesPerPage = 5; // Assuming each page has 5 sentences
|
|
3980
|
-
var sentences = text.split(/[.!?]+/).filter(function (sentence) { return sentence.trim() !== ''; });
|
|
3981
|
-
var pageCount = Math.ceil(sentences.length / sentencesPerPage);
|
|
3982
|
-
return pageCount;
|
|
3983
|
-
}
|
|
3651
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
3984
3652
|
|
|
3985
|
-
|
|
3986
|
-
|
|
3987
|
-
|
|
3988
|
-
|
|
3989
|
-
|
|
3990
|
-
|
|
3991
|
-
return text.split(/\n\s*\n/).filter(function (paragraph) { return paragraph.trim() !== ''; }).length;
|
|
3992
|
-
}
|
|
3653
|
+
Unless required by applicable law or agreed to in writing, software
|
|
3654
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
3655
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
3656
|
+
See the License for the specific language governing permissions and
|
|
3657
|
+
limitations under the License.
|
|
3658
|
+
*/
|
|
3993
3659
|
|
|
3994
3660
|
/**
|
|
3995
|
-
*
|
|
3661
|
+
* @@@
|
|
3996
3662
|
*
|
|
3663
|
+
* @param input @@@
|
|
3664
|
+
* @returns @@@
|
|
3997
3665
|
* @public exported from `@promptbook/utils`
|
|
3998
3666
|
*/
|
|
3999
|
-
function
|
|
4000
|
-
|
|
3667
|
+
function removeDiacritics(input) {
|
|
3668
|
+
/*eslint no-control-regex: "off"*/
|
|
3669
|
+
return input.replace(/[^\u0000-\u007E]/g, function (a) {
|
|
3670
|
+
return DIACRITIC_VARIANTS_LETTERS[a] || a;
|
|
3671
|
+
});
|
|
4001
3672
|
}
|
|
4002
3673
|
/**
|
|
4003
|
-
*
|
|
4004
|
-
*
|
|
4005
|
-
* @public exported from `@promptbook/utils`
|
|
3674
|
+
* TODO: [Ж] Variant for cyrillic (and in general non-latin) letters
|
|
4006
3675
|
*/
|
|
4007
|
-
function countSentences(text) {
|
|
4008
|
-
return splitIntoSentences(text).length;
|
|
4009
|
-
}
|
|
4010
3676
|
|
|
4011
3677
|
/**
|
|
4012
3678
|
* Counts number of words in the text
|
|
@@ -4104,18 +3770,19 @@
|
|
|
4104
3770
|
*/
|
|
4105
3771
|
function executeAttempts(options) {
|
|
4106
3772
|
return __awaiter(this, void 0, void 0, function () {
|
|
4107
|
-
var jokerParameterNames, priority, maxAttempts, preparedContent, parameters, template, preparedPipeline, tools,
|
|
3773
|
+
var jokerParameterNames, priority, maxAttempts, preparedContent, parameters, template, preparedPipeline, tools, $executionReport, pipelineIdentification, maxExecutionAttempts, $ongoingTemplateResult, _llms, llmTools, _loop_1, attempt, state_1;
|
|
4108
3774
|
return __generator(this, function (_a) {
|
|
4109
3775
|
switch (_a.label) {
|
|
4110
3776
|
case 0:
|
|
4111
|
-
jokerParameterNames = options.jokerParameterNames, priority = options.priority, maxAttempts = options.maxAttempts, preparedContent = options.preparedContent, parameters = options.parameters, template = options.template, preparedPipeline = options.preparedPipeline, tools = options.tools,
|
|
4112
|
-
maxExecutionAttempts = settings.maxExecutionAttempts;
|
|
3777
|
+
jokerParameterNames = options.jokerParameterNames, priority = options.priority, maxAttempts = options.maxAttempts, preparedContent = options.preparedContent, parameters = options.parameters, template = options.template, preparedPipeline = options.preparedPipeline, tools = options.tools, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification, maxExecutionAttempts = options.maxExecutionAttempts;
|
|
4113
3778
|
$ongoingTemplateResult = {
|
|
4114
3779
|
$result: null,
|
|
4115
3780
|
$resultString: null,
|
|
4116
3781
|
$expectError: null,
|
|
4117
3782
|
$scriptPipelineExecutionErrors: [],
|
|
4118
3783
|
};
|
|
3784
|
+
_llms = arrayableToArray(tools.llm);
|
|
3785
|
+
llmTools = _llms.length === 1 ? _llms[0] : joinLlmExecutionTools.apply(void 0, __spreadArray([], __read(_llms), false));
|
|
4119
3786
|
_loop_1 = function (attempt) {
|
|
4120
3787
|
var isJokerAttempt, jokerParameterName, _b, modelRequirements, _c, _d, _e, _f, _g, scriptTools, _h, error_1, e_1_1, _j, _k, _l, functionName, postprocessingError, _m, _o, scriptTools, _p, error_2, e_2_1, e_3_1, error_3;
|
|
4121
3788
|
var e_1, _q, e_3, _r, e_2, _s;
|
|
@@ -4184,7 +3851,9 @@
|
|
|
4184
3851
|
return [3 /*break*/, 9];
|
|
4185
3852
|
case 4:
|
|
4186
3853
|
_d = $ongoingTemplateResult;
|
|
4187
|
-
return [4 /*yield*/, llmTools.callChatModel(
|
|
3854
|
+
return [4 /*yield*/, llmTools.callChatModel(
|
|
3855
|
+
// <- TODO: [🧁] Check that `callChatModel` is defined
|
|
3856
|
+
$deepFreeze($ongoingTemplateResult.$prompt))];
|
|
4188
3857
|
case 5:
|
|
4189
3858
|
_d.$chatResult = _t.sent();
|
|
4190
3859
|
// TODO: [🍬] Destroy chatThread
|
|
@@ -4193,7 +3862,9 @@
|
|
|
4193
3862
|
return [3 /*break*/, 10];
|
|
4194
3863
|
case 6:
|
|
4195
3864
|
_e = $ongoingTemplateResult;
|
|
4196
|
-
return [4 /*yield*/, llmTools.callCompletionModel(
|
|
3865
|
+
return [4 /*yield*/, llmTools.callCompletionModel(
|
|
3866
|
+
// <- TODO: [🧁] Check that `callCompletionModel` is defined
|
|
3867
|
+
$deepFreeze($ongoingTemplateResult.$prompt))];
|
|
4197
3868
|
case 7:
|
|
4198
3869
|
_e.$completionResult = _t.sent();
|
|
4199
3870
|
$ongoingTemplateResult.$result = $ongoingTemplateResult.$completionResult;
|
|
@@ -4469,12 +4140,12 @@
|
|
|
4469
4140
|
*/
|
|
4470
4141
|
function executeFormatSubvalues(options) {
|
|
4471
4142
|
return __awaiter(this, void 0, void 0, function () {
|
|
4472
|
-
var template, jokerParameterNames, parameters, priority,
|
|
4143
|
+
var template, jokerParameterNames, parameters, priority, csvSettings, pipelineIdentification, parameterValue, formatDefinition, subvalueDefinition, formatSettings, resultString;
|
|
4473
4144
|
var _this = this;
|
|
4474
4145
|
return __generator(this, function (_a) {
|
|
4475
4146
|
switch (_a.label) {
|
|
4476
4147
|
case 0:
|
|
4477
|
-
template = options.template, jokerParameterNames = options.jokerParameterNames, parameters = options.parameters, priority = options.priority,
|
|
4148
|
+
template = options.template, jokerParameterNames = options.jokerParameterNames, parameters = options.parameters, priority = options.priority, csvSettings = options.csvSettings, pipelineIdentification = options.pipelineIdentification;
|
|
4478
4149
|
if (template.foreach === undefined) {
|
|
4479
4150
|
return [2 /*return*/, /* not await */ executeAttempts(options)];
|
|
4480
4151
|
}
|
|
@@ -4504,7 +4175,7 @@
|
|
|
4504
4175
|
.join('\n')), "\n\n [\u26F7] This should never happen because subformat name should be validated during parsing\n\n ").concat(block(pipelineIdentification), "\n "); }));
|
|
4505
4176
|
}
|
|
4506
4177
|
if (formatDefinition.formatName === 'CSV') {
|
|
4507
|
-
formatSettings =
|
|
4178
|
+
formatSettings = csvSettings;
|
|
4508
4179
|
// <- TODO: [🤹♂️] More universal, make simmilar pattern for other formats for example \n vs \r\n in text
|
|
4509
4180
|
}
|
|
4510
4181
|
return [4 /*yield*/, subvalueDefinition.mapValues(parameterValue, template.foreach.outputSubparameterName, formatSettings, function (subparameters, index) { return __awaiter(_this, void 0, void 0, function () {
|
|
@@ -4657,13 +4328,12 @@
|
|
|
4657
4328
|
*/
|
|
4658
4329
|
function executeTemplate(options) {
|
|
4659
4330
|
return __awaiter(this, void 0, void 0, function () {
|
|
4660
|
-
var currentTemplate, preparedPipeline, parametersToPass, tools,
|
|
4661
|
-
var e_1,
|
|
4662
|
-
return __generator(this, function (
|
|
4663
|
-
switch (
|
|
4331
|
+
var currentTemplate, preparedPipeline, parametersToPass, tools, onProgress, $executionReport, pipelineIdentification, _a, maxExecutionAttempts, name, title, priority, usedParameterNames, dependentParameterNames, definedParameters, _b, _c, _d, definedParameterNames, parameters, _loop_1, _e, _f, parameterName, maxAttempts, jokerParameterNames, preparedContent, resultString;
|
|
4332
|
+
var e_1, _g, _h;
|
|
4333
|
+
return __generator(this, function (_j) {
|
|
4334
|
+
switch (_j.label) {
|
|
4664
4335
|
case 0:
|
|
4665
|
-
currentTemplate = options.currentTemplate, preparedPipeline = options.preparedPipeline, parametersToPass = options.parametersToPass, tools = options.tools,
|
|
4666
|
-
maxExecutionAttempts = settings.maxExecutionAttempts;
|
|
4336
|
+
currentTemplate = options.currentTemplate, preparedPipeline = options.preparedPipeline, parametersToPass = options.parametersToPass, tools = options.tools, onProgress = options.onProgress, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification, _a = options.maxExecutionAttempts, maxExecutionAttempts = _a === void 0 ? DEFAULT_MAX_EXECUTION_ATTEMPTS : _a;
|
|
4667
4337
|
name = "pipeline-executor-frame-".concat(currentTemplate.name);
|
|
4668
4338
|
title = currentTemplate.title;
|
|
4669
4339
|
priority = preparedPipeline.templates.length - preparedPipeline.templates.indexOf(currentTemplate);
|
|
@@ -4678,7 +4348,7 @@
|
|
|
4678
4348
|
// <- [🍸]
|
|
4679
4349
|
})];
|
|
4680
4350
|
case 1:
|
|
4681
|
-
|
|
4351
|
+
_j.sent();
|
|
4682
4352
|
usedParameterNames = extractParameterNamesFromTemplate(currentTemplate);
|
|
4683
4353
|
dependentParameterNames = new Set(currentTemplate.dependentParameterNames);
|
|
4684
4354
|
// TODO: [👩🏾🤝👩🏻] Use here `mapAvailableToExpectedParameters`
|
|
@@ -4689,15 +4359,15 @@
|
|
|
4689
4359
|
.map(function (name) { return "{".concat(name, "}"); })
|
|
4690
4360
|
.join(', '), "\n\n ").concat(block(pipelineIdentification), "\n\n "); }));
|
|
4691
4361
|
}
|
|
4692
|
-
|
|
4693
|
-
|
|
4362
|
+
_c = (_b = Object).freeze;
|
|
4363
|
+
_d = [{}];
|
|
4694
4364
|
return [4 /*yield*/, getReservedParametersForTemplate({
|
|
4695
4365
|
preparedPipeline: preparedPipeline,
|
|
4696
4366
|
template: currentTemplate,
|
|
4697
4367
|
pipelineIdentification: pipelineIdentification,
|
|
4698
4368
|
})];
|
|
4699
4369
|
case 2:
|
|
4700
|
-
definedParameters =
|
|
4370
|
+
definedParameters = _c.apply(_b, [__assign.apply(void 0, [__assign.apply(void 0, _d.concat([(_j.sent())])), parametersToPass])]);
|
|
4701
4371
|
definedParameterNames = new Set(Object.keys(definedParameters));
|
|
4702
4372
|
parameters = {};
|
|
4703
4373
|
_loop_1 = function (parameterName) {
|
|
@@ -4717,15 +4387,15 @@
|
|
|
4717
4387
|
try {
|
|
4718
4388
|
// Note: [2] Check that all used parameters are defined and removing unused parameters for this template
|
|
4719
4389
|
// TODO: [👩🏾🤝👩🏻] Use here `mapAvailableToExpectedParameters`
|
|
4720
|
-
for (
|
|
4721
|
-
parameterName =
|
|
4390
|
+
for (_e = __values(Array.from(union(definedParameterNames, usedParameterNames, dependentParameterNames))), _f = _e.next(); !_f.done; _f = _e.next()) {
|
|
4391
|
+
parameterName = _f.value;
|
|
4722
4392
|
_loop_1(parameterName);
|
|
4723
4393
|
}
|
|
4724
4394
|
}
|
|
4725
4395
|
catch (e_1_1) { e_1 = { error: e_1_1 }; }
|
|
4726
4396
|
finally {
|
|
4727
4397
|
try {
|
|
4728
|
-
if (
|
|
4398
|
+
if (_f && !_f.done && (_g = _e.return)) _g.call(_e);
|
|
4729
4399
|
}
|
|
4730
4400
|
finally { if (e_1) throw e_1.error; }
|
|
4731
4401
|
}
|
|
@@ -4745,13 +4415,11 @@
|
|
|
4745
4415
|
template: currentTemplate,
|
|
4746
4416
|
preparedPipeline: preparedPipeline,
|
|
4747
4417
|
tools: tools,
|
|
4748
|
-
llmTools: llmTools,
|
|
4749
|
-
settings: settings,
|
|
4750
4418
|
$executionReport: $executionReport,
|
|
4751
4419
|
pipelineIdentification: pipelineIdentification,
|
|
4752
4420
|
})];
|
|
4753
4421
|
case 3:
|
|
4754
|
-
resultString =
|
|
4422
|
+
resultString = _j.sent();
|
|
4755
4423
|
return [4 /*yield*/, onProgress({
|
|
4756
4424
|
name: name,
|
|
4757
4425
|
title: title,
|
|
@@ -4763,12 +4431,12 @@
|
|
|
4763
4431
|
// <- [🍸]
|
|
4764
4432
|
})];
|
|
4765
4433
|
case 4:
|
|
4766
|
-
|
|
4767
|
-
return [2 /*return*/, Object.freeze((
|
|
4768
|
-
|
|
4434
|
+
_j.sent();
|
|
4435
|
+
return [2 /*return*/, Object.freeze((_h = {},
|
|
4436
|
+
_h[currentTemplate.resultingParameterName] =
|
|
4769
4437
|
// <- Note: [👩👩👧] No need to detect parameter collision here because pipeline checks logic consistency during construction
|
|
4770
4438
|
resultString,
|
|
4771
|
-
|
|
4439
|
+
_h))];
|
|
4772
4440
|
}
|
|
4773
4441
|
});
|
|
4774
4442
|
});
|
|
@@ -4827,18 +4495,15 @@
|
|
|
4827
4495
|
*/
|
|
4828
4496
|
function executePipeline(options) {
|
|
4829
4497
|
return __awaiter(this, void 0, void 0, function () {
|
|
4830
|
-
var inputParameters, tools, onProgress, pipeline, setPreparedPipeline, pipelineIdentification,
|
|
4498
|
+
var inputParameters, tools, onProgress, pipeline, setPreparedPipeline, pipelineIdentification, maxParallelCount, rootDirname, _a, isVerbose, preparedPipeline, errors, warnings, executionReport, isReturned, _b, _c, parameter, e_1_1, _loop_1, _d, _e, parameterName, state_1, e_2_1, parametersToPass, resovedParameterNames_1, unresovedTemplates_1, resolving_1, loopLimit, _loop_2, error_1, usage_1, outputParameters_1, usage, outputParameters;
|
|
4831
4499
|
var e_1, _f, e_2, _g;
|
|
4832
4500
|
return __generator(this, function (_h) {
|
|
4833
4501
|
switch (_h.label) {
|
|
4834
4502
|
case 0:
|
|
4835
|
-
inputParameters = options.inputParameters, tools = options.tools, onProgress = options.onProgress, pipeline = options.pipeline, setPreparedPipeline = options.setPreparedPipeline, pipelineIdentification = options.pipelineIdentification,
|
|
4836
|
-
maxParallelCount = settings.maxParallelCount, rootDirname = settings.rootDirname, _a = settings.isVerbose, isVerbose = _a === void 0 ? IS_VERBOSE : _a;
|
|
4503
|
+
inputParameters = options.inputParameters, tools = options.tools, onProgress = options.onProgress, pipeline = options.pipeline, setPreparedPipeline = options.setPreparedPipeline, pipelineIdentification = options.pipelineIdentification, maxParallelCount = options.maxParallelCount, rootDirname = options.rootDirname, _a = options.isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a;
|
|
4837
4504
|
preparedPipeline = options.preparedPipeline;
|
|
4838
|
-
llmTools = joinLlmExecutionTools.apply(void 0, __spreadArray([], __read(arrayableToArray(tools.llm)), false));
|
|
4839
4505
|
if (!(preparedPipeline === undefined)) return [3 /*break*/, 2];
|
|
4840
|
-
return [4 /*yield*/, preparePipeline(pipeline, {
|
|
4841
|
-
llmTools: llmTools,
|
|
4506
|
+
return [4 /*yield*/, preparePipeline(pipeline, tools, {
|
|
4842
4507
|
rootDirname: rootDirname,
|
|
4843
4508
|
isVerbose: isVerbose,
|
|
4844
4509
|
maxParallelCount: maxParallelCount,
|
|
@@ -5020,13 +4685,7 @@
|
|
|
5020
4685
|
return [3 /*break*/, 4];
|
|
5021
4686
|
case 3:
|
|
5022
4687
|
unresovedTemplates_1 = unresovedTemplates_1.filter(function (template) { return template !== currentTemplate; });
|
|
5023
|
-
work_1 = executeTemplate({
|
|
5024
|
-
currentTemplate: currentTemplate,
|
|
5025
|
-
preparedPipeline: preparedPipeline,
|
|
5026
|
-
parametersToPass: parametersToPass,
|
|
5027
|
-
tools: tools,
|
|
5028
|
-
llmTools: llmTools,
|
|
5029
|
-
onProgress: function (progress) {
|
|
4688
|
+
work_1 = executeTemplate(__assign(__assign({}, options), { currentTemplate: currentTemplate, preparedPipeline: preparedPipeline, parametersToPass: parametersToPass, tools: tools, onProgress: function (progress) {
|
|
5030
4689
|
if (isReturned) {
|
|
5031
4690
|
throw new UnexpectedError(spaceTrim.spaceTrim(function (block) { return "\n Can not call `onProgress` after pipeline execution is finished\n\n ".concat(block(pipelineIdentification), "\n\n ").concat(block(JSON.stringify(progress, null, 4)
|
|
5032
4691
|
.split('\n')
|
|
@@ -5036,11 +4695,7 @@
|
|
|
5036
4695
|
if (onProgress) {
|
|
5037
4696
|
onProgress(progress);
|
|
5038
4697
|
}
|
|
5039
|
-
},
|
|
5040
|
-
settings: settings,
|
|
5041
|
-
$executionReport: executionReport,
|
|
5042
|
-
pipelineIdentification: spaceTrim.spaceTrim(function (block) { return "\n ".concat(block(pipelineIdentification), "\n Template name: ").concat(currentTemplate.name, "\n Template title: ").concat(currentTemplate.title, "\n "); }),
|
|
5043
|
-
})
|
|
4698
|
+
}, $executionReport: executionReport, pipelineIdentification: spaceTrim.spaceTrim(function (block) { return "\n ".concat(block(pipelineIdentification), "\n Template name: ").concat(currentTemplate.name, "\n Template title: ").concat(currentTemplate.title, "\n "); }) }))
|
|
5044
4699
|
.then(function (newParametersToPass) {
|
|
5045
4700
|
parametersToPass = __assign(__assign({}, newParametersToPass), parametersToPass);
|
|
5046
4701
|
resovedParameterNames_1 = __spreadArray(__spreadArray([], __read(resovedParameterNames_1), false), [currentTemplate.resultingParameterName], false);
|
|
@@ -5144,8 +4799,7 @@
|
|
|
5144
4799
|
*/
|
|
5145
4800
|
function createPipelineExecutor(options) {
|
|
5146
4801
|
var _this = this;
|
|
5147
|
-
var pipeline = options.pipeline, tools = options.tools, _a = options.
|
|
5148
|
-
var _b = settings.maxExecutionAttempts, maxExecutionAttempts = _b === void 0 ? MAX_EXECUTION_ATTEMPTS : _b, _c = settings.maxParallelCount, maxParallelCount = _c === void 0 ? MAX_PARALLEL_COUNT : _c, _d = settings.csvSettings, csvSettings = _d === void 0 ? DEFAULT_CSV_SETTINGS : _d, _e = settings.isVerbose, isVerbose = _e === void 0 ? IS_VERBOSE : _e, _f = settings.isNotPreparedWarningSupressed, isNotPreparedWarningSupressed = _f === void 0 ? false : _f, _g = settings.rootDirname, rootDirname = _g === void 0 ? null : _g;
|
|
4802
|
+
var pipeline = options.pipeline, tools = options.tools, _a = options.maxExecutionAttempts, maxExecutionAttempts = _a === void 0 ? DEFAULT_MAX_EXECUTION_ATTEMPTS : _a, _b = options.maxParallelCount, maxParallelCount = _b === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _b, _c = options.csvSettings, csvSettings = _c === void 0 ? DEFAULT_CSV_SETTINGS : _c, _d = options.isVerbose, isVerbose = _d === void 0 ? DEFAULT_IS_VERBOSE : _d, _e = options.isNotPreparedWarningSupressed, isNotPreparedWarningSupressed = _e === void 0 ? false : _e, _f = options.rootDirname, rootDirname = _f === void 0 ? null : _f;
|
|
5149
4803
|
validatePipeline(pipeline);
|
|
5150
4804
|
var pipelineIdentification = (function () {
|
|
5151
4805
|
// Note: This is a 😐 implementation of [🚞]
|
|
@@ -5179,14 +4833,12 @@
|
|
|
5179
4833
|
tools: tools,
|
|
5180
4834
|
onProgress: onProgress,
|
|
5181
4835
|
pipelineIdentification: spaceTrim.spaceTrim(function (block) { return "\n ".concat(block(pipelineIdentification), "\n ").concat(runCount === 1 ? '' : "Run #".concat(runCount), "\n "); }),
|
|
5182
|
-
|
|
5183
|
-
|
|
5184
|
-
|
|
5185
|
-
|
|
5186
|
-
|
|
5187
|
-
|
|
5188
|
-
rootDirname: rootDirname,
|
|
5189
|
-
},
|
|
4836
|
+
maxExecutionAttempts: maxExecutionAttempts,
|
|
4837
|
+
maxParallelCount: maxParallelCount,
|
|
4838
|
+
csvSettings: csvSettings,
|
|
4839
|
+
isVerbose: isVerbose,
|
|
4840
|
+
isNotPreparedWarningSupressed: isNotPreparedWarningSupressed,
|
|
4841
|
+
rootDirname: rootDirname,
|
|
5190
4842
|
})];
|
|
5191
4843
|
});
|
|
5192
4844
|
}); };
|
|
@@ -5197,678 +4849,531 @@
|
|
|
5197
4849
|
*/
|
|
5198
4850
|
|
|
5199
4851
|
/**
|
|
5200
|
-
*
|
|
4852
|
+
* Prepares the persona for the pipeline
|
|
4853
|
+
*
|
|
4854
|
+
* @see https://github.com/webgptorg/promptbook/discussions/22
|
|
4855
|
+
* @public exported from `@promptbook/core`
|
|
4856
|
+
*/
|
|
4857
|
+
function preparePersona(personaDescription, tools, options) {
|
|
4858
|
+
return __awaiter(this, void 0, void 0, function () {
|
|
4859
|
+
var _a, isVerbose, collection, preparePersonaExecutor, _b, _llms, llmTools, availableModels, availableModelNames, result, outputParameters, modelRequirementsRaw, modelRequirements, modelName, systemMessage, temperature;
|
|
4860
|
+
var _c;
|
|
4861
|
+
return __generator(this, function (_d) {
|
|
4862
|
+
switch (_d.label) {
|
|
4863
|
+
case 0:
|
|
4864
|
+
_a = options.isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a;
|
|
4865
|
+
if (tools === undefined || tools.llm === undefined) {
|
|
4866
|
+
throw new MissingToolsError('LLM tools are required for preparing persona');
|
|
4867
|
+
}
|
|
4868
|
+
collection = createCollectionFromJson.apply(void 0, __spreadArray([], __read(PipelineCollection), false));
|
|
4869
|
+
_b = createPipelineExecutor;
|
|
4870
|
+
_c = {};
|
|
4871
|
+
return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-persona.ptbk.md')];
|
|
4872
|
+
case 1:
|
|
4873
|
+
preparePersonaExecutor = _b.apply(void 0, [(_c.pipeline = _d.sent(),
|
|
4874
|
+
_c.tools = tools,
|
|
4875
|
+
_c)]);
|
|
4876
|
+
_llms = arrayableToArray(tools.llm);
|
|
4877
|
+
llmTools = _llms.length === 1 ? _llms[0] : joinLlmExecutionTools.apply(void 0, __spreadArray([], __read(_llms), false));
|
|
4878
|
+
return [4 /*yield*/, llmTools.listModels()];
|
|
4879
|
+
case 2:
|
|
4880
|
+
availableModels = _d.sent();
|
|
4881
|
+
availableModelNames = availableModels
|
|
4882
|
+
.filter(function (_a) {
|
|
4883
|
+
var modelVariant = _a.modelVariant;
|
|
4884
|
+
return modelVariant === 'CHAT';
|
|
4885
|
+
})
|
|
4886
|
+
.map(function (_a) {
|
|
4887
|
+
var modelName = _a.modelName;
|
|
4888
|
+
return modelName;
|
|
4889
|
+
})
|
|
4890
|
+
.join(',');
|
|
4891
|
+
return [4 /*yield*/, preparePersonaExecutor({ availableModelNames: availableModelNames, personaDescription: personaDescription })];
|
|
4892
|
+
case 3:
|
|
4893
|
+
result = _d.sent();
|
|
4894
|
+
assertsExecutionSuccessful(result);
|
|
4895
|
+
outputParameters = result.outputParameters;
|
|
4896
|
+
modelRequirementsRaw = outputParameters.modelRequirements;
|
|
4897
|
+
modelRequirements = JSON.parse(modelRequirementsRaw);
|
|
4898
|
+
if (isVerbose) {
|
|
4899
|
+
console.info("PERSONA ".concat(personaDescription), modelRequirements);
|
|
4900
|
+
}
|
|
4901
|
+
modelName = modelRequirements.modelName, systemMessage = modelRequirements.systemMessage, temperature = modelRequirements.temperature;
|
|
4902
|
+
return [2 /*return*/, {
|
|
4903
|
+
modelVariant: 'CHAT',
|
|
4904
|
+
modelName: modelName,
|
|
4905
|
+
systemMessage: systemMessage,
|
|
4906
|
+
temperature: temperature,
|
|
4907
|
+
}];
|
|
4908
|
+
}
|
|
4909
|
+
});
|
|
4910
|
+
});
|
|
4911
|
+
}
|
|
4912
|
+
/**
|
|
4913
|
+
* TODO: [🔃][main] !!!!! If the persona was prepared with different version or different set of models, prepare it once again
|
|
4914
|
+
* TODO: [🏢] !! Check validity of `modelName` in pipeline
|
|
4915
|
+
* TODO: [🏢] !! Check validity of `systemMessage` in pipeline
|
|
4916
|
+
* TODO: [🏢] !! Check validity of `temperature` in pipeline
|
|
4917
|
+
*/
|
|
4918
|
+
|
|
4919
|
+
/**
|
|
4920
|
+
* This error indicates that the promptbook can not retrieve knowledge from external sources
|
|
5201
4921
|
*
|
|
5202
|
-
* @see `documentationUrl` for more details
|
|
5203
4922
|
* @public exported from `@promptbook/core`
|
|
5204
4923
|
*/
|
|
5205
|
-
var
|
|
5206
|
-
|
|
5207
|
-
|
|
5208
|
-
|
|
5209
|
-
|
|
5210
|
-
|
|
5211
|
-
|
|
5212
|
-
|
|
5213
|
-
|
|
5214
|
-
|
|
5215
|
-
|
|
5216
|
-
*/
|
|
5217
|
-
scrape: function (source, options) {
|
|
5218
|
-
return __awaiter(this, void 0, void 0, function () {
|
|
5219
|
-
var llmTools, _a, maxParallelCount, _b, isVerbose, collection, prepareKnowledgeFromMarkdownExecutor, _c, prepareTitleExecutor, _d, prepareKeywordsExecutor, _e, knowledgeContent, result, outputParameters, knowledgePiecesRaw, knowledgeTextPieces, knowledge;
|
|
5220
|
-
var _f, _g, _h;
|
|
5221
|
-
var _this = this;
|
|
5222
|
-
return __generator(this, function (_j) {
|
|
5223
|
-
switch (_j.label) {
|
|
5224
|
-
case 0:
|
|
5225
|
-
llmTools = options.llmTools, _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? MAX_PARALLEL_COUNT : _a, _b = options.isVerbose, isVerbose = _b === void 0 ? IS_VERBOSE : _b;
|
|
5226
|
-
if (llmTools === undefined) {
|
|
5227
|
-
throw new MissingToolsError('LLM tools are required for scraping external files');
|
|
5228
|
-
// <- Note: This scraper is used in all other scrapers, so saying "external files" not "markdown files"
|
|
5229
|
-
}
|
|
5230
|
-
TODO_USE(maxParallelCount); // <- [🪂]
|
|
5231
|
-
collection = createCollectionFromJson.apply(void 0, __spreadArray([], __read(PipelineCollection), false));
|
|
5232
|
-
_c = createPipelineExecutor;
|
|
5233
|
-
_f = {};
|
|
5234
|
-
return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.ptbk.md')];
|
|
5235
|
-
case 1:
|
|
5236
|
-
prepareKnowledgeFromMarkdownExecutor = _c.apply(void 0, [(_f.pipeline = _j.sent(),
|
|
5237
|
-
_f.tools = {
|
|
5238
|
-
llm: llmTools,
|
|
5239
|
-
},
|
|
5240
|
-
_f)]);
|
|
5241
|
-
_d = createPipelineExecutor;
|
|
5242
|
-
_g = {};
|
|
5243
|
-
return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-knowledge-title.ptbk.md')];
|
|
5244
|
-
case 2:
|
|
5245
|
-
prepareTitleExecutor = _d.apply(void 0, [(_g.pipeline = _j.sent(),
|
|
5246
|
-
_g.tools = {
|
|
5247
|
-
llm: llmTools,
|
|
5248
|
-
},
|
|
5249
|
-
_g)]);
|
|
5250
|
-
_e = createPipelineExecutor;
|
|
5251
|
-
_h = {};
|
|
5252
|
-
return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-knowledge-keywords.ptbk.md')];
|
|
5253
|
-
case 3:
|
|
5254
|
-
prepareKeywordsExecutor = _e.apply(void 0, [(_h.pipeline = _j.sent(),
|
|
5255
|
-
_h.tools = {
|
|
5256
|
-
llm: llmTools,
|
|
5257
|
-
},
|
|
5258
|
-
_h)]);
|
|
5259
|
-
return [4 /*yield*/, source.asText()];
|
|
5260
|
-
case 4:
|
|
5261
|
-
knowledgeContent = _j.sent();
|
|
5262
|
-
return [4 /*yield*/, prepareKnowledgeFromMarkdownExecutor({ knowledgeContent: knowledgeContent })];
|
|
5263
|
-
case 5:
|
|
5264
|
-
result = _j.sent();
|
|
5265
|
-
assertsExecutionSuccessful(result);
|
|
5266
|
-
outputParameters = result.outputParameters;
|
|
5267
|
-
knowledgePiecesRaw = outputParameters.knowledgePieces;
|
|
5268
|
-
knowledgeTextPieces = (knowledgePiecesRaw || '').split('\n---\n');
|
|
5269
|
-
// <- TODO: [main] !!!!! Smarter split and filter out empty pieces
|
|
5270
|
-
if (isVerbose) {
|
|
5271
|
-
console.info('knowledgeTextPieces:', knowledgeTextPieces);
|
|
5272
|
-
}
|
|
5273
|
-
return [4 /*yield*/, Promise.all(
|
|
5274
|
-
// TODO: [🪂] !! Do not send all at once but in chunks
|
|
5275
|
-
knowledgeTextPieces.map(function (knowledgeTextPiece, i) { return __awaiter(_this, void 0, void 0, function () {
|
|
5276
|
-
var name, title, knowledgePieceContent, keywords, index, titleResult, _a, titleRaw, keywordsResult, _b, keywordsRaw, embeddingResult, error_1;
|
|
5277
|
-
return __generator(this, function (_c) {
|
|
5278
|
-
switch (_c.label) {
|
|
5279
|
-
case 0:
|
|
5280
|
-
name = "piece-".concat(i);
|
|
5281
|
-
title = spaceTrim__default["default"](knowledgeTextPiece.substring(0, 100));
|
|
5282
|
-
knowledgePieceContent = spaceTrim__default["default"](knowledgeTextPiece);
|
|
5283
|
-
keywords = [];
|
|
5284
|
-
index = [];
|
|
5285
|
-
_c.label = 1;
|
|
5286
|
-
case 1:
|
|
5287
|
-
_c.trys.push([1, 7, , 8]);
|
|
5288
|
-
return [4 /*yield*/, prepareTitleExecutor({ knowledgePieceContent: knowledgePieceContent })];
|
|
5289
|
-
case 2:
|
|
5290
|
-
titleResult = _c.sent();
|
|
5291
|
-
_a = titleResult.outputParameters.title, titleRaw = _a === void 0 ? 'Untitled' : _a;
|
|
5292
|
-
title = spaceTrim__default["default"](titleRaw) /* <- TODO: Maybe do in pipeline */;
|
|
5293
|
-
name = titleToName(title);
|
|
5294
|
-
return [4 /*yield*/, prepareKeywordsExecutor({ knowledgePieceContent: knowledgePieceContent })];
|
|
5295
|
-
case 3:
|
|
5296
|
-
keywordsResult = _c.sent();
|
|
5297
|
-
_b = keywordsResult.outputParameters.keywords, keywordsRaw = _b === void 0 ? '' : _b;
|
|
5298
|
-
keywords = (keywordsRaw || '')
|
|
5299
|
-
.split(',')
|
|
5300
|
-
.map(function (keyword) { return keyword.trim(); })
|
|
5301
|
-
.filter(function (keyword) { return keyword !== ''; });
|
|
5302
|
-
if (isVerbose) {
|
|
5303
|
-
console.info("Keywords for \"".concat(title, "\":"), keywords);
|
|
5304
|
-
}
|
|
5305
|
-
if (!!llmTools.callEmbeddingModel) return [3 /*break*/, 4];
|
|
5306
|
-
// TODO: [🟥] Detect browser / node and make it colorfull
|
|
5307
|
-
console.error('No callEmbeddingModel function provided');
|
|
5308
|
-
return [3 /*break*/, 6];
|
|
5309
|
-
case 4: return [4 /*yield*/, llmTools.callEmbeddingModel({
|
|
5310
|
-
title: "Embedding for ".concat(title) /* <- Note: No impact on embedding result itself, just for logging */,
|
|
5311
|
-
parameters: {},
|
|
5312
|
-
content: knowledgePieceContent,
|
|
5313
|
-
modelRequirements: {
|
|
5314
|
-
modelVariant: 'EMBEDDING',
|
|
5315
|
-
},
|
|
5316
|
-
})];
|
|
5317
|
-
case 5:
|
|
5318
|
-
embeddingResult = _c.sent();
|
|
5319
|
-
index.push({
|
|
5320
|
-
modelName: embeddingResult.modelName,
|
|
5321
|
-
position: embeddingResult.content,
|
|
5322
|
-
});
|
|
5323
|
-
_c.label = 6;
|
|
5324
|
-
case 6: return [3 /*break*/, 8];
|
|
5325
|
-
case 7:
|
|
5326
|
-
error_1 = _c.sent();
|
|
5327
|
-
// Note: Here is expected error:
|
|
5328
|
-
// > PipelineExecutionError: You have not provided any `LlmExecutionTools` that support model variant "EMBEDDING
|
|
5329
|
-
if (!(error_1 instanceof PipelineExecutionError)) {
|
|
5330
|
-
throw error_1;
|
|
5331
|
-
}
|
|
5332
|
-
// TODO: [🟥] Detect browser / node and make it colorfull
|
|
5333
|
-
console.error(error_1, "<- Note: This error is not critical to prepare the pipeline, just knowledge pieces won't have embeddings");
|
|
5334
|
-
return [3 /*break*/, 8];
|
|
5335
|
-
case 8: return [2 /*return*/, {
|
|
5336
|
-
name: name,
|
|
5337
|
-
title: title,
|
|
5338
|
-
content: knowledgePieceContent,
|
|
5339
|
-
keywords: keywords,
|
|
5340
|
-
index: index,
|
|
5341
|
-
// <- TODO: [☀] sources,
|
|
5342
|
-
}];
|
|
5343
|
-
}
|
|
5344
|
-
});
|
|
5345
|
-
}); }))];
|
|
5346
|
-
case 6:
|
|
5347
|
-
knowledge = _j.sent();
|
|
5348
|
-
return [2 /*return*/, knowledge];
|
|
5349
|
-
}
|
|
5350
|
-
});
|
|
5351
|
-
});
|
|
5352
|
-
},
|
|
5353
|
-
} /* TODO: [🦷] as const */;
|
|
4924
|
+
var KnowledgeScrapeError = /** @class */ (function (_super) {
|
|
4925
|
+
__extends(KnowledgeScrapeError, _super);
|
|
4926
|
+
function KnowledgeScrapeError(message) {
|
|
4927
|
+
var _this = _super.call(this, message) || this;
|
|
4928
|
+
_this.name = 'KnowledgeScrapeError';
|
|
4929
|
+
Object.setPrototypeOf(_this, KnowledgeScrapeError.prototype);
|
|
4930
|
+
return _this;
|
|
4931
|
+
}
|
|
4932
|
+
return KnowledgeScrapeError;
|
|
4933
|
+
}(Error));
|
|
4934
|
+
|
|
5354
4935
|
/**
|
|
5355
|
-
*
|
|
5356
|
-
*
|
|
5357
|
-
*
|
|
5358
|
-
*
|
|
4936
|
+
* @@@
|
|
4937
|
+
*
|
|
4938
|
+
* Note: `$` is used to indicate that this function is not a pure function - it access global scope
|
|
4939
|
+
*
|
|
4940
|
+
* @private internal function of `$Register`
|
|
5359
4941
|
*/
|
|
4942
|
+
function $getGlobalScope() {
|
|
4943
|
+
return Function('return this')();
|
|
4944
|
+
}
|
|
5360
4945
|
|
|
5361
4946
|
/**
|
|
5362
|
-
*
|
|
4947
|
+
* @@@
|
|
5363
4948
|
*
|
|
5364
|
-
* @
|
|
5365
|
-
* @
|
|
4949
|
+
* @param text @@@
|
|
4950
|
+
* @returns @@@
|
|
4951
|
+
* @example 'HELLO_WORLD'
|
|
4952
|
+
* @example 'I_LOVE_PROMPTBOOK'
|
|
4953
|
+
* @public exported from `@promptbook/utils`
|
|
5366
4954
|
*/
|
|
5367
|
-
|
|
5368
|
-
|
|
5369
|
-
|
|
5370
|
-
|
|
5371
|
-
|
|
5372
|
-
|
|
5373
|
-
|
|
5374
|
-
|
|
5375
|
-
|
|
5376
|
-
|
|
5377
|
-
|
|
5378
|
-
|
|
5379
|
-
|
|
5380
|
-
|
|
5381
|
-
|
|
5382
|
-
|
|
5383
|
-
|
|
5384
|
-
|
|
5385
|
-
|
|
5386
|
-
|
|
5387
|
-
|
|
5388
|
-
|
|
5389
|
-
|
|
5390
|
-
|
|
5391
|
-
|
|
5392
|
-
|
|
5393
|
-
|
|
5394
|
-
|
|
5395
|
-
|
|
5396
|
-
|
|
5397
|
-
|
|
5398
|
-
|
|
5399
|
-
|
|
5400
|
-
|
|
5401
|
-
|
|
5402
|
-
|
|
5403
|
-
|
|
5404
|
-
|
|
5405
|
-
|
|
5406
|
-
|
|
5407
|
-
|
|
5408
|
-
|
|
5409
|
-
|
|
5410
|
-
|
|
5411
|
-
|
|
5412
|
-
|
|
5413
|
-
|
|
5414
|
-
|
|
5415
|
-
// TODO: !!!!!! [🕊] Make execCommand standard (?node-)util of the promptbook
|
|
5416
|
-
_e.sent();
|
|
5417
|
-
return [4 /*yield*/, $isFileExisting(cacheFilehandler.filename)];
|
|
5418
|
-
case 4:
|
|
5419
|
-
// Note: [0]
|
|
5420
|
-
if (!(_e.sent())) {
|
|
5421
|
-
throw new UnexpectedError(spaceTrim__default["default"](function (block) { return "\n File that was supposed to be created by Pandoc does not exist for unknown reason\n\n Expected file:\n ".concat(block(cacheFilehandler.filename), "\n\n Command:\n > ").concat(block(command_1), "\n\n "); }));
|
|
5422
|
-
}
|
|
5423
|
-
_e.label = 5;
|
|
5424
|
-
case 5: return [2 /*return*/, cacheFilehandler];
|
|
5425
|
-
}
|
|
5426
|
-
});
|
|
5427
|
-
});
|
|
5428
|
-
},
|
|
5429
|
-
/**
|
|
5430
|
-
* Scrapes the docx file and returns the knowledge pieces or `null` if it can't scrape it
|
|
5431
|
-
*/
|
|
5432
|
-
scrape: function (source, options) {
|
|
5433
|
-
return __awaiter(this, void 0, void 0, function () {
|
|
5434
|
-
var cacheFilehandler, markdownSource, knowledge;
|
|
5435
|
-
return __generator(this, function (_a) {
|
|
5436
|
-
switch (_a.label) {
|
|
5437
|
-
case 0: return [4 /*yield*/, documentScraper.$convert(source, options)];
|
|
5438
|
-
case 1:
|
|
5439
|
-
cacheFilehandler = _a.sent();
|
|
5440
|
-
markdownSource = {
|
|
5441
|
-
source: source.source,
|
|
5442
|
-
filename: cacheFilehandler.filename,
|
|
5443
|
-
url: null,
|
|
5444
|
-
mimeType: 'text/markdown',
|
|
5445
|
-
asText: function () {
|
|
5446
|
-
return __awaiter(this, void 0, void 0, function () {
|
|
5447
|
-
return __generator(this, function (_a) {
|
|
5448
|
-
switch (_a.label) {
|
|
5449
|
-
case 0: return [4 /*yield*/, promises.readFile(cacheFilehandler.filename, 'utf-8')];
|
|
5450
|
-
case 1:
|
|
5451
|
-
// Note: [0] In $convert we check that the file exists
|
|
5452
|
-
return [2 /*return*/, _a.sent()];
|
|
5453
|
-
}
|
|
5454
|
-
});
|
|
5455
|
-
});
|
|
5456
|
-
},
|
|
5457
|
-
asJson: function () {
|
|
5458
|
-
throw new UnexpectedError('Did not expect that `markdownScraper` would need to get the content `asJson`');
|
|
5459
|
-
},
|
|
5460
|
-
asBlob: function () {
|
|
5461
|
-
throw new UnexpectedError('Did not expect that `markdownScraper` would need to get the content `asBlob`');
|
|
5462
|
-
},
|
|
5463
|
-
};
|
|
5464
|
-
knowledge = markdownScraper.scrape(markdownSource, options);
|
|
5465
|
-
return [4 /*yield*/, cacheFilehandler.destroy()];
|
|
5466
|
-
case 2:
|
|
5467
|
-
_a.sent();
|
|
5468
|
-
return [2 /*return*/, knowledge];
|
|
5469
|
-
}
|
|
5470
|
-
});
|
|
5471
|
-
});
|
|
5472
|
-
},
|
|
5473
|
-
} /* TODO: [🦷] as const */;
|
|
4955
|
+
function normalizeTo_SCREAMING_CASE(text) {
|
|
4956
|
+
var e_1, _a;
|
|
4957
|
+
var charType;
|
|
4958
|
+
var lastCharType = 'OTHER';
|
|
4959
|
+
var normalizedName = '';
|
|
4960
|
+
try {
|
|
4961
|
+
for (var text_1 = __values(text), text_1_1 = text_1.next(); !text_1_1.done; text_1_1 = text_1.next()) {
|
|
4962
|
+
var char = text_1_1.value;
|
|
4963
|
+
var normalizedChar = void 0;
|
|
4964
|
+
if (/^[a-z]$/.test(char)) {
|
|
4965
|
+
charType = 'LOWERCASE';
|
|
4966
|
+
normalizedChar = char.toUpperCase();
|
|
4967
|
+
}
|
|
4968
|
+
else if (/^[A-Z]$/.test(char)) {
|
|
4969
|
+
charType = 'UPPERCASE';
|
|
4970
|
+
normalizedChar = char;
|
|
4971
|
+
}
|
|
4972
|
+
else if (/^[0-9]$/.test(char)) {
|
|
4973
|
+
charType = 'NUMBER';
|
|
4974
|
+
normalizedChar = char;
|
|
4975
|
+
}
|
|
4976
|
+
else {
|
|
4977
|
+
charType = 'OTHER';
|
|
4978
|
+
normalizedChar = '_';
|
|
4979
|
+
}
|
|
4980
|
+
if (charType !== lastCharType &&
|
|
4981
|
+
!(lastCharType === 'UPPERCASE' && charType === 'LOWERCASE') &&
|
|
4982
|
+
!(lastCharType === 'NUMBER') &&
|
|
4983
|
+
!(charType === 'NUMBER')) {
|
|
4984
|
+
normalizedName += '_';
|
|
4985
|
+
}
|
|
4986
|
+
normalizedName += normalizedChar;
|
|
4987
|
+
lastCharType = charType;
|
|
4988
|
+
}
|
|
4989
|
+
}
|
|
4990
|
+
catch (e_1_1) { e_1 = { error: e_1_1 }; }
|
|
4991
|
+
finally {
|
|
4992
|
+
try {
|
|
4993
|
+
if (text_1_1 && !text_1_1.done && (_a = text_1.return)) _a.call(text_1);
|
|
4994
|
+
}
|
|
4995
|
+
finally { if (e_1) throw e_1.error; }
|
|
4996
|
+
}
|
|
4997
|
+
normalizedName = normalizedName.replace(/_+/g, '_');
|
|
4998
|
+
normalizedName = normalizedName.replace(/_?\/_?/g, '/');
|
|
4999
|
+
normalizedName = normalizedName.replace(/^_/, '');
|
|
5000
|
+
normalizedName = normalizedName.replace(/_$/, '');
|
|
5001
|
+
return normalizedName;
|
|
5002
|
+
}
|
|
5474
5003
|
/**
|
|
5475
|
-
* TODO:
|
|
5476
|
-
*
|
|
5477
|
-
*
|
|
5478
|
-
*
|
|
5479
|
-
*
|
|
5004
|
+
* TODO: Tests
|
|
5005
|
+
* > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: 'Moje tabule' })).toEqual('/VtG7sR9rRJqwNEdM2/Moje tabule');
|
|
5006
|
+
* > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: 'ěščřžžýáíúů' })).toEqual('/VtG7sR9rRJqwNEdM2/escrzyaieuu');
|
|
5007
|
+
* > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: ' ahoj ' })).toEqual('/VtG7sR9rRJqwNEdM2/ahoj');
|
|
5008
|
+
* > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: ' ahoj_ahojAhoj ahoj ' })).toEqual('/VtG7sR9rRJqwNEdM2/ahoj-ahoj-ahoj-ahoj');
|
|
5009
|
+
* TODO: [🌺] Use some intermediate util splitWords
|
|
5480
5010
|
*/
|
|
5481
5011
|
|
|
5482
5012
|
/**
|
|
5483
|
-
*
|
|
5013
|
+
* @@@
|
|
5484
5014
|
*
|
|
5485
|
-
* @
|
|
5486
|
-
* @
|
|
5015
|
+
* @param text @@@
|
|
5016
|
+
* @returns @@@
|
|
5017
|
+
* @example 'hello_world'
|
|
5018
|
+
* @example 'i_love_promptbook'
|
|
5019
|
+
* @public exported from `@promptbook/utils`
|
|
5487
5020
|
*/
|
|
5488
|
-
|
|
5489
|
-
|
|
5490
|
-
|
|
5491
|
-
|
|
5492
|
-
mimeTypes: ['application/msword', 'text/rtf'],
|
|
5493
|
-
/**
|
|
5494
|
-
* Link to documentation
|
|
5495
|
-
*/
|
|
5496
|
-
documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
|
|
5497
|
-
/**
|
|
5498
|
-
* Convert the `.doc` or `.rtf` to `.doc` file and returns intermediate source
|
|
5499
|
-
*
|
|
5500
|
-
* Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object
|
|
5501
|
-
*/
|
|
5502
|
-
$convert: function (source, options) {
|
|
5503
|
-
return __awaiter(this, void 0, void 0, function () {
|
|
5504
|
-
var _a, externalProgramsPaths, rootDirname, _b, cacheDirname, _c, isCacheCleaned, _d, isVerbose, extension, cacheFilehandler, documentSourceOutdirPathForLibreOffice_1, command_1, files_1, file;
|
|
5505
|
-
return __generator(this, function (_e) {
|
|
5506
|
-
switch (_e.label) {
|
|
5507
|
-
case 0:
|
|
5508
|
-
_a = options.externalProgramsPaths, externalProgramsPaths = _a === void 0 ? {} : _a, rootDirname = options.rootDirname, _b = options.cacheDirname, cacheDirname = _b === void 0 ? SCRAPE_CACHE_DIRNAME : _b, _c = options.isCacheCleaned, isCacheCleaned = _c === void 0 ? false : _c, _d = options.isVerbose, isVerbose = _d === void 0 ? IS_VERBOSE : _d;
|
|
5509
|
-
if (!$isRunningInNode()) {
|
|
5510
|
-
throw new KnowledgeScrapeError('Scraping .doc files is only supported in Node environment');
|
|
5511
|
-
}
|
|
5512
|
-
if (externalProgramsPaths.libreOfficePath === undefined) {
|
|
5513
|
-
throw new MissingToolsError('LibreOffice is required for scraping .doc and .rtf files');
|
|
5514
|
-
}
|
|
5515
|
-
if (source.filename === null) {
|
|
5516
|
-
// TODO: [🧠] Maybe save file as temporary
|
|
5517
|
-
throw new KnowledgeScrapeError('When parsing .doc or .rtf file, it must be real file in the file system');
|
|
5518
|
-
}
|
|
5519
|
-
extension = getFileExtension(source.filename);
|
|
5520
|
-
return [4 /*yield*/, getScraperIntermediateSource(source, {
|
|
5521
|
-
rootDirname: rootDirname,
|
|
5522
|
-
cacheDirname: cacheDirname,
|
|
5523
|
-
isCacheCleaned: isCacheCleaned,
|
|
5524
|
-
extension: 'docx',
|
|
5525
|
-
isVerbose: isVerbose,
|
|
5526
|
-
})];
|
|
5527
|
-
case 1:
|
|
5528
|
-
cacheFilehandler = _e.sent();
|
|
5529
|
-
if (isVerbose) {
|
|
5530
|
-
console.info("documentScraper: Converting .".concat(extension, " -> .docx"));
|
|
5531
|
-
}
|
|
5532
|
-
return [4 /*yield*/, $isFileExisting(cacheFilehandler.filename)];
|
|
5533
|
-
case 2:
|
|
5534
|
-
if (!!(_e.sent())) return [3 /*break*/, 8];
|
|
5535
|
-
documentSourceOutdirPathForLibreOffice_1 = path.join(path.dirname(cacheFilehandler.filename), 'libreoffice')
|
|
5536
|
-
.split('\\')
|
|
5537
|
-
.join('/');
|
|
5538
|
-
command_1 = "\"".concat(externalProgramsPaths.libreOfficePath, "\" --headless --convert-to docx \"").concat(source.filename, "\" --outdir \"").concat(documentSourceOutdirPathForLibreOffice_1, "\"");
|
|
5539
|
-
// TODO: !!!!!! [🕊] Make execCommand standard (?node-)util of the promptbook - this should trigger build polution error
|
|
5540
|
-
return [4 /*yield*/, $execCommand(command_1)];
|
|
5541
|
-
case 3:
|
|
5542
|
-
// TODO: !!!!!! [🕊] Make execCommand standard (?node-)util of the promptbook - this should trigger build polution error
|
|
5543
|
-
_e.sent();
|
|
5544
|
-
return [4 /*yield*/, promises.readdir(documentSourceOutdirPathForLibreOffice_1)];
|
|
5545
|
-
case 4:
|
|
5546
|
-
files_1 = _e.sent();
|
|
5547
|
-
if (files_1.length !== 1) {
|
|
5548
|
-
throw new UnexpectedError(spaceTrim__default["default"](function (block) { return "\n Expected exactly 1 file in the LibreOffice output directory, got ".concat(files_1.length, "\n\n The temporary folder:\n ").concat(block(documentSourceOutdirPathForLibreOffice_1), "\n\n Command:\n > ").concat(block(command_1), "\n "); }));
|
|
5549
|
-
}
|
|
5550
|
-
file = files_1[0];
|
|
5551
|
-
return [4 /*yield*/, promises.rename(path.join(documentSourceOutdirPathForLibreOffice_1, file), cacheFilehandler.filename)];
|
|
5552
|
-
case 5:
|
|
5553
|
-
_e.sent();
|
|
5554
|
-
return [4 /*yield*/, promises.rmdir(documentSourceOutdirPathForLibreOffice_1)];
|
|
5555
|
-
case 6:
|
|
5556
|
-
_e.sent();
|
|
5557
|
-
return [4 /*yield*/, $isFileExisting(cacheFilehandler.filename)];
|
|
5558
|
-
case 7:
|
|
5559
|
-
if (!(_e.sent())) {
|
|
5560
|
-
throw new UnexpectedError(spaceTrim__default["default"](function (block) { return "\n File that was supposed to be created by LibreOffice does not exist for unknown reason\n\n Expected file:\n ".concat(block(cacheFilehandler.filename), "\n\n The temporary folder:\n ").concat(block(documentSourceOutdirPathForLibreOffice_1), "\n\n Command:\n > ").concat(block(command_1), "\n\n "); }));
|
|
5561
|
-
}
|
|
5562
|
-
_e.label = 8;
|
|
5563
|
-
case 8: return [2 /*return*/, cacheFilehandler];
|
|
5564
|
-
}
|
|
5565
|
-
});
|
|
5566
|
-
});
|
|
5567
|
-
},
|
|
5568
|
-
/**
|
|
5569
|
-
* Scrapes the `.doc` or `.rtf` file and returns the knowledge pieces or `null` if it can't scrape it
|
|
5570
|
-
*/
|
|
5571
|
-
scrape: function (source, options) {
|
|
5572
|
-
return __awaiter(this, void 0, void 0, function () {
|
|
5573
|
-
var cacheFilehandler, markdownSource, knowledge;
|
|
5574
|
-
return __generator(this, function (_a) {
|
|
5575
|
-
switch (_a.label) {
|
|
5576
|
-
case 0: return [4 /*yield*/, legacyDocumentScraper.$convert(source, options)];
|
|
5577
|
-
case 1:
|
|
5578
|
-
cacheFilehandler = _a.sent();
|
|
5579
|
-
markdownSource = {
|
|
5580
|
-
source: source.source,
|
|
5581
|
-
filename: cacheFilehandler.filename,
|
|
5582
|
-
url: null,
|
|
5583
|
-
mimeType: 'text/markdown',
|
|
5584
|
-
asText: function () {
|
|
5585
|
-
throw new UnexpectedError('Did not expect that `documentScraper` would need to get the content `asText`');
|
|
5586
|
-
},
|
|
5587
|
-
asJson: function () {
|
|
5588
|
-
throw new UnexpectedError('Did not expect that `documentScraper` would need to get the content `asJson`');
|
|
5589
|
-
},
|
|
5590
|
-
asBlob: function () {
|
|
5591
|
-
throw new UnexpectedError('Did not expect that `documentScraper` would need to get the content `asBlob`');
|
|
5592
|
-
},
|
|
5593
|
-
};
|
|
5594
|
-
knowledge = documentScraper.scrape(markdownSource, options);
|
|
5595
|
-
return [4 /*yield*/, cacheFilehandler.destroy()];
|
|
5596
|
-
case 2:
|
|
5597
|
-
_a.sent();
|
|
5598
|
-
return [2 /*return*/, knowledge];
|
|
5599
|
-
}
|
|
5600
|
-
});
|
|
5601
|
-
});
|
|
5602
|
-
},
|
|
5603
|
-
} /* TODO: [🦷] as const */;
|
|
5021
|
+
function normalizeTo_snake_case(text) {
|
|
5022
|
+
return normalizeTo_SCREAMING_CASE(text).toLowerCase();
|
|
5023
|
+
}
|
|
5024
|
+
|
|
5604
5025
|
/**
|
|
5605
|
-
*
|
|
5606
|
-
*
|
|
5607
|
-
*
|
|
5608
|
-
*
|
|
5609
|
-
*
|
|
5026
|
+
* Register is @@@
|
|
5027
|
+
*
|
|
5028
|
+
* Note: `$` is used to indicate that this function is not a pure function - it accesses and adds variables in global scope.
|
|
5029
|
+
*
|
|
5030
|
+
* @private internal utility, exported are only signleton instances of this class
|
|
5610
5031
|
*/
|
|
5032
|
+
var $Register = /** @class */ (function () {
|
|
5033
|
+
function $Register(registerName) {
|
|
5034
|
+
this.registerName = registerName;
|
|
5035
|
+
var storageName = "_promptbook_".concat(normalizeTo_snake_case(registerName));
|
|
5036
|
+
var globalScope = $getGlobalScope();
|
|
5037
|
+
if (globalScope[storageName] === undefined) {
|
|
5038
|
+
globalScope[storageName] = [];
|
|
5039
|
+
}
|
|
5040
|
+
else if (!Array.isArray(globalScope[storageName])) {
|
|
5041
|
+
throw new UnexpectedError("Expected (global) ".concat(storageName, " to be an array, but got ").concat(typeof globalScope[storageName]));
|
|
5042
|
+
}
|
|
5043
|
+
this.storage = globalScope[storageName];
|
|
5044
|
+
}
|
|
5045
|
+
$Register.prototype.list = function () {
|
|
5046
|
+
// <- TODO: ReadonlyDeep<ReadonlyArray<TRegistered>>
|
|
5047
|
+
return this.storage;
|
|
5048
|
+
};
|
|
5049
|
+
$Register.prototype.register = function (registered) {
|
|
5050
|
+
var packageName = registered.packageName, className = registered.className;
|
|
5051
|
+
var existingRegistrationIndex = this.storage.findIndex(function (item) { return item.packageName === packageName && item.className === className; });
|
|
5052
|
+
var existingRegistration = this.storage[existingRegistrationIndex];
|
|
5053
|
+
if (!existingRegistration) {
|
|
5054
|
+
this.storage.push(registered);
|
|
5055
|
+
}
|
|
5056
|
+
else {
|
|
5057
|
+
this.storage[existingRegistrationIndex] = registered;
|
|
5058
|
+
}
|
|
5059
|
+
return {
|
|
5060
|
+
registerName: this.registerName,
|
|
5061
|
+
packageName: packageName,
|
|
5062
|
+
className: className,
|
|
5063
|
+
get isDestroyed() {
|
|
5064
|
+
return false;
|
|
5065
|
+
},
|
|
5066
|
+
destroy: function () {
|
|
5067
|
+
throw new NotYetImplementedError("Registration to ".concat(this.registerName, " is permanent in this version of Promptbook"));
|
|
5068
|
+
},
|
|
5069
|
+
};
|
|
5070
|
+
};
|
|
5071
|
+
return $Register;
|
|
5072
|
+
}());
|
|
5611
5073
|
|
|
5612
5074
|
/**
|
|
5613
|
-
*
|
|
5075
|
+
* @@@
|
|
5614
5076
|
*
|
|
5615
|
-
*
|
|
5077
|
+
* Note: `$` is used to indicate that this interacts with the global scope
|
|
5078
|
+
* @singleton Only one instance of each register is created per build, but thare can be more @@@
|
|
5616
5079
|
* @public exported from `@promptbook/core`
|
|
5617
5080
|
*/
|
|
5618
|
-
var
|
|
5619
|
-
/**
|
|
5620
|
-
* Mime types that this scraper can handle
|
|
5621
|
-
*/
|
|
5622
|
-
mimeTypes: ['application/pdf'],
|
|
5623
|
-
/**
|
|
5624
|
-
* Link to documentation
|
|
5625
|
-
*/
|
|
5626
|
-
documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
|
|
5627
|
-
/**
|
|
5628
|
-
* Converts the `.pdf` file to `.md` file and returns intermediate source
|
|
5629
|
-
*/
|
|
5630
|
-
$convert: function (source, options) {
|
|
5631
|
-
return __awaiter(this, void 0, void 0, function () {
|
|
5632
|
-
return __generator(this, function (_a) {
|
|
5633
|
-
TODO_USE(source);
|
|
5634
|
-
TODO_USE(options);
|
|
5635
|
-
throw new NotYetImplementedError('PDF conversion not yet implemented');
|
|
5636
|
-
});
|
|
5637
|
-
});
|
|
5638
|
-
},
|
|
5639
|
-
/**
|
|
5640
|
-
* Scrapes the `.pdf` file and returns the knowledge pieces or `null` if it can't scrape it
|
|
5641
|
-
*/
|
|
5642
|
-
scrape: function (source, options) {
|
|
5643
|
-
return __awaiter(this, void 0, void 0, function () {
|
|
5644
|
-
return __generator(this, function (_a) {
|
|
5645
|
-
TODO_USE(source);
|
|
5646
|
-
TODO_USE(options);
|
|
5647
|
-
/*
|
|
5648
|
-
const {
|
|
5649
|
-
externalProgramsPaths = {},
|
|
5650
|
-
cacheDirname = SCRAPE_CACHE_DIRNAME,
|
|
5651
|
-
isCacheCleaned = false,
|
|
5652
|
-
isVerbose = IS_VERBOSE,
|
|
5653
|
-
} = options;
|
|
5654
|
-
*/
|
|
5655
|
-
throw new NotYetImplementedError('PDF scraping not yet implemented');
|
|
5656
|
-
});
|
|
5657
|
-
});
|
|
5658
|
-
},
|
|
5659
|
-
} /* TODO: [🦷] as const */;
|
|
5081
|
+
var $scrapersMetadataRegister = new $Register('scrapers_metadata');
|
|
5660
5082
|
/**
|
|
5661
|
-
* TODO: [
|
|
5662
|
-
* TODO: [🦖] Make some system for putting scrapers to separete packages
|
|
5663
|
-
* TODO: [🪂] Do it in parallel 11:11
|
|
5664
|
-
* TODO: [🦷] Ideally use `as const satisfies Converter & Scraper` BUT this combination throws errors
|
|
5665
|
-
* Note: No need to aggregate usage here, it is done by intercepting the llmTools
|
|
5083
|
+
* TODO: [®] DRY Register logic
|
|
5666
5084
|
*/
|
|
5667
5085
|
|
|
5668
5086
|
/**
|
|
5669
|
-
*
|
|
5087
|
+
* @@@
|
|
5670
5088
|
*
|
|
5671
|
-
*
|
|
5672
|
-
* @
|
|
5089
|
+
* Note: `$` is used to indicate that this interacts with the global scope
|
|
5090
|
+
* @singleton Only one instance of each register is created per build, but thare can be more @@@
|
|
5091
|
+
* @public exported from `@promptbook/core`
|
|
5673
5092
|
*/
|
|
5674
|
-
var
|
|
5675
|
-
flavor: 'github', // <- TODO: !!!!!! Explicitly specify the flavor of promptbook markdown
|
|
5676
|
-
/*
|
|
5677
|
-
> import showdownHighlight from 'showdown-highlight';
|
|
5678
|
-
> extensions: [
|
|
5679
|
-
> showdownHighlight({
|
|
5680
|
-
> // Whether to add the classes to the <pre> tag, default is false
|
|
5681
|
-
> pre: true,
|
|
5682
|
-
> // Whether to use hljs' auto language detection, default is true
|
|
5683
|
-
> auto_detection: true,
|
|
5684
|
-
> }),
|
|
5685
|
-
> ],
|
|
5686
|
-
*/
|
|
5687
|
-
});
|
|
5093
|
+
var $scrapersRegister = new $Register('scraper_constructors');
|
|
5688
5094
|
/**
|
|
5689
|
-
* TODO:
|
|
5690
|
-
* TODO: !!!!!! Lazy-make converter
|
|
5095
|
+
* TODO: [®] DRY Register logic
|
|
5691
5096
|
*/
|
|
5692
5097
|
|
|
5693
5098
|
/**
|
|
5694
|
-
*
|
|
5099
|
+
* Creates a message with all registered scrapers
|
|
5695
5100
|
*
|
|
5696
|
-
*
|
|
5697
|
-
*
|
|
5101
|
+
* Note: This function is used to create a (error) message when there is no scraper for particular mime type
|
|
5102
|
+
*
|
|
5103
|
+
* @private internal function of `createScrapersFromConfiguration` and `createScrapersFromEnv`
|
|
5698
5104
|
*/
|
|
5699
|
-
|
|
5700
|
-
|
|
5701
|
-
* Mime types that this scraper can handle
|
|
5702
|
-
*/
|
|
5703
|
-
mimeTypes: ['text/html'],
|
|
5704
|
-
/**
|
|
5705
|
-
* Link to documentation
|
|
5706
|
-
*/
|
|
5707
|
-
documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
|
|
5105
|
+
function $registeredScrapersMessage() {
|
|
5106
|
+
var e_1, _a, e_2, _b;
|
|
5708
5107
|
/**
|
|
5709
|
-
*
|
|
5710
|
-
*
|
|
5711
|
-
* Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object
|
|
5108
|
+
* Mixes registered scrapers from $scrapersMetadataRegister and $scrapersRegister
|
|
5712
5109
|
*/
|
|
5713
|
-
|
|
5714
|
-
|
|
5715
|
-
|
|
5716
|
-
|
|
5717
|
-
|
|
5718
|
-
|
|
5719
|
-
|
|
5720
|
-
|
|
5721
|
-
|
|
5722
|
-
|
|
5723
|
-
|
|
5724
|
-
|
|
5725
|
-
|
|
5726
|
-
|
|
5727
|
-
|
|
5728
|
-
|
|
5729
|
-
|
|
5730
|
-
|
|
5731
|
-
|
|
5732
|
-
|
|
5733
|
-
|
|
5734
|
-
|
|
5735
|
-
|
|
5736
|
-
|
|
5737
|
-
|
|
5738
|
-
|
|
5739
|
-
|
|
5740
|
-
|
|
5741
|
-
|
|
5742
|
-
|
|
5743
|
-
|
|
5744
|
-
|
|
5745
|
-
|
|
5746
|
-
|
|
5747
|
-
|
|
5748
|
-
|
|
5749
|
-
|
|
5750
|
-
|
|
5751
|
-
|
|
5752
|
-
|
|
5753
|
-
|
|
5754
|
-
|
|
5755
|
-
|
|
5756
|
-
|
|
5757
|
-
|
|
5758
|
-
_e.sent();
|
|
5759
|
-
markdown = markdownConverter.makeMarkdown(html, jsdom$1.window.document);
|
|
5760
|
-
return [2 /*return*/, __assign(__assign({}, cacheFilehandler), { markdown: markdown })];
|
|
5761
|
-
}
|
|
5762
|
-
});
|
|
5110
|
+
var all = [];
|
|
5111
|
+
var _loop_1 = function (packageName, className) {
|
|
5112
|
+
if (all.some(function (item) { return item.packageName === packageName && item.className === className; })) {
|
|
5113
|
+
return "continue";
|
|
5114
|
+
}
|
|
5115
|
+
all.push({ packageName: packageName, className: className });
|
|
5116
|
+
};
|
|
5117
|
+
try {
|
|
5118
|
+
for (var _c = __values($scrapersMetadataRegister.list()), _d = _c.next(); !_d.done; _d = _c.next()) {
|
|
5119
|
+
var _e = _d.value, packageName = _e.packageName, className = _e.className;
|
|
5120
|
+
_loop_1(packageName, className);
|
|
5121
|
+
}
|
|
5122
|
+
}
|
|
5123
|
+
catch (e_1_1) { e_1 = { error: e_1_1 }; }
|
|
5124
|
+
finally {
|
|
5125
|
+
try {
|
|
5126
|
+
if (_d && !_d.done && (_a = _c.return)) _a.call(_c);
|
|
5127
|
+
}
|
|
5128
|
+
finally { if (e_1) throw e_1.error; }
|
|
5129
|
+
}
|
|
5130
|
+
var _loop_2 = function (packageName, className) {
|
|
5131
|
+
if (all.some(function (item) { return item.packageName === packageName && item.className === className; })) {
|
|
5132
|
+
return "continue";
|
|
5133
|
+
}
|
|
5134
|
+
all.push({ packageName: packageName, className: className });
|
|
5135
|
+
};
|
|
5136
|
+
try {
|
|
5137
|
+
for (var _f = __values($scrapersRegister.list()), _g = _f.next(); !_g.done; _g = _f.next()) {
|
|
5138
|
+
var _h = _g.value, packageName = _h.packageName, className = _h.className;
|
|
5139
|
+
_loop_2(packageName, className);
|
|
5140
|
+
}
|
|
5141
|
+
}
|
|
5142
|
+
catch (e_2_1) { e_2 = { error: e_2_1 }; }
|
|
5143
|
+
finally {
|
|
5144
|
+
try {
|
|
5145
|
+
if (_g && !_g.done && (_b = _f.return)) _b.call(_f);
|
|
5146
|
+
}
|
|
5147
|
+
finally { if (e_2) throw e_2.error; }
|
|
5148
|
+
}
|
|
5149
|
+
var metadata = all.map(function (metadata) {
|
|
5150
|
+
var isMetadataAviailable = $scrapersMetadataRegister
|
|
5151
|
+
.list()
|
|
5152
|
+
.find(function (_a) {
|
|
5153
|
+
var packageName = _a.packageName, className = _a.className;
|
|
5154
|
+
return metadata.packageName === packageName && metadata.className === className;
|
|
5763
5155
|
});
|
|
5764
|
-
|
|
5765
|
-
|
|
5766
|
-
|
|
5767
|
-
|
|
5768
|
-
|
|
5769
|
-
return __awaiter(this, void 0, void 0, function () {
|
|
5770
|
-
var cacheFilehandler, markdownSource, knowledge;
|
|
5771
|
-
return __generator(this, function (_a) {
|
|
5772
|
-
switch (_a.label) {
|
|
5773
|
-
case 0: return [4 /*yield*/, websiteScraper.$convert(source, options)];
|
|
5774
|
-
case 1:
|
|
5775
|
-
cacheFilehandler = _a.sent();
|
|
5776
|
-
markdownSource = {
|
|
5777
|
-
source: source.source,
|
|
5778
|
-
filename: cacheFilehandler.filename,
|
|
5779
|
-
url: null,
|
|
5780
|
-
mimeType: 'text/markdown',
|
|
5781
|
-
asText: function () {
|
|
5782
|
-
return cacheFilehandler.markdown;
|
|
5783
|
-
},
|
|
5784
|
-
asJson: function () {
|
|
5785
|
-
throw new UnexpectedError('Did not expect that `markdownScraper` would need to get the content `asJson`');
|
|
5786
|
-
},
|
|
5787
|
-
asBlob: function () {
|
|
5788
|
-
throw new UnexpectedError('Did not expect that `markdownScraper` would need to get the content `asBlob`');
|
|
5789
|
-
},
|
|
5790
|
-
};
|
|
5791
|
-
knowledge = markdownScraper.scrape(markdownSource, options);
|
|
5792
|
-
return [4 /*yield*/, cacheFilehandler.destroy()];
|
|
5793
|
-
case 2:
|
|
5794
|
-
_a.sent();
|
|
5795
|
-
return [2 /*return*/, knowledge];
|
|
5796
|
-
}
|
|
5797
|
-
});
|
|
5156
|
+
var isInstalled = $scrapersRegister
|
|
5157
|
+
.list()
|
|
5158
|
+
.find(function (_a) {
|
|
5159
|
+
var packageName = _a.packageName, className = _a.className;
|
|
5160
|
+
return metadata.packageName === packageName && metadata.className === className;
|
|
5798
5161
|
});
|
|
5799
|
-
|
|
5800
|
-
|
|
5162
|
+
return __assign(__assign({}, metadata), { isMetadataAviailable: isMetadataAviailable, isInstalled: isInstalled });
|
|
5163
|
+
});
|
|
5164
|
+
if (metadata.length === 0) {
|
|
5165
|
+
return "No scrapers are available";
|
|
5166
|
+
}
|
|
5167
|
+
return spaceTrim__default["default"](function (block) { return "\n Available scrapers are:\n ".concat(block(metadata
|
|
5168
|
+
.map(function (_a, i) {
|
|
5169
|
+
var packageName = _a.packageName, className = _a.className, isMetadataAviailable = _a.isMetadataAviailable, isInstalled = _a.isInstalled;
|
|
5170
|
+
var more;
|
|
5171
|
+
if (just(false)) {
|
|
5172
|
+
more = '';
|
|
5173
|
+
}
|
|
5174
|
+
else if (!isMetadataAviailable && !isInstalled) {
|
|
5175
|
+
// TODO: [�][�] Maybe do allow to do auto-install if package not registered and not found
|
|
5176
|
+
more = "(not installed and no metadata, looks like a unexpected behavior)";
|
|
5177
|
+
}
|
|
5178
|
+
else if (isMetadataAviailable && !isInstalled) {
|
|
5179
|
+
// TODO: [�][�]
|
|
5180
|
+
more = "(not installed)";
|
|
5181
|
+
}
|
|
5182
|
+
else if (!isMetadataAviailable && isInstalled) {
|
|
5183
|
+
more = "(no metadata, looks like a unexpected behavior)";
|
|
5184
|
+
}
|
|
5185
|
+
else if (isMetadataAviailable && isInstalled) {
|
|
5186
|
+
more = "(installed)";
|
|
5187
|
+
}
|
|
5188
|
+
else {
|
|
5189
|
+
more = "(unknown state, looks like a unexpected behavior)";
|
|
5190
|
+
}
|
|
5191
|
+
return "".concat(i + 1, ") `").concat(className, "` from `").concat(packageName, "` ").concat(more);
|
|
5192
|
+
})
|
|
5193
|
+
.join('\n')), "\n "); });
|
|
5194
|
+
}
|
|
5801
5195
|
/**
|
|
5802
|
-
* TODO:
|
|
5803
|
-
* TODO: [👣] Scraped website in .md can act as cache item - there is no need to run conversion each time
|
|
5804
|
-
* TODO: [🦖] Make some system for putting scrapers to separete packages
|
|
5805
|
-
* TODO: [🪂] Do it in parallel 11:11
|
|
5806
|
-
* TODO: [🦷] Ideally use `as const satisfies Converter & Scraper` BUT this combination throws errors
|
|
5807
|
-
* Note: No need to aggregate usage here, it is done by intercepting the llmTools
|
|
5196
|
+
* TODO: [®] DRY Register logic
|
|
5808
5197
|
*/
|
|
5809
5198
|
|
|
5810
|
-
// TODO: [🦖] !!!!!! Pass scrapers as dependency,
|
|
5811
5199
|
/**
|
|
5812
5200
|
* @@@
|
|
5813
5201
|
*
|
|
5814
|
-
* @
|
|
5815
|
-
*
|
|
5202
|
+
* @param text @@@
|
|
5203
|
+
* @returns @@@
|
|
5204
|
+
* @example 'hello-world'
|
|
5205
|
+
* @example 'i-love-promptbook'
|
|
5206
|
+
* @public exported from `@promptbook/utils`
|
|
5816
5207
|
*/
|
|
5817
|
-
|
|
5818
|
-
|
|
5819
|
-
|
|
5820
|
-
|
|
5821
|
-
|
|
5822
|
-
|
|
5823
|
-
|
|
5824
|
-
|
|
5208
|
+
function normalizeToKebabCase(text) {
|
|
5209
|
+
var e_1, _a;
|
|
5210
|
+
text = removeDiacritics(text);
|
|
5211
|
+
var charType;
|
|
5212
|
+
var lastCharType = 'OTHER';
|
|
5213
|
+
var normalizedName = '';
|
|
5214
|
+
try {
|
|
5215
|
+
for (var text_1 = __values(text), text_1_1 = text_1.next(); !text_1_1.done; text_1_1 = text_1.next()) {
|
|
5216
|
+
var char = text_1_1.value;
|
|
5217
|
+
var normalizedChar = void 0;
|
|
5218
|
+
if (/^[a-z]$/.test(char)) {
|
|
5219
|
+
charType = 'LOWERCASE';
|
|
5220
|
+
normalizedChar = char;
|
|
5221
|
+
}
|
|
5222
|
+
else if (/^[A-Z]$/.test(char)) {
|
|
5223
|
+
charType = 'UPPERCASE';
|
|
5224
|
+
normalizedChar = char.toLowerCase();
|
|
5225
|
+
}
|
|
5226
|
+
else if (/^[0-9]$/.test(char)) {
|
|
5227
|
+
charType = 'NUMBER';
|
|
5228
|
+
normalizedChar = char;
|
|
5229
|
+
}
|
|
5230
|
+
else {
|
|
5231
|
+
charType = 'OTHER';
|
|
5232
|
+
normalizedChar = '-';
|
|
5233
|
+
}
|
|
5234
|
+
if (charType !== lastCharType &&
|
|
5235
|
+
!(lastCharType === 'UPPERCASE' && charType === 'LOWERCASE') &&
|
|
5236
|
+
!(lastCharType === 'NUMBER') &&
|
|
5237
|
+
!(charType === 'NUMBER')) {
|
|
5238
|
+
normalizedName += '-';
|
|
5239
|
+
}
|
|
5240
|
+
normalizedName += normalizedChar;
|
|
5241
|
+
lastCharType = charType;
|
|
5242
|
+
}
|
|
5243
|
+
}
|
|
5244
|
+
catch (e_1_1) { e_1 = { error: e_1_1 }; }
|
|
5245
|
+
finally {
|
|
5246
|
+
try {
|
|
5247
|
+
if (text_1_1 && !text_1_1.done && (_a = text_1.return)) _a.call(text_1);
|
|
5248
|
+
}
|
|
5249
|
+
finally { if (e_1) throw e_1.error; }
|
|
5250
|
+
}
|
|
5251
|
+
normalizedName = normalizedName.split(/-+/g).join('-');
|
|
5252
|
+
normalizedName = normalizedName.split(/-?\/-?/g).join('/');
|
|
5253
|
+
normalizedName = normalizedName.replace(/^-/, '');
|
|
5254
|
+
normalizedName = normalizedName.replace(/-$/, '');
|
|
5255
|
+
return normalizedName;
|
|
5256
|
+
}
|
|
5825
5257
|
|
|
5826
5258
|
/**
|
|
5827
5259
|
* Creates unique name for the source
|
|
5828
5260
|
*
|
|
5829
5261
|
* @private within the repository
|
|
5830
5262
|
*/
|
|
5831
|
-
function sourceContentToName(sourceContent) {
|
|
5832
|
-
|
|
5833
|
-
|
|
5834
|
-
|
|
5835
|
-
.
|
|
5836
|
-
|
|
5837
|
-
|
|
5838
|
-
var
|
|
5839
|
-
var
|
|
5840
|
-
|
|
5841
|
-
|
|
5842
|
-
|
|
5263
|
+
function sourceContentToName(sourceContent) {
|
|
5264
|
+
var hash = cryptoJs.SHA256(hexEncoder__default["default"].parse(JSON.stringify(sourceContent)))
|
|
5265
|
+
// <- TODO: [🥬] Encapsulate sha256 to some private utility function
|
|
5266
|
+
.toString( /* hex */)
|
|
5267
|
+
.substring(0, 20);
|
|
5268
|
+
// <- TODO: [🥬] Make some system for hashes and ids of promptbook
|
|
5269
|
+
var semanticName = normalizeToKebabCase(sourceContent.substring(0, 20));
|
|
5270
|
+
var pieces = ['source', semanticName, hash].filter(function (piece) { return piece !== ''; });
|
|
5271
|
+
var name = pieces.join('-').split('--').join('-');
|
|
5272
|
+
// <- TODO: Use MAX_FILENAME_LENGTH
|
|
5273
|
+
return name;
|
|
5274
|
+
}
|
|
5275
|
+
/**
|
|
5276
|
+
* TODO: [🐱🐉][🧠] Make some smart crop NOT source-i-m-pavol-a-develop-... BUT source-i-m-pavol-a-developer-...
|
|
5277
|
+
*/
|
|
5278
|
+
|
|
5279
|
+
/**
|
|
5280
|
+
* Convert file extension to mime type
|
|
5281
|
+
*
|
|
5282
|
+
* @private within the repository
|
|
5283
|
+
*/
|
|
5284
|
+
function extensionToMimeType(value) {
|
|
5285
|
+
return mimeTypes.lookup(value) || 'application/octet-stream';
|
|
5286
|
+
}
|
|
5287
|
+
|
|
5288
|
+
/**
|
|
5289
|
+
* Get the file extension from a file name
|
|
5290
|
+
*
|
|
5291
|
+
* @private within the repository
|
|
5292
|
+
*/
|
|
5293
|
+
function getFileExtension(value) {
|
|
5294
|
+
var match = value.match(/\.([0-9a-z]+)(?:[?#]|$)/i);
|
|
5295
|
+
return match ? match[1].toLowerCase() : null;
|
|
5296
|
+
}
|
|
5297
|
+
|
|
5298
|
+
/**
|
|
5299
|
+
* Checks if the file exists
|
|
5300
|
+
*
|
|
5301
|
+
* @private within the repository
|
|
5302
|
+
*/
|
|
5303
|
+
function isFileExisting(filename, fs) {
|
|
5304
|
+
return __awaiter(this, void 0, void 0, function () {
|
|
5305
|
+
var isReadAccessAllowed, isFile;
|
|
5306
|
+
return __generator(this, function (_a) {
|
|
5307
|
+
switch (_a.label) {
|
|
5308
|
+
case 0: return [4 /*yield*/, fs
|
|
5309
|
+
.access(filename, fs.constants.R_OK)
|
|
5310
|
+
.then(function () { return true; })
|
|
5311
|
+
.catch(function () { return false; })];
|
|
5312
|
+
case 1:
|
|
5313
|
+
isReadAccessAllowed = _a.sent();
|
|
5314
|
+
if (!isReadAccessAllowed) {
|
|
5315
|
+
return [2 /*return*/, false];
|
|
5316
|
+
}
|
|
5317
|
+
return [4 /*yield*/, fs
|
|
5318
|
+
.stat(filename)
|
|
5319
|
+
.then(function (fileStat) { return fileStat.isFile(); })
|
|
5320
|
+
.catch(function () { return false; })];
|
|
5321
|
+
case 2:
|
|
5322
|
+
isFile = _a.sent();
|
|
5323
|
+
return [2 /*return*/, isFile];
|
|
5324
|
+
}
|
|
5325
|
+
});
|
|
5326
|
+
});
|
|
5843
5327
|
}
|
|
5844
5328
|
/**
|
|
5845
|
-
*
|
|
5329
|
+
* Note: Not [~🟢~] because it is not directly dependent on `fs
|
|
5330
|
+
* TODO: [🐠] This can be a validator - with variants that return true/false and variants that throw errors with meaningless messages
|
|
5331
|
+
* TODO: [🖇] What about symlinks?
|
|
5846
5332
|
*/
|
|
5847
5333
|
|
|
5848
5334
|
/**
|
|
5849
|
-
*
|
|
5335
|
+
* Tests if given string is valid URL.
|
|
5850
5336
|
*
|
|
5851
|
-
*
|
|
5337
|
+
* Note: This does not check if the file exists only if the path is valid
|
|
5338
|
+
* @public exported from `@promptbook/utils`
|
|
5852
5339
|
*/
|
|
5853
|
-
function
|
|
5854
|
-
|
|
5340
|
+
function isValidFilePath(filename) {
|
|
5341
|
+
if (typeof filename !== 'string') {
|
|
5342
|
+
return false;
|
|
5343
|
+
}
|
|
5344
|
+
var filenameSlashes = filename.split('\\').join('/');
|
|
5345
|
+
// Absolute Unix path: /hello.txt
|
|
5346
|
+
if (/^(\/)/i.test(filenameSlashes)) {
|
|
5347
|
+
return true;
|
|
5348
|
+
}
|
|
5349
|
+
// Absolute Windows path: /hello.txt
|
|
5350
|
+
if (/^([A-Z]{1,2}:\/?)\//i.test(filenameSlashes)) {
|
|
5351
|
+
return true;
|
|
5352
|
+
}
|
|
5353
|
+
// Relative path: ./hello.txt
|
|
5354
|
+
if (/^(\.\.?\/)+/i.test(filenameSlashes)) {
|
|
5355
|
+
return true;
|
|
5356
|
+
}
|
|
5357
|
+
return false;
|
|
5855
5358
|
}
|
|
5856
5359
|
|
|
5857
5360
|
/**
|
|
5858
5361
|
* @@@
|
|
5859
5362
|
*
|
|
5860
|
-
* @
|
|
5363
|
+
* @public exported from `@promptbook/core`
|
|
5861
5364
|
*/
|
|
5862
|
-
function makeKnowledgeSourceHandler(knowledgeSource, options) {
|
|
5365
|
+
function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
|
|
5863
5366
|
var _a;
|
|
5864
5367
|
return __awaiter(this, void 0, void 0, function () {
|
|
5865
|
-
var sourceContent, name, _b, _c, rootDirname, _d,
|
|
5368
|
+
var sourceContent, name, _b, _c, rootDirname, _d,
|
|
5369
|
+
// <- TODO: process.cwd() if running in Node.js
|
|
5370
|
+
isVerbose, url, response_1, mimeType, filename_1, fileExtension, mimeType;
|
|
5866
5371
|
return __generator(this, function (_e) {
|
|
5867
5372
|
switch (_e.label) {
|
|
5868
5373
|
case 0:
|
|
5869
5374
|
sourceContent = knowledgeSource.sourceContent;
|
|
5870
5375
|
name = knowledgeSource.name;
|
|
5871
|
-
_b = options || {}, _c = _b.rootDirname, rootDirname = _c === void 0 ? null : _c, _d = _b.isVerbose, isVerbose = _d === void 0 ?
|
|
5376
|
+
_b = options || {}, _c = _b.rootDirname, rootDirname = _c === void 0 ? null : _c, _d = _b.isVerbose, isVerbose = _d === void 0 ? DEFAULT_IS_VERBOSE : _d;
|
|
5872
5377
|
TODO_USE(isVerbose);
|
|
5873
5378
|
if (!name) {
|
|
5874
5379
|
name = sourceContentToName(sourceContent);
|
|
@@ -5884,19 +5389,14 @@
|
|
|
5884
5389
|
filename: null,
|
|
5885
5390
|
url: url,
|
|
5886
5391
|
mimeType: mimeType,
|
|
5887
|
-
|
|
5888
|
-
|
|
5889
|
-
|
|
5890
|
-
|
|
5891
|
-
|
|
5892
|
-
|
|
5893
|
-
|
|
5894
|
-
|
|
5895
|
-
return [2 /*return*/, content];
|
|
5896
|
-
}
|
|
5897
|
-
});
|
|
5898
|
-
});
|
|
5899
|
-
},
|
|
5392
|
+
/*
|
|
5393
|
+
TODO: [🥽]
|
|
5394
|
+
> async asBlob() {
|
|
5395
|
+
> // TODO: [👨🏻🤝👨🏻] This can be called multiple times BUT when called second time, response in already consumed
|
|
5396
|
+
> const content = await response.blob();
|
|
5397
|
+
> return content;
|
|
5398
|
+
> },
|
|
5399
|
+
*/
|
|
5900
5400
|
asJson: function () {
|
|
5901
5401
|
return __awaiter(this, void 0, void 0, function () {
|
|
5902
5402
|
var content;
|
|
@@ -5926,8 +5426,9 @@
|
|
|
5926
5426
|
}];
|
|
5927
5427
|
case 2:
|
|
5928
5428
|
if (!(isValidFilePath(sourceContent) || /\.[a-z]{1,10}$/i.exec(sourceContent))) return [3 /*break*/, 4];
|
|
5929
|
-
if (
|
|
5930
|
-
throw new EnvironmentMismatchError('
|
|
5429
|
+
if (tools.fs === undefined) {
|
|
5430
|
+
throw new EnvironmentMismatchError('Can not import file knowledge without filesystem tools');
|
|
5431
|
+
// <- TODO: [🧠] What is the best error type here`
|
|
5931
5432
|
}
|
|
5932
5433
|
if (rootDirname === null) {
|
|
5933
5434
|
throw new EnvironmentMismatchError('Can not import file knowledge in non-file pipeline');
|
|
@@ -5935,35 +5436,31 @@
|
|
|
5935
5436
|
}
|
|
5936
5437
|
filename_1 = path.join(rootDirname, sourceContent).split('\\').join('/');
|
|
5937
5438
|
fileExtension = getFileExtension(filename_1);
|
|
5938
|
-
|
|
5939
|
-
return [4 /*yield*/,
|
|
5439
|
+
mimeType = extensionToMimeType(fileExtension || '');
|
|
5440
|
+
return [4 /*yield*/, isFileExisting(filename_1, tools.fs)];
|
|
5940
5441
|
case 3:
|
|
5941
5442
|
if (!(_e.sent())) {
|
|
5942
5443
|
throw new NotFoundError(spaceTrim__default["default"](function (block) { return "\n Can not make source handler for file which does not exist:\n\n File:\n ".concat(block(filename_1), "\n "); }));
|
|
5943
5444
|
}
|
|
5944
|
-
// TODO:
|
|
5445
|
+
// TODO: [🧠][😿] Test security file - file is scoped to the project (BUT maybe do this in `filesystemTools`)
|
|
5945
5446
|
return [2 /*return*/, {
|
|
5946
5447
|
source: name,
|
|
5947
5448
|
filename: filename_1,
|
|
5948
5449
|
url: null,
|
|
5949
|
-
mimeType:
|
|
5950
|
-
|
|
5951
|
-
|
|
5952
|
-
|
|
5953
|
-
|
|
5954
|
-
|
|
5955
|
-
|
|
5956
|
-
|
|
5957
|
-
|
|
5958
|
-
|
|
5959
|
-
|
|
5960
|
-
|
|
5961
|
-
|
|
5962
|
-
|
|
5963
|
-
}
|
|
5964
|
-
});
|
|
5965
|
-
});
|
|
5966
|
-
},
|
|
5450
|
+
mimeType: mimeType,
|
|
5451
|
+
/*
|
|
5452
|
+
TODO: [🥽]
|
|
5453
|
+
> async asBlob() {
|
|
5454
|
+
> const content = await tools.fs!.readFile(filename);
|
|
5455
|
+
> return new Blob(
|
|
5456
|
+
> [
|
|
5457
|
+
> content,
|
|
5458
|
+
> // <- TODO: [🥽] This is NOT tested, test it
|
|
5459
|
+
> ],
|
|
5460
|
+
> { type: mimeType },
|
|
5461
|
+
> );
|
|
5462
|
+
> },
|
|
5463
|
+
*/
|
|
5967
5464
|
asJson: function () {
|
|
5968
5465
|
return __awaiter(this, void 0, void 0, function () {
|
|
5969
5466
|
var _a, _b;
|
|
@@ -5971,7 +5468,7 @@
|
|
|
5971
5468
|
switch (_c.label) {
|
|
5972
5469
|
case 0:
|
|
5973
5470
|
_b = (_a = JSON).parse;
|
|
5974
|
-
return [4 /*yield*/,
|
|
5471
|
+
return [4 /*yield*/, tools.fs.readFile(filename_1, 'utf-8')];
|
|
5975
5472
|
case 1: return [2 /*return*/, _b.apply(_a, [_c.sent()])];
|
|
5976
5473
|
}
|
|
5977
5474
|
});
|
|
@@ -5981,7 +5478,7 @@
|
|
|
5981
5478
|
return __awaiter(this, void 0, void 0, function () {
|
|
5982
5479
|
return __generator(this, function (_a) {
|
|
5983
5480
|
switch (_a.label) {
|
|
5984
|
-
case 0: return [4 /*yield*/,
|
|
5481
|
+
case 0: return [4 /*yield*/, tools.fs.readFile(filename_1, 'utf-8')];
|
|
5985
5482
|
case 1: return [2 /*return*/, _a.sent()];
|
|
5986
5483
|
}
|
|
5987
5484
|
});
|
|
@@ -5999,9 +5496,14 @@
|
|
|
5999
5496
|
asJson: function () {
|
|
6000
5497
|
throw new UnexpectedError('Did not expect that `markdownScraper` would need to get the content `asJson`');
|
|
6001
5498
|
},
|
|
6002
|
-
|
|
6003
|
-
|
|
6004
|
-
|
|
5499
|
+
/*
|
|
5500
|
+
TODO: [🥽]
|
|
5501
|
+
> asBlob() {
|
|
5502
|
+
> throw new UnexpectedError(
|
|
5503
|
+
> 'Did not expect that `markdownScraper` would need to get the content `asBlob`',
|
|
5504
|
+
> );
|
|
5505
|
+
> },
|
|
5506
|
+
*/
|
|
6005
5507
|
}];
|
|
6006
5508
|
}
|
|
6007
5509
|
});
|
|
@@ -6014,262 +5516,106 @@
|
|
|
6014
5516
|
* @see https://github.com/webgptorg/promptbook/discussions/41
|
|
6015
5517
|
* @public exported from `@promptbook/core`
|
|
6016
5518
|
*/
|
|
6017
|
-
function prepareKnowledgePieces(knowledgeSources, options) {
|
|
5519
|
+
function prepareKnowledgePieces(knowledgeSources, tools, options) {
|
|
6018
5520
|
return __awaiter(this, void 0, void 0, function () {
|
|
6019
5521
|
var _a, maxParallelCount, rootDirname, _b, isVerbose, knowledgePreparedUnflatten, knowledgePrepared;
|
|
6020
5522
|
var _this = this;
|
|
6021
5523
|
return __generator(this, function (_c) {
|
|
6022
5524
|
switch (_c.label) {
|
|
6023
5525
|
case 0:
|
|
6024
|
-
_a = options.maxParallelCount, maxParallelCount = _a === void 0 ?
|
|
5526
|
+
_a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a, rootDirname = options.rootDirname, _b = options.isVerbose, isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b;
|
|
6025
5527
|
knowledgePreparedUnflatten = new Array(knowledgeSources.length);
|
|
6026
5528
|
return [4 /*yield*/, forEachAsync(knowledgeSources, { maxParallelCount: maxParallelCount }, function (knowledgeSource, index) { return __awaiter(_this, void 0, void 0, function () {
|
|
6027
|
-
var partialPieces, sourceHandler,
|
|
6028
|
-
var e_1,
|
|
6029
|
-
return __generator(this, function (
|
|
6030
|
-
switch (
|
|
5529
|
+
var partialPieces, sourceHandler, _a, _b, scraper, partialPiecesUnchecked, e_1_1, pieces;
|
|
5530
|
+
var e_1, _c;
|
|
5531
|
+
return __generator(this, function (_d) {
|
|
5532
|
+
switch (_d.label) {
|
|
6031
5533
|
case 0:
|
|
6032
5534
|
partialPieces = null;
|
|
6033
|
-
return [4 /*yield*/, makeKnowledgeSourceHandler(knowledgeSource, { rootDirname: rootDirname, isVerbose: isVerbose })];
|
|
5535
|
+
return [4 /*yield*/, makeKnowledgeSourceHandler(knowledgeSource, tools, { rootDirname: rootDirname, isVerbose: isVerbose })];
|
|
6034
5536
|
case 1:
|
|
6035
|
-
sourceHandler =
|
|
6036
|
-
|
|
5537
|
+
sourceHandler = _d.sent();
|
|
5538
|
+
_d.label = 2;
|
|
6037
5539
|
case 2:
|
|
6038
|
-
|
|
6039
|
-
|
|
6040
|
-
|
|
5540
|
+
_d.trys.push([2, 7, 8, 9]);
|
|
5541
|
+
_a = __values(arrayableToArray(tools.scrapers)), _b = _a.next();
|
|
5542
|
+
_d.label = 3;
|
|
6041
5543
|
case 3:
|
|
6042
|
-
if (!!
|
|
6043
|
-
scraper =
|
|
6044
|
-
if (!scraper.mimeTypes.includes(sourceHandler.mimeType)
|
|
5544
|
+
if (!!_b.done) return [3 /*break*/, 6];
|
|
5545
|
+
scraper = _b.value;
|
|
5546
|
+
if (!scraper.metadata.mimeTypes.includes(sourceHandler.mimeType)
|
|
6045
5547
|
// <- TODO: [🦔] Implement mime-type wildcards
|
|
6046
5548
|
) {
|
|
6047
5549
|
return [3 /*break*/, 5];
|
|
6048
5550
|
}
|
|
6049
|
-
return [4 /*yield*/, scraper.scrape(sourceHandler
|
|
5551
|
+
return [4 /*yield*/, scraper.scrape(sourceHandler)];
|
|
6050
5552
|
case 4:
|
|
6051
|
-
partialPiecesUnchecked =
|
|
5553
|
+
partialPiecesUnchecked = _d.sent();
|
|
6052
5554
|
if (partialPiecesUnchecked !== null) {
|
|
6053
|
-
partialPieces = partialPiecesUnchecked;
|
|
5555
|
+
partialPieces = __spreadArray([], __read(partialPiecesUnchecked), false);
|
|
5556
|
+
// <- TODO: [🪓] Here should be no need for spreading new array, just `partialPieces = partialPiecesUnchecked`
|
|
6054
5557
|
return [3 /*break*/, 6];
|
|
6055
5558
|
}
|
|
6056
|
-
|
|
5559
|
+
_d.label = 5;
|
|
6057
5560
|
case 5:
|
|
6058
|
-
|
|
5561
|
+
_b = _a.next();
|
|
6059
5562
|
return [3 /*break*/, 3];
|
|
6060
5563
|
case 6: return [3 /*break*/, 9];
|
|
6061
5564
|
case 7:
|
|
6062
|
-
e_1_1 =
|
|
5565
|
+
e_1_1 = _d.sent();
|
|
6063
5566
|
e_1 = { error: e_1_1 };
|
|
6064
5567
|
return [3 /*break*/, 9];
|
|
6065
5568
|
case 8:
|
|
6066
5569
|
try {
|
|
6067
|
-
if (
|
|
5570
|
+
if (_b && !_b.done && (_c = _a.return)) _c.call(_a);
|
|
6068
5571
|
}
|
|
6069
5572
|
finally { if (e_1) throw e_1.error; }
|
|
6070
5573
|
return [7 /*endfinally*/];
|
|
6071
5574
|
case 9:
|
|
6072
5575
|
if (partialPieces === null) {
|
|
6073
|
-
throw new KnowledgeScrapeError(spaceTrim__default["default"](function (block) { return "\n Cannot scrape knowledge from source: ".concat(knowledgeSource.sourceContent, "\n\n No scraper found for the mime type \"").concat(sourceHandler.mimeType, "\"\n\n
|
|
6074
|
-
.map(function (mimeType) { return "- ".concat(mimeType); })
|
|
6075
|
-
.join('\n')), "\n\n\n "); }));
|
|
5576
|
+
throw new KnowledgeScrapeError(spaceTrim__default["default"](function (block) { return "\n Cannot scrape knowledge from source: ".concat(knowledgeSource.sourceContent, "\n\n No scraper found for the mime type \"").concat(sourceHandler.mimeType, "\"\n\n ").concat(block($registeredScrapersMessage()), "\n\n\n "); }));
|
|
6076
5577
|
}
|
|
6077
5578
|
pieces = partialPieces.map(function (partialPiece) { return (__assign(__assign({}, partialPiece), { sources: [
|
|
6078
5579
|
{
|
|
6079
5580
|
name: knowledgeSource.name,
|
|
6080
5581
|
// line, column <- TODO: [☀]
|
|
6081
5582
|
// <- TODO: [❎]
|
|
6082
|
-
},
|
|
6083
|
-
] })); });
|
|
6084
|
-
knowledgePreparedUnflatten[index] = pieces;
|
|
6085
|
-
return [2 /*return*/];
|
|
6086
|
-
}
|
|
6087
|
-
});
|
|
6088
|
-
}); })];
|
|
6089
|
-
case 1:
|
|
6090
|
-
_c.sent();
|
|
6091
|
-
knowledgePrepared = knowledgePreparedUnflatten.flat();
|
|
6092
|
-
return [2 /*return*/, knowledgePrepared];
|
|
6093
|
-
}
|
|
6094
|
-
});
|
|
6095
|
-
});
|
|
6096
|
-
}
|
|
6097
|
-
/*
|
|
6098
|
-
TODO: [🧊] This is how it can look in future
|
|
6099
|
-
> type PrepareKnowledgeKnowledge = {
|
|
6100
|
-
> /**
|
|
6101
|
-
> * Unprepared knowledge
|
|
6102
|
-
> * /
|
|
6103
|
-
> readonly knowledgeSources: Array<KnowledgeSourceJson>;
|
|
6104
|
-
> };
|
|
6105
|
-
>
|
|
6106
|
-
> export async function prepareKnowledgePieces(
|
|
6107
|
-
> knowledge: PrepareKnowledgeKnowledge,
|
|
6108
|
-
> options: PrepareAndScrapeOptions,
|
|
6109
|
-
> ):
|
|
6110
|
-
*/
|
|
6111
|
-
/**
|
|
6112
|
-
* TODO: [🧊] In future one preparation can take data from previous preparation and save tokens and time
|
|
6113
|
-
* Put `knowledgePieces` into `PrepareKnowledgeOptions`
|
|
6114
|
-
* TODO: [🪂] More than max things can run in parallel by acident [1,[2a,2b,_],[3a,3b,_]]
|
|
6115
|
-
* TODO: [🧠][❎] Do here propper M:N mapping
|
|
6116
|
-
* [x] One source can make multiple pieces
|
|
6117
|
-
* [ ] One piece can have multiple sources
|
|
6118
|
-
*/
|
|
6119
|
-
|
|
6120
|
-
/**
|
|
6121
|
-
* Intercepts LLM tools and counts total usage of the tools
|
|
6122
|
-
*
|
|
6123
|
-
* @param llmTools LLM tools to be intercepted with usage counting
|
|
6124
|
-
* @returns LLM tools with same functionality with added total cost counting
|
|
6125
|
-
* @public exported from `@promptbook/core`
|
|
6126
|
-
*/
|
|
6127
|
-
function countTotalUsage(llmTools) {
|
|
6128
|
-
var _this = this;
|
|
6129
|
-
var totalUsage = ZERO_USAGE;
|
|
6130
|
-
var proxyTools = {
|
|
6131
|
-
get title() {
|
|
6132
|
-
// TODO: [🧠] Maybe put here some suffix
|
|
6133
|
-
return llmTools.title;
|
|
6134
|
-
},
|
|
6135
|
-
get description() {
|
|
6136
|
-
// TODO: [🧠] Maybe put here some suffix
|
|
6137
|
-
return llmTools.description;
|
|
6138
|
-
},
|
|
6139
|
-
checkConfiguration: function () {
|
|
6140
|
-
return __awaiter(this, void 0, void 0, function () {
|
|
6141
|
-
return __generator(this, function (_a) {
|
|
6142
|
-
return [2 /*return*/, /* not await */ llmTools.checkConfiguration()];
|
|
6143
|
-
});
|
|
6144
|
-
});
|
|
6145
|
-
},
|
|
6146
|
-
listModels: function () {
|
|
6147
|
-
return /* not await */ llmTools.listModels();
|
|
6148
|
-
},
|
|
6149
|
-
getTotalUsage: function () {
|
|
6150
|
-
// <- Note: [🥫] Not using getter `get totalUsage` but `getTotalUsage` to allow this object to be proxied
|
|
6151
|
-
return totalUsage;
|
|
6152
|
-
},
|
|
6153
|
-
};
|
|
6154
|
-
if (llmTools.callChatModel !== undefined) {
|
|
6155
|
-
proxyTools.callChatModel = function (prompt) { return __awaiter(_this, void 0, void 0, function () {
|
|
6156
|
-
var promptResult;
|
|
6157
|
-
return __generator(this, function (_a) {
|
|
6158
|
-
switch (_a.label) {
|
|
6159
|
-
case 0: return [4 /*yield*/, llmTools.callChatModel(prompt)];
|
|
6160
|
-
case 1:
|
|
6161
|
-
promptResult = _a.sent();
|
|
6162
|
-
totalUsage = addUsage(totalUsage, promptResult.usage);
|
|
6163
|
-
return [2 /*return*/, promptResult];
|
|
6164
|
-
}
|
|
6165
|
-
});
|
|
6166
|
-
}); };
|
|
6167
|
-
}
|
|
6168
|
-
if (llmTools.callCompletionModel !== undefined) {
|
|
6169
|
-
proxyTools.callCompletionModel = function (prompt) { return __awaiter(_this, void 0, void 0, function () {
|
|
6170
|
-
var promptResult;
|
|
6171
|
-
return __generator(this, function (_a) {
|
|
6172
|
-
switch (_a.label) {
|
|
6173
|
-
case 0: return [4 /*yield*/, llmTools.callCompletionModel(prompt)];
|
|
6174
|
-
case 1:
|
|
6175
|
-
promptResult = _a.sent();
|
|
6176
|
-
totalUsage = addUsage(totalUsage, promptResult.usage);
|
|
6177
|
-
return [2 /*return*/, promptResult];
|
|
6178
|
-
}
|
|
6179
|
-
});
|
|
6180
|
-
}); };
|
|
6181
|
-
}
|
|
6182
|
-
if (llmTools.callEmbeddingModel !== undefined) {
|
|
6183
|
-
proxyTools.callEmbeddingModel = function (prompt) { return __awaiter(_this, void 0, void 0, function () {
|
|
6184
|
-
var promptResult;
|
|
6185
|
-
return __generator(this, function (_a) {
|
|
6186
|
-
switch (_a.label) {
|
|
6187
|
-
case 0: return [4 /*yield*/, llmTools.callEmbeddingModel(prompt)];
|
|
6188
|
-
case 1:
|
|
6189
|
-
promptResult = _a.sent();
|
|
6190
|
-
totalUsage = addUsage(totalUsage, promptResult.usage);
|
|
6191
|
-
return [2 /*return*/, promptResult];
|
|
6192
|
-
}
|
|
6193
|
-
});
|
|
6194
|
-
}); };
|
|
6195
|
-
}
|
|
6196
|
-
// <- Note: [🤖]
|
|
6197
|
-
return proxyTools;
|
|
6198
|
-
}
|
|
6199
|
-
/**
|
|
6200
|
-
* TODO: [🧠][💸] Maybe make some common abstraction `interceptLlmTools` and use here (or use javascript Proxy?)
|
|
6201
|
-
* TODO: [🧠] Is there some meaningfull way how to test this util
|
|
6202
|
-
* TODO: [🧠][🌯] Maybe a way how to hide ability to `get totalUsage`
|
|
6203
|
-
* > const [llmToolsWithUsage,getUsage] = countTotalUsage(llmTools);
|
|
6204
|
-
* TODO: [👷♂️] @@@ Manual about construction of llmTools
|
|
6205
|
-
*/
|
|
6206
|
-
|
|
6207
|
-
/**
|
|
6208
|
-
* Prepares the persona for the pipeline
|
|
6209
|
-
*
|
|
6210
|
-
* @see https://github.com/webgptorg/promptbook/discussions/22
|
|
6211
|
-
* @public exported from `@promptbook/core`
|
|
6212
|
-
*/
|
|
6213
|
-
function preparePersona(personaDescription, options) {
|
|
6214
|
-
return __awaiter(this, void 0, void 0, function () {
|
|
6215
|
-
var llmTools, _a, isVerbose, collection, preparePersonaExecutor, _b, availableModels, availableModelNames, result, outputParameters, modelRequirementsRaw, modelRequirements, modelName, systemMessage, temperature;
|
|
6216
|
-
var _c;
|
|
6217
|
-
return __generator(this, function (_d) {
|
|
6218
|
-
switch (_d.label) {
|
|
6219
|
-
case 0:
|
|
6220
|
-
llmTools = options.llmTools, _a = options.isVerbose, isVerbose = _a === void 0 ? IS_VERBOSE : _a;
|
|
6221
|
-
if (llmTools === undefined) {
|
|
6222
|
-
throw new MissingToolsError('LLM tools are required for preparing persona');
|
|
6223
|
-
}
|
|
6224
|
-
collection = createCollectionFromJson.apply(void 0, __spreadArray([], __read(PipelineCollection), false));
|
|
6225
|
-
_b = createPipelineExecutor;
|
|
6226
|
-
_c = {};
|
|
6227
|
-
return [4 /*yield*/, collection.getPipelineByUrl('https://promptbook.studio/promptbook/prepare-persona.ptbk.md')];
|
|
6228
|
-
case 1:
|
|
6229
|
-
preparePersonaExecutor = _b.apply(void 0, [(_c.pipeline = _d.sent(),
|
|
6230
|
-
_c.tools = {
|
|
6231
|
-
llm: llmTools,
|
|
6232
|
-
},
|
|
6233
|
-
_c)]);
|
|
6234
|
-
return [4 /*yield*/, llmTools.listModels()];
|
|
6235
|
-
case 2:
|
|
6236
|
-
availableModels = _d.sent();
|
|
6237
|
-
availableModelNames = availableModels
|
|
6238
|
-
.filter(function (_a) {
|
|
6239
|
-
var modelVariant = _a.modelVariant;
|
|
6240
|
-
return modelVariant === 'CHAT';
|
|
6241
|
-
})
|
|
6242
|
-
.map(function (_a) {
|
|
6243
|
-
var modelName = _a.modelName;
|
|
6244
|
-
return modelName;
|
|
6245
|
-
})
|
|
6246
|
-
.join(',');
|
|
6247
|
-
return [4 /*yield*/, preparePersonaExecutor({ availableModelNames: availableModelNames, personaDescription: personaDescription })];
|
|
6248
|
-
case 3:
|
|
6249
|
-
result = _d.sent();
|
|
6250
|
-
assertsExecutionSuccessful(result);
|
|
6251
|
-
outputParameters = result.outputParameters;
|
|
6252
|
-
modelRequirementsRaw = outputParameters.modelRequirements;
|
|
6253
|
-
modelRequirements = JSON.parse(modelRequirementsRaw);
|
|
6254
|
-
if (isVerbose) {
|
|
6255
|
-
console.info("PERSONA ".concat(personaDescription), modelRequirements);
|
|
6256
|
-
}
|
|
6257
|
-
modelName = modelRequirements.modelName, systemMessage = modelRequirements.systemMessage, temperature = modelRequirements.temperature;
|
|
6258
|
-
return [2 /*return*/, {
|
|
6259
|
-
modelVariant: 'CHAT',
|
|
6260
|
-
modelName: modelName,
|
|
6261
|
-
systemMessage: systemMessage,
|
|
6262
|
-
temperature: temperature,
|
|
6263
|
-
}];
|
|
5583
|
+
},
|
|
5584
|
+
] })); });
|
|
5585
|
+
knowledgePreparedUnflatten[index] = pieces;
|
|
5586
|
+
return [2 /*return*/];
|
|
5587
|
+
}
|
|
5588
|
+
});
|
|
5589
|
+
}); })];
|
|
5590
|
+
case 1:
|
|
5591
|
+
_c.sent();
|
|
5592
|
+
knowledgePrepared = knowledgePreparedUnflatten.flat();
|
|
5593
|
+
return [2 /*return*/, knowledgePrepared];
|
|
6264
5594
|
}
|
|
6265
5595
|
});
|
|
6266
5596
|
});
|
|
6267
5597
|
}
|
|
5598
|
+
/*
|
|
5599
|
+
TODO: [🧊] This is how it can look in future
|
|
5600
|
+
> type PrepareKnowledgeKnowledge = {
|
|
5601
|
+
> /**
|
|
5602
|
+
> * Unprepared knowledge
|
|
5603
|
+
> * /
|
|
5604
|
+
> readonly knowledgeSources: ReadonlyArray<KnowledgeSourceJson>;
|
|
5605
|
+
> };
|
|
5606
|
+
>
|
|
5607
|
+
> export async function prepareKnowledgePieces(
|
|
5608
|
+
> knowledge: PrepareKnowledgeKnowledge,
|
|
5609
|
+
> options: PrepareAndScrapeOptions,
|
|
5610
|
+
> ):
|
|
5611
|
+
*/
|
|
6268
5612
|
/**
|
|
6269
|
-
* TODO: [
|
|
6270
|
-
*
|
|
6271
|
-
* TODO: [
|
|
6272
|
-
* TODO: [
|
|
5613
|
+
* TODO: [🧊] In future one preparation can take data from previous preparation and save tokens and time
|
|
5614
|
+
* Put `knowledgePieces` into `PrepareKnowledgeOptions`
|
|
5615
|
+
* TODO: [🪂] More than max things can run in parallel by acident [1,[2a,2b,_],[3a,3b,_]]
|
|
5616
|
+
* TODO: [🧠][❎] Do here propper M:N mapping
|
|
5617
|
+
* [x] One source can make multiple pieces
|
|
5618
|
+
* [ ] One piece can have multiple sources
|
|
6273
5619
|
*/
|
|
6274
5620
|
|
|
6275
5621
|
/**
|
|
@@ -6306,20 +5652,18 @@
|
|
|
6306
5652
|
*
|
|
6307
5653
|
* @public exported from `@promptbook/core`
|
|
6308
5654
|
*/
|
|
6309
|
-
function prepareTemplates(pipeline, options) {
|
|
5655
|
+
function prepareTemplates(pipeline, tools, options) {
|
|
6310
5656
|
return __awaiter(this, void 0, void 0, function () {
|
|
6311
5657
|
var _a, maxParallelCount, templates, parameters, knowledgePiecesCount, templatesPrepared;
|
|
6312
5658
|
var _this = this;
|
|
6313
5659
|
return __generator(this, function (_b) {
|
|
6314
5660
|
switch (_b.label) {
|
|
6315
5661
|
case 0:
|
|
6316
|
-
_a = options.maxParallelCount, maxParallelCount = _a === void 0 ?
|
|
5662
|
+
_a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a;
|
|
6317
5663
|
templates = pipeline.templates, parameters = pipeline.parameters, knowledgePiecesCount = pipeline.knowledgePiecesCount;
|
|
6318
5664
|
// TODO: [main] !!!!! Apply samples to each template (if missing and is for the template defined)
|
|
6319
5665
|
TODO_USE(parameters);
|
|
6320
|
-
templatesPrepared = new Array(
|
|
6321
|
-
// <- TODO: [🧱] Implement in a functional (not new Class) way
|
|
6322
|
-
templates.length);
|
|
5666
|
+
templatesPrepared = new Array(templates.length);
|
|
6323
5667
|
return [4 /*yield*/, forEachAsync(templates, { maxParallelCount: maxParallelCount /* <- TODO: [🪂] When there are subtasks, this maximul limit can be broken */ }, function (template, index) { return __awaiter(_this, void 0, void 0, function () {
|
|
6324
5668
|
var dependentParameterNames, preparedContent, preparedTemplate;
|
|
6325
5669
|
return __generator(this, function (_a) {
|
|
@@ -6362,14 +5706,14 @@
|
|
|
6362
5706
|
* Note: When the pipeline is already prepared, it returns the same pipeline
|
|
6363
5707
|
* @public exported from `@promptbook/core`
|
|
6364
5708
|
*/
|
|
6365
|
-
function preparePipeline(pipeline, options) {
|
|
5709
|
+
function preparePipeline(pipeline, tools, options) {
|
|
6366
5710
|
return __awaiter(this, void 0, void 0, function () {
|
|
6367
|
-
var
|
|
5711
|
+
var rootDirname, _a, maxParallelCount, _b, isVerbose, parameters, templates,
|
|
6368
5712
|
/*
|
|
6369
5713
|
<- TODO: [🧠][🪑] `promptbookVersion` */
|
|
6370
5714
|
knowledgeSources /*
|
|
6371
5715
|
<- TODO: [🧊] `knowledgePieces` */, personas /*
|
|
6372
|
-
<- TODO: [🧊] `preparations` */, llmToolsWithUsage, currentPreparation, preparations, preparedPersonas, knowledgeSourcesPrepared, partialknowledgePiecesPrepared, knowledgePiecesPrepared, templatesPrepared /* TODO: parameters: parametersPrepared*/;
|
|
5716
|
+
<- TODO: [🧊] `preparations` */, _llms, llmTools, llmToolsWithUsage, currentPreparation, preparations, preparedPersonas, knowledgeSourcesPrepared, partialknowledgePiecesPrepared, knowledgePiecesPrepared, templatesPrepared /* TODO: parameters: parametersPrepared*/;
|
|
6373
5717
|
var _this = this;
|
|
6374
5718
|
return __generator(this, function (_c) {
|
|
6375
5719
|
switch (_c.label) {
|
|
@@ -6377,11 +5721,13 @@
|
|
|
6377
5721
|
if (isPipelinePrepared(pipeline)) {
|
|
6378
5722
|
return [2 /*return*/, pipeline];
|
|
6379
5723
|
}
|
|
6380
|
-
|
|
5724
|
+
rootDirname = options.rootDirname, _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a, _b = options.isVerbose, isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b;
|
|
6381
5725
|
parameters = pipeline.parameters, templates = pipeline.templates, knowledgeSources = pipeline.knowledgeSources, personas = pipeline.personas;
|
|
6382
|
-
if (
|
|
5726
|
+
if (tools === undefined || tools.llm === undefined) {
|
|
6383
5727
|
throw new MissingToolsError('LLM tools are required for preparing the pipeline');
|
|
6384
5728
|
}
|
|
5729
|
+
_llms = arrayableToArray(tools.llm);
|
|
5730
|
+
llmTools = _llms.length === 1 ? _llms[0] : joinLlmExecutionTools.apply(void 0, __spreadArray([], __read(_llms), false));
|
|
6385
5731
|
llmToolsWithUsage = countTotalUsage(llmTools);
|
|
6386
5732
|
currentPreparation = {
|
|
6387
5733
|
id: 1,
|
|
@@ -6394,15 +5740,12 @@
|
|
|
6394
5740
|
// <- TODO: [🧊]
|
|
6395
5741
|
currentPreparation,
|
|
6396
5742
|
];
|
|
6397
|
-
preparedPersonas = new Array(
|
|
6398
|
-
// <- TODO: [🧱] Implement in a functional (not new Class) way
|
|
6399
|
-
personas.length);
|
|
5743
|
+
preparedPersonas = new Array(personas.length);
|
|
6400
5744
|
return [4 /*yield*/, forEachAsync(personas, { maxParallelCount: maxParallelCount /* <- TODO: [🪂] When there are subtasks, this maximul limit can be broken */ }, function (persona, index) { return __awaiter(_this, void 0, void 0, function () {
|
|
6401
5745
|
var modelRequirements, preparedPersona;
|
|
6402
5746
|
return __generator(this, function (_a) {
|
|
6403
5747
|
switch (_a.label) {
|
|
6404
|
-
case 0: return [4 /*yield*/, preparePersona(persona.description, {
|
|
6405
|
-
llmTools: llmToolsWithUsage,
|
|
5748
|
+
case 0: return [4 /*yield*/, preparePersona(persona.description, __assign(__assign({}, tools), { llm: llmToolsWithUsage }), {
|
|
6406
5749
|
rootDirname: rootDirname,
|
|
6407
5750
|
maxParallelCount: maxParallelCount /* <- TODO: [🪂] */,
|
|
6408
5751
|
isVerbose: isVerbose,
|
|
@@ -6418,7 +5761,7 @@
|
|
|
6418
5761
|
case 1:
|
|
6419
5762
|
_c.sent();
|
|
6420
5763
|
knowledgeSourcesPrepared = knowledgeSources.map(function (source) { return (__assign(__assign({}, source), { preparationIds: [/* TODO: [🧊] -> */ currentPreparation.id] })); });
|
|
6421
|
-
return [4 /*yield*/, prepareKnowledgePieces(knowledgeSources /* <- TODO: [🧊] {knowledgeSources, knowledgePieces} */, __assign(__assign({},
|
|
5764
|
+
return [4 /*yield*/, prepareKnowledgePieces(knowledgeSources /* <- TODO: [🧊] {knowledgeSources, knowledgePieces} */, __assign(__assign({}, tools), { llm: llmToolsWithUsage }), __assign(__assign({}, options), { rootDirname: rootDirname, maxParallelCount: maxParallelCount /* <- TODO: [🪂] */, isVerbose: isVerbose }))];
|
|
6422
5765
|
case 2:
|
|
6423
5766
|
partialknowledgePiecesPrepared = _c.sent();
|
|
6424
5767
|
knowledgePiecesPrepared = partialknowledgePiecesPrepared.map(function (piece) { return (__assign(__assign({}, piece), { preparationIds: [/* TODO: [🧊] -> */ currentPreparation.id] })); });
|
|
@@ -6426,8 +5769,7 @@
|
|
|
6426
5769
|
parameters: parameters,
|
|
6427
5770
|
templates: templates,
|
|
6428
5771
|
knowledgePiecesCount: knowledgePiecesPrepared.length,
|
|
6429
|
-
}, {
|
|
6430
|
-
llmTools: llmToolsWithUsage,
|
|
5772
|
+
}, __assign(__assign({}, tools), { llm: llmToolsWithUsage }), {
|
|
6431
5773
|
rootDirname: rootDirname,
|
|
6432
5774
|
maxParallelCount: maxParallelCount /* <- TODO: [🪂] */,
|
|
6433
5775
|
isVerbose: isVerbose,
|
|
@@ -6437,7 +5779,9 @@
|
|
|
6437
5779
|
// ----- /Templates preparation -----
|
|
6438
5780
|
// Note: Count total usage
|
|
6439
5781
|
currentPreparation.usage = llmToolsWithUsage.getTotalUsage();
|
|
6440
|
-
return [2 /*return*/, $asDeeplyFrozenSerializableJson('Prepared PipelineJson', __assign(__assign({}, clonePipeline(pipeline)), { templates:
|
|
5782
|
+
return [2 /*return*/, $asDeeplyFrozenSerializableJson('Prepared PipelineJson', __assign(__assign({}, clonePipeline(pipeline)), { templates: __spreadArray([], __read(templatesPrepared), false),
|
|
5783
|
+
// <- TODO: [🪓] Here should be no need for spreading new array, just ` templates: templatesPrepared`
|
|
5784
|
+
knowledgeSources: knowledgeSourcesPrepared, knowledgePieces: knowledgePiecesPrepared, personas: preparedPersonas, preparations: __spreadArray([], __read(preparations), false) }))];
|
|
6441
5785
|
}
|
|
6442
5786
|
});
|
|
6443
5787
|
});
|
|
@@ -6484,6 +5828,7 @@
|
|
|
6484
5828
|
'KNOWLEDGE ./hejny-cv.md',
|
|
6485
5829
|
'KNOWLEDGE ./hejny-cv.pdf',
|
|
6486
5830
|
'KNOWLEDGE ./hejny-cv.docx',
|
|
5831
|
+
// <- TODO: [😿] Allow ONLY files scoped in the (sub)directory NOT ../ and test it
|
|
6487
5832
|
],
|
|
6488
5833
|
/**
|
|
6489
5834
|
* Parses the KNOWLEDGE command
|
|
@@ -6956,72 +6301,6 @@
|
|
|
6956
6301
|
},
|
|
6957
6302
|
};
|
|
6958
6303
|
|
|
6959
|
-
/**
|
|
6960
|
-
* @@@
|
|
6961
|
-
*
|
|
6962
|
-
* @param text @@@
|
|
6963
|
-
* @returns @@@
|
|
6964
|
-
* @example 'HELLO_WORLD'
|
|
6965
|
-
* @example 'I_LOVE_PROMPTBOOK'
|
|
6966
|
-
* @public exported from `@promptbook/utils`
|
|
6967
|
-
*/
|
|
6968
|
-
function normalizeTo_SCREAMING_CASE(text) {
|
|
6969
|
-
var e_1, _a;
|
|
6970
|
-
var charType;
|
|
6971
|
-
var lastCharType = 'OTHER';
|
|
6972
|
-
var normalizedName = '';
|
|
6973
|
-
try {
|
|
6974
|
-
for (var text_1 = __values(text), text_1_1 = text_1.next(); !text_1_1.done; text_1_1 = text_1.next()) {
|
|
6975
|
-
var char = text_1_1.value;
|
|
6976
|
-
var normalizedChar = void 0;
|
|
6977
|
-
if (/^[a-z]$/.test(char)) {
|
|
6978
|
-
charType = 'LOWERCASE';
|
|
6979
|
-
normalizedChar = char.toUpperCase();
|
|
6980
|
-
}
|
|
6981
|
-
else if (/^[A-Z]$/.test(char)) {
|
|
6982
|
-
charType = 'UPPERCASE';
|
|
6983
|
-
normalizedChar = char;
|
|
6984
|
-
}
|
|
6985
|
-
else if (/^[0-9]$/.test(char)) {
|
|
6986
|
-
charType = 'NUMBER';
|
|
6987
|
-
normalizedChar = char;
|
|
6988
|
-
}
|
|
6989
|
-
else {
|
|
6990
|
-
charType = 'OTHER';
|
|
6991
|
-
normalizedChar = '_';
|
|
6992
|
-
}
|
|
6993
|
-
if (charType !== lastCharType &&
|
|
6994
|
-
!(lastCharType === 'UPPERCASE' && charType === 'LOWERCASE') &&
|
|
6995
|
-
!(lastCharType === 'NUMBER') &&
|
|
6996
|
-
!(charType === 'NUMBER')) {
|
|
6997
|
-
normalizedName += '_';
|
|
6998
|
-
}
|
|
6999
|
-
normalizedName += normalizedChar;
|
|
7000
|
-
lastCharType = charType;
|
|
7001
|
-
}
|
|
7002
|
-
}
|
|
7003
|
-
catch (e_1_1) { e_1 = { error: e_1_1 }; }
|
|
7004
|
-
finally {
|
|
7005
|
-
try {
|
|
7006
|
-
if (text_1_1 && !text_1_1.done && (_a = text_1.return)) _a.call(text_1);
|
|
7007
|
-
}
|
|
7008
|
-
finally { if (e_1) throw e_1.error; }
|
|
7009
|
-
}
|
|
7010
|
-
normalizedName = normalizedName.replace(/_+/g, '_');
|
|
7011
|
-
normalizedName = normalizedName.replace(/_?\/_?/g, '/');
|
|
7012
|
-
normalizedName = normalizedName.replace(/^_/, '');
|
|
7013
|
-
normalizedName = normalizedName.replace(/_$/, '');
|
|
7014
|
-
return normalizedName;
|
|
7015
|
-
}
|
|
7016
|
-
/**
|
|
7017
|
-
* TODO: Tests
|
|
7018
|
-
* > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: 'Moje tabule' })).toEqual('/VtG7sR9rRJqwNEdM2/Moje tabule');
|
|
7019
|
-
* > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: 'ěščřžžýáíúů' })).toEqual('/VtG7sR9rRJqwNEdM2/escrzyaieuu');
|
|
7020
|
-
* > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: ' ahoj ' })).toEqual('/VtG7sR9rRJqwNEdM2/ahoj');
|
|
7021
|
-
* > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: ' ahoj_ahojAhoj ahoj ' })).toEqual('/VtG7sR9rRJqwNEdM2/ahoj-ahoj-ahoj-ahoj');
|
|
7022
|
-
* TODO: [🌺] Use some intermediate util splitWords
|
|
7023
|
-
*/
|
|
7024
|
-
|
|
7025
6304
|
/**
|
|
7026
6305
|
* @@@
|
|
7027
6306
|
*
|
|
@@ -7086,6 +6365,22 @@
|
|
|
7086
6365
|
* TODO: [🌺] Use some intermediate util splitWords
|
|
7087
6366
|
*/
|
|
7088
6367
|
|
|
6368
|
+
/**
|
|
6369
|
+
* Removes emojis from a string and fix whitespaces
|
|
6370
|
+
*
|
|
6371
|
+
* @param text with emojis
|
|
6372
|
+
* @returns text without emojis
|
|
6373
|
+
* @public exported from `@promptbook/utils`
|
|
6374
|
+
*/
|
|
6375
|
+
function removeEmojis(text) {
|
|
6376
|
+
// Replace emojis (and also ZWJ sequence) with hyphens
|
|
6377
|
+
text = text.replace(/(\p{Extended_Pictographic})\p{Modifier_Symbol}/gu, '$1');
|
|
6378
|
+
text = text.replace(/(\p{Extended_Pictographic})[\u{FE00}-\u{FE0F}]/gu, '$1');
|
|
6379
|
+
text = text.replace(/(\p{Extended_Pictographic})(\u{200D}\p{Extended_Pictographic})*/gu, '$1');
|
|
6380
|
+
text = text.replace(/\p{Extended_Pictographic}/gu, '');
|
|
6381
|
+
return text;
|
|
6382
|
+
}
|
|
6383
|
+
|
|
7089
6384
|
/**
|
|
7090
6385
|
* Removes quotes from a string
|
|
7091
6386
|
*
|
|
@@ -8833,6 +8128,30 @@
|
|
|
8833
8128
|
return spaceTrim.spaceTrim(content.replace(/<!--(.*?)-->/gs, ''));
|
|
8834
8129
|
}
|
|
8835
8130
|
|
|
8131
|
+
/**
|
|
8132
|
+
* @@@
|
|
8133
|
+
*
|
|
8134
|
+
* @param value @@@
|
|
8135
|
+
* @returns @@@
|
|
8136
|
+
* @example @@@
|
|
8137
|
+
* @public exported from `@promptbook/utils`
|
|
8138
|
+
*/
|
|
8139
|
+
function titleToName(value) {
|
|
8140
|
+
if (isValidUrl(value)) {
|
|
8141
|
+
value = value.replace(/^https?:\/\//, '');
|
|
8142
|
+
value = value.replace(/\.html$/, '');
|
|
8143
|
+
}
|
|
8144
|
+
else if (isValidFilePath(value)) {
|
|
8145
|
+
value = path.basename(value);
|
|
8146
|
+
// Note: Keeping extension in the name
|
|
8147
|
+
}
|
|
8148
|
+
value = value.split('/').join('-');
|
|
8149
|
+
value = removeEmojis(value);
|
|
8150
|
+
value = normalizeToKebabCase(value);
|
|
8151
|
+
// TODO: [🧠] Maybe warn or add some padding to short name which are not good identifiers
|
|
8152
|
+
return value;
|
|
8153
|
+
}
|
|
8154
|
+
|
|
8836
8155
|
/**
|
|
8837
8156
|
* Compile pipeline from string (markdown) format to JSON format synchronously
|
|
8838
8157
|
*
|
|
@@ -9167,21 +8486,21 @@
|
|
|
9167
8486
|
* Note: This function acts as compilation process
|
|
9168
8487
|
*
|
|
9169
8488
|
* @param pipelineString {Promptbook} in string markdown format (.ptbk.md)
|
|
8489
|
+
* @param tools - Tools for the preparation and scraping - if not provided together with `llm`, the preparation will be skipped
|
|
9170
8490
|
* @param options - Options and tools for the compilation
|
|
9171
8491
|
* @returns {Promptbook} compiled in JSON format (.ptbk.json)
|
|
9172
8492
|
* @throws {ParseError} if the promptbook string is not valid
|
|
9173
8493
|
* @public exported from `@promptbook/core`
|
|
9174
8494
|
*/
|
|
9175
|
-
function pipelineStringToJson(pipelineString, options) {
|
|
8495
|
+
function pipelineStringToJson(pipelineString, tools, options) {
|
|
9176
8496
|
return __awaiter(this, void 0, void 0, function () {
|
|
9177
|
-
var
|
|
8497
|
+
var pipelineJson;
|
|
9178
8498
|
return __generator(this, function (_a) {
|
|
9179
8499
|
switch (_a.label) {
|
|
9180
8500
|
case 0:
|
|
9181
|
-
llmTools = (options || {}).llmTools;
|
|
9182
8501
|
pipelineJson = pipelineStringToJsonSync(pipelineString);
|
|
9183
|
-
if (!(
|
|
9184
|
-
return [4 /*yield*/, preparePipeline(pipelineJson, options || {
|
|
8502
|
+
if (!(tools !== undefined && tools.llm !== undefined)) return [3 /*break*/, 2];
|
|
8503
|
+
return [4 /*yield*/, preparePipeline(pipelineJson, tools, options || {
|
|
9185
8504
|
rootDirname: null,
|
|
9186
8505
|
})];
|
|
9187
8506
|
case 1:
|
|
@@ -9498,56 +8817,6 @@
|
|
|
9498
8817
|
* TODO: [🏛] Maybe make some markdown builder
|
|
9499
8818
|
*/
|
|
9500
8819
|
|
|
9501
|
-
/**
|
|
9502
|
-
* @@@
|
|
9503
|
-
*
|
|
9504
|
-
* Note: `$` is used to indicate that this function is not a pure function - it access global scope
|
|
9505
|
-
*
|
|
9506
|
-
* @private internal function of `$Register`
|
|
9507
|
-
*/
|
|
9508
|
-
function $getGlobalScope() {
|
|
9509
|
-
return Function('return this')();
|
|
9510
|
-
}
|
|
9511
|
-
|
|
9512
|
-
/**
|
|
9513
|
-
* Register is @@@
|
|
9514
|
-
*
|
|
9515
|
-
* Note: `$` is used to indicate that this function is not a pure function - it accesses and adds variables in global scope.
|
|
9516
|
-
*
|
|
9517
|
-
* @private internal utility, exported are only signleton instances of this class
|
|
9518
|
-
*/
|
|
9519
|
-
var $Register = /** @class */ (function () {
|
|
9520
|
-
function $Register(storageName) {
|
|
9521
|
-
this.storageName = storageName;
|
|
9522
|
-
storageName = "_promptbook_".concat(storageName);
|
|
9523
|
-
var globalScope = $getGlobalScope();
|
|
9524
|
-
if (globalScope[storageName] === undefined) {
|
|
9525
|
-
globalScope[storageName] = [];
|
|
9526
|
-
}
|
|
9527
|
-
else if (!Array.isArray(globalScope[storageName])) {
|
|
9528
|
-
throw new UnexpectedError("Expected (global) ".concat(storageName, " to be an array, but got ").concat(typeof globalScope[storageName]));
|
|
9529
|
-
}
|
|
9530
|
-
this.storage = globalScope[storageName];
|
|
9531
|
-
}
|
|
9532
|
-
$Register.prototype.list = function () {
|
|
9533
|
-
// <- TODO: ReadonlyDeep<Array<TRegistered>>
|
|
9534
|
-
return this.storage;
|
|
9535
|
-
};
|
|
9536
|
-
$Register.prototype.register = function (registered) {
|
|
9537
|
-
// <- TODO: What to return here
|
|
9538
|
-
var packageName = registered.packageName, className = registered.className;
|
|
9539
|
-
var existingRegistrationIndex = this.storage.findIndex(function (item) { return item.packageName === packageName && item.className === className; });
|
|
9540
|
-
var existingRegistration = this.storage[existingRegistrationIndex];
|
|
9541
|
-
if (!existingRegistration) {
|
|
9542
|
-
this.storage.push(registered);
|
|
9543
|
-
}
|
|
9544
|
-
else {
|
|
9545
|
-
this.storage[existingRegistrationIndex] = registered;
|
|
9546
|
-
}
|
|
9547
|
-
};
|
|
9548
|
-
return $Register;
|
|
9549
|
-
}());
|
|
9550
|
-
|
|
9551
8820
|
/**
|
|
9552
8821
|
* @@@
|
|
9553
8822
|
*
|
|
@@ -9556,6 +8825,9 @@
|
|
|
9556
8825
|
* @public exported from `@promptbook/core`
|
|
9557
8826
|
*/
|
|
9558
8827
|
var $llmToolsMetadataRegister = new $Register('llm_tools_metadata');
|
|
8828
|
+
/**
|
|
8829
|
+
* TODO: [®] DRY Register logic
|
|
8830
|
+
*/
|
|
9559
8831
|
|
|
9560
8832
|
/**
|
|
9561
8833
|
* @@@
|
|
@@ -9565,13 +8837,16 @@
|
|
|
9565
8837
|
* @public exported from `@promptbook/core`
|
|
9566
8838
|
*/
|
|
9567
8839
|
var $llmToolsRegister = new $Register('llm_execution_tools_constructors');
|
|
8840
|
+
/**
|
|
8841
|
+
* TODO: [®] DRY Register logic
|
|
8842
|
+
*/
|
|
9568
8843
|
|
|
9569
8844
|
/**
|
|
9570
8845
|
* Creates a message with all registered LLM tools
|
|
9571
8846
|
*
|
|
9572
8847
|
* Note: This function is used to create a (error) message when there is no constructor for some LLM provider
|
|
9573
8848
|
*
|
|
9574
|
-
* @private internal function of `createLlmToolsFromConfiguration` and `
|
|
8849
|
+
* @private internal function of `createLlmToolsFromConfiguration` and `$provideLlmToolsFromEnv`
|
|
9575
8850
|
*/
|
|
9576
8851
|
function $registeredLlmToolsMessage() {
|
|
9577
8852
|
var e_1, _a, e_2, _b;
|
|
@@ -9663,6 +8938,9 @@
|
|
|
9663
8938
|
})
|
|
9664
8939
|
.join('\n')), "\n "); });
|
|
9665
8940
|
}
|
|
8941
|
+
/**
|
|
8942
|
+
* TODO: [®] DRY Register logic
|
|
8943
|
+
*/
|
|
9666
8944
|
|
|
9667
8945
|
/**
|
|
9668
8946
|
* @@@
|
|
@@ -9674,7 +8952,7 @@
|
|
|
9674
8952
|
*/
|
|
9675
8953
|
function createLlmToolsFromConfiguration(configuration, options) {
|
|
9676
8954
|
if (options === void 0) { options = {}; }
|
|
9677
|
-
var _a = options.isVerbose, isVerbose = _a === void 0 ?
|
|
8955
|
+
var _a = options.isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a, userId = options.userId;
|
|
9678
8956
|
var llmTools = configuration.map(function (llmConfiguration) {
|
|
9679
8957
|
var registeredItem = $llmToolsRegister
|
|
9680
8958
|
.list()
|
|
@@ -9685,17 +8963,18 @@
|
|
|
9685
8963
|
if (registeredItem === undefined) {
|
|
9686
8964
|
throw new Error(spaceTrim__default["default"](function (block) { return "\n There is no constructor for LLM provider `".concat(llmConfiguration.className, "` from `").concat(llmConfiguration.packageName, "`\n\n You have probably forgotten install and import the provider package.\n To fix this issue, you can:\n\n Install:\n\n > npm install ").concat(llmConfiguration.packageName, "\n\n And import:\n\n > import '").concat(llmConfiguration.packageName, "';\n\n\n ").concat(block($registeredLlmToolsMessage()), "\n "); }));
|
|
9687
8965
|
}
|
|
9688
|
-
return registeredItem(__assign({ isVerbose: isVerbose }, llmConfiguration.options));
|
|
8966
|
+
return registeredItem(__assign({ isVerbose: isVerbose, userId: userId }, llmConfiguration.options));
|
|
9689
8967
|
});
|
|
9690
8968
|
return joinLlmExecutionTools.apply(void 0, __spreadArray([], __read(llmTools), false));
|
|
9691
8969
|
}
|
|
9692
8970
|
/**
|
|
9693
|
-
* TODO: [🎌]
|
|
8971
|
+
* TODO: [🎌] Together with `createLlmToolsFromConfiguration` + 'EXECUTION_TOOLS_CLASSES' gets to `@promptbook/core` ALL model providers, make this more efficient
|
|
9694
8972
|
* TODO: [🧠][🎌] Dynamically install required providers
|
|
9695
8973
|
* TODO: @@@ write discussion about this - wizzard
|
|
9696
8974
|
* TODO: [🧠][🍛] Which name is better `createLlmToolsFromConfig` or `createLlmToolsFromConfiguration`?
|
|
9697
8975
|
* TODO: [🧠] Is there some meaningfull way how to test this util
|
|
9698
8976
|
* TODO: This should be maybe not under `_common` but under `utils`
|
|
8977
|
+
* TODO: [®] DRY Register logic
|
|
9699
8978
|
*/
|
|
9700
8979
|
|
|
9701
8980
|
/**
|
|
@@ -9774,11 +9053,7 @@
|
|
|
9774
9053
|
function cacheLlmTools(llmTools, options) {
|
|
9775
9054
|
var _this = this;
|
|
9776
9055
|
if (options === void 0) { options = {}; }
|
|
9777
|
-
var _a = options.storage, storage = _a === void 0 ? new MemoryStorage() : _a,
|
|
9778
|
-
// <- TODO: [🧱] Implement in a functional (not new Class) way
|
|
9779
|
-
_b = options.isReloaded,
|
|
9780
|
-
// <- TODO: [🧱] Implement in a functional (not new Class) way
|
|
9781
|
-
isReloaded = _b === void 0 ? false : _b;
|
|
9056
|
+
var _a = options.storage, storage = _a === void 0 ? new MemoryStorage() : _a, _b = options.isCacheReloaded, isCacheReloaded = _b === void 0 ? false : _b;
|
|
9782
9057
|
var proxyTools = __assign(__assign({}, llmTools), {
|
|
9783
9058
|
// <- Note: [🥫]
|
|
9784
9059
|
get title() {
|
|
@@ -9800,7 +9075,7 @@
|
|
|
9800
9075
|
key = titleToName(prompt.title.substring(0, MAX_FILENAME_LENGTH - 10) +
|
|
9801
9076
|
'-' +
|
|
9802
9077
|
sha256__default["default"](hexEncoder__default["default"].parse(JSON.stringify(prompt.parameters))).toString( /* hex */));
|
|
9803
|
-
if (!!
|
|
9078
|
+
if (!!isCacheReloaded) return [3 /*break*/, 2];
|
|
9804
9079
|
return [4 /*yield*/, storage.getItem(key)];
|
|
9805
9080
|
case 1:
|
|
9806
9081
|
_a = _c.sent();
|
|
@@ -9928,9 +9203,9 @@
|
|
|
9928
9203
|
*/
|
|
9929
9204
|
|
|
9930
9205
|
/**
|
|
9931
|
-
*
|
|
9206
|
+
* Registration of LLM provider metadata
|
|
9932
9207
|
*
|
|
9933
|
-
*
|
|
9208
|
+
* Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available LLM tools
|
|
9934
9209
|
*
|
|
9935
9210
|
* @public exported from `@promptbook/core`
|
|
9936
9211
|
* @public exported from `@promptbook/cli`
|
|
@@ -9969,9 +9244,9 @@
|
|
|
9969
9244
|
});
|
|
9970
9245
|
|
|
9971
9246
|
/**
|
|
9972
|
-
*
|
|
9247
|
+
* Registration of LLM provider metadata
|
|
9973
9248
|
*
|
|
9974
|
-
*
|
|
9249
|
+
* Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available LLM tools
|
|
9975
9250
|
*
|
|
9976
9251
|
* @public exported from `@promptbook/core`
|
|
9977
9252
|
* @public exported from `@promptbook/cli`
|
|
@@ -10016,9 +9291,9 @@
|
|
|
10016
9291
|
});
|
|
10017
9292
|
|
|
10018
9293
|
/**
|
|
10019
|
-
*
|
|
9294
|
+
* Registration of LLM provider metadata
|
|
10020
9295
|
*
|
|
10021
|
-
*
|
|
9296
|
+
* Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available LLM tools
|
|
10022
9297
|
*
|
|
10023
9298
|
* @public exported from `@promptbook/core`
|
|
10024
9299
|
* @public exported from `@promptbook/cli`
|
|
@@ -10097,6 +9372,130 @@
|
|
|
10097
9372
|
},
|
|
10098
9373
|
});
|
|
10099
9374
|
|
|
9375
|
+
/**
|
|
9376
|
+
* Metadata of the scraper
|
|
9377
|
+
*
|
|
9378
|
+
* @private within the scraper directory
|
|
9379
|
+
*/
|
|
9380
|
+
var legacyDocumentScraperMetadata = $deepFreeze({
|
|
9381
|
+
title: 'LegacyDocument scraper',
|
|
9382
|
+
packageName: '@promptbook/legacy-documents',
|
|
9383
|
+
className: 'LegacyDocumentScraper',
|
|
9384
|
+
mimeTypes: ['application/msword', 'text/rtf'],
|
|
9385
|
+
documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
|
|
9386
|
+
isAvilableInBrowser: false,
|
|
9387
|
+
requiredExecutables: [
|
|
9388
|
+
'Pandoc',
|
|
9389
|
+
'LibreOffice',
|
|
9390
|
+
// <- TODO: [🧠] Should be 'LibreOffice' here, its dependency of dependency
|
|
9391
|
+
],
|
|
9392
|
+
}); /* <- TODO: [🤛] */
|
|
9393
|
+
/**
|
|
9394
|
+
* Registration of known scraper metadata
|
|
9395
|
+
*
|
|
9396
|
+
* Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available known scrapers
|
|
9397
|
+
*
|
|
9398
|
+
* @public exported from `@promptbook/core`
|
|
9399
|
+
* @public exported from `@promptbook/cli`
|
|
9400
|
+
*/
|
|
9401
|
+
var _LegacyDocumentScraperMetadataRegistration = $scrapersMetadataRegister.register(legacyDocumentScraperMetadata);
|
|
9402
|
+
|
|
9403
|
+
/**
|
|
9404
|
+
* Metadata of the scraper
|
|
9405
|
+
*
|
|
9406
|
+
* @private within the scraper directory
|
|
9407
|
+
*/
|
|
9408
|
+
var documentScraperMetadata = $deepFreeze({
|
|
9409
|
+
title: 'Document scraper',
|
|
9410
|
+
packageName: '@promptbook/documents',
|
|
9411
|
+
className: 'DocumentScraper',
|
|
9412
|
+
mimeTypes: ['application/vnd.openxmlformats-officedocument.wordprocessingml.document'],
|
|
9413
|
+
documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
|
|
9414
|
+
isAvilableInBrowser: false,
|
|
9415
|
+
requiredExecutables: ['Pandoc'],
|
|
9416
|
+
}); /* <- TODO: [🤛] */
|
|
9417
|
+
/**
|
|
9418
|
+
* Registration of known scraper metadata
|
|
9419
|
+
*
|
|
9420
|
+
* Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available known scrapers
|
|
9421
|
+
*
|
|
9422
|
+
* @public exported from `@promptbook/core`
|
|
9423
|
+
* @public exported from `@promptbook/cli`
|
|
9424
|
+
*/
|
|
9425
|
+
var _DocumentScraperMetadataRegistration = $scrapersMetadataRegister.register(documentScraperMetadata);
|
|
9426
|
+
|
|
9427
|
+
/**
|
|
9428
|
+
* Metadata of the scraper
|
|
9429
|
+
*
|
|
9430
|
+
* @private within the scraper directory
|
|
9431
|
+
*/
|
|
9432
|
+
var markdownScraperMetadata = $deepFreeze({
|
|
9433
|
+
title: 'Markdown scraper',
|
|
9434
|
+
packageName: '@promptbook/markdown-utils',
|
|
9435
|
+
className: 'MarkdownScraper',
|
|
9436
|
+
mimeTypes: ['text/markdown', 'text/plain'],
|
|
9437
|
+
documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
|
|
9438
|
+
isAvilableInBrowser: true,
|
|
9439
|
+
requiredExecutables: [],
|
|
9440
|
+
}); /* <- TODO: [🤛] */
|
|
9441
|
+
/**
|
|
9442
|
+
* Registration of known scraper metadata
|
|
9443
|
+
*
|
|
9444
|
+
* Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available known scrapers
|
|
9445
|
+
*
|
|
9446
|
+
* @public exported from `@promptbook/core`
|
|
9447
|
+
* @public exported from `@promptbook/cli`
|
|
9448
|
+
*/
|
|
9449
|
+
var _MarkdownScraperMetadataRegistration = $scrapersMetadataRegister.register(markdownScraperMetadata);
|
|
9450
|
+
|
|
9451
|
+
/**
|
|
9452
|
+
* Metadata of the scraper
|
|
9453
|
+
*
|
|
9454
|
+
* @private within the scraper directory
|
|
9455
|
+
*/
|
|
9456
|
+
var pdfScraperMetadata = $deepFreeze({
|
|
9457
|
+
title: 'Pdf scraper',
|
|
9458
|
+
packageName: '@promptbook/pdf',
|
|
9459
|
+
className: 'PdfScraper',
|
|
9460
|
+
mimeTypes: ['application/pdf'],
|
|
9461
|
+
documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
|
|
9462
|
+
isAvilableInBrowser: true,
|
|
9463
|
+
requiredExecutables: [],
|
|
9464
|
+
}); /* <- TODO: [🤛] */
|
|
9465
|
+
/**
|
|
9466
|
+
* Registration of known scraper metadata
|
|
9467
|
+
*
|
|
9468
|
+
* Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available known scrapers
|
|
9469
|
+
*
|
|
9470
|
+
* @public exported from `@promptbook/core`
|
|
9471
|
+
* @public exported from `@promptbook/cli`
|
|
9472
|
+
*/
|
|
9473
|
+
var _PdfScraperMetadataRegistration = $scrapersMetadataRegister.register(pdfScraperMetadata);
|
|
9474
|
+
|
|
9475
|
+
/**
|
|
9476
|
+
* Metadata of the scraper
|
|
9477
|
+
*
|
|
9478
|
+
* @private within the scraper directory
|
|
9479
|
+
*/
|
|
9480
|
+
var websiteScraperMetadata = $deepFreeze({
|
|
9481
|
+
title: 'Website scraper',
|
|
9482
|
+
packageName: '@promptbook/website-crawler',
|
|
9483
|
+
className: 'WebsiteScraper',
|
|
9484
|
+
mimeTypes: ['text/html'],
|
|
9485
|
+
documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
|
|
9486
|
+
isAvilableInBrowser: false,
|
|
9487
|
+
requiredExecutables: [],
|
|
9488
|
+
}); /* <- TODO: [🤛] */
|
|
9489
|
+
/**
|
|
9490
|
+
* Registration of known scraper metadata
|
|
9491
|
+
*
|
|
9492
|
+
* Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available known scrapers
|
|
9493
|
+
*
|
|
9494
|
+
* @public exported from `@promptbook/core`
|
|
9495
|
+
* @public exported from `@promptbook/cli`
|
|
9496
|
+
*/
|
|
9497
|
+
var _WebsiteScraperMetadataRegistration = $scrapersMetadataRegister.register(websiteScraperMetadata);
|
|
9498
|
+
|
|
10100
9499
|
/**
|
|
10101
9500
|
* This class behaves like LocalStorage but separates keys by prefix
|
|
10102
9501
|
*
|
|
@@ -10449,6 +9848,8 @@
|
|
|
10449
9848
|
|
|
10450
9849
|
exports.$llmToolsMetadataRegister = $llmToolsMetadataRegister;
|
|
10451
9850
|
exports.$llmToolsRegister = $llmToolsRegister;
|
|
9851
|
+
exports.$scrapersMetadataRegister = $scrapersMetadataRegister;
|
|
9852
|
+
exports.$scrapersRegister = $scrapersRegister;
|
|
10452
9853
|
exports.AbstractFormatError = AbstractFormatError;
|
|
10453
9854
|
exports.CLAIM = CLAIM;
|
|
10454
9855
|
exports.CallbackInterfaceTools = CallbackInterfaceTools;
|
|
@@ -10456,29 +9857,33 @@
|
|
|
10456
9857
|
exports.CsvFormatDefinition = CsvFormatDefinition;
|
|
10457
9858
|
exports.CsvFormatError = CsvFormatError;
|
|
10458
9859
|
exports.DEFAULT_CSV_SETTINGS = DEFAULT_CSV_SETTINGS;
|
|
9860
|
+
exports.DEFAULT_EXECUTIONS_CACHE_DIRNAME = DEFAULT_EXECUTIONS_CACHE_DIRNAME;
|
|
9861
|
+
exports.DEFAULT_INTERMEDIATE_FILES_STRATEGY = DEFAULT_INTERMEDIATE_FILES_STRATEGY;
|
|
9862
|
+
exports.DEFAULT_IS_AUTO_INSTALLED = DEFAULT_IS_AUTO_INSTALLED;
|
|
9863
|
+
exports.DEFAULT_IS_VERBOSE = DEFAULT_IS_VERBOSE;
|
|
9864
|
+
exports.DEFAULT_MAX_EXECUTION_ATTEMPTS = DEFAULT_MAX_EXECUTION_ATTEMPTS;
|
|
9865
|
+
exports.DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_DEPTH = DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_DEPTH;
|
|
9866
|
+
exports.DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_TOTAL = DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_TOTAL;
|
|
9867
|
+
exports.DEFAULT_MAX_PARALLEL_COUNT = DEFAULT_MAX_PARALLEL_COUNT;
|
|
9868
|
+
exports.DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME = DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME;
|
|
10459
9869
|
exports.DEFAULT_REMOTE_URL = DEFAULT_REMOTE_URL;
|
|
10460
9870
|
exports.DEFAULT_REMOTE_URL_PATH = DEFAULT_REMOTE_URL_PATH;
|
|
9871
|
+
exports.DEFAULT_SCRAPE_CACHE_DIRNAME = DEFAULT_SCRAPE_CACHE_DIRNAME;
|
|
10461
9872
|
exports.ERRORS = ERRORS;
|
|
10462
|
-
exports.EXECUTIONS_CACHE_DIRNAME = EXECUTIONS_CACHE_DIRNAME;
|
|
10463
9873
|
exports.EXPECTATION_UNITS = EXPECTATION_UNITS;
|
|
10464
9874
|
exports.EnvironmentMismatchError = EnvironmentMismatchError;
|
|
10465
9875
|
exports.ExecutionReportStringOptionsDefaults = ExecutionReportStringOptionsDefaults;
|
|
10466
9876
|
exports.ExpectError = ExpectError;
|
|
10467
|
-
exports.IS_VERBOSE = IS_VERBOSE;
|
|
10468
9877
|
exports.KnowledgeScrapeError = KnowledgeScrapeError;
|
|
10469
9878
|
exports.LimitReachedError = LimitReachedError;
|
|
10470
9879
|
exports.MANDATORY_CSV_SETTINGS = MANDATORY_CSV_SETTINGS;
|
|
10471
|
-
exports.MAX_EXECUTION_ATTEMPTS = MAX_EXECUTION_ATTEMPTS;
|
|
10472
9880
|
exports.MAX_FILENAME_LENGTH = MAX_FILENAME_LENGTH;
|
|
10473
|
-
exports.MAX_KNOWLEDGE_SOURCES_SCRAPING_DEPTH = MAX_KNOWLEDGE_SOURCES_SCRAPING_DEPTH;
|
|
10474
|
-
exports.MAX_KNOWLEDGE_SOURCES_SCRAPING_TOTAL = MAX_KNOWLEDGE_SOURCES_SCRAPING_TOTAL;
|
|
10475
|
-
exports.MAX_PARALLEL_COUNT = MAX_PARALLEL_COUNT;
|
|
10476
9881
|
exports.MODEL_VARIANTS = MODEL_VARIANTS;
|
|
10477
9882
|
exports.MemoryStorage = MemoryStorage;
|
|
10478
9883
|
exports.MissingToolsError = MissingToolsError;
|
|
9884
|
+
exports.MultipleLlmExecutionTools = MultipleLlmExecutionTools;
|
|
10479
9885
|
exports.NotFoundError = NotFoundError;
|
|
10480
9886
|
exports.NotYetImplementedError = NotYetImplementedError;
|
|
10481
|
-
exports.PIPELINE_COLLECTION_BASE_FILENAME = PIPELINE_COLLECTION_BASE_FILENAME;
|
|
10482
9887
|
exports.PROMPTBOOK_VERSION = PROMPTBOOK_VERSION;
|
|
10483
9888
|
exports.ParseError = ParseError;
|
|
10484
9889
|
exports.PipelineExecutionError = PipelineExecutionError;
|
|
@@ -10486,7 +9891,6 @@
|
|
|
10486
9891
|
exports.PipelineUrlError = PipelineUrlError;
|
|
10487
9892
|
exports.PrefixStorage = PrefixStorage;
|
|
10488
9893
|
exports.RESERVED_PARAMETER_NAMES = RESERVED_PARAMETER_NAMES;
|
|
10489
|
-
exports.SCRAPE_CACHE_DIRNAME = SCRAPE_CACHE_DIRNAME;
|
|
10490
9894
|
exports.TemplateTypes = TemplateTypes;
|
|
10491
9895
|
exports.TextFormatDefinition = TextFormatDefinition;
|
|
10492
9896
|
exports.UNCERTAIN_USAGE = UNCERTAIN_USAGE;
|
|
@@ -10494,8 +9898,13 @@
|
|
|
10494
9898
|
exports.ZERO_USAGE = ZERO_USAGE;
|
|
10495
9899
|
exports._AnthropicClaudeMetadataRegistration = _AnthropicClaudeMetadataRegistration;
|
|
10496
9900
|
exports._AzureOpenAiMetadataRegistration = _AzureOpenAiMetadataRegistration;
|
|
9901
|
+
exports._DocumentScraperMetadataRegistration = _DocumentScraperMetadataRegistration;
|
|
9902
|
+
exports._LegacyDocumentScraperMetadataRegistration = _LegacyDocumentScraperMetadataRegistration;
|
|
9903
|
+
exports._MarkdownScraperMetadataRegistration = _MarkdownScraperMetadataRegistration;
|
|
10497
9904
|
exports._OpenAiAssistantMetadataRegistration = _OpenAiAssistantMetadataRegistration;
|
|
10498
9905
|
exports._OpenAiMetadataRegistration = _OpenAiMetadataRegistration;
|
|
9906
|
+
exports._PdfScraperMetadataRegistration = _PdfScraperMetadataRegistration;
|
|
9907
|
+
exports._WebsiteScraperMetadataRegistration = _WebsiteScraperMetadataRegistration;
|
|
10499
9908
|
exports.addUsage = addUsage;
|
|
10500
9909
|
exports.assertsExecutionSuccessful = assertsExecutionSuccessful;
|
|
10501
9910
|
exports.cacheLlmTools = cacheLlmTools;
|
|
@@ -10507,16 +9916,13 @@
|
|
|
10507
9916
|
exports.createLlmToolsFromConfiguration = createLlmToolsFromConfiguration;
|
|
10508
9917
|
exports.createPipelineExecutor = createPipelineExecutor;
|
|
10509
9918
|
exports.createSubcollection = createSubcollection;
|
|
10510
|
-
exports.documentScraper = documentScraper;
|
|
10511
9919
|
exports.embeddingVectorToString = embeddingVectorToString;
|
|
10512
9920
|
exports.executionReportJsonToString = executionReportJsonToString;
|
|
10513
9921
|
exports.isPassingExpectations = isPassingExpectations;
|
|
10514
9922
|
exports.isPipelinePrepared = isPipelinePrepared;
|
|
10515
9923
|
exports.joinLlmExecutionTools = joinLlmExecutionTools;
|
|
10516
|
-
exports.legacyDocumentScraper = legacyDocumentScraper;
|
|
10517
9924
|
exports.limitTotalUsage = limitTotalUsage;
|
|
10518
|
-
exports.
|
|
10519
|
-
exports.pdfScraper = pdfScraper;
|
|
9925
|
+
exports.makeKnowledgeSourceHandler = makeKnowledgeSourceHandler;
|
|
10520
9926
|
exports.pipelineJsonToString = pipelineJsonToString;
|
|
10521
9927
|
exports.pipelineStringToJson = pipelineStringToJson;
|
|
10522
9928
|
exports.pipelineStringToJsonSync = pipelineStringToJsonSync;
|
|
@@ -10530,7 +9936,6 @@
|
|
|
10530
9936
|
exports.usageToHuman = usageToHuman;
|
|
10531
9937
|
exports.usageToWorktime = usageToWorktime;
|
|
10532
9938
|
exports.validatePipeline = validatePipeline;
|
|
10533
|
-
exports.websiteScraper = websiteScraper;
|
|
10534
9939
|
|
|
10535
9940
|
Object.defineProperty(exports, '__esModule', { value: true });
|
|
10536
9941
|
|