@promptbook/markdown-utils 0.71.0-17 → 0.71.0-19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -4
- package/esm/index.es.js +59 -73
- package/esm/index.es.js.map +1 -1
- package/esm/typings/src/_packages/core.index.d.ts +24 -18
- package/esm/typings/src/_packages/node.index.d.ts +2 -0
- package/esm/typings/src/_packages/types.index.d.ts +10 -10
- package/esm/typings/src/_packages/utils.index.d.ts +2 -2
- package/esm/typings/src/collection/PipelineCollection.d.ts +1 -1
- package/esm/typings/src/collection/SimplePipelineCollection.d.ts +2 -2
- package/esm/typings/src/collection/collectionToJson.d.ts +1 -1
- package/esm/typings/src/collection/constructors/createCollectionFromJson.d.ts +1 -1
- package/esm/typings/src/collection/constructors/createCollectionFromPromise.d.ts +1 -1
- package/esm/typings/src/commands/_common/types/CommandParser.d.ts +5 -5
- package/esm/typings/src/config.d.ts +21 -14
- package/esm/typings/src/execution/EmbeddingVector.d.ts +1 -1
- package/esm/typings/src/execution/Executables.d.ts +18 -0
- package/esm/typings/src/execution/ExecutionTools.d.ts +9 -3
- package/esm/typings/src/execution/LlmExecutionTools.d.ts +1 -1
- package/esm/typings/src/execution/PipelineExecutorResult.d.ts +2 -2
- package/esm/typings/src/execution/assertsExecutionSuccessful.d.ts +1 -0
- package/esm/typings/src/execution/createPipelineExecutor/00-CreatePipelineExecutorOptions.d.ts +29 -6
- package/esm/typings/src/execution/createPipelineExecutor/10-executePipeline.d.ts +2 -11
- package/esm/typings/src/execution/createPipelineExecutor/20-executeTemplate.d.ts +4 -13
- package/esm/typings/src/execution/createPipelineExecutor/40-executeAttempts.d.ts +9 -14
- package/esm/typings/src/execution/translation/automatic-translate/automatic-translators/LindatAutomaticTranslator.d.ts +11 -3
- package/esm/typings/src/execution/utils/addUsage.d.ts +1 -1
- package/esm/typings/src/execution/utils/forEachAsync.d.ts +1 -1
- package/esm/typings/src/formats/_common/FormatDefinition.d.ts +2 -2
- package/esm/typings/src/formats/_common/FormatSubvalueDefinition.d.ts +2 -2
- package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsForCli.d.ts +2 -2
- package/esm/typings/src/llm-providers/_common/register/LlmToolsConfiguration.d.ts +1 -1
- package/esm/typings/src/llm-providers/_common/utils/cache/CacheLlmToolsOptions.d.ts +4 -1
- package/esm/typings/src/llm-providers/anthropic-claude/AnthropicClaudeExecutionTools.d.ts +1 -1
- package/esm/typings/src/llm-providers/anthropic-claude/anthropic-claude-models.d.ts +1 -1
- package/esm/typings/src/llm-providers/azure-openai/AzureOpenAiExecutionTools.d.ts +1 -1
- package/esm/typings/src/llm-providers/mocked/$fakeTextToExpectations.d.ts +1 -1
- package/esm/typings/src/llm-providers/mocked/MockedEchoLlmExecutionTools.d.ts +1 -1
- package/esm/typings/src/llm-providers/mocked/MockedFackedLlmExecutionTools.d.ts +1 -1
- package/esm/typings/src/llm-providers/multiple/MultipleLlmExecutionTools.d.ts +4 -5
- package/esm/typings/src/llm-providers/multiple/joinLlmExecutionTools.d.ts +1 -1
- package/esm/typings/src/llm-providers/openai/OpenAiExecutionTools.d.ts +1 -1
- package/esm/typings/src/llm-providers/openai/openai-models.d.ts +1 -1
- package/esm/typings/src/llm-providers/remote/RemoteLlmExecutionTools.d.ts +1 -1
- package/esm/typings/src/llm-providers/remote/interfaces/PromptbookServer_ListModels_Response.d.ts +4 -4
- package/esm/typings/src/llm-providers/remote/interfaces/PromptbookServer_Prompt_Response.d.ts +3 -3
- package/esm/typings/src/prepare/PrepareAndScrapeOptions.d.ts +4 -23
- package/esm/typings/src/prepare/prepareTemplates.d.ts +1 -1
- package/esm/typings/src/scrapers/_common/Scraper.d.ts +2 -2
- package/esm/typings/src/scrapers/_common/prepareKnowledgePieces.d.ts +1 -1
- package/esm/typings/src/scrapers/_common/register/$provideExecutablesForNode.d.ts +12 -0
- package/esm/typings/src/scrapers/_common/register/$provideScrapersForBrowser.d.ts +2 -2
- package/esm/typings/src/scrapers/_common/register/$provideScrapersForNode.d.ts +2 -2
- package/esm/typings/src/scrapers/_common/register/ScraperAndConverterMetadata.d.ts +2 -2
- package/esm/typings/src/scrapers/_common/utils/getScraperIntermediateSource.d.ts +2 -2
- package/esm/typings/src/scrapers/_common/utils/makeKnowledgeSourceHandler.d.ts +1 -1
- package/esm/typings/src/scrapers/document/DocumentScraper.d.ts +2 -2
- package/esm/typings/src/scrapers/document/createDocumentScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/document/register-metadata.d.ts +1 -1
- package/esm/typings/src/scrapers/document-legacy/LegacyDocumentScraper.d.ts +3 -3
- package/esm/typings/src/scrapers/document-legacy/createLegacyDocumentScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/document-legacy/register-metadata.d.ts +1 -1
- package/esm/typings/src/scrapers/markdown/MarkdownScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/markdown/createMarkdownScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/markdown/register-metadata.d.ts +1 -1
- package/esm/typings/src/scrapers/pdf/PdfScraper.d.ts +2 -2
- package/esm/typings/src/scrapers/pdf/createPdfScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/pdf/register-metadata.d.ts +1 -1
- package/esm/typings/src/scrapers/website/WebsiteScraper.d.ts +6 -3
- package/esm/typings/src/scrapers/website/createWebsiteScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/website/register-metadata.d.ts +1 -1
- package/esm/typings/src/scrapers/website/utils/createShowdownConverter.d.ts +7 -0
- package/esm/typings/src/scripting/javascript/utils/preserve.d.ts +1 -1
- package/esm/typings/src/storage/file-cache-storage/utils/nameToSubfolderPath.d.ts +1 -1
- package/esm/typings/src/types/Arrayable.d.ts +1 -1
- package/esm/typings/src/types/IntermediateFilesStrategy.d.ts +7 -0
- package/esm/typings/src/types/PipelineJson/KnowledgePieceJson.d.ts +4 -4
- package/esm/typings/src/types/PipelineJson/KnowledgeSourceJson.d.ts +1 -1
- package/esm/typings/src/types/PipelineJson/PersonaJson.d.ts +1 -1
- package/esm/typings/src/types/PipelineJson/TemplateJsonCommon.d.ts +2 -2
- package/esm/typings/src/types/Prompt.d.ts +1 -1
- package/esm/typings/src/types/execution-report/ExecutionReportJson.d.ts +1 -1
- package/esm/typings/src/utils/$Register.d.ts +1 -1
- package/esm/typings/src/utils/FromtoItems.d.ts +1 -1
- package/esm/typings/src/utils/arrayableToArray.d.ts +1 -1
- package/esm/typings/src/utils/emojis.d.ts +1 -1
- package/esm/typings/src/utils/execCommand/$execCommand.d.ts +2 -2
- package/esm/typings/src/utils/execCommand/{IExecCommandOptions.d.ts → ExecCommandOptions.d.ts} +2 -6
- package/esm/typings/src/utils/execCommand/execCommandNormalizeOptions.d.ts +3 -3
- package/esm/typings/src/utils/expectation-counters/countSentences.d.ts +1 -1
- package/esm/typings/src/utils/markdown/extractAllBlocksFromMarkdown.d.ts +1 -1
- package/esm/typings/src/utils/markdown/splitMarkdownIntoSections.d.ts +1 -1
- package/esm/typings/src/utils/normalization/IKeywords.d.ts +2 -2
- package/esm/typings/src/utils/normalization/parseKeywords.d.ts +2 -2
- package/esm/typings/src/utils/normalization/parseKeywordsFromString.d.ts +2 -2
- package/esm/typings/src/utils/normalization/searchKeywords.d.ts +2 -2
- package/esm/typings/src/utils/organization/TODO_USE.d.ts +1 -1
- package/esm/typings/src/utils/organization/keepUnused.d.ts +1 -1
- package/esm/typings/src/utils/random/$randomSeed.d.ts +1 -1
- package/esm/typings/src/utils/sets/intersection.d.ts +1 -1
- package/esm/typings/src/utils/sets/union.d.ts +1 -1
- package/esm/typings/src/utils/unwrapResult.d.ts +4 -4
- package/package.json +3 -2
- package/umd/index.umd.js +59 -73
- package/umd/index.umd.js.map +1 -1
- package/esm/typings/src/execution/createPipelineExecutor/00-CreatePipelineExecutorSettings.d.ts +0 -29
- package/esm/typings/src/scrapers/website/utils/markdownConverter.d.ts +0 -12
- /package/esm/typings/src/scrapers/website/utils/{markdownConverter.test.d.ts → createShowdownConverter.test.d.ts} +0 -0
package/README.md
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
#  Promptbook
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
Build responsible, controlled and transparent applications on top of LLM models!
|
|
6
6
|
|
|
7
7
|
|
|
8
8
|
|
|
@@ -21,11 +21,9 @@ Supercharge your use of large language models
|
|
|
21
21
|
- ✨ **Support of [OpenAI o1 model](https://openai.com/o1/)**
|
|
22
22
|
|
|
23
23
|
<blockquote style="color: #ff8811">
|
|
24
|
-
<b>⚠ Warning:</b> This is a pre-release version of the library. It is not yet ready for production use. Please look at <a href="https://www.npmjs.com/package/@promptbook/core?activeTab=versions">latest stable release</a>.
|
|
24
|
+
<b>⚠ Warning:</b> This is a pre-release version of the library. It is not yet ready for production use. Please look at <a href="https://www.npmjs.com/package/@promptbook/core?activeTab=versions">latest stable release</a>.
|
|
25
25
|
</blockquote>
|
|
26
26
|
|
|
27
|
-
|
|
28
|
-
|
|
29
27
|
## 📦 Package `@promptbook/markdown-utils`
|
|
30
28
|
|
|
31
29
|
- Promptbooks are [divided into several](#-packages) packages, all are published from [single monorepo](https://github.com/webgptorg/promptbook).
|
package/esm/index.es.js
CHANGED
|
@@ -12,7 +12,7 @@ import { unparse, parse } from 'papaparse';
|
|
|
12
12
|
/**
|
|
13
13
|
* The version of the Promptbook library
|
|
14
14
|
*/
|
|
15
|
-
var PROMPTBOOK_VERSION = '0.71.0-
|
|
15
|
+
var PROMPTBOOK_VERSION = '0.71.0-18';
|
|
16
16
|
// TODO: [main] !!!! List here all the versions and annotate + put into script
|
|
17
17
|
|
|
18
18
|
/*! *****************************************************************************
|
|
@@ -413,7 +413,6 @@ function pipelineJsonToString(pipelineJson) {
|
|
|
413
413
|
pipelineString += '\n\n';
|
|
414
414
|
pipelineString += description;
|
|
415
415
|
}
|
|
416
|
-
// TODO:> const commands: Array<Command>
|
|
417
416
|
var commands = [];
|
|
418
417
|
if (pipelineUrl) {
|
|
419
418
|
commands.push("PIPELINE URL ".concat(pipelineUrl));
|
|
@@ -469,7 +468,6 @@ function pipelineJsonToString(pipelineJson) {
|
|
|
469
468
|
pipelineString += '\n\n';
|
|
470
469
|
pipelineString += description_1;
|
|
471
470
|
}
|
|
472
|
-
// TODO:> const commands: Array<Command>
|
|
473
471
|
var commands_1 = [];
|
|
474
472
|
var contentLanguage = 'text';
|
|
475
473
|
if (templateType === 'PROMPT_TEMPLATE') {
|
|
@@ -835,18 +833,19 @@ var LOOP_LIMIT = 1000;
|
|
|
835
833
|
* @private within the repository - too low-level in comparison with other `MAX_...`
|
|
836
834
|
*/
|
|
837
835
|
var IMMEDIATE_TIME = 10;
|
|
836
|
+
// <- TODO: [😡] Change to 'VISIBLE'
|
|
838
837
|
/**
|
|
839
838
|
* The maximum number of (LLM) tasks running in parallel
|
|
840
839
|
*
|
|
841
840
|
* @public exported from `@promptbook/core`
|
|
842
841
|
*/
|
|
843
|
-
var
|
|
842
|
+
var DEFAULT_MAX_PARALLEL_COUNT = 5; // <- TODO: [🤹♂️]
|
|
844
843
|
/**
|
|
845
844
|
* The maximum number of attempts to execute LLM task before giving up
|
|
846
845
|
*
|
|
847
846
|
* @public exported from `@promptbook/core`
|
|
848
847
|
*/
|
|
849
|
-
var
|
|
848
|
+
var DEFAULT_MAX_EXECUTION_ATTEMPTS = 3; // <- TODO: [🤹♂️]
|
|
850
849
|
/**
|
|
851
850
|
* Nonce which is used for replacing things in strings
|
|
852
851
|
*
|
|
@@ -898,7 +897,7 @@ var DEFAULT_CSV_SETTINGS = Object.freeze({
|
|
|
898
897
|
*
|
|
899
898
|
* @public exported from `@promptbook/core`
|
|
900
899
|
*/
|
|
901
|
-
var
|
|
900
|
+
var DEFAULT_IS_VERBOSE = false;
|
|
902
901
|
/**
|
|
903
902
|
* @@@
|
|
904
903
|
*
|
|
@@ -2091,6 +2090,7 @@ function assertsExecutionSuccessful(executionResult) {
|
|
|
2091
2090
|
}
|
|
2092
2091
|
}
|
|
2093
2092
|
/**
|
|
2093
|
+
* TODO: [🐚] This function should be removed OR changed OR be completely rewritten
|
|
2094
2094
|
* TODO: [🧠] Can this return type be better typed than void
|
|
2095
2095
|
*/
|
|
2096
2096
|
|
|
@@ -2431,8 +2431,7 @@ function countTotalUsage(llmTools) {
|
|
|
2431
2431
|
* Multiple LLM Execution Tools is a proxy server that uses multiple execution tools internally and exposes the executor interface externally.
|
|
2432
2432
|
*
|
|
2433
2433
|
* Note: Internal utility of `joinLlmExecutionTools` but exposed type
|
|
2434
|
-
* @public exported from `@promptbook/
|
|
2435
|
-
* TODO: !!!!!! Export as runtime class not just type
|
|
2434
|
+
* @public exported from `@promptbook/core`
|
|
2436
2435
|
*/
|
|
2437
2436
|
var MultipleLlmExecutionTools = /** @class */ (function () {
|
|
2438
2437
|
/**
|
|
@@ -2752,7 +2751,7 @@ function preparePersona(personaDescription, tools, options) {
|
|
|
2752
2751
|
return __generator(this, function (_d) {
|
|
2753
2752
|
switch (_d.label) {
|
|
2754
2753
|
case 0:
|
|
2755
|
-
_a = options.isVerbose, isVerbose = _a === void 0 ?
|
|
2754
|
+
_a = options.isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a;
|
|
2756
2755
|
if (tools === undefined || tools.llm === undefined) {
|
|
2757
2756
|
throw new MissingToolsError('LLM tools are required for preparing persona');
|
|
2758
2757
|
}
|
|
@@ -2934,7 +2933,7 @@ var $Register = /** @class */ (function () {
|
|
|
2934
2933
|
this.storage = globalScope[storageName];
|
|
2935
2934
|
}
|
|
2936
2935
|
$Register.prototype.list = function () {
|
|
2937
|
-
// <- TODO: ReadonlyDeep<
|
|
2936
|
+
// <- TODO: ReadonlyDeep<ReadonlyArray<TRegistered>>
|
|
2938
2937
|
return this.storage;
|
|
2939
2938
|
};
|
|
2940
2939
|
$Register.prototype.register = function (registered) {
|
|
@@ -2986,7 +2985,7 @@ var $scrapersRegister = new $Register('scraper_constructors');
|
|
|
2986
2985
|
* TODO: [®] DRY Register logic
|
|
2987
2986
|
*/
|
|
2988
2987
|
|
|
2989
|
-
// TODO: !!!!!! Maybe delete this function
|
|
2988
|
+
// TODO: !!!!!!last - Maybe delete this function
|
|
2990
2989
|
/**
|
|
2991
2990
|
* Creates a message with all registered scrapers
|
|
2992
2991
|
*
|
|
@@ -3094,7 +3093,6 @@ function $registeredScrapersMessage() {
|
|
|
3094
3093
|
* @private within the repository
|
|
3095
3094
|
*/
|
|
3096
3095
|
function sourceContentToName(sourceContent) {
|
|
3097
|
-
// TODO: !!!!!! Better name for source than gibberish hash
|
|
3098
3096
|
var hash = SHA256(hexEncoder.parse(JSON.stringify(sourceContent)))
|
|
3099
3097
|
// <- TODO: [🥬] Encapsulate sha256 to some private utility function
|
|
3100
3098
|
.toString( /* hex */)
|
|
@@ -3168,7 +3166,7 @@ function isFileExisting(filename, fs) {
|
|
|
3168
3166
|
/**
|
|
3169
3167
|
* @@@
|
|
3170
3168
|
*
|
|
3171
|
-
* @
|
|
3169
|
+
* @public exported from `@promptbook/core`
|
|
3172
3170
|
*/
|
|
3173
3171
|
function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
|
|
3174
3172
|
var _a;
|
|
@@ -3181,7 +3179,7 @@ function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
|
|
|
3181
3179
|
case 0:
|
|
3182
3180
|
sourceContent = knowledgeSource.sourceContent;
|
|
3183
3181
|
name = knowledgeSource.name;
|
|
3184
|
-
_b = options || {}, _c = _b.rootDirname, rootDirname = _c === void 0 ? null : _c, _d = _b.isVerbose, isVerbose = _d === void 0 ?
|
|
3182
|
+
_b = options || {}, _c = _b.rootDirname, rootDirname = _c === void 0 ? null : _c, _d = _b.isVerbose, isVerbose = _d === void 0 ? DEFAULT_IS_VERBOSE : _d;
|
|
3185
3183
|
TODO_USE(isVerbose);
|
|
3186
3184
|
if (!name) {
|
|
3187
3185
|
name = sourceContentToName(sourceContent);
|
|
@@ -3271,7 +3269,7 @@ function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
|
|
|
3271
3269
|
content = _a.sent();
|
|
3272
3270
|
return [2 /*return*/, new Blob([
|
|
3273
3271
|
content,
|
|
3274
|
-
// <- TODO: !!!!!!
|
|
3272
|
+
// <- TODO: !!!!!! Test that this is working
|
|
3275
3273
|
], { type: mimeType_1 })];
|
|
3276
3274
|
}
|
|
3277
3275
|
});
|
|
@@ -3334,7 +3332,7 @@ function prepareKnowledgePieces(knowledgeSources, tools, options) {
|
|
|
3334
3332
|
return __generator(this, function (_c) {
|
|
3335
3333
|
switch (_c.label) {
|
|
3336
3334
|
case 0:
|
|
3337
|
-
_a = options.maxParallelCount, maxParallelCount = _a === void 0 ?
|
|
3335
|
+
_a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a, rootDirname = options.rootDirname, _b = options.isVerbose, isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b;
|
|
3338
3336
|
knowledgePreparedUnflatten = new Array(knowledgeSources.length);
|
|
3339
3337
|
return [4 /*yield*/, forEachAsync(knowledgeSources, { maxParallelCount: maxParallelCount }, function (knowledgeSource, index) { return __awaiter(_this, void 0, void 0, function () {
|
|
3340
3338
|
var partialPieces, sourceHandler, _a, _b, scraper, partialPiecesUnchecked, e_1_1, pieces;
|
|
@@ -3363,7 +3361,8 @@ function prepareKnowledgePieces(knowledgeSources, tools, options) {
|
|
|
3363
3361
|
case 4:
|
|
3364
3362
|
partialPiecesUnchecked = _d.sent();
|
|
3365
3363
|
if (partialPiecesUnchecked !== null) {
|
|
3366
|
-
partialPieces = partialPiecesUnchecked;
|
|
3364
|
+
partialPieces = __spreadArray([], __read(partialPiecesUnchecked), false);
|
|
3365
|
+
// <- TODO: [🪓] Here should be no need for spreading new array, just `partialPieces = partialPiecesUnchecked`
|
|
3367
3366
|
return [3 /*break*/, 6];
|
|
3368
3367
|
}
|
|
3369
3368
|
_d.label = 5;
|
|
@@ -3411,7 +3410,7 @@ TODO: [🧊] This is how it can look in future
|
|
|
3411
3410
|
> /**
|
|
3412
3411
|
> * Unprepared knowledge
|
|
3413
3412
|
> * /
|
|
3414
|
-
> readonly knowledgeSources:
|
|
3413
|
+
> readonly knowledgeSources: ReadonlyArray<KnowledgeSourceJson>;
|
|
3415
3414
|
> };
|
|
3416
3415
|
>
|
|
3417
3416
|
> export async function prepareKnowledgePieces(
|
|
@@ -3469,7 +3468,7 @@ function prepareTemplates(pipeline, tools, options) {
|
|
|
3469
3468
|
return __generator(this, function (_b) {
|
|
3470
3469
|
switch (_b.label) {
|
|
3471
3470
|
case 0:
|
|
3472
|
-
_a = options.maxParallelCount, maxParallelCount = _a === void 0 ?
|
|
3471
|
+
_a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a;
|
|
3473
3472
|
templates = pipeline.templates, parameters = pipeline.parameters, knowledgePiecesCount = pipeline.knowledgePiecesCount;
|
|
3474
3473
|
// TODO: [main] !!!!! Apply samples to each template (if missing and is for the template defined)
|
|
3475
3474
|
TODO_USE(parameters);
|
|
@@ -3531,7 +3530,7 @@ function preparePipeline(pipeline, tools, options) {
|
|
|
3531
3530
|
if (isPipelinePrepared(pipeline)) {
|
|
3532
3531
|
return [2 /*return*/, pipeline];
|
|
3533
3532
|
}
|
|
3534
|
-
rootDirname = options.rootDirname, _a = options.maxParallelCount, maxParallelCount = _a === void 0 ?
|
|
3533
|
+
rootDirname = options.rootDirname, _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a, _b = options.isVerbose, isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b;
|
|
3535
3534
|
parameters = pipeline.parameters, templates = pipeline.templates, knowledgeSources = pipeline.knowledgeSources, personas = pipeline.personas;
|
|
3536
3535
|
if (tools === undefined || tools.llm === undefined) {
|
|
3537
3536
|
throw new MissingToolsError('LLM tools are required for preparing the pipeline');
|
|
@@ -3589,7 +3588,9 @@ function preparePipeline(pipeline, tools, options) {
|
|
|
3589
3588
|
// ----- /Templates preparation -----
|
|
3590
3589
|
// Note: Count total usage
|
|
3591
3590
|
currentPreparation.usage = llmToolsWithUsage.getTotalUsage();
|
|
3592
|
-
return [2 /*return*/, $asDeeplyFrozenSerializableJson('Prepared PipelineJson', __assign(__assign({}, clonePipeline(pipeline)), { templates:
|
|
3591
|
+
return [2 /*return*/, $asDeeplyFrozenSerializableJson('Prepared PipelineJson', __assign(__assign({}, clonePipeline(pipeline)), { templates: __spreadArray([], __read(templatesPrepared), false),
|
|
3592
|
+
// <- TODO: [🪓] Here should be no need for spreading new array, just ` templates: templatesPrepared`
|
|
3593
|
+
knowledgeSources: knowledgeSourcesPrepared, knowledgePieces: knowledgePiecesPrepared, personas: preparedPersonas, preparations: __spreadArray([], __read(preparations), false) }))];
|
|
3593
3594
|
}
|
|
3594
3595
|
});
|
|
3595
3596
|
});
|
|
@@ -4423,12 +4424,11 @@ function checkExpectations(expectations, value) {
|
|
|
4423
4424
|
*/
|
|
4424
4425
|
function executeAttempts(options) {
|
|
4425
4426
|
return __awaiter(this, void 0, void 0, function () {
|
|
4426
|
-
var jokerParameterNames, priority, maxAttempts, preparedContent, parameters, template, preparedPipeline, tools,
|
|
4427
|
+
var jokerParameterNames, priority, maxAttempts, preparedContent, parameters, template, preparedPipeline, tools, $executionReport, pipelineIdentification, maxExecutionAttempts, $ongoingTemplateResult, _llms, llmTools, _loop_1, attempt, state_1;
|
|
4427
4428
|
return __generator(this, function (_a) {
|
|
4428
4429
|
switch (_a.label) {
|
|
4429
4430
|
case 0:
|
|
4430
|
-
jokerParameterNames = options.jokerParameterNames, priority = options.priority, maxAttempts = options.maxAttempts, preparedContent = options.preparedContent, parameters = options.parameters, template = options.template, preparedPipeline = options.preparedPipeline, tools = options.tools,
|
|
4431
|
-
maxExecutionAttempts = settings.maxExecutionAttempts;
|
|
4431
|
+
jokerParameterNames = options.jokerParameterNames, priority = options.priority, maxAttempts = options.maxAttempts, preparedContent = options.preparedContent, parameters = options.parameters, template = options.template, preparedPipeline = options.preparedPipeline, tools = options.tools, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification, maxExecutionAttempts = options.maxExecutionAttempts;
|
|
4432
4432
|
$ongoingTemplateResult = {
|
|
4433
4433
|
$result: null,
|
|
4434
4434
|
$resultString: null,
|
|
@@ -4794,12 +4794,12 @@ function executeAttempts(options) {
|
|
|
4794
4794
|
*/
|
|
4795
4795
|
function executeFormatSubvalues(options) {
|
|
4796
4796
|
return __awaiter(this, void 0, void 0, function () {
|
|
4797
|
-
var template, jokerParameterNames, parameters, priority,
|
|
4797
|
+
var template, jokerParameterNames, parameters, priority, csvSettings, pipelineIdentification, parameterValue, formatDefinition, subvalueDefinition, formatSettings, resultString;
|
|
4798
4798
|
var _this = this;
|
|
4799
4799
|
return __generator(this, function (_a) {
|
|
4800
4800
|
switch (_a.label) {
|
|
4801
4801
|
case 0:
|
|
4802
|
-
template = options.template, jokerParameterNames = options.jokerParameterNames, parameters = options.parameters, priority = options.priority,
|
|
4802
|
+
template = options.template, jokerParameterNames = options.jokerParameterNames, parameters = options.parameters, priority = options.priority, csvSettings = options.csvSettings, pipelineIdentification = options.pipelineIdentification;
|
|
4803
4803
|
if (template.foreach === undefined) {
|
|
4804
4804
|
return [2 /*return*/, /* not await */ executeAttempts(options)];
|
|
4805
4805
|
}
|
|
@@ -4829,7 +4829,7 @@ function executeFormatSubvalues(options) {
|
|
|
4829
4829
|
.join('\n')), "\n\n [\u26F7] This should never happen because subformat name should be validated during parsing\n\n ").concat(block(pipelineIdentification), "\n "); }));
|
|
4830
4830
|
}
|
|
4831
4831
|
if (formatDefinition.formatName === 'CSV') {
|
|
4832
|
-
formatSettings =
|
|
4832
|
+
formatSettings = csvSettings;
|
|
4833
4833
|
// <- TODO: [🤹♂️] More universal, make simmilar pattern for other formats for example \n vs \r\n in text
|
|
4834
4834
|
}
|
|
4835
4835
|
return [4 /*yield*/, subvalueDefinition.mapValues(parameterValue, template.foreach.outputSubparameterName, formatSettings, function (subparameters, index) { return __awaiter(_this, void 0, void 0, function () {
|
|
@@ -4982,13 +4982,12 @@ function getReservedParametersForTemplate(options) {
|
|
|
4982
4982
|
*/
|
|
4983
4983
|
function executeTemplate(options) {
|
|
4984
4984
|
return __awaiter(this, void 0, void 0, function () {
|
|
4985
|
-
var currentTemplate, preparedPipeline, parametersToPass, tools, onProgress,
|
|
4986
|
-
var e_1,
|
|
4987
|
-
return __generator(this, function (
|
|
4988
|
-
switch (
|
|
4985
|
+
var currentTemplate, preparedPipeline, parametersToPass, tools, onProgress, $executionReport, pipelineIdentification, _a, maxExecutionAttempts, name, title, priority, usedParameterNames, dependentParameterNames, definedParameters, _b, _c, _d, definedParameterNames, parameters, _loop_1, _e, _f, parameterName, maxAttempts, jokerParameterNames, preparedContent, resultString;
|
|
4986
|
+
var e_1, _g, _h;
|
|
4987
|
+
return __generator(this, function (_j) {
|
|
4988
|
+
switch (_j.label) {
|
|
4989
4989
|
case 0:
|
|
4990
|
-
currentTemplate = options.currentTemplate, preparedPipeline = options.preparedPipeline, parametersToPass = options.parametersToPass, tools = options.tools, onProgress = options.onProgress,
|
|
4991
|
-
maxExecutionAttempts = settings.maxExecutionAttempts;
|
|
4990
|
+
currentTemplate = options.currentTemplate, preparedPipeline = options.preparedPipeline, parametersToPass = options.parametersToPass, tools = options.tools, onProgress = options.onProgress, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification, _a = options.maxExecutionAttempts, maxExecutionAttempts = _a === void 0 ? DEFAULT_MAX_EXECUTION_ATTEMPTS : _a;
|
|
4992
4991
|
name = "pipeline-executor-frame-".concat(currentTemplate.name);
|
|
4993
4992
|
title = currentTemplate.title;
|
|
4994
4993
|
priority = preparedPipeline.templates.length - preparedPipeline.templates.indexOf(currentTemplate);
|
|
@@ -5003,7 +5002,7 @@ function executeTemplate(options) {
|
|
|
5003
5002
|
// <- [🍸]
|
|
5004
5003
|
})];
|
|
5005
5004
|
case 1:
|
|
5006
|
-
|
|
5005
|
+
_j.sent();
|
|
5007
5006
|
usedParameterNames = extractParameterNamesFromTemplate(currentTemplate);
|
|
5008
5007
|
dependentParameterNames = new Set(currentTemplate.dependentParameterNames);
|
|
5009
5008
|
// TODO: [👩🏾🤝👩🏻] Use here `mapAvailableToExpectedParameters`
|
|
@@ -5014,15 +5013,15 @@ function executeTemplate(options) {
|
|
|
5014
5013
|
.map(function (name) { return "{".concat(name, "}"); })
|
|
5015
5014
|
.join(', '), "\n\n ").concat(block(pipelineIdentification), "\n\n "); }));
|
|
5016
5015
|
}
|
|
5017
|
-
|
|
5018
|
-
|
|
5016
|
+
_c = (_b = Object).freeze;
|
|
5017
|
+
_d = [{}];
|
|
5019
5018
|
return [4 /*yield*/, getReservedParametersForTemplate({
|
|
5020
5019
|
preparedPipeline: preparedPipeline,
|
|
5021
5020
|
template: currentTemplate,
|
|
5022
5021
|
pipelineIdentification: pipelineIdentification,
|
|
5023
5022
|
})];
|
|
5024
5023
|
case 2:
|
|
5025
|
-
definedParameters =
|
|
5024
|
+
definedParameters = _c.apply(_b, [__assign.apply(void 0, [__assign.apply(void 0, _d.concat([(_j.sent())])), parametersToPass])]);
|
|
5026
5025
|
definedParameterNames = new Set(Object.keys(definedParameters));
|
|
5027
5026
|
parameters = {};
|
|
5028
5027
|
_loop_1 = function (parameterName) {
|
|
@@ -5042,15 +5041,15 @@ function executeTemplate(options) {
|
|
|
5042
5041
|
try {
|
|
5043
5042
|
// Note: [2] Check that all used parameters are defined and removing unused parameters for this template
|
|
5044
5043
|
// TODO: [👩🏾🤝👩🏻] Use here `mapAvailableToExpectedParameters`
|
|
5045
|
-
for (
|
|
5046
|
-
parameterName =
|
|
5044
|
+
for (_e = __values(Array.from(union(definedParameterNames, usedParameterNames, dependentParameterNames))), _f = _e.next(); !_f.done; _f = _e.next()) {
|
|
5045
|
+
parameterName = _f.value;
|
|
5047
5046
|
_loop_1(parameterName);
|
|
5048
5047
|
}
|
|
5049
5048
|
}
|
|
5050
5049
|
catch (e_1_1) { e_1 = { error: e_1_1 }; }
|
|
5051
5050
|
finally {
|
|
5052
5051
|
try {
|
|
5053
|
-
if (
|
|
5052
|
+
if (_f && !_f.done && (_g = _e.return)) _g.call(_e);
|
|
5054
5053
|
}
|
|
5055
5054
|
finally { if (e_1) throw e_1.error; }
|
|
5056
5055
|
}
|
|
@@ -5070,12 +5069,11 @@ function executeTemplate(options) {
|
|
|
5070
5069
|
template: currentTemplate,
|
|
5071
5070
|
preparedPipeline: preparedPipeline,
|
|
5072
5071
|
tools: tools,
|
|
5073
|
-
settings: settings,
|
|
5074
5072
|
$executionReport: $executionReport,
|
|
5075
5073
|
pipelineIdentification: pipelineIdentification,
|
|
5076
5074
|
})];
|
|
5077
5075
|
case 3:
|
|
5078
|
-
resultString =
|
|
5076
|
+
resultString = _j.sent();
|
|
5079
5077
|
return [4 /*yield*/, onProgress({
|
|
5080
5078
|
name: name,
|
|
5081
5079
|
title: title,
|
|
@@ -5087,12 +5085,12 @@ function executeTemplate(options) {
|
|
|
5087
5085
|
// <- [🍸]
|
|
5088
5086
|
})];
|
|
5089
5087
|
case 4:
|
|
5090
|
-
|
|
5091
|
-
return [2 /*return*/, Object.freeze((
|
|
5092
|
-
|
|
5088
|
+
_j.sent();
|
|
5089
|
+
return [2 /*return*/, Object.freeze((_h = {},
|
|
5090
|
+
_h[currentTemplate.resultingParameterName] =
|
|
5093
5091
|
// <- Note: [👩👩👧] No need to detect parameter collision here because pipeline checks logic consistency during construction
|
|
5094
5092
|
resultString,
|
|
5095
|
-
|
|
5093
|
+
_h))];
|
|
5096
5094
|
}
|
|
5097
5095
|
});
|
|
5098
5096
|
});
|
|
@@ -5151,13 +5149,12 @@ function filterJustOutputParameters(options) {
|
|
|
5151
5149
|
*/
|
|
5152
5150
|
function executePipeline(options) {
|
|
5153
5151
|
return __awaiter(this, void 0, void 0, function () {
|
|
5154
|
-
var inputParameters, tools, onProgress, pipeline, setPreparedPipeline, pipelineIdentification,
|
|
5152
|
+
var inputParameters, tools, onProgress, pipeline, setPreparedPipeline, pipelineIdentification, maxParallelCount, rootDirname, _a, isVerbose, preparedPipeline, errors, warnings, executionReport, isReturned, _b, _c, parameter, e_1_1, _loop_1, _d, _e, parameterName, state_1, e_2_1, parametersToPass, resovedParameterNames_1, unresovedTemplates_1, resolving_1, loopLimit, _loop_2, error_1, usage_1, outputParameters_1, usage, outputParameters;
|
|
5155
5153
|
var e_1, _f, e_2, _g;
|
|
5156
5154
|
return __generator(this, function (_h) {
|
|
5157
5155
|
switch (_h.label) {
|
|
5158
5156
|
case 0:
|
|
5159
|
-
inputParameters = options.inputParameters, tools = options.tools, onProgress = options.onProgress, pipeline = options.pipeline, setPreparedPipeline = options.setPreparedPipeline, pipelineIdentification = options.pipelineIdentification,
|
|
5160
|
-
maxParallelCount = settings.maxParallelCount, rootDirname = settings.rootDirname, _a = settings.isVerbose, isVerbose = _a === void 0 ? IS_VERBOSE : _a;
|
|
5157
|
+
inputParameters = options.inputParameters, tools = options.tools, onProgress = options.onProgress, pipeline = options.pipeline, setPreparedPipeline = options.setPreparedPipeline, pipelineIdentification = options.pipelineIdentification, maxParallelCount = options.maxParallelCount, rootDirname = options.rootDirname, _a = options.isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a;
|
|
5161
5158
|
preparedPipeline = options.preparedPipeline;
|
|
5162
5159
|
if (!(preparedPipeline === undefined)) return [3 /*break*/, 2];
|
|
5163
5160
|
return [4 /*yield*/, preparePipeline(pipeline, tools, {
|
|
@@ -5342,12 +5339,7 @@ function executePipeline(options) {
|
|
|
5342
5339
|
return [3 /*break*/, 4];
|
|
5343
5340
|
case 3:
|
|
5344
5341
|
unresovedTemplates_1 = unresovedTemplates_1.filter(function (template) { return template !== currentTemplate; });
|
|
5345
|
-
work_1 = executeTemplate({
|
|
5346
|
-
currentTemplate: currentTemplate,
|
|
5347
|
-
preparedPipeline: preparedPipeline,
|
|
5348
|
-
parametersToPass: parametersToPass,
|
|
5349
|
-
tools: tools,
|
|
5350
|
-
onProgress: function (progress) {
|
|
5342
|
+
work_1 = executeTemplate(__assign(__assign({}, options), { currentTemplate: currentTemplate, preparedPipeline: preparedPipeline, parametersToPass: parametersToPass, tools: tools, onProgress: function (progress) {
|
|
5351
5343
|
if (isReturned) {
|
|
5352
5344
|
throw new UnexpectedError(spaceTrim$1(function (block) { return "\n Can not call `onProgress` after pipeline execution is finished\n\n ".concat(block(pipelineIdentification), "\n\n ").concat(block(JSON.stringify(progress, null, 4)
|
|
5353
5345
|
.split('\n')
|
|
@@ -5357,11 +5349,7 @@ function executePipeline(options) {
|
|
|
5357
5349
|
if (onProgress) {
|
|
5358
5350
|
onProgress(progress);
|
|
5359
5351
|
}
|
|
5360
|
-
},
|
|
5361
|
-
settings: settings,
|
|
5362
|
-
$executionReport: executionReport,
|
|
5363
|
-
pipelineIdentification: spaceTrim$1(function (block) { return "\n ".concat(block(pipelineIdentification), "\n Template name: ").concat(currentTemplate.name, "\n Template title: ").concat(currentTemplate.title, "\n "); }),
|
|
5364
|
-
})
|
|
5352
|
+
}, $executionReport: executionReport, pipelineIdentification: spaceTrim$1(function (block) { return "\n ".concat(block(pipelineIdentification), "\n Template name: ").concat(currentTemplate.name, "\n Template title: ").concat(currentTemplate.title, "\n "); }) }))
|
|
5365
5353
|
.then(function (newParametersToPass) {
|
|
5366
5354
|
parametersToPass = __assign(__assign({}, newParametersToPass), parametersToPass);
|
|
5367
5355
|
resovedParameterNames_1 = __spreadArray(__spreadArray([], __read(resovedParameterNames_1), false), [currentTemplate.resultingParameterName], false);
|
|
@@ -5465,8 +5453,7 @@ function executePipeline(options) {
|
|
|
5465
5453
|
*/
|
|
5466
5454
|
function createPipelineExecutor(options) {
|
|
5467
5455
|
var _this = this;
|
|
5468
|
-
var pipeline = options.pipeline, tools = options.tools, _a = options.
|
|
5469
|
-
var _b = settings.maxExecutionAttempts, maxExecutionAttempts = _b === void 0 ? MAX_EXECUTION_ATTEMPTS : _b, _c = settings.maxParallelCount, maxParallelCount = _c === void 0 ? MAX_PARALLEL_COUNT : _c, _d = settings.csvSettings, csvSettings = _d === void 0 ? DEFAULT_CSV_SETTINGS : _d, _e = settings.isVerbose, isVerbose = _e === void 0 ? IS_VERBOSE : _e, _f = settings.isNotPreparedWarningSupressed, isNotPreparedWarningSupressed = _f === void 0 ? false : _f, _g = settings.rootDirname, rootDirname = _g === void 0 ? null : _g;
|
|
5456
|
+
var pipeline = options.pipeline, tools = options.tools, _a = options.maxExecutionAttempts, maxExecutionAttempts = _a === void 0 ? DEFAULT_MAX_EXECUTION_ATTEMPTS : _a, _b = options.maxParallelCount, maxParallelCount = _b === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _b, _c = options.csvSettings, csvSettings = _c === void 0 ? DEFAULT_CSV_SETTINGS : _c, _d = options.isVerbose, isVerbose = _d === void 0 ? DEFAULT_IS_VERBOSE : _d, _e = options.isNotPreparedWarningSupressed, isNotPreparedWarningSupressed = _e === void 0 ? false : _e, _f = options.rootDirname, rootDirname = _f === void 0 ? null : _f;
|
|
5470
5457
|
validatePipeline(pipeline);
|
|
5471
5458
|
var pipelineIdentification = (function () {
|
|
5472
5459
|
// Note: This is a 😐 implementation of [🚞]
|
|
@@ -5500,14 +5487,12 @@ function createPipelineExecutor(options) {
|
|
|
5500
5487
|
tools: tools,
|
|
5501
5488
|
onProgress: onProgress,
|
|
5502
5489
|
pipelineIdentification: spaceTrim$1(function (block) { return "\n ".concat(block(pipelineIdentification), "\n ").concat(runCount === 1 ? '' : "Run #".concat(runCount), "\n "); }),
|
|
5503
|
-
|
|
5504
|
-
|
|
5505
|
-
|
|
5506
|
-
|
|
5507
|
-
|
|
5508
|
-
|
|
5509
|
-
rootDirname: rootDirname,
|
|
5510
|
-
},
|
|
5490
|
+
maxExecutionAttempts: maxExecutionAttempts,
|
|
5491
|
+
maxParallelCount: maxParallelCount,
|
|
5492
|
+
csvSettings: csvSettings,
|
|
5493
|
+
isVerbose: isVerbose,
|
|
5494
|
+
isNotPreparedWarningSupressed: isNotPreparedWarningSupressed,
|
|
5495
|
+
rootDirname: rootDirname,
|
|
5511
5496
|
})];
|
|
5512
5497
|
});
|
|
5513
5498
|
}); };
|
|
@@ -5529,7 +5514,7 @@ var markdownScraperMetadata = $deepFreeze({
|
|
|
5529
5514
|
mimeTypes: ['text/markdown', 'text/plain'],
|
|
5530
5515
|
documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
|
|
5531
5516
|
isAvilableInBrowser: true,
|
|
5532
|
-
requiredExecutables: [
|
|
5517
|
+
requiredExecutables: [],
|
|
5533
5518
|
}); /* <- TODO: [🤛] */
|
|
5534
5519
|
/**
|
|
5535
5520
|
* Registration of known scraper metadata
|
|
@@ -5573,7 +5558,7 @@ var MarkdownScraper = /** @class */ (function () {
|
|
|
5573
5558
|
return __generator(this, function (_k) {
|
|
5574
5559
|
switch (_k.label) {
|
|
5575
5560
|
case 0:
|
|
5576
|
-
_a = this.options, _b = _a.maxParallelCount, maxParallelCount = _b === void 0 ?
|
|
5561
|
+
_a = this.options, _b = _a.maxParallelCount, maxParallelCount = _b === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? DEFAULT_IS_VERBOSE : _c;
|
|
5577
5562
|
llm = this.tools.llm;
|
|
5578
5563
|
if (llm === undefined) {
|
|
5579
5564
|
throw new MissingToolsError('LLM tools are required for scraping external files');
|
|
@@ -5672,7 +5657,8 @@ var MarkdownScraper = /** @class */ (function () {
|
|
|
5672
5657
|
embeddingResult = _c.sent();
|
|
5673
5658
|
index.push({
|
|
5674
5659
|
modelName: embeddingResult.modelName,
|
|
5675
|
-
position: embeddingResult.content,
|
|
5660
|
+
position: __spreadArray([], __read(embeddingResult.content), false),
|
|
5661
|
+
// <- TODO: [🪓] Here should be no need for spreading new array, just `position: embeddingResult.content`
|
|
5676
5662
|
});
|
|
5677
5663
|
_c.label = 6;
|
|
5678
5664
|
case 6: return [3 /*break*/, 8];
|