@promptbook/markdown-utils 0.71.0-18 → 0.71.0-19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +0 -6
- package/esm/index.es.js +56 -68
- package/esm/index.es.js.map +1 -1
- package/esm/typings/src/_packages/core.index.d.ts +22 -18
- package/esm/typings/src/_packages/types.index.d.ts +2 -2
- package/esm/typings/src/collection/PipelineCollection.d.ts +1 -1
- package/esm/typings/src/collection/SimplePipelineCollection.d.ts +2 -2
- package/esm/typings/src/collection/collectionToJson.d.ts +1 -1
- package/esm/typings/src/collection/constructors/createCollectionFromJson.d.ts +1 -1
- package/esm/typings/src/collection/constructors/createCollectionFromPromise.d.ts +1 -1
- package/esm/typings/src/commands/_common/types/CommandParser.d.ts +5 -5
- package/esm/typings/src/config.d.ts +21 -14
- package/esm/typings/src/execution/EmbeddingVector.d.ts +1 -1
- package/esm/typings/src/execution/LlmExecutionTools.d.ts +1 -1
- package/esm/typings/src/execution/PipelineExecutorResult.d.ts +2 -2
- package/esm/typings/src/execution/assertsExecutionSuccessful.d.ts +1 -0
- package/esm/typings/src/execution/createPipelineExecutor/00-CreatePipelineExecutorOptions.d.ts +29 -6
- package/esm/typings/src/execution/createPipelineExecutor/10-executePipeline.d.ts +2 -11
- package/esm/typings/src/execution/createPipelineExecutor/20-executeTemplate.d.ts +4 -13
- package/esm/typings/src/execution/createPipelineExecutor/40-executeAttempts.d.ts +9 -14
- package/esm/typings/src/execution/utils/addUsage.d.ts +1 -1
- package/esm/typings/src/execution/utils/forEachAsync.d.ts +1 -1
- package/esm/typings/src/formats/_common/FormatDefinition.d.ts +2 -2
- package/esm/typings/src/formats/_common/FormatSubvalueDefinition.d.ts +2 -2
- package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsForCli.d.ts +2 -2
- package/esm/typings/src/llm-providers/_common/register/LlmToolsConfiguration.d.ts +1 -1
- package/esm/typings/src/llm-providers/_common/utils/cache/CacheLlmToolsOptions.d.ts +4 -1
- package/esm/typings/src/llm-providers/anthropic-claude/AnthropicClaudeExecutionTools.d.ts +1 -1
- package/esm/typings/src/llm-providers/anthropic-claude/anthropic-claude-models.d.ts +1 -1
- package/esm/typings/src/llm-providers/azure-openai/AzureOpenAiExecutionTools.d.ts +1 -1
- package/esm/typings/src/llm-providers/mocked/$fakeTextToExpectations.d.ts +1 -1
- package/esm/typings/src/llm-providers/mocked/MockedEchoLlmExecutionTools.d.ts +1 -1
- package/esm/typings/src/llm-providers/mocked/MockedFackedLlmExecutionTools.d.ts +1 -1
- package/esm/typings/src/llm-providers/multiple/MultipleLlmExecutionTools.d.ts +3 -3
- package/esm/typings/src/llm-providers/multiple/joinLlmExecutionTools.d.ts +1 -1
- package/esm/typings/src/llm-providers/openai/OpenAiExecutionTools.d.ts +1 -1
- package/esm/typings/src/llm-providers/openai/openai-models.d.ts +1 -1
- package/esm/typings/src/llm-providers/remote/RemoteLlmExecutionTools.d.ts +1 -1
- package/esm/typings/src/llm-providers/remote/interfaces/PromptbookServer_ListModels_Response.d.ts +1 -1
- package/esm/typings/src/prepare/PrepareAndScrapeOptions.d.ts +4 -4
- package/esm/typings/src/prepare/prepareTemplates.d.ts +1 -1
- package/esm/typings/src/scrapers/_common/Scraper.d.ts +1 -1
- package/esm/typings/src/scrapers/_common/prepareKnowledgePieces.d.ts +1 -1
- package/esm/typings/src/scrapers/_common/register/$provideScrapersForBrowser.d.ts +1 -1
- package/esm/typings/src/scrapers/_common/register/$provideScrapersForNode.d.ts +1 -1
- package/esm/typings/src/scrapers/_common/register/ScraperAndConverterMetadata.d.ts +2 -2
- package/esm/typings/src/scrapers/_common/utils/getScraperIntermediateSource.d.ts +2 -2
- package/esm/typings/src/scrapers/_common/utils/makeKnowledgeSourceHandler.d.ts +1 -1
- package/esm/typings/src/scrapers/document/DocumentScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/document/createDocumentScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/document/register-metadata.d.ts +1 -1
- package/esm/typings/src/scrapers/document-legacy/LegacyDocumentScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/document-legacy/createLegacyDocumentScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/document-legacy/register-metadata.d.ts +1 -1
- package/esm/typings/src/scrapers/markdown/MarkdownScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/markdown/createMarkdownScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/markdown/register-metadata.d.ts +1 -1
- package/esm/typings/src/scrapers/pdf/PdfScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/pdf/createPdfScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/pdf/register-metadata.d.ts +1 -1
- package/esm/typings/src/scrapers/website/WebsiteScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/website/createWebsiteScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/website/register-metadata.d.ts +1 -1
- package/esm/typings/src/scripting/javascript/utils/preserve.d.ts +1 -1
- package/esm/typings/src/storage/file-cache-storage/utils/nameToSubfolderPath.d.ts +1 -1
- package/esm/typings/src/types/Arrayable.d.ts +1 -1
- package/esm/typings/src/types/IntermediateFilesStrategy.d.ts +7 -0
- package/esm/typings/src/types/PipelineJson/KnowledgePieceJson.d.ts +4 -4
- package/esm/typings/src/types/PipelineJson/KnowledgeSourceJson.d.ts +1 -1
- package/esm/typings/src/types/PipelineJson/PersonaJson.d.ts +1 -1
- package/esm/typings/src/types/Prompt.d.ts +1 -1
- package/esm/typings/src/types/execution-report/ExecutionReportJson.d.ts +1 -1
- package/esm/typings/src/utils/$Register.d.ts +1 -1
- package/esm/typings/src/utils/FromtoItems.d.ts +1 -1
- package/esm/typings/src/utils/arrayableToArray.d.ts +1 -1
- package/esm/typings/src/utils/emojis.d.ts +1 -1
- package/esm/typings/src/utils/expectation-counters/countSentences.d.ts +1 -1
- package/esm/typings/src/utils/markdown/extractAllBlocksFromMarkdown.d.ts +1 -1
- package/esm/typings/src/utils/markdown/splitMarkdownIntoSections.d.ts +1 -1
- package/esm/typings/src/utils/organization/TODO_USE.d.ts +1 -1
- package/esm/typings/src/utils/organization/keepUnused.d.ts +1 -1
- package/esm/typings/src/utils/random/$randomSeed.d.ts +1 -1
- package/esm/typings/src/utils/sets/intersection.d.ts +1 -1
- package/esm/typings/src/utils/sets/union.d.ts +1 -1
- package/package.json +1 -1
- package/umd/index.umd.js +56 -68
- package/umd/index.umd.js.map +1 -1
- package/esm/typings/src/execution/createPipelineExecutor/00-CreatePipelineExecutorSettings.d.ts +0 -29
package/README.md
CHANGED
|
@@ -20,12 +20,6 @@ Build responsible, controlled and transparent applications on top of LLM models!
|
|
|
20
20
|
|
|
21
21
|
- ✨ **Support of [OpenAI o1 model](https://openai.com/o1/)**
|
|
22
22
|
|
|
23
|
-
<blockquote style="color: #ff8811">
|
|
24
|
-
<b>⚠ Warning:</b> This is a pre-release version of the library. It is not yet ready for production use. Please look at <a href="https://www.npmjs.com/package/@promptbook/core?activeTab=versions">latest stable release</a>.
|
|
25
|
-
</blockquote>
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
23
|
<blockquote style="color: #ff8811">
|
|
30
24
|
<b>⚠ Warning:</b> This is a pre-release version of the library. It is not yet ready for production use. Please look at <a href="https://www.npmjs.com/package/@promptbook/core?activeTab=versions">latest stable release</a>.
|
|
31
25
|
</blockquote>
|
package/esm/index.es.js
CHANGED
|
@@ -12,7 +12,7 @@ import { unparse, parse } from 'papaparse';
|
|
|
12
12
|
/**
|
|
13
13
|
* The version of the Promptbook library
|
|
14
14
|
*/
|
|
15
|
-
var PROMPTBOOK_VERSION = '0.71.0-
|
|
15
|
+
var PROMPTBOOK_VERSION = '0.71.0-18';
|
|
16
16
|
// TODO: [main] !!!! List here all the versions and annotate + put into script
|
|
17
17
|
|
|
18
18
|
/*! *****************************************************************************
|
|
@@ -413,7 +413,6 @@ function pipelineJsonToString(pipelineJson) {
|
|
|
413
413
|
pipelineString += '\n\n';
|
|
414
414
|
pipelineString += description;
|
|
415
415
|
}
|
|
416
|
-
// TODO:> const commands: Array<Command>
|
|
417
416
|
var commands = [];
|
|
418
417
|
if (pipelineUrl) {
|
|
419
418
|
commands.push("PIPELINE URL ".concat(pipelineUrl));
|
|
@@ -469,7 +468,6 @@ function pipelineJsonToString(pipelineJson) {
|
|
|
469
468
|
pipelineString += '\n\n';
|
|
470
469
|
pipelineString += description_1;
|
|
471
470
|
}
|
|
472
|
-
// TODO:> const commands: Array<Command>
|
|
473
471
|
var commands_1 = [];
|
|
474
472
|
var contentLanguage = 'text';
|
|
475
473
|
if (templateType === 'PROMPT_TEMPLATE') {
|
|
@@ -835,18 +833,19 @@ var LOOP_LIMIT = 1000;
|
|
|
835
833
|
* @private within the repository - too low-level in comparison with other `MAX_...`
|
|
836
834
|
*/
|
|
837
835
|
var IMMEDIATE_TIME = 10;
|
|
836
|
+
// <- TODO: [😡] Change to 'VISIBLE'
|
|
838
837
|
/**
|
|
839
838
|
* The maximum number of (LLM) tasks running in parallel
|
|
840
839
|
*
|
|
841
840
|
* @public exported from `@promptbook/core`
|
|
842
841
|
*/
|
|
843
|
-
var
|
|
842
|
+
var DEFAULT_MAX_PARALLEL_COUNT = 5; // <- TODO: [🤹♂️]
|
|
844
843
|
/**
|
|
845
844
|
* The maximum number of attempts to execute LLM task before giving up
|
|
846
845
|
*
|
|
847
846
|
* @public exported from `@promptbook/core`
|
|
848
847
|
*/
|
|
849
|
-
var
|
|
848
|
+
var DEFAULT_MAX_EXECUTION_ATTEMPTS = 3; // <- TODO: [🤹♂️]
|
|
850
849
|
/**
|
|
851
850
|
* Nonce which is used for replacing things in strings
|
|
852
851
|
*
|
|
@@ -898,7 +897,7 @@ var DEFAULT_CSV_SETTINGS = Object.freeze({
|
|
|
898
897
|
*
|
|
899
898
|
* @public exported from `@promptbook/core`
|
|
900
899
|
*/
|
|
901
|
-
var
|
|
900
|
+
var DEFAULT_IS_VERBOSE = false;
|
|
902
901
|
/**
|
|
903
902
|
* @@@
|
|
904
903
|
*
|
|
@@ -2091,6 +2090,7 @@ function assertsExecutionSuccessful(executionResult) {
|
|
|
2091
2090
|
}
|
|
2092
2091
|
}
|
|
2093
2092
|
/**
|
|
2093
|
+
* TODO: [🐚] This function should be removed OR changed OR be completely rewritten
|
|
2094
2094
|
* TODO: [🧠] Can this return type be better typed than void
|
|
2095
2095
|
*/
|
|
2096
2096
|
|
|
@@ -2751,7 +2751,7 @@ function preparePersona(personaDescription, tools, options) {
|
|
|
2751
2751
|
return __generator(this, function (_d) {
|
|
2752
2752
|
switch (_d.label) {
|
|
2753
2753
|
case 0:
|
|
2754
|
-
_a = options.isVerbose, isVerbose = _a === void 0 ?
|
|
2754
|
+
_a = options.isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a;
|
|
2755
2755
|
if (tools === undefined || tools.llm === undefined) {
|
|
2756
2756
|
throw new MissingToolsError('LLM tools are required for preparing persona');
|
|
2757
2757
|
}
|
|
@@ -2933,7 +2933,7 @@ var $Register = /** @class */ (function () {
|
|
|
2933
2933
|
this.storage = globalScope[storageName];
|
|
2934
2934
|
}
|
|
2935
2935
|
$Register.prototype.list = function () {
|
|
2936
|
-
// <- TODO: ReadonlyDeep<
|
|
2936
|
+
// <- TODO: ReadonlyDeep<ReadonlyArray<TRegistered>>
|
|
2937
2937
|
return this.storage;
|
|
2938
2938
|
};
|
|
2939
2939
|
$Register.prototype.register = function (registered) {
|
|
@@ -3166,7 +3166,7 @@ function isFileExisting(filename, fs) {
|
|
|
3166
3166
|
/**
|
|
3167
3167
|
* @@@
|
|
3168
3168
|
*
|
|
3169
|
-
* @
|
|
3169
|
+
* @public exported from `@promptbook/core`
|
|
3170
3170
|
*/
|
|
3171
3171
|
function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
|
|
3172
3172
|
var _a;
|
|
@@ -3179,7 +3179,7 @@ function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
|
|
|
3179
3179
|
case 0:
|
|
3180
3180
|
sourceContent = knowledgeSource.sourceContent;
|
|
3181
3181
|
name = knowledgeSource.name;
|
|
3182
|
-
_b = options || {}, _c = _b.rootDirname, rootDirname = _c === void 0 ? null : _c, _d = _b.isVerbose, isVerbose = _d === void 0 ?
|
|
3182
|
+
_b = options || {}, _c = _b.rootDirname, rootDirname = _c === void 0 ? null : _c, _d = _b.isVerbose, isVerbose = _d === void 0 ? DEFAULT_IS_VERBOSE : _d;
|
|
3183
3183
|
TODO_USE(isVerbose);
|
|
3184
3184
|
if (!name) {
|
|
3185
3185
|
name = sourceContentToName(sourceContent);
|
|
@@ -3332,7 +3332,7 @@ function prepareKnowledgePieces(knowledgeSources, tools, options) {
|
|
|
3332
3332
|
return __generator(this, function (_c) {
|
|
3333
3333
|
switch (_c.label) {
|
|
3334
3334
|
case 0:
|
|
3335
|
-
_a = options.maxParallelCount, maxParallelCount = _a === void 0 ?
|
|
3335
|
+
_a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a, rootDirname = options.rootDirname, _b = options.isVerbose, isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b;
|
|
3336
3336
|
knowledgePreparedUnflatten = new Array(knowledgeSources.length);
|
|
3337
3337
|
return [4 /*yield*/, forEachAsync(knowledgeSources, { maxParallelCount: maxParallelCount }, function (knowledgeSource, index) { return __awaiter(_this, void 0, void 0, function () {
|
|
3338
3338
|
var partialPieces, sourceHandler, _a, _b, scraper, partialPiecesUnchecked, e_1_1, pieces;
|
|
@@ -3361,7 +3361,8 @@ function prepareKnowledgePieces(knowledgeSources, tools, options) {
|
|
|
3361
3361
|
case 4:
|
|
3362
3362
|
partialPiecesUnchecked = _d.sent();
|
|
3363
3363
|
if (partialPiecesUnchecked !== null) {
|
|
3364
|
-
partialPieces = partialPiecesUnchecked;
|
|
3364
|
+
partialPieces = __spreadArray([], __read(partialPiecesUnchecked), false);
|
|
3365
|
+
// <- TODO: [🪓] Here should be no need for spreading new array, just `partialPieces = partialPiecesUnchecked`
|
|
3365
3366
|
return [3 /*break*/, 6];
|
|
3366
3367
|
}
|
|
3367
3368
|
_d.label = 5;
|
|
@@ -3409,7 +3410,7 @@ TODO: [🧊] This is how it can look in future
|
|
|
3409
3410
|
> /**
|
|
3410
3411
|
> * Unprepared knowledge
|
|
3411
3412
|
> * /
|
|
3412
|
-
> readonly knowledgeSources:
|
|
3413
|
+
> readonly knowledgeSources: ReadonlyArray<KnowledgeSourceJson>;
|
|
3413
3414
|
> };
|
|
3414
3415
|
>
|
|
3415
3416
|
> export async function prepareKnowledgePieces(
|
|
@@ -3467,7 +3468,7 @@ function prepareTemplates(pipeline, tools, options) {
|
|
|
3467
3468
|
return __generator(this, function (_b) {
|
|
3468
3469
|
switch (_b.label) {
|
|
3469
3470
|
case 0:
|
|
3470
|
-
_a = options.maxParallelCount, maxParallelCount = _a === void 0 ?
|
|
3471
|
+
_a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a;
|
|
3471
3472
|
templates = pipeline.templates, parameters = pipeline.parameters, knowledgePiecesCount = pipeline.knowledgePiecesCount;
|
|
3472
3473
|
// TODO: [main] !!!!! Apply samples to each template (if missing and is for the template defined)
|
|
3473
3474
|
TODO_USE(parameters);
|
|
@@ -3529,7 +3530,7 @@ function preparePipeline(pipeline, tools, options) {
|
|
|
3529
3530
|
if (isPipelinePrepared(pipeline)) {
|
|
3530
3531
|
return [2 /*return*/, pipeline];
|
|
3531
3532
|
}
|
|
3532
|
-
rootDirname = options.rootDirname, _a = options.maxParallelCount, maxParallelCount = _a === void 0 ?
|
|
3533
|
+
rootDirname = options.rootDirname, _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a, _b = options.isVerbose, isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b;
|
|
3533
3534
|
parameters = pipeline.parameters, templates = pipeline.templates, knowledgeSources = pipeline.knowledgeSources, personas = pipeline.personas;
|
|
3534
3535
|
if (tools === undefined || tools.llm === undefined) {
|
|
3535
3536
|
throw new MissingToolsError('LLM tools are required for preparing the pipeline');
|
|
@@ -3587,7 +3588,9 @@ function preparePipeline(pipeline, tools, options) {
|
|
|
3587
3588
|
// ----- /Templates preparation -----
|
|
3588
3589
|
// Note: Count total usage
|
|
3589
3590
|
currentPreparation.usage = llmToolsWithUsage.getTotalUsage();
|
|
3590
|
-
return [2 /*return*/, $asDeeplyFrozenSerializableJson('Prepared PipelineJson', __assign(__assign({}, clonePipeline(pipeline)), { templates:
|
|
3591
|
+
return [2 /*return*/, $asDeeplyFrozenSerializableJson('Prepared PipelineJson', __assign(__assign({}, clonePipeline(pipeline)), { templates: __spreadArray([], __read(templatesPrepared), false),
|
|
3592
|
+
// <- TODO: [🪓] Here should be no need for spreading new array, just ` templates: templatesPrepared`
|
|
3593
|
+
knowledgeSources: knowledgeSourcesPrepared, knowledgePieces: knowledgePiecesPrepared, personas: preparedPersonas, preparations: __spreadArray([], __read(preparations), false) }))];
|
|
3591
3594
|
}
|
|
3592
3595
|
});
|
|
3593
3596
|
});
|
|
@@ -4421,12 +4424,11 @@ function checkExpectations(expectations, value) {
|
|
|
4421
4424
|
*/
|
|
4422
4425
|
function executeAttempts(options) {
|
|
4423
4426
|
return __awaiter(this, void 0, void 0, function () {
|
|
4424
|
-
var jokerParameterNames, priority, maxAttempts, preparedContent, parameters, template, preparedPipeline, tools,
|
|
4427
|
+
var jokerParameterNames, priority, maxAttempts, preparedContent, parameters, template, preparedPipeline, tools, $executionReport, pipelineIdentification, maxExecutionAttempts, $ongoingTemplateResult, _llms, llmTools, _loop_1, attempt, state_1;
|
|
4425
4428
|
return __generator(this, function (_a) {
|
|
4426
4429
|
switch (_a.label) {
|
|
4427
4430
|
case 0:
|
|
4428
|
-
jokerParameterNames = options.jokerParameterNames, priority = options.priority, maxAttempts = options.maxAttempts, preparedContent = options.preparedContent, parameters = options.parameters, template = options.template, preparedPipeline = options.preparedPipeline, tools = options.tools,
|
|
4429
|
-
maxExecutionAttempts = settings.maxExecutionAttempts;
|
|
4431
|
+
jokerParameterNames = options.jokerParameterNames, priority = options.priority, maxAttempts = options.maxAttempts, preparedContent = options.preparedContent, parameters = options.parameters, template = options.template, preparedPipeline = options.preparedPipeline, tools = options.tools, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification, maxExecutionAttempts = options.maxExecutionAttempts;
|
|
4430
4432
|
$ongoingTemplateResult = {
|
|
4431
4433
|
$result: null,
|
|
4432
4434
|
$resultString: null,
|
|
@@ -4792,12 +4794,12 @@ function executeAttempts(options) {
|
|
|
4792
4794
|
*/
|
|
4793
4795
|
function executeFormatSubvalues(options) {
|
|
4794
4796
|
return __awaiter(this, void 0, void 0, function () {
|
|
4795
|
-
var template, jokerParameterNames, parameters, priority,
|
|
4797
|
+
var template, jokerParameterNames, parameters, priority, csvSettings, pipelineIdentification, parameterValue, formatDefinition, subvalueDefinition, formatSettings, resultString;
|
|
4796
4798
|
var _this = this;
|
|
4797
4799
|
return __generator(this, function (_a) {
|
|
4798
4800
|
switch (_a.label) {
|
|
4799
4801
|
case 0:
|
|
4800
|
-
template = options.template, jokerParameterNames = options.jokerParameterNames, parameters = options.parameters, priority = options.priority,
|
|
4802
|
+
template = options.template, jokerParameterNames = options.jokerParameterNames, parameters = options.parameters, priority = options.priority, csvSettings = options.csvSettings, pipelineIdentification = options.pipelineIdentification;
|
|
4801
4803
|
if (template.foreach === undefined) {
|
|
4802
4804
|
return [2 /*return*/, /* not await */ executeAttempts(options)];
|
|
4803
4805
|
}
|
|
@@ -4827,7 +4829,7 @@ function executeFormatSubvalues(options) {
|
|
|
4827
4829
|
.join('\n')), "\n\n [\u26F7] This should never happen because subformat name should be validated during parsing\n\n ").concat(block(pipelineIdentification), "\n "); }));
|
|
4828
4830
|
}
|
|
4829
4831
|
if (formatDefinition.formatName === 'CSV') {
|
|
4830
|
-
formatSettings =
|
|
4832
|
+
formatSettings = csvSettings;
|
|
4831
4833
|
// <- TODO: [🤹♂️] More universal, make simmilar pattern for other formats for example \n vs \r\n in text
|
|
4832
4834
|
}
|
|
4833
4835
|
return [4 /*yield*/, subvalueDefinition.mapValues(parameterValue, template.foreach.outputSubparameterName, formatSettings, function (subparameters, index) { return __awaiter(_this, void 0, void 0, function () {
|
|
@@ -4980,13 +4982,12 @@ function getReservedParametersForTemplate(options) {
|
|
|
4980
4982
|
*/
|
|
4981
4983
|
function executeTemplate(options) {
|
|
4982
4984
|
return __awaiter(this, void 0, void 0, function () {
|
|
4983
|
-
var currentTemplate, preparedPipeline, parametersToPass, tools, onProgress,
|
|
4984
|
-
var e_1,
|
|
4985
|
-
return __generator(this, function (
|
|
4986
|
-
switch (
|
|
4985
|
+
var currentTemplate, preparedPipeline, parametersToPass, tools, onProgress, $executionReport, pipelineIdentification, _a, maxExecutionAttempts, name, title, priority, usedParameterNames, dependentParameterNames, definedParameters, _b, _c, _d, definedParameterNames, parameters, _loop_1, _e, _f, parameterName, maxAttempts, jokerParameterNames, preparedContent, resultString;
|
|
4986
|
+
var e_1, _g, _h;
|
|
4987
|
+
return __generator(this, function (_j) {
|
|
4988
|
+
switch (_j.label) {
|
|
4987
4989
|
case 0:
|
|
4988
|
-
currentTemplate = options.currentTemplate, preparedPipeline = options.preparedPipeline, parametersToPass = options.parametersToPass, tools = options.tools, onProgress = options.onProgress,
|
|
4989
|
-
maxExecutionAttempts = settings.maxExecutionAttempts;
|
|
4990
|
+
currentTemplate = options.currentTemplate, preparedPipeline = options.preparedPipeline, parametersToPass = options.parametersToPass, tools = options.tools, onProgress = options.onProgress, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification, _a = options.maxExecutionAttempts, maxExecutionAttempts = _a === void 0 ? DEFAULT_MAX_EXECUTION_ATTEMPTS : _a;
|
|
4990
4991
|
name = "pipeline-executor-frame-".concat(currentTemplate.name);
|
|
4991
4992
|
title = currentTemplate.title;
|
|
4992
4993
|
priority = preparedPipeline.templates.length - preparedPipeline.templates.indexOf(currentTemplate);
|
|
@@ -5001,7 +5002,7 @@ function executeTemplate(options) {
|
|
|
5001
5002
|
// <- [🍸]
|
|
5002
5003
|
})];
|
|
5003
5004
|
case 1:
|
|
5004
|
-
|
|
5005
|
+
_j.sent();
|
|
5005
5006
|
usedParameterNames = extractParameterNamesFromTemplate(currentTemplate);
|
|
5006
5007
|
dependentParameterNames = new Set(currentTemplate.dependentParameterNames);
|
|
5007
5008
|
// TODO: [👩🏾🤝👩🏻] Use here `mapAvailableToExpectedParameters`
|
|
@@ -5012,15 +5013,15 @@ function executeTemplate(options) {
|
|
|
5012
5013
|
.map(function (name) { return "{".concat(name, "}"); })
|
|
5013
5014
|
.join(', '), "\n\n ").concat(block(pipelineIdentification), "\n\n "); }));
|
|
5014
5015
|
}
|
|
5015
|
-
|
|
5016
|
-
|
|
5016
|
+
_c = (_b = Object).freeze;
|
|
5017
|
+
_d = [{}];
|
|
5017
5018
|
return [4 /*yield*/, getReservedParametersForTemplate({
|
|
5018
5019
|
preparedPipeline: preparedPipeline,
|
|
5019
5020
|
template: currentTemplate,
|
|
5020
5021
|
pipelineIdentification: pipelineIdentification,
|
|
5021
5022
|
})];
|
|
5022
5023
|
case 2:
|
|
5023
|
-
definedParameters =
|
|
5024
|
+
definedParameters = _c.apply(_b, [__assign.apply(void 0, [__assign.apply(void 0, _d.concat([(_j.sent())])), parametersToPass])]);
|
|
5024
5025
|
definedParameterNames = new Set(Object.keys(definedParameters));
|
|
5025
5026
|
parameters = {};
|
|
5026
5027
|
_loop_1 = function (parameterName) {
|
|
@@ -5040,15 +5041,15 @@ function executeTemplate(options) {
|
|
|
5040
5041
|
try {
|
|
5041
5042
|
// Note: [2] Check that all used parameters are defined and removing unused parameters for this template
|
|
5042
5043
|
// TODO: [👩🏾🤝👩🏻] Use here `mapAvailableToExpectedParameters`
|
|
5043
|
-
for (
|
|
5044
|
-
parameterName =
|
|
5044
|
+
for (_e = __values(Array.from(union(definedParameterNames, usedParameterNames, dependentParameterNames))), _f = _e.next(); !_f.done; _f = _e.next()) {
|
|
5045
|
+
parameterName = _f.value;
|
|
5045
5046
|
_loop_1(parameterName);
|
|
5046
5047
|
}
|
|
5047
5048
|
}
|
|
5048
5049
|
catch (e_1_1) { e_1 = { error: e_1_1 }; }
|
|
5049
5050
|
finally {
|
|
5050
5051
|
try {
|
|
5051
|
-
if (
|
|
5052
|
+
if (_f && !_f.done && (_g = _e.return)) _g.call(_e);
|
|
5052
5053
|
}
|
|
5053
5054
|
finally { if (e_1) throw e_1.error; }
|
|
5054
5055
|
}
|
|
@@ -5068,12 +5069,11 @@ function executeTemplate(options) {
|
|
|
5068
5069
|
template: currentTemplate,
|
|
5069
5070
|
preparedPipeline: preparedPipeline,
|
|
5070
5071
|
tools: tools,
|
|
5071
|
-
settings: settings,
|
|
5072
5072
|
$executionReport: $executionReport,
|
|
5073
5073
|
pipelineIdentification: pipelineIdentification,
|
|
5074
5074
|
})];
|
|
5075
5075
|
case 3:
|
|
5076
|
-
resultString =
|
|
5076
|
+
resultString = _j.sent();
|
|
5077
5077
|
return [4 /*yield*/, onProgress({
|
|
5078
5078
|
name: name,
|
|
5079
5079
|
title: title,
|
|
@@ -5085,12 +5085,12 @@ function executeTemplate(options) {
|
|
|
5085
5085
|
// <- [🍸]
|
|
5086
5086
|
})];
|
|
5087
5087
|
case 4:
|
|
5088
|
-
|
|
5089
|
-
return [2 /*return*/, Object.freeze((
|
|
5090
|
-
|
|
5088
|
+
_j.sent();
|
|
5089
|
+
return [2 /*return*/, Object.freeze((_h = {},
|
|
5090
|
+
_h[currentTemplate.resultingParameterName] =
|
|
5091
5091
|
// <- Note: [👩👩👧] No need to detect parameter collision here because pipeline checks logic consistency during construction
|
|
5092
5092
|
resultString,
|
|
5093
|
-
|
|
5093
|
+
_h))];
|
|
5094
5094
|
}
|
|
5095
5095
|
});
|
|
5096
5096
|
});
|
|
@@ -5149,13 +5149,12 @@ function filterJustOutputParameters(options) {
|
|
|
5149
5149
|
*/
|
|
5150
5150
|
function executePipeline(options) {
|
|
5151
5151
|
return __awaiter(this, void 0, void 0, function () {
|
|
5152
|
-
var inputParameters, tools, onProgress, pipeline, setPreparedPipeline, pipelineIdentification,
|
|
5152
|
+
var inputParameters, tools, onProgress, pipeline, setPreparedPipeline, pipelineIdentification, maxParallelCount, rootDirname, _a, isVerbose, preparedPipeline, errors, warnings, executionReport, isReturned, _b, _c, parameter, e_1_1, _loop_1, _d, _e, parameterName, state_1, e_2_1, parametersToPass, resovedParameterNames_1, unresovedTemplates_1, resolving_1, loopLimit, _loop_2, error_1, usage_1, outputParameters_1, usage, outputParameters;
|
|
5153
5153
|
var e_1, _f, e_2, _g;
|
|
5154
5154
|
return __generator(this, function (_h) {
|
|
5155
5155
|
switch (_h.label) {
|
|
5156
5156
|
case 0:
|
|
5157
|
-
inputParameters = options.inputParameters, tools = options.tools, onProgress = options.onProgress, pipeline = options.pipeline, setPreparedPipeline = options.setPreparedPipeline, pipelineIdentification = options.pipelineIdentification,
|
|
5158
|
-
maxParallelCount = settings.maxParallelCount, rootDirname = settings.rootDirname, _a = settings.isVerbose, isVerbose = _a === void 0 ? IS_VERBOSE : _a;
|
|
5157
|
+
inputParameters = options.inputParameters, tools = options.tools, onProgress = options.onProgress, pipeline = options.pipeline, setPreparedPipeline = options.setPreparedPipeline, pipelineIdentification = options.pipelineIdentification, maxParallelCount = options.maxParallelCount, rootDirname = options.rootDirname, _a = options.isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a;
|
|
5159
5158
|
preparedPipeline = options.preparedPipeline;
|
|
5160
5159
|
if (!(preparedPipeline === undefined)) return [3 /*break*/, 2];
|
|
5161
5160
|
return [4 /*yield*/, preparePipeline(pipeline, tools, {
|
|
@@ -5340,12 +5339,7 @@ function executePipeline(options) {
|
|
|
5340
5339
|
return [3 /*break*/, 4];
|
|
5341
5340
|
case 3:
|
|
5342
5341
|
unresovedTemplates_1 = unresovedTemplates_1.filter(function (template) { return template !== currentTemplate; });
|
|
5343
|
-
work_1 = executeTemplate({
|
|
5344
|
-
currentTemplate: currentTemplate,
|
|
5345
|
-
preparedPipeline: preparedPipeline,
|
|
5346
|
-
parametersToPass: parametersToPass,
|
|
5347
|
-
tools: tools,
|
|
5348
|
-
onProgress: function (progress) {
|
|
5342
|
+
work_1 = executeTemplate(__assign(__assign({}, options), { currentTemplate: currentTemplate, preparedPipeline: preparedPipeline, parametersToPass: parametersToPass, tools: tools, onProgress: function (progress) {
|
|
5349
5343
|
if (isReturned) {
|
|
5350
5344
|
throw new UnexpectedError(spaceTrim$1(function (block) { return "\n Can not call `onProgress` after pipeline execution is finished\n\n ".concat(block(pipelineIdentification), "\n\n ").concat(block(JSON.stringify(progress, null, 4)
|
|
5351
5345
|
.split('\n')
|
|
@@ -5355,11 +5349,7 @@ function executePipeline(options) {
|
|
|
5355
5349
|
if (onProgress) {
|
|
5356
5350
|
onProgress(progress);
|
|
5357
5351
|
}
|
|
5358
|
-
},
|
|
5359
|
-
settings: settings,
|
|
5360
|
-
$executionReport: executionReport,
|
|
5361
|
-
pipelineIdentification: spaceTrim$1(function (block) { return "\n ".concat(block(pipelineIdentification), "\n Template name: ").concat(currentTemplate.name, "\n Template title: ").concat(currentTemplate.title, "\n "); }),
|
|
5362
|
-
})
|
|
5352
|
+
}, $executionReport: executionReport, pipelineIdentification: spaceTrim$1(function (block) { return "\n ".concat(block(pipelineIdentification), "\n Template name: ").concat(currentTemplate.name, "\n Template title: ").concat(currentTemplate.title, "\n "); }) }))
|
|
5363
5353
|
.then(function (newParametersToPass) {
|
|
5364
5354
|
parametersToPass = __assign(__assign({}, newParametersToPass), parametersToPass);
|
|
5365
5355
|
resovedParameterNames_1 = __spreadArray(__spreadArray([], __read(resovedParameterNames_1), false), [currentTemplate.resultingParameterName], false);
|
|
@@ -5463,8 +5453,7 @@ function executePipeline(options) {
|
|
|
5463
5453
|
*/
|
|
5464
5454
|
function createPipelineExecutor(options) {
|
|
5465
5455
|
var _this = this;
|
|
5466
|
-
var pipeline = options.pipeline, tools = options.tools, _a = options.
|
|
5467
|
-
var _b = settings.maxExecutionAttempts, maxExecutionAttempts = _b === void 0 ? MAX_EXECUTION_ATTEMPTS : _b, _c = settings.maxParallelCount, maxParallelCount = _c === void 0 ? MAX_PARALLEL_COUNT : _c, _d = settings.csvSettings, csvSettings = _d === void 0 ? DEFAULT_CSV_SETTINGS : _d, _e = settings.isVerbose, isVerbose = _e === void 0 ? IS_VERBOSE : _e, _f = settings.isNotPreparedWarningSupressed, isNotPreparedWarningSupressed = _f === void 0 ? false : _f, _g = settings.rootDirname, rootDirname = _g === void 0 ? null : _g;
|
|
5456
|
+
var pipeline = options.pipeline, tools = options.tools, _a = options.maxExecutionAttempts, maxExecutionAttempts = _a === void 0 ? DEFAULT_MAX_EXECUTION_ATTEMPTS : _a, _b = options.maxParallelCount, maxParallelCount = _b === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _b, _c = options.csvSettings, csvSettings = _c === void 0 ? DEFAULT_CSV_SETTINGS : _c, _d = options.isVerbose, isVerbose = _d === void 0 ? DEFAULT_IS_VERBOSE : _d, _e = options.isNotPreparedWarningSupressed, isNotPreparedWarningSupressed = _e === void 0 ? false : _e, _f = options.rootDirname, rootDirname = _f === void 0 ? null : _f;
|
|
5468
5457
|
validatePipeline(pipeline);
|
|
5469
5458
|
var pipelineIdentification = (function () {
|
|
5470
5459
|
// Note: This is a 😐 implementation of [🚞]
|
|
@@ -5498,14 +5487,12 @@ function createPipelineExecutor(options) {
|
|
|
5498
5487
|
tools: tools,
|
|
5499
5488
|
onProgress: onProgress,
|
|
5500
5489
|
pipelineIdentification: spaceTrim$1(function (block) { return "\n ".concat(block(pipelineIdentification), "\n ").concat(runCount === 1 ? '' : "Run #".concat(runCount), "\n "); }),
|
|
5501
|
-
|
|
5502
|
-
|
|
5503
|
-
|
|
5504
|
-
|
|
5505
|
-
|
|
5506
|
-
|
|
5507
|
-
rootDirname: rootDirname,
|
|
5508
|
-
},
|
|
5490
|
+
maxExecutionAttempts: maxExecutionAttempts,
|
|
5491
|
+
maxParallelCount: maxParallelCount,
|
|
5492
|
+
csvSettings: csvSettings,
|
|
5493
|
+
isVerbose: isVerbose,
|
|
5494
|
+
isNotPreparedWarningSupressed: isNotPreparedWarningSupressed,
|
|
5495
|
+
rootDirname: rootDirname,
|
|
5509
5496
|
})];
|
|
5510
5497
|
});
|
|
5511
5498
|
}); };
|
|
@@ -5527,7 +5514,7 @@ var markdownScraperMetadata = $deepFreeze({
|
|
|
5527
5514
|
mimeTypes: ['text/markdown', 'text/plain'],
|
|
5528
5515
|
documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
|
|
5529
5516
|
isAvilableInBrowser: true,
|
|
5530
|
-
requiredExecutables: [
|
|
5517
|
+
requiredExecutables: [],
|
|
5531
5518
|
}); /* <- TODO: [🤛] */
|
|
5532
5519
|
/**
|
|
5533
5520
|
* Registration of known scraper metadata
|
|
@@ -5571,7 +5558,7 @@ var MarkdownScraper = /** @class */ (function () {
|
|
|
5571
5558
|
return __generator(this, function (_k) {
|
|
5572
5559
|
switch (_k.label) {
|
|
5573
5560
|
case 0:
|
|
5574
|
-
_a = this.options, _b = _a.maxParallelCount, maxParallelCount = _b === void 0 ?
|
|
5561
|
+
_a = this.options, _b = _a.maxParallelCount, maxParallelCount = _b === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? DEFAULT_IS_VERBOSE : _c;
|
|
5575
5562
|
llm = this.tools.llm;
|
|
5576
5563
|
if (llm === undefined) {
|
|
5577
5564
|
throw new MissingToolsError('LLM tools are required for scraping external files');
|
|
@@ -5670,7 +5657,8 @@ var MarkdownScraper = /** @class */ (function () {
|
|
|
5670
5657
|
embeddingResult = _c.sent();
|
|
5671
5658
|
index.push({
|
|
5672
5659
|
modelName: embeddingResult.modelName,
|
|
5673
|
-
position: embeddingResult.content,
|
|
5660
|
+
position: __spreadArray([], __read(embeddingResult.content), false),
|
|
5661
|
+
// <- TODO: [🪓] Here should be no need for spreading new array, just `position: embeddingResult.content`
|
|
5674
5662
|
});
|
|
5675
5663
|
_c.label = 6;
|
|
5676
5664
|
case 6: return [3 /*break*/, 8];
|