@promptbook/cli 0.72.0-14 → 0.72.0-15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/esm/index.es.js +544 -516
- package/esm/index.es.js.map +1 -1
- package/esm/typings/src/_packages/core.index.d.ts +22 -18
- package/esm/typings/src/_packages/node.index.d.ts +2 -0
- package/esm/typings/src/_packages/types.index.d.ts +10 -10
- package/esm/typings/src/_packages/utils.index.d.ts +2 -2
- package/esm/typings/src/collection/PipelineCollection.d.ts +1 -1
- package/esm/typings/src/collection/SimplePipelineCollection.d.ts +2 -2
- package/esm/typings/src/collection/collectionToJson.d.ts +1 -1
- package/esm/typings/src/collection/constructors/createCollectionFromJson.d.ts +1 -1
- package/esm/typings/src/collection/constructors/createCollectionFromPromise.d.ts +1 -1
- package/esm/typings/src/commands/_common/types/CommandParser.d.ts +5 -5
- package/esm/typings/src/config.d.ts +21 -14
- package/esm/typings/src/execution/EmbeddingVector.d.ts +1 -1
- package/esm/typings/src/execution/Executables.d.ts +18 -0
- package/esm/typings/src/execution/ExecutionTools.d.ts +9 -3
- package/esm/typings/src/execution/LlmExecutionTools.d.ts +1 -1
- package/esm/typings/src/execution/PipelineExecutorResult.d.ts +2 -2
- package/esm/typings/src/execution/assertsExecutionSuccessful.d.ts +1 -0
- package/esm/typings/src/execution/createPipelineExecutor/00-CreatePipelineExecutorOptions.d.ts +29 -6
- package/esm/typings/src/execution/createPipelineExecutor/10-executePipeline.d.ts +2 -11
- package/esm/typings/src/execution/createPipelineExecutor/20-executeTemplate.d.ts +4 -13
- package/esm/typings/src/execution/createPipelineExecutor/40-executeAttempts.d.ts +9 -14
- package/esm/typings/src/execution/translation/automatic-translate/automatic-translators/LindatAutomaticTranslator.d.ts +11 -3
- package/esm/typings/src/execution/utils/addUsage.d.ts +1 -1
- package/esm/typings/src/execution/utils/forEachAsync.d.ts +1 -1
- package/esm/typings/src/formats/_common/FormatDefinition.d.ts +2 -2
- package/esm/typings/src/formats/_common/FormatSubvalueDefinition.d.ts +2 -2
- package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsForCli.d.ts +2 -2
- package/esm/typings/src/llm-providers/_common/register/LlmToolsConfiguration.d.ts +1 -1
- package/esm/typings/src/llm-providers/_common/utils/cache/CacheLlmToolsOptions.d.ts +4 -1
- package/esm/typings/src/llm-providers/anthropic-claude/AnthropicClaudeExecutionTools.d.ts +1 -1
- package/esm/typings/src/llm-providers/anthropic-claude/anthropic-claude-models.d.ts +1 -1
- package/esm/typings/src/llm-providers/azure-openai/AzureOpenAiExecutionTools.d.ts +1 -1
- package/esm/typings/src/llm-providers/mocked/$fakeTextToExpectations.d.ts +1 -1
- package/esm/typings/src/llm-providers/mocked/MockedEchoLlmExecutionTools.d.ts +1 -1
- package/esm/typings/src/llm-providers/mocked/MockedFackedLlmExecutionTools.d.ts +1 -1
- package/esm/typings/src/llm-providers/multiple/MultipleLlmExecutionTools.d.ts +4 -5
- package/esm/typings/src/llm-providers/multiple/joinLlmExecutionTools.d.ts +1 -1
- package/esm/typings/src/llm-providers/openai/OpenAiExecutionTools.d.ts +1 -1
- package/esm/typings/src/llm-providers/openai/openai-models.d.ts +1 -1
- package/esm/typings/src/llm-providers/remote/RemoteLlmExecutionTools.d.ts +1 -1
- package/esm/typings/src/llm-providers/remote/interfaces/PromptbookServer_ListModels_Response.d.ts +3 -3
- package/esm/typings/src/llm-providers/remote/interfaces/PromptbookServer_Prompt_Response.d.ts +2 -2
- package/esm/typings/src/prepare/PrepareAndScrapeOptions.d.ts +4 -23
- package/esm/typings/src/prepare/prepareTemplates.d.ts +1 -1
- package/esm/typings/src/scrapers/_common/Scraper.d.ts +1 -5
- package/esm/typings/src/scrapers/_common/prepareKnowledgePieces.d.ts +1 -1
- package/esm/typings/src/scrapers/_common/register/$provideExecutablesForNode.d.ts +12 -0
- package/esm/typings/src/scrapers/_common/register/$provideScrapersForBrowser.d.ts +2 -2
- package/esm/typings/src/scrapers/_common/register/$provideScrapersForNode.d.ts +2 -2
- package/esm/typings/src/scrapers/_common/register/ScraperAndConverterMetadata.d.ts +2 -2
- package/esm/typings/src/scrapers/_common/utils/getScraperIntermediateSource.d.ts +2 -2
- package/esm/typings/src/scrapers/document/DocumentScraper.d.ts +2 -2
- package/esm/typings/src/scrapers/document/createDocumentScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/document/register-metadata.d.ts +1 -1
- package/esm/typings/src/scrapers/document-legacy/LegacyDocumentScraper.d.ts +3 -3
- package/esm/typings/src/scrapers/document-legacy/createLegacyDocumentScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/document-legacy/register-metadata.d.ts +1 -1
- package/esm/typings/src/scrapers/markdown/MarkdownScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/markdown/createMarkdownScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/markdown/register-metadata.d.ts +1 -1
- package/esm/typings/src/scrapers/pdf/PdfScraper.d.ts +2 -2
- package/esm/typings/src/scrapers/pdf/createPdfScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/pdf/register-metadata.d.ts +1 -1
- package/esm/typings/src/scrapers/website/WebsiteScraper.d.ts +6 -3
- package/esm/typings/src/scrapers/website/createWebsiteScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/website/register-metadata.d.ts +1 -1
- package/esm/typings/src/scrapers/website/utils/createShowdownConverter.d.ts +7 -0
- package/esm/typings/src/scripting/javascript/utils/preserve.d.ts +1 -1
- package/esm/typings/src/storage/file-cache-storage/utils/nameToSubfolderPath.d.ts +1 -1
- package/esm/typings/src/types/Arrayable.d.ts +1 -1
- package/esm/typings/src/types/IntermediateFilesStrategy.d.ts +7 -0
- package/esm/typings/src/types/PipelineJson/KnowledgePieceJson.d.ts +4 -4
- package/esm/typings/src/types/PipelineJson/KnowledgeSourceJson.d.ts +1 -1
- package/esm/typings/src/types/PipelineJson/PersonaJson.d.ts +1 -1
- package/esm/typings/src/types/PipelineJson/TemplateJsonCommon.d.ts +2 -2
- package/esm/typings/src/types/Prompt.d.ts +1 -1
- package/esm/typings/src/types/execution-report/ExecutionReportJson.d.ts +1 -1
- package/esm/typings/src/utils/$Register.d.ts +1 -1
- package/esm/typings/src/utils/FromtoItems.d.ts +1 -1
- package/esm/typings/src/utils/arrayableToArray.d.ts +1 -1
- package/esm/typings/src/utils/emojis.d.ts +1 -1
- package/esm/typings/src/utils/execCommand/$execCommand.d.ts +2 -2
- package/esm/typings/src/utils/execCommand/{IExecCommandOptions.d.ts → ExecCommandOptions.d.ts} +2 -6
- package/esm/typings/src/utils/execCommand/execCommandNormalizeOptions.d.ts +3 -3
- package/esm/typings/src/utils/expectation-counters/countSentences.d.ts +1 -1
- package/esm/typings/src/utils/markdown/extractAllBlocksFromMarkdown.d.ts +1 -1
- package/esm/typings/src/utils/markdown/splitMarkdownIntoSections.d.ts +1 -1
- package/esm/typings/src/utils/normalization/IKeywords.d.ts +2 -2
- package/esm/typings/src/utils/normalization/parseKeywords.d.ts +2 -2
- package/esm/typings/src/utils/normalization/parseKeywordsFromString.d.ts +2 -2
- package/esm/typings/src/utils/normalization/searchKeywords.d.ts +2 -2
- package/esm/typings/src/utils/organization/TODO_USE.d.ts +1 -1
- package/esm/typings/src/utils/organization/keepUnused.d.ts +1 -1
- package/esm/typings/src/utils/random/$randomSeed.d.ts +1 -1
- package/esm/typings/src/utils/sets/intersection.d.ts +1 -1
- package/esm/typings/src/utils/sets/union.d.ts +1 -1
- package/esm/typings/src/utils/unwrapResult.d.ts +4 -4
- package/package.json +3 -2
- package/umd/index.umd.js +544 -516
- package/umd/index.umd.js.map +1 -1
- package/esm/typings/src/execution/createPipelineExecutor/00-CreatePipelineExecutorSettings.d.ts +0 -29
- package/esm/typings/src/scrapers/website/utils/markdownConverter.d.ts +0 -12
- /package/esm/typings/src/scrapers/website/utils/{markdownConverter.test.d.ts → createShowdownConverter.test.d.ts} +0 -0
package/esm/index.es.js
CHANGED
|
@@ -26,7 +26,7 @@ import { Converter } from 'showdown';
|
|
|
26
26
|
/**
|
|
27
27
|
* The version of the Promptbook library
|
|
28
28
|
*/
|
|
29
|
-
var PROMPTBOOK_VERSION = '0.72.0-
|
|
29
|
+
var PROMPTBOOK_VERSION = '0.72.0-14';
|
|
30
30
|
// TODO: [main] !!!! List here all the versions and annotate + put into script
|
|
31
31
|
|
|
32
32
|
/*! *****************************************************************************
|
|
@@ -399,7 +399,8 @@ var GENERATOR_WARNING = "\u26A0\uFE0F WARNING: This code has been generated so t
|
|
|
399
399
|
*
|
|
400
400
|
* @public exported from `@promptbook/core`
|
|
401
401
|
*/
|
|
402
|
-
var CLAIM = "
|
|
402
|
+
var CLAIM = "Build responsible, controlled and transparent applications on top of LLM models!";
|
|
403
|
+
// <- TODO: [🐊] Pick the best claim
|
|
403
404
|
/**
|
|
404
405
|
* Warning message for the generated sections and files files
|
|
405
406
|
*
|
|
@@ -432,23 +433,30 @@ var CONNECTION_RETRIES_LIMIT = 5;
|
|
|
432
433
|
*/
|
|
433
434
|
var IMMEDIATE_TIME = 10;
|
|
434
435
|
/**
|
|
435
|
-
* The maximum
|
|
436
|
+
* The maximum length of the (generated) filename
|
|
436
437
|
*
|
|
437
438
|
* @public exported from `@promptbook/core`
|
|
438
439
|
*/
|
|
439
|
-
var
|
|
440
|
+
var MAX_FILENAME_LENGTH = 30;
|
|
440
441
|
/**
|
|
441
|
-
*
|
|
442
|
+
* Strategy for caching the intermediate results for knowledge sources
|
|
442
443
|
*
|
|
443
444
|
* @public exported from `@promptbook/core`
|
|
444
445
|
*/
|
|
445
|
-
var
|
|
446
|
+
var DEFAULT_INTERMEDIATE_FILES_STRATEGY = 'HIDE_AND_KEEP';
|
|
447
|
+
// <- TODO: [😡] Change to 'VISIBLE'
|
|
446
448
|
/**
|
|
447
|
-
* The maximum
|
|
449
|
+
* The maximum number of (LLM) tasks running in parallel
|
|
448
450
|
*
|
|
449
451
|
* @public exported from `@promptbook/core`
|
|
450
452
|
*/
|
|
451
|
-
var
|
|
453
|
+
var DEFAULT_MAX_PARALLEL_COUNT = 5; // <- TODO: [🤹♂️]
|
|
454
|
+
/**
|
|
455
|
+
* The maximum number of attempts to execute LLM task before giving up
|
|
456
|
+
*
|
|
457
|
+
* @public exported from `@promptbook/core`
|
|
458
|
+
*/
|
|
459
|
+
var DEFAULT_MAX_EXECUTION_ATTEMPTS = 3; // <- TODO: [🤹♂️]
|
|
452
460
|
/**
|
|
453
461
|
* Where to store the cache of executions for promptbook CLI
|
|
454
462
|
*
|
|
@@ -456,7 +464,7 @@ var MAX_FILENAME_LENGTH = 30;
|
|
|
456
464
|
*
|
|
457
465
|
* @public exported from `@promptbook/core`
|
|
458
466
|
*/
|
|
459
|
-
var
|
|
467
|
+
var DEFAULT_EXECUTIONS_CACHE_DIRNAME = '/.promptbook/executions-cache';
|
|
460
468
|
/**
|
|
461
469
|
* Where to store the scrape cache
|
|
462
470
|
*
|
|
@@ -464,13 +472,13 @@ var EXECUTIONS_CACHE_DIRNAME = '/.promptbook/executions-cache';
|
|
|
464
472
|
*
|
|
465
473
|
* @public exported from `@promptbook/core`
|
|
466
474
|
*/
|
|
467
|
-
var
|
|
475
|
+
var DEFAULT_SCRAPE_CACHE_DIRNAME = '/.promptbook/scrape-cache';
|
|
468
476
|
/**
|
|
469
477
|
* The name of the builded pipeline collection made by CLI `ptbk make` and for lookup in `createCollectionFromDirectory`
|
|
470
478
|
*
|
|
471
479
|
* @public exported from `@promptbook/core`
|
|
472
480
|
*/
|
|
473
|
-
var
|
|
481
|
+
var DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME = "index";
|
|
474
482
|
/**
|
|
475
483
|
* Nonce which is used for replacing things in strings
|
|
476
484
|
*
|
|
@@ -534,13 +542,13 @@ var DEFAULT_CSV_SETTINGS = Object.freeze({
|
|
|
534
542
|
*
|
|
535
543
|
* @public exported from `@promptbook/core`
|
|
536
544
|
*/
|
|
537
|
-
var
|
|
545
|
+
var DEFAULT_IS_VERBOSE = false;
|
|
538
546
|
/**
|
|
539
547
|
* @@@
|
|
540
548
|
*
|
|
541
549
|
* @public exported from `@promptbook/core`
|
|
542
550
|
*/
|
|
543
|
-
var
|
|
551
|
+
var DEFAULT_IS_AUTO_INSTALLED = false;
|
|
544
552
|
/**
|
|
545
553
|
* @@@
|
|
546
554
|
*
|
|
@@ -613,271 +621,6 @@ function initializeHelloCommand(program) {
|
|
|
613
621
|
* Note: [🟡] Code in this file should never be published outside of `@promptbook/cli`
|
|
614
622
|
*/
|
|
615
623
|
|
|
616
|
-
/**
|
|
617
|
-
* Just marks a place of place where should be something implemented
|
|
618
|
-
* No side effects.
|
|
619
|
-
*
|
|
620
|
-
* Note: It can be usefull suppressing eslint errors of unused variables
|
|
621
|
-
*
|
|
622
|
-
* @param value any values
|
|
623
|
-
* @returns void
|
|
624
|
-
* @private within the repository
|
|
625
|
-
*/
|
|
626
|
-
function TODO_USE() {
|
|
627
|
-
var value = [];
|
|
628
|
-
for (var _i = 0; _i < arguments.length; _i++) {
|
|
629
|
-
value[_i] = arguments[_i];
|
|
630
|
-
}
|
|
631
|
-
}
|
|
632
|
-
|
|
633
|
-
/**
|
|
634
|
-
* @@@
|
|
635
|
-
*
|
|
636
|
-
* @public exported from `@promptbook/node`
|
|
637
|
-
*/
|
|
638
|
-
function $provideFilesystemForNode(options) {
|
|
639
|
-
if (!$isRunningInNode()) {
|
|
640
|
-
throw new EnvironmentMismatchError('Function `$provideFilesystemForNode` works only in Node.js environment');
|
|
641
|
-
}
|
|
642
|
-
var _a = (options || {}).isVerbose, isVerbose = _a === void 0 ? IS_VERBOSE : _a;
|
|
643
|
-
TODO_USE(isVerbose);
|
|
644
|
-
return {
|
|
645
|
-
stat: stat,
|
|
646
|
-
access: access,
|
|
647
|
-
constants: constants,
|
|
648
|
-
readFile: readFile,
|
|
649
|
-
readdir: readdir,
|
|
650
|
-
};
|
|
651
|
-
}
|
|
652
|
-
/**
|
|
653
|
-
* Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
|
|
654
|
-
*/
|
|
655
|
-
|
|
656
|
-
/**
|
|
657
|
-
* This error type indicates that some part of the code is not implemented yet
|
|
658
|
-
*
|
|
659
|
-
* @public exported from `@promptbook/core`
|
|
660
|
-
*/
|
|
661
|
-
var NotYetImplementedError = /** @class */ (function (_super) {
|
|
662
|
-
__extends(NotYetImplementedError, _super);
|
|
663
|
-
function NotYetImplementedError(message) {
|
|
664
|
-
var _this = _super.call(this, spaceTrim(function (block) { return "\n ".concat(block(message), "\n\n Note: This feature is not implemented yet but it will be soon.\n\n If you want speed up the implementation or just read more, look here:\n https://github.com/webgptorg/promptbook\n\n Or contact us on me@pavolhejny.com\n\n "); })) || this;
|
|
665
|
-
_this.name = 'NotYetImplementedError';
|
|
666
|
-
Object.setPrototypeOf(_this, NotYetImplementedError.prototype);
|
|
667
|
-
return _this;
|
|
668
|
-
}
|
|
669
|
-
return NotYetImplementedError;
|
|
670
|
-
}(Error));
|
|
671
|
-
|
|
672
|
-
/**
|
|
673
|
-
* @@@
|
|
674
|
-
*
|
|
675
|
-
* Note: `$` is used to indicate that this function is not a pure function - it access global scope
|
|
676
|
-
*
|
|
677
|
-
* @private internal function of `$Register`
|
|
678
|
-
*/
|
|
679
|
-
function $getGlobalScope() {
|
|
680
|
-
return Function('return this')();
|
|
681
|
-
}
|
|
682
|
-
|
|
683
|
-
/**
|
|
684
|
-
* @@@
|
|
685
|
-
*
|
|
686
|
-
* @param text @@@
|
|
687
|
-
* @returns @@@
|
|
688
|
-
* @example 'HELLO_WORLD'
|
|
689
|
-
* @example 'I_LOVE_PROMPTBOOK'
|
|
690
|
-
* @public exported from `@promptbook/utils`
|
|
691
|
-
*/
|
|
692
|
-
function normalizeTo_SCREAMING_CASE(text) {
|
|
693
|
-
var e_1, _a;
|
|
694
|
-
var charType;
|
|
695
|
-
var lastCharType = 'OTHER';
|
|
696
|
-
var normalizedName = '';
|
|
697
|
-
try {
|
|
698
|
-
for (var text_1 = __values(text), text_1_1 = text_1.next(); !text_1_1.done; text_1_1 = text_1.next()) {
|
|
699
|
-
var char = text_1_1.value;
|
|
700
|
-
var normalizedChar = void 0;
|
|
701
|
-
if (/^[a-z]$/.test(char)) {
|
|
702
|
-
charType = 'LOWERCASE';
|
|
703
|
-
normalizedChar = char.toUpperCase();
|
|
704
|
-
}
|
|
705
|
-
else if (/^[A-Z]$/.test(char)) {
|
|
706
|
-
charType = 'UPPERCASE';
|
|
707
|
-
normalizedChar = char;
|
|
708
|
-
}
|
|
709
|
-
else if (/^[0-9]$/.test(char)) {
|
|
710
|
-
charType = 'NUMBER';
|
|
711
|
-
normalizedChar = char;
|
|
712
|
-
}
|
|
713
|
-
else {
|
|
714
|
-
charType = 'OTHER';
|
|
715
|
-
normalizedChar = '_';
|
|
716
|
-
}
|
|
717
|
-
if (charType !== lastCharType &&
|
|
718
|
-
!(lastCharType === 'UPPERCASE' && charType === 'LOWERCASE') &&
|
|
719
|
-
!(lastCharType === 'NUMBER') &&
|
|
720
|
-
!(charType === 'NUMBER')) {
|
|
721
|
-
normalizedName += '_';
|
|
722
|
-
}
|
|
723
|
-
normalizedName += normalizedChar;
|
|
724
|
-
lastCharType = charType;
|
|
725
|
-
}
|
|
726
|
-
}
|
|
727
|
-
catch (e_1_1) { e_1 = { error: e_1_1 }; }
|
|
728
|
-
finally {
|
|
729
|
-
try {
|
|
730
|
-
if (text_1_1 && !text_1_1.done && (_a = text_1.return)) _a.call(text_1);
|
|
731
|
-
}
|
|
732
|
-
finally { if (e_1) throw e_1.error; }
|
|
733
|
-
}
|
|
734
|
-
normalizedName = normalizedName.replace(/_+/g, '_');
|
|
735
|
-
normalizedName = normalizedName.replace(/_?\/_?/g, '/');
|
|
736
|
-
normalizedName = normalizedName.replace(/^_/, '');
|
|
737
|
-
normalizedName = normalizedName.replace(/_$/, '');
|
|
738
|
-
return normalizedName;
|
|
739
|
-
}
|
|
740
|
-
/**
|
|
741
|
-
* TODO: Tests
|
|
742
|
-
* > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: 'Moje tabule' })).toEqual('/VtG7sR9rRJqwNEdM2/Moje tabule');
|
|
743
|
-
* > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: 'ěščřžžýáíúů' })).toEqual('/VtG7sR9rRJqwNEdM2/escrzyaieuu');
|
|
744
|
-
* > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: ' ahoj ' })).toEqual('/VtG7sR9rRJqwNEdM2/ahoj');
|
|
745
|
-
* > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: ' ahoj_ahojAhoj ahoj ' })).toEqual('/VtG7sR9rRJqwNEdM2/ahoj-ahoj-ahoj-ahoj');
|
|
746
|
-
* TODO: [🌺] Use some intermediate util splitWords
|
|
747
|
-
*/
|
|
748
|
-
|
|
749
|
-
/**
|
|
750
|
-
* @@@
|
|
751
|
-
*
|
|
752
|
-
* @param text @@@
|
|
753
|
-
* @returns @@@
|
|
754
|
-
* @example 'hello_world'
|
|
755
|
-
* @example 'i_love_promptbook'
|
|
756
|
-
* @public exported from `@promptbook/utils`
|
|
757
|
-
*/
|
|
758
|
-
function normalizeTo_snake_case(text) {
|
|
759
|
-
return normalizeTo_SCREAMING_CASE(text).toLowerCase();
|
|
760
|
-
}
|
|
761
|
-
|
|
762
|
-
/**
|
|
763
|
-
* Register is @@@
|
|
764
|
-
*
|
|
765
|
-
* Note: `$` is used to indicate that this function is not a pure function - it accesses and adds variables in global scope.
|
|
766
|
-
*
|
|
767
|
-
* @private internal utility, exported are only signleton instances of this class
|
|
768
|
-
*/
|
|
769
|
-
var $Register = /** @class */ (function () {
|
|
770
|
-
function $Register(registerName) {
|
|
771
|
-
this.registerName = registerName;
|
|
772
|
-
var storageName = "_promptbook_".concat(normalizeTo_snake_case(registerName));
|
|
773
|
-
var globalScope = $getGlobalScope();
|
|
774
|
-
if (globalScope[storageName] === undefined) {
|
|
775
|
-
globalScope[storageName] = [];
|
|
776
|
-
}
|
|
777
|
-
else if (!Array.isArray(globalScope[storageName])) {
|
|
778
|
-
throw new UnexpectedError("Expected (global) ".concat(storageName, " to be an array, but got ").concat(typeof globalScope[storageName]));
|
|
779
|
-
}
|
|
780
|
-
this.storage = globalScope[storageName];
|
|
781
|
-
}
|
|
782
|
-
$Register.prototype.list = function () {
|
|
783
|
-
// <- TODO: ReadonlyDeep<Array<TRegistered>>
|
|
784
|
-
return this.storage;
|
|
785
|
-
};
|
|
786
|
-
$Register.prototype.register = function (registered) {
|
|
787
|
-
var packageName = registered.packageName, className = registered.className;
|
|
788
|
-
var existingRegistrationIndex = this.storage.findIndex(function (item) { return item.packageName === packageName && item.className === className; });
|
|
789
|
-
var existingRegistration = this.storage[existingRegistrationIndex];
|
|
790
|
-
if (!existingRegistration) {
|
|
791
|
-
this.storage.push(registered);
|
|
792
|
-
}
|
|
793
|
-
else {
|
|
794
|
-
this.storage[existingRegistrationIndex] = registered;
|
|
795
|
-
}
|
|
796
|
-
return {
|
|
797
|
-
registerName: this.registerName,
|
|
798
|
-
packageName: packageName,
|
|
799
|
-
className: className,
|
|
800
|
-
get isDestroyed() {
|
|
801
|
-
return false;
|
|
802
|
-
},
|
|
803
|
-
destroy: function () {
|
|
804
|
-
throw new NotYetImplementedError("Registration to ".concat(this.registerName, " is permanent in this version of Promptbook"));
|
|
805
|
-
},
|
|
806
|
-
};
|
|
807
|
-
};
|
|
808
|
-
return $Register;
|
|
809
|
-
}());
|
|
810
|
-
|
|
811
|
-
/**
|
|
812
|
-
* @@@
|
|
813
|
-
*
|
|
814
|
-
* Note: `$` is used to indicate that this interacts with the global scope
|
|
815
|
-
* @singleton Only one instance of each register is created per build, but thare can be more @@@
|
|
816
|
-
* @public exported from `@promptbook/core`
|
|
817
|
-
*/
|
|
818
|
-
var $scrapersRegister = new $Register('scraper_constructors');
|
|
819
|
-
/**
|
|
820
|
-
* TODO: [®] DRY Register logic
|
|
821
|
-
*/
|
|
822
|
-
|
|
823
|
-
/**
|
|
824
|
-
* !!!!!!
|
|
825
|
-
*
|
|
826
|
-
* 1) @@@
|
|
827
|
-
* 2) @@@
|
|
828
|
-
*
|
|
829
|
-
* @public exported from `@promptbook/node`
|
|
830
|
-
*/
|
|
831
|
-
function $provideScrapersForNode(tools, options) {
|
|
832
|
-
return __awaiter(this, void 0, void 0, function () {
|
|
833
|
-
var _a, _b, isAutoInstalled, _c, isVerbose, scrapers, _d, _e, scraperFactory, scraper, e_1_1;
|
|
834
|
-
var e_1, _f;
|
|
835
|
-
return __generator(this, function (_g) {
|
|
836
|
-
switch (_g.label) {
|
|
837
|
-
case 0:
|
|
838
|
-
if (!$isRunningInNode()) {
|
|
839
|
-
throw new EnvironmentMismatchError('Function `$getScrapersForNode` works only in Node.js environment');
|
|
840
|
-
}
|
|
841
|
-
_a = options || {}, _b = _a.isAutoInstalled, isAutoInstalled = _b === void 0 ? IS_AUTO_INSTALLED : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? IS_VERBOSE : _c;
|
|
842
|
-
TODO_USE(isAutoInstalled);
|
|
843
|
-
TODO_USE(isVerbose);
|
|
844
|
-
scrapers = [];
|
|
845
|
-
_g.label = 1;
|
|
846
|
-
case 1:
|
|
847
|
-
_g.trys.push([1, 6, 7, 8]);
|
|
848
|
-
_d = __values($scrapersRegister.list()), _e = _d.next();
|
|
849
|
-
_g.label = 2;
|
|
850
|
-
case 2:
|
|
851
|
-
if (!!_e.done) return [3 /*break*/, 5];
|
|
852
|
-
scraperFactory = _e.value;
|
|
853
|
-
return [4 /*yield*/, scraperFactory(tools, options || {})];
|
|
854
|
-
case 3:
|
|
855
|
-
scraper = _g.sent();
|
|
856
|
-
scrapers.push(scraper);
|
|
857
|
-
_g.label = 4;
|
|
858
|
-
case 4:
|
|
859
|
-
_e = _d.next();
|
|
860
|
-
return [3 /*break*/, 2];
|
|
861
|
-
case 5: return [3 /*break*/, 8];
|
|
862
|
-
case 6:
|
|
863
|
-
e_1_1 = _g.sent();
|
|
864
|
-
e_1 = { error: e_1_1 };
|
|
865
|
-
return [3 /*break*/, 8];
|
|
866
|
-
case 7:
|
|
867
|
-
try {
|
|
868
|
-
if (_e && !_e.done && (_f = _d.return)) _f.call(_d);
|
|
869
|
-
}
|
|
870
|
-
finally { if (e_1) throw e_1.error; }
|
|
871
|
-
return [7 /*endfinally*/];
|
|
872
|
-
case 8: return [2 /*return*/, scrapers];
|
|
873
|
-
}
|
|
874
|
-
});
|
|
875
|
-
});
|
|
876
|
-
}
|
|
877
|
-
/**
|
|
878
|
-
* Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
|
|
879
|
-
*/
|
|
880
|
-
|
|
881
624
|
/**
|
|
882
625
|
* Converts PipelineCollection to serialized JSON
|
|
883
626
|
*
|
|
@@ -963,7 +706,6 @@ function pipelineJsonToString(pipelineJson) {
|
|
|
963
706
|
pipelineString += '\n\n';
|
|
964
707
|
pipelineString += description;
|
|
965
708
|
}
|
|
966
|
-
// TODO:> const commands: Array<Command>
|
|
967
709
|
var commands = [];
|
|
968
710
|
if (pipelineUrl) {
|
|
969
711
|
commands.push("PIPELINE URL ".concat(pipelineUrl));
|
|
@@ -1019,7 +761,6 @@ function pipelineJsonToString(pipelineJson) {
|
|
|
1019
761
|
pipelineString += '\n\n';
|
|
1020
762
|
pipelineString += description_1;
|
|
1021
763
|
}
|
|
1022
|
-
// TODO:> const commands: Array<Command>
|
|
1023
764
|
var commands_1 = [];
|
|
1024
765
|
var contentLanguage = 'text';
|
|
1025
766
|
if (templateType === 'PROMPT_TEMPLATE') {
|
|
@@ -1318,8 +1059,7 @@ var PipelineExecutionError = /** @class */ (function (_super) {
|
|
|
1318
1059
|
* Multiple LLM Execution Tools is a proxy server that uses multiple execution tools internally and exposes the executor interface externally.
|
|
1319
1060
|
*
|
|
1320
1061
|
* Note: Internal utility of `joinLlmExecutionTools` but exposed type
|
|
1321
|
-
* @public exported from `@promptbook/
|
|
1322
|
-
* TODO: !!!!!! Export as runtime class not just type
|
|
1062
|
+
* @public exported from `@promptbook/core`
|
|
1323
1063
|
*/
|
|
1324
1064
|
var MultipleLlmExecutionTools = /** @class */ (function () {
|
|
1325
1065
|
/**
|
|
@@ -2478,6 +2218,22 @@ var LimitReachedError = /** @class */ (function (_super) {
|
|
|
2478
2218
|
return LimitReachedError;
|
|
2479
2219
|
}(Error));
|
|
2480
2220
|
|
|
2221
|
+
/**
|
|
2222
|
+
* This error type indicates that some part of the code is not implemented yet
|
|
2223
|
+
*
|
|
2224
|
+
* @public exported from `@promptbook/core`
|
|
2225
|
+
*/
|
|
2226
|
+
var NotYetImplementedError = /** @class */ (function (_super) {
|
|
2227
|
+
__extends(NotYetImplementedError, _super);
|
|
2228
|
+
function NotYetImplementedError(message) {
|
|
2229
|
+
var _this = _super.call(this, spaceTrim(function (block) { return "\n ".concat(block(message), "\n\n Note: This feature is not implemented yet but it will be soon.\n\n If you want speed up the implementation or just read more, look here:\n https://github.com/webgptorg/promptbook\n\n Or contact us on me@pavolhejny.com\n\n "); })) || this;
|
|
2230
|
+
_this.name = 'NotYetImplementedError';
|
|
2231
|
+
Object.setPrototypeOf(_this, NotYetImplementedError.prototype);
|
|
2232
|
+
return _this;
|
|
2233
|
+
}
|
|
2234
|
+
return NotYetImplementedError;
|
|
2235
|
+
}(Error));
|
|
2236
|
+
|
|
2481
2237
|
/**
|
|
2482
2238
|
* Index of all custom errors
|
|
2483
2239
|
*
|
|
@@ -2539,6 +2295,7 @@ function assertsExecutionSuccessful(executionResult) {
|
|
|
2539
2295
|
}
|
|
2540
2296
|
}
|
|
2541
2297
|
/**
|
|
2298
|
+
* TODO: [🐚] This function should be removed OR changed OR be completely rewritten
|
|
2542
2299
|
* TODO: [🧠] Can this return type be better typed than void
|
|
2543
2300
|
*/
|
|
2544
2301
|
|
|
@@ -2796,6 +2553,23 @@ function union() {
|
|
|
2796
2553
|
return union;
|
|
2797
2554
|
}
|
|
2798
2555
|
|
|
2556
|
+
/**
|
|
2557
|
+
* Just marks a place of place where should be something implemented
|
|
2558
|
+
* No side effects.
|
|
2559
|
+
*
|
|
2560
|
+
* Note: It can be usefull suppressing eslint errors of unused variables
|
|
2561
|
+
*
|
|
2562
|
+
* @param value any values
|
|
2563
|
+
* @returns void
|
|
2564
|
+
* @private within the repository
|
|
2565
|
+
*/
|
|
2566
|
+
function TODO_USE() {
|
|
2567
|
+
var value = [];
|
|
2568
|
+
for (var _i = 0; _i < arguments.length; _i++) {
|
|
2569
|
+
value[_i] = arguments[_i];
|
|
2570
|
+
}
|
|
2571
|
+
}
|
|
2572
|
+
|
|
2799
2573
|
/**
|
|
2800
2574
|
* This error indicates problems parsing the format value
|
|
2801
2575
|
*
|
|
@@ -3828,12 +3602,11 @@ function checkExpectations(expectations, value) {
|
|
|
3828
3602
|
*/
|
|
3829
3603
|
function executeAttempts(options) {
|
|
3830
3604
|
return __awaiter(this, void 0, void 0, function () {
|
|
3831
|
-
var jokerParameterNames, priority, maxAttempts, preparedContent, parameters, template, preparedPipeline, tools,
|
|
3605
|
+
var jokerParameterNames, priority, maxAttempts, preparedContent, parameters, template, preparedPipeline, tools, $executionReport, pipelineIdentification, maxExecutionAttempts, $ongoingTemplateResult, _llms, llmTools, _loop_1, attempt, state_1;
|
|
3832
3606
|
return __generator(this, function (_a) {
|
|
3833
3607
|
switch (_a.label) {
|
|
3834
3608
|
case 0:
|
|
3835
|
-
jokerParameterNames = options.jokerParameterNames, priority = options.priority, maxAttempts = options.maxAttempts, preparedContent = options.preparedContent, parameters = options.parameters, template = options.template, preparedPipeline = options.preparedPipeline, tools = options.tools,
|
|
3836
|
-
maxExecutionAttempts = settings.maxExecutionAttempts;
|
|
3609
|
+
jokerParameterNames = options.jokerParameterNames, priority = options.priority, maxAttempts = options.maxAttempts, preparedContent = options.preparedContent, parameters = options.parameters, template = options.template, preparedPipeline = options.preparedPipeline, tools = options.tools, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification, maxExecutionAttempts = options.maxExecutionAttempts;
|
|
3837
3610
|
$ongoingTemplateResult = {
|
|
3838
3611
|
$result: null,
|
|
3839
3612
|
$resultString: null,
|
|
@@ -4199,12 +3972,12 @@ function executeAttempts(options) {
|
|
|
4199
3972
|
*/
|
|
4200
3973
|
function executeFormatSubvalues(options) {
|
|
4201
3974
|
return __awaiter(this, void 0, void 0, function () {
|
|
4202
|
-
var template, jokerParameterNames, parameters, priority,
|
|
3975
|
+
var template, jokerParameterNames, parameters, priority, csvSettings, pipelineIdentification, parameterValue, formatDefinition, subvalueDefinition, formatSettings, resultString;
|
|
4203
3976
|
var _this = this;
|
|
4204
3977
|
return __generator(this, function (_a) {
|
|
4205
3978
|
switch (_a.label) {
|
|
4206
3979
|
case 0:
|
|
4207
|
-
template = options.template, jokerParameterNames = options.jokerParameterNames, parameters = options.parameters, priority = options.priority,
|
|
3980
|
+
template = options.template, jokerParameterNames = options.jokerParameterNames, parameters = options.parameters, priority = options.priority, csvSettings = options.csvSettings, pipelineIdentification = options.pipelineIdentification;
|
|
4208
3981
|
if (template.foreach === undefined) {
|
|
4209
3982
|
return [2 /*return*/, /* not await */ executeAttempts(options)];
|
|
4210
3983
|
}
|
|
@@ -4234,7 +4007,7 @@ function executeFormatSubvalues(options) {
|
|
|
4234
4007
|
.join('\n')), "\n\n [\u26F7] This should never happen because subformat name should be validated during parsing\n\n ").concat(block(pipelineIdentification), "\n "); }));
|
|
4235
4008
|
}
|
|
4236
4009
|
if (formatDefinition.formatName === 'CSV') {
|
|
4237
|
-
formatSettings =
|
|
4010
|
+
formatSettings = csvSettings;
|
|
4238
4011
|
// <- TODO: [🤹♂️] More universal, make simmilar pattern for other formats for example \n vs \r\n in text
|
|
4239
4012
|
}
|
|
4240
4013
|
return [4 /*yield*/, subvalueDefinition.mapValues(parameterValue, template.foreach.outputSubparameterName, formatSettings, function (subparameters, index) { return __awaiter(_this, void 0, void 0, function () {
|
|
@@ -4387,13 +4160,12 @@ function getReservedParametersForTemplate(options) {
|
|
|
4387
4160
|
*/
|
|
4388
4161
|
function executeTemplate(options) {
|
|
4389
4162
|
return __awaiter(this, void 0, void 0, function () {
|
|
4390
|
-
var currentTemplate, preparedPipeline, parametersToPass, tools, onProgress,
|
|
4391
|
-
var e_1,
|
|
4392
|
-
return __generator(this, function (
|
|
4393
|
-
switch (
|
|
4163
|
+
var currentTemplate, preparedPipeline, parametersToPass, tools, onProgress, $executionReport, pipelineIdentification, _a, maxExecutionAttempts, name, title, priority, usedParameterNames, dependentParameterNames, definedParameters, _b, _c, _d, definedParameterNames, parameters, _loop_1, _e, _f, parameterName, maxAttempts, jokerParameterNames, preparedContent, resultString;
|
|
4164
|
+
var e_1, _g, _h;
|
|
4165
|
+
return __generator(this, function (_j) {
|
|
4166
|
+
switch (_j.label) {
|
|
4394
4167
|
case 0:
|
|
4395
|
-
currentTemplate = options.currentTemplate, preparedPipeline = options.preparedPipeline, parametersToPass = options.parametersToPass, tools = options.tools, onProgress = options.onProgress,
|
|
4396
|
-
maxExecutionAttempts = settings.maxExecutionAttempts;
|
|
4168
|
+
currentTemplate = options.currentTemplate, preparedPipeline = options.preparedPipeline, parametersToPass = options.parametersToPass, tools = options.tools, onProgress = options.onProgress, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification, _a = options.maxExecutionAttempts, maxExecutionAttempts = _a === void 0 ? DEFAULT_MAX_EXECUTION_ATTEMPTS : _a;
|
|
4397
4169
|
name = "pipeline-executor-frame-".concat(currentTemplate.name);
|
|
4398
4170
|
title = currentTemplate.title;
|
|
4399
4171
|
priority = preparedPipeline.templates.length - preparedPipeline.templates.indexOf(currentTemplate);
|
|
@@ -4408,7 +4180,7 @@ function executeTemplate(options) {
|
|
|
4408
4180
|
// <- [🍸]
|
|
4409
4181
|
})];
|
|
4410
4182
|
case 1:
|
|
4411
|
-
|
|
4183
|
+
_j.sent();
|
|
4412
4184
|
usedParameterNames = extractParameterNamesFromTemplate(currentTemplate);
|
|
4413
4185
|
dependentParameterNames = new Set(currentTemplate.dependentParameterNames);
|
|
4414
4186
|
// TODO: [👩🏾🤝👩🏻] Use here `mapAvailableToExpectedParameters`
|
|
@@ -4419,15 +4191,15 @@ function executeTemplate(options) {
|
|
|
4419
4191
|
.map(function (name) { return "{".concat(name, "}"); })
|
|
4420
4192
|
.join(', '), "\n\n ").concat(block(pipelineIdentification), "\n\n "); }));
|
|
4421
4193
|
}
|
|
4422
|
-
|
|
4423
|
-
|
|
4194
|
+
_c = (_b = Object).freeze;
|
|
4195
|
+
_d = [{}];
|
|
4424
4196
|
return [4 /*yield*/, getReservedParametersForTemplate({
|
|
4425
4197
|
preparedPipeline: preparedPipeline,
|
|
4426
4198
|
template: currentTemplate,
|
|
4427
4199
|
pipelineIdentification: pipelineIdentification,
|
|
4428
4200
|
})];
|
|
4429
4201
|
case 2:
|
|
4430
|
-
definedParameters =
|
|
4202
|
+
definedParameters = _c.apply(_b, [__assign.apply(void 0, [__assign.apply(void 0, _d.concat([(_j.sent())])), parametersToPass])]);
|
|
4431
4203
|
definedParameterNames = new Set(Object.keys(definedParameters));
|
|
4432
4204
|
parameters = {};
|
|
4433
4205
|
_loop_1 = function (parameterName) {
|
|
@@ -4447,15 +4219,15 @@ function executeTemplate(options) {
|
|
|
4447
4219
|
try {
|
|
4448
4220
|
// Note: [2] Check that all used parameters are defined and removing unused parameters for this template
|
|
4449
4221
|
// TODO: [👩🏾🤝👩🏻] Use here `mapAvailableToExpectedParameters`
|
|
4450
|
-
for (
|
|
4451
|
-
parameterName =
|
|
4222
|
+
for (_e = __values(Array.from(union(definedParameterNames, usedParameterNames, dependentParameterNames))), _f = _e.next(); !_f.done; _f = _e.next()) {
|
|
4223
|
+
parameterName = _f.value;
|
|
4452
4224
|
_loop_1(parameterName);
|
|
4453
4225
|
}
|
|
4454
4226
|
}
|
|
4455
4227
|
catch (e_1_1) { e_1 = { error: e_1_1 }; }
|
|
4456
4228
|
finally {
|
|
4457
4229
|
try {
|
|
4458
|
-
if (
|
|
4230
|
+
if (_f && !_f.done && (_g = _e.return)) _g.call(_e);
|
|
4459
4231
|
}
|
|
4460
4232
|
finally { if (e_1) throw e_1.error; }
|
|
4461
4233
|
}
|
|
@@ -4475,12 +4247,11 @@ function executeTemplate(options) {
|
|
|
4475
4247
|
template: currentTemplate,
|
|
4476
4248
|
preparedPipeline: preparedPipeline,
|
|
4477
4249
|
tools: tools,
|
|
4478
|
-
settings: settings,
|
|
4479
4250
|
$executionReport: $executionReport,
|
|
4480
4251
|
pipelineIdentification: pipelineIdentification,
|
|
4481
4252
|
})];
|
|
4482
4253
|
case 3:
|
|
4483
|
-
resultString =
|
|
4254
|
+
resultString = _j.sent();
|
|
4484
4255
|
return [4 /*yield*/, onProgress({
|
|
4485
4256
|
name: name,
|
|
4486
4257
|
title: title,
|
|
@@ -4492,12 +4263,12 @@ function executeTemplate(options) {
|
|
|
4492
4263
|
// <- [🍸]
|
|
4493
4264
|
})];
|
|
4494
4265
|
case 4:
|
|
4495
|
-
|
|
4496
|
-
return [2 /*return*/, Object.freeze((
|
|
4497
|
-
|
|
4266
|
+
_j.sent();
|
|
4267
|
+
return [2 /*return*/, Object.freeze((_h = {},
|
|
4268
|
+
_h[currentTemplate.resultingParameterName] =
|
|
4498
4269
|
// <- Note: [👩👩👧] No need to detect parameter collision here because pipeline checks logic consistency during construction
|
|
4499
4270
|
resultString,
|
|
4500
|
-
|
|
4271
|
+
_h))];
|
|
4501
4272
|
}
|
|
4502
4273
|
});
|
|
4503
4274
|
});
|
|
@@ -4556,13 +4327,12 @@ function filterJustOutputParameters(options) {
|
|
|
4556
4327
|
*/
|
|
4557
4328
|
function executePipeline(options) {
|
|
4558
4329
|
return __awaiter(this, void 0, void 0, function () {
|
|
4559
|
-
var inputParameters, tools, onProgress, pipeline, setPreparedPipeline, pipelineIdentification,
|
|
4330
|
+
var inputParameters, tools, onProgress, pipeline, setPreparedPipeline, pipelineIdentification, maxParallelCount, rootDirname, _a, isVerbose, preparedPipeline, errors, warnings, executionReport, isReturned, _b, _c, parameter, e_1_1, _loop_1, _d, _e, parameterName, state_1, e_2_1, parametersToPass, resovedParameterNames_1, unresovedTemplates_1, resolving_1, loopLimit, _loop_2, error_1, usage_1, outputParameters_1, usage, outputParameters;
|
|
4560
4331
|
var e_1, _f, e_2, _g;
|
|
4561
4332
|
return __generator(this, function (_h) {
|
|
4562
4333
|
switch (_h.label) {
|
|
4563
4334
|
case 0:
|
|
4564
|
-
inputParameters = options.inputParameters, tools = options.tools, onProgress = options.onProgress, pipeline = options.pipeline, setPreparedPipeline = options.setPreparedPipeline, pipelineIdentification = options.pipelineIdentification,
|
|
4565
|
-
maxParallelCount = settings.maxParallelCount, rootDirname = settings.rootDirname, _a = settings.isVerbose, isVerbose = _a === void 0 ? IS_VERBOSE : _a;
|
|
4335
|
+
inputParameters = options.inputParameters, tools = options.tools, onProgress = options.onProgress, pipeline = options.pipeline, setPreparedPipeline = options.setPreparedPipeline, pipelineIdentification = options.pipelineIdentification, maxParallelCount = options.maxParallelCount, rootDirname = options.rootDirname, _a = options.isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a;
|
|
4566
4336
|
preparedPipeline = options.preparedPipeline;
|
|
4567
4337
|
if (!(preparedPipeline === undefined)) return [3 /*break*/, 2];
|
|
4568
4338
|
return [4 /*yield*/, preparePipeline(pipeline, tools, {
|
|
@@ -4747,12 +4517,7 @@ function executePipeline(options) {
|
|
|
4747
4517
|
return [3 /*break*/, 4];
|
|
4748
4518
|
case 3:
|
|
4749
4519
|
unresovedTemplates_1 = unresovedTemplates_1.filter(function (template) { return template !== currentTemplate; });
|
|
4750
|
-
work_1 = executeTemplate({
|
|
4751
|
-
currentTemplate: currentTemplate,
|
|
4752
|
-
preparedPipeline: preparedPipeline,
|
|
4753
|
-
parametersToPass: parametersToPass,
|
|
4754
|
-
tools: tools,
|
|
4755
|
-
onProgress: function (progress) {
|
|
4520
|
+
work_1 = executeTemplate(__assign(__assign({}, options), { currentTemplate: currentTemplate, preparedPipeline: preparedPipeline, parametersToPass: parametersToPass, tools: tools, onProgress: function (progress) {
|
|
4756
4521
|
if (isReturned) {
|
|
4757
4522
|
throw new UnexpectedError(spaceTrim(function (block) { return "\n Can not call `onProgress` after pipeline execution is finished\n\n ".concat(block(pipelineIdentification), "\n\n ").concat(block(JSON.stringify(progress, null, 4)
|
|
4758
4523
|
.split('\n')
|
|
@@ -4762,11 +4527,7 @@ function executePipeline(options) {
|
|
|
4762
4527
|
if (onProgress) {
|
|
4763
4528
|
onProgress(progress);
|
|
4764
4529
|
}
|
|
4765
|
-
},
|
|
4766
|
-
settings: settings,
|
|
4767
|
-
$executionReport: executionReport,
|
|
4768
|
-
pipelineIdentification: spaceTrim(function (block) { return "\n ".concat(block(pipelineIdentification), "\n Template name: ").concat(currentTemplate.name, "\n Template title: ").concat(currentTemplate.title, "\n "); }),
|
|
4769
|
-
})
|
|
4530
|
+
}, $executionReport: executionReport, pipelineIdentification: spaceTrim(function (block) { return "\n ".concat(block(pipelineIdentification), "\n Template name: ").concat(currentTemplate.name, "\n Template title: ").concat(currentTemplate.title, "\n "); }) }))
|
|
4770
4531
|
.then(function (newParametersToPass) {
|
|
4771
4532
|
parametersToPass = __assign(__assign({}, newParametersToPass), parametersToPass);
|
|
4772
4533
|
resovedParameterNames_1 = __spreadArray(__spreadArray([], __read(resovedParameterNames_1), false), [currentTemplate.resultingParameterName], false);
|
|
@@ -4870,8 +4631,7 @@ function executePipeline(options) {
|
|
|
4870
4631
|
*/
|
|
4871
4632
|
function createPipelineExecutor(options) {
|
|
4872
4633
|
var _this = this;
|
|
4873
|
-
var pipeline = options.pipeline, tools = options.tools, _a = options.
|
|
4874
|
-
var _b = settings.maxExecutionAttempts, maxExecutionAttempts = _b === void 0 ? MAX_EXECUTION_ATTEMPTS : _b, _c = settings.maxParallelCount, maxParallelCount = _c === void 0 ? MAX_PARALLEL_COUNT : _c, _d = settings.csvSettings, csvSettings = _d === void 0 ? DEFAULT_CSV_SETTINGS : _d, _e = settings.isVerbose, isVerbose = _e === void 0 ? IS_VERBOSE : _e, _f = settings.isNotPreparedWarningSupressed, isNotPreparedWarningSupressed = _f === void 0 ? false : _f, _g = settings.rootDirname, rootDirname = _g === void 0 ? null : _g;
|
|
4634
|
+
var pipeline = options.pipeline, tools = options.tools, _a = options.maxExecutionAttempts, maxExecutionAttempts = _a === void 0 ? DEFAULT_MAX_EXECUTION_ATTEMPTS : _a, _b = options.maxParallelCount, maxParallelCount = _b === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _b, _c = options.csvSettings, csvSettings = _c === void 0 ? DEFAULT_CSV_SETTINGS : _c, _d = options.isVerbose, isVerbose = _d === void 0 ? DEFAULT_IS_VERBOSE : _d, _e = options.isNotPreparedWarningSupressed, isNotPreparedWarningSupressed = _e === void 0 ? false : _e, _f = options.rootDirname, rootDirname = _f === void 0 ? null : _f;
|
|
4875
4635
|
validatePipeline(pipeline);
|
|
4876
4636
|
var pipelineIdentification = (function () {
|
|
4877
4637
|
// Note: This is a 😐 implementation of [🚞]
|
|
@@ -4905,14 +4665,12 @@ function createPipelineExecutor(options) {
|
|
|
4905
4665
|
tools: tools,
|
|
4906
4666
|
onProgress: onProgress,
|
|
4907
4667
|
pipelineIdentification: spaceTrim(function (block) { return "\n ".concat(block(pipelineIdentification), "\n ").concat(runCount === 1 ? '' : "Run #".concat(runCount), "\n "); }),
|
|
4908
|
-
|
|
4909
|
-
|
|
4910
|
-
|
|
4911
|
-
|
|
4912
|
-
|
|
4913
|
-
|
|
4914
|
-
rootDirname: rootDirname,
|
|
4915
|
-
},
|
|
4668
|
+
maxExecutionAttempts: maxExecutionAttempts,
|
|
4669
|
+
maxParallelCount: maxParallelCount,
|
|
4670
|
+
csvSettings: csvSettings,
|
|
4671
|
+
isVerbose: isVerbose,
|
|
4672
|
+
isNotPreparedWarningSupressed: isNotPreparedWarningSupressed,
|
|
4673
|
+
rootDirname: rootDirname,
|
|
4916
4674
|
})];
|
|
4917
4675
|
});
|
|
4918
4676
|
}); };
|
|
@@ -4935,7 +4693,7 @@ function preparePersona(personaDescription, tools, options) {
|
|
|
4935
4693
|
return __generator(this, function (_d) {
|
|
4936
4694
|
switch (_d.label) {
|
|
4937
4695
|
case 0:
|
|
4938
|
-
_a = options.isVerbose, isVerbose = _a === void 0 ?
|
|
4696
|
+
_a = options.isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a;
|
|
4939
4697
|
if (tools === undefined || tools.llm === undefined) {
|
|
4940
4698
|
throw new MissingToolsError('LLM tools are required for preparing persona');
|
|
4941
4699
|
}
|
|
@@ -4991,20 +4749,171 @@ function preparePersona(personaDescription, tools, options) {
|
|
|
4991
4749
|
*/
|
|
4992
4750
|
|
|
4993
4751
|
/**
|
|
4994
|
-
* This error indicates that the promptbook can not retrieve knowledge from external sources
|
|
4752
|
+
* This error indicates that the promptbook can not retrieve knowledge from external sources
|
|
4753
|
+
*
|
|
4754
|
+
* @public exported from `@promptbook/core`
|
|
4755
|
+
*/
|
|
4756
|
+
var KnowledgeScrapeError = /** @class */ (function (_super) {
|
|
4757
|
+
__extends(KnowledgeScrapeError, _super);
|
|
4758
|
+
function KnowledgeScrapeError(message) {
|
|
4759
|
+
var _this = _super.call(this, message) || this;
|
|
4760
|
+
_this.name = 'KnowledgeScrapeError';
|
|
4761
|
+
Object.setPrototypeOf(_this, KnowledgeScrapeError.prototype);
|
|
4762
|
+
return _this;
|
|
4763
|
+
}
|
|
4764
|
+
return KnowledgeScrapeError;
|
|
4765
|
+
}(Error));
|
|
4766
|
+
|
|
4767
|
+
/**
|
|
4768
|
+
* @@@
|
|
4769
|
+
*
|
|
4770
|
+
* Note: `$` is used to indicate that this function is not a pure function - it access global scope
|
|
4771
|
+
*
|
|
4772
|
+
* @private internal function of `$Register`
|
|
4773
|
+
*/
|
|
4774
|
+
function $getGlobalScope() {
|
|
4775
|
+
return Function('return this')();
|
|
4776
|
+
}
|
|
4777
|
+
|
|
4778
|
+
/**
|
|
4779
|
+
* @@@
|
|
4780
|
+
*
|
|
4781
|
+
* @param text @@@
|
|
4782
|
+
* @returns @@@
|
|
4783
|
+
* @example 'HELLO_WORLD'
|
|
4784
|
+
* @example 'I_LOVE_PROMPTBOOK'
|
|
4785
|
+
* @public exported from `@promptbook/utils`
|
|
4786
|
+
*/
|
|
4787
|
+
function normalizeTo_SCREAMING_CASE(text) {
|
|
4788
|
+
var e_1, _a;
|
|
4789
|
+
var charType;
|
|
4790
|
+
var lastCharType = 'OTHER';
|
|
4791
|
+
var normalizedName = '';
|
|
4792
|
+
try {
|
|
4793
|
+
for (var text_1 = __values(text), text_1_1 = text_1.next(); !text_1_1.done; text_1_1 = text_1.next()) {
|
|
4794
|
+
var char = text_1_1.value;
|
|
4795
|
+
var normalizedChar = void 0;
|
|
4796
|
+
if (/^[a-z]$/.test(char)) {
|
|
4797
|
+
charType = 'LOWERCASE';
|
|
4798
|
+
normalizedChar = char.toUpperCase();
|
|
4799
|
+
}
|
|
4800
|
+
else if (/^[A-Z]$/.test(char)) {
|
|
4801
|
+
charType = 'UPPERCASE';
|
|
4802
|
+
normalizedChar = char;
|
|
4803
|
+
}
|
|
4804
|
+
else if (/^[0-9]$/.test(char)) {
|
|
4805
|
+
charType = 'NUMBER';
|
|
4806
|
+
normalizedChar = char;
|
|
4807
|
+
}
|
|
4808
|
+
else {
|
|
4809
|
+
charType = 'OTHER';
|
|
4810
|
+
normalizedChar = '_';
|
|
4811
|
+
}
|
|
4812
|
+
if (charType !== lastCharType &&
|
|
4813
|
+
!(lastCharType === 'UPPERCASE' && charType === 'LOWERCASE') &&
|
|
4814
|
+
!(lastCharType === 'NUMBER') &&
|
|
4815
|
+
!(charType === 'NUMBER')) {
|
|
4816
|
+
normalizedName += '_';
|
|
4817
|
+
}
|
|
4818
|
+
normalizedName += normalizedChar;
|
|
4819
|
+
lastCharType = charType;
|
|
4820
|
+
}
|
|
4821
|
+
}
|
|
4822
|
+
catch (e_1_1) { e_1 = { error: e_1_1 }; }
|
|
4823
|
+
finally {
|
|
4824
|
+
try {
|
|
4825
|
+
if (text_1_1 && !text_1_1.done && (_a = text_1.return)) _a.call(text_1);
|
|
4826
|
+
}
|
|
4827
|
+
finally { if (e_1) throw e_1.error; }
|
|
4828
|
+
}
|
|
4829
|
+
normalizedName = normalizedName.replace(/_+/g, '_');
|
|
4830
|
+
normalizedName = normalizedName.replace(/_?\/_?/g, '/');
|
|
4831
|
+
normalizedName = normalizedName.replace(/^_/, '');
|
|
4832
|
+
normalizedName = normalizedName.replace(/_$/, '');
|
|
4833
|
+
return normalizedName;
|
|
4834
|
+
}
|
|
4835
|
+
/**
|
|
4836
|
+
* TODO: Tests
|
|
4837
|
+
* > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: 'Moje tabule' })).toEqual('/VtG7sR9rRJqwNEdM2/Moje tabule');
|
|
4838
|
+
* > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: 'ěščřžžýáíúů' })).toEqual('/VtG7sR9rRJqwNEdM2/escrzyaieuu');
|
|
4839
|
+
* > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: ' ahoj ' })).toEqual('/VtG7sR9rRJqwNEdM2/ahoj');
|
|
4840
|
+
* > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: ' ahoj_ahojAhoj ahoj ' })).toEqual('/VtG7sR9rRJqwNEdM2/ahoj-ahoj-ahoj-ahoj');
|
|
4841
|
+
* TODO: [🌺] Use some intermediate util splitWords
|
|
4842
|
+
*/
|
|
4843
|
+
|
|
4844
|
+
/**
|
|
4845
|
+
* @@@
|
|
4846
|
+
*
|
|
4847
|
+
* @param text @@@
|
|
4848
|
+
* @returns @@@
|
|
4849
|
+
* @example 'hello_world'
|
|
4850
|
+
* @example 'i_love_promptbook'
|
|
4851
|
+
* @public exported from `@promptbook/utils`
|
|
4852
|
+
*/
|
|
4853
|
+
function normalizeTo_snake_case(text) {
|
|
4854
|
+
return normalizeTo_SCREAMING_CASE(text).toLowerCase();
|
|
4855
|
+
}
|
|
4856
|
+
|
|
4857
|
+
/**
|
|
4858
|
+
* Register is @@@
|
|
4859
|
+
*
|
|
4860
|
+
* Note: `$` is used to indicate that this function is not a pure function - it accesses and adds variables in global scope.
|
|
4861
|
+
*
|
|
4862
|
+
* @private internal utility, exported are only signleton instances of this class
|
|
4863
|
+
*/
|
|
4864
|
+
var $Register = /** @class */ (function () {
|
|
4865
|
+
function $Register(registerName) {
|
|
4866
|
+
this.registerName = registerName;
|
|
4867
|
+
var storageName = "_promptbook_".concat(normalizeTo_snake_case(registerName));
|
|
4868
|
+
var globalScope = $getGlobalScope();
|
|
4869
|
+
if (globalScope[storageName] === undefined) {
|
|
4870
|
+
globalScope[storageName] = [];
|
|
4871
|
+
}
|
|
4872
|
+
else if (!Array.isArray(globalScope[storageName])) {
|
|
4873
|
+
throw new UnexpectedError("Expected (global) ".concat(storageName, " to be an array, but got ").concat(typeof globalScope[storageName]));
|
|
4874
|
+
}
|
|
4875
|
+
this.storage = globalScope[storageName];
|
|
4876
|
+
}
|
|
4877
|
+
$Register.prototype.list = function () {
|
|
4878
|
+
// <- TODO: ReadonlyDeep<ReadonlyArray<TRegistered>>
|
|
4879
|
+
return this.storage;
|
|
4880
|
+
};
|
|
4881
|
+
$Register.prototype.register = function (registered) {
|
|
4882
|
+
var packageName = registered.packageName, className = registered.className;
|
|
4883
|
+
var existingRegistrationIndex = this.storage.findIndex(function (item) { return item.packageName === packageName && item.className === className; });
|
|
4884
|
+
var existingRegistration = this.storage[existingRegistrationIndex];
|
|
4885
|
+
if (!existingRegistration) {
|
|
4886
|
+
this.storage.push(registered);
|
|
4887
|
+
}
|
|
4888
|
+
else {
|
|
4889
|
+
this.storage[existingRegistrationIndex] = registered;
|
|
4890
|
+
}
|
|
4891
|
+
return {
|
|
4892
|
+
registerName: this.registerName,
|
|
4893
|
+
packageName: packageName,
|
|
4894
|
+
className: className,
|
|
4895
|
+
get isDestroyed() {
|
|
4896
|
+
return false;
|
|
4897
|
+
},
|
|
4898
|
+
destroy: function () {
|
|
4899
|
+
throw new NotYetImplementedError("Registration to ".concat(this.registerName, " is permanent in this version of Promptbook"));
|
|
4900
|
+
},
|
|
4901
|
+
};
|
|
4902
|
+
};
|
|
4903
|
+
return $Register;
|
|
4904
|
+
}());
|
|
4905
|
+
|
|
4906
|
+
/**
|
|
4907
|
+
* @@@
|
|
4995
4908
|
*
|
|
4909
|
+
* Note: `$` is used to indicate that this interacts with the global scope
|
|
4910
|
+
* @singleton Only one instance of each register is created per build, but thare can be more @@@
|
|
4996
4911
|
* @public exported from `@promptbook/core`
|
|
4997
4912
|
*/
|
|
4998
|
-
var
|
|
4999
|
-
|
|
5000
|
-
|
|
5001
|
-
|
|
5002
|
-
_this.name = 'KnowledgeScrapeError';
|
|
5003
|
-
Object.setPrototypeOf(_this, KnowledgeScrapeError.prototype);
|
|
5004
|
-
return _this;
|
|
5005
|
-
}
|
|
5006
|
-
return KnowledgeScrapeError;
|
|
5007
|
-
}(Error));
|
|
4913
|
+
var $scrapersMetadataRegister = new $Register('scrapers_metadata');
|
|
4914
|
+
/**
|
|
4915
|
+
* TODO: [®] DRY Register logic
|
|
4916
|
+
*/
|
|
5008
4917
|
|
|
5009
4918
|
/**
|
|
5010
4919
|
* @@@
|
|
@@ -5013,12 +4922,12 @@ var KnowledgeScrapeError = /** @class */ (function (_super) {
|
|
|
5013
4922
|
* @singleton Only one instance of each register is created per build, but thare can be more @@@
|
|
5014
4923
|
* @public exported from `@promptbook/core`
|
|
5015
4924
|
*/
|
|
5016
|
-
var $
|
|
4925
|
+
var $scrapersRegister = new $Register('scraper_constructors');
|
|
5017
4926
|
/**
|
|
5018
4927
|
* TODO: [®] DRY Register logic
|
|
5019
4928
|
*/
|
|
5020
4929
|
|
|
5021
|
-
// TODO: !!!!!! Maybe delete this function
|
|
4930
|
+
// TODO: !!!!!!last - Maybe delete this function
|
|
5022
4931
|
/**
|
|
5023
4932
|
* Creates a message with all registered scrapers
|
|
5024
4933
|
*
|
|
@@ -5185,7 +5094,6 @@ function normalizeToKebabCase(text) {
|
|
|
5185
5094
|
* @private within the repository
|
|
5186
5095
|
*/
|
|
5187
5096
|
function sourceContentToName(sourceContent) {
|
|
5188
|
-
// TODO: !!!!!! Better name for source than gibberish hash
|
|
5189
5097
|
var hash = SHA256(hexEncoder.parse(JSON.stringify(sourceContent)))
|
|
5190
5098
|
// <- TODO: [🥬] Encapsulate sha256 to some private utility function
|
|
5191
5099
|
.toString( /* hex */)
|
|
@@ -5298,7 +5206,7 @@ function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
|
|
|
5298
5206
|
case 0:
|
|
5299
5207
|
sourceContent = knowledgeSource.sourceContent;
|
|
5300
5208
|
name = knowledgeSource.name;
|
|
5301
|
-
_b = options || {}, _c = _b.rootDirname, rootDirname = _c === void 0 ? null : _c, _d = _b.isVerbose, isVerbose = _d === void 0 ?
|
|
5209
|
+
_b = options || {}, _c = _b.rootDirname, rootDirname = _c === void 0 ? null : _c, _d = _b.isVerbose, isVerbose = _d === void 0 ? DEFAULT_IS_VERBOSE : _d;
|
|
5302
5210
|
TODO_USE(isVerbose);
|
|
5303
5211
|
if (!name) {
|
|
5304
5212
|
name = sourceContentToName(sourceContent);
|
|
@@ -5388,7 +5296,7 @@ function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
|
|
|
5388
5296
|
content = _a.sent();
|
|
5389
5297
|
return [2 /*return*/, new Blob([
|
|
5390
5298
|
content,
|
|
5391
|
-
// <- TODO: !!!!!!
|
|
5299
|
+
// <- TODO: !!!!!! Test that this is working
|
|
5392
5300
|
], { type: mimeType_1 })];
|
|
5393
5301
|
}
|
|
5394
5302
|
});
|
|
@@ -5451,7 +5359,7 @@ function prepareKnowledgePieces(knowledgeSources, tools, options) {
|
|
|
5451
5359
|
return __generator(this, function (_c) {
|
|
5452
5360
|
switch (_c.label) {
|
|
5453
5361
|
case 0:
|
|
5454
|
-
_a = options.maxParallelCount, maxParallelCount = _a === void 0 ?
|
|
5362
|
+
_a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a, rootDirname = options.rootDirname, _b = options.isVerbose, isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b;
|
|
5455
5363
|
knowledgePreparedUnflatten = new Array(knowledgeSources.length);
|
|
5456
5364
|
return [4 /*yield*/, forEachAsync(knowledgeSources, { maxParallelCount: maxParallelCount }, function (knowledgeSource, index) { return __awaiter(_this, void 0, void 0, function () {
|
|
5457
5365
|
var partialPieces, sourceHandler, _a, _b, scraper, partialPiecesUnchecked, e_1_1, pieces;
|
|
@@ -5480,7 +5388,8 @@ function prepareKnowledgePieces(knowledgeSources, tools, options) {
|
|
|
5480
5388
|
case 4:
|
|
5481
5389
|
partialPiecesUnchecked = _d.sent();
|
|
5482
5390
|
if (partialPiecesUnchecked !== null) {
|
|
5483
|
-
partialPieces = partialPiecesUnchecked;
|
|
5391
|
+
partialPieces = __spreadArray([], __read(partialPiecesUnchecked), false);
|
|
5392
|
+
// <- TODO: [🪓] Here should be no need for spreading new array, just `partialPieces = partialPiecesUnchecked`
|
|
5484
5393
|
return [3 /*break*/, 6];
|
|
5485
5394
|
}
|
|
5486
5395
|
_d.label = 5;
|
|
@@ -5528,7 +5437,7 @@ TODO: [🧊] This is how it can look in future
|
|
|
5528
5437
|
> /**
|
|
5529
5438
|
> * Unprepared knowledge
|
|
5530
5439
|
> * /
|
|
5531
|
-
> readonly knowledgeSources:
|
|
5440
|
+
> readonly knowledgeSources: ReadonlyArray<KnowledgeSourceJson>;
|
|
5532
5441
|
> };
|
|
5533
5442
|
>
|
|
5534
5443
|
> export async function prepareKnowledgePieces(
|
|
@@ -5586,7 +5495,7 @@ function prepareTemplates(pipeline, tools, options) {
|
|
|
5586
5495
|
return __generator(this, function (_b) {
|
|
5587
5496
|
switch (_b.label) {
|
|
5588
5497
|
case 0:
|
|
5589
|
-
_a = options.maxParallelCount, maxParallelCount = _a === void 0 ?
|
|
5498
|
+
_a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a;
|
|
5590
5499
|
templates = pipeline.templates, parameters = pipeline.parameters, knowledgePiecesCount = pipeline.knowledgePiecesCount;
|
|
5591
5500
|
// TODO: [main] !!!!! Apply samples to each template (if missing and is for the template defined)
|
|
5592
5501
|
TODO_USE(parameters);
|
|
@@ -5648,7 +5557,7 @@ function preparePipeline(pipeline, tools, options) {
|
|
|
5648
5557
|
if (isPipelinePrepared(pipeline)) {
|
|
5649
5558
|
return [2 /*return*/, pipeline];
|
|
5650
5559
|
}
|
|
5651
|
-
rootDirname = options.rootDirname, _a = options.maxParallelCount, maxParallelCount = _a === void 0 ?
|
|
5560
|
+
rootDirname = options.rootDirname, _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a, _b = options.isVerbose, isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b;
|
|
5652
5561
|
parameters = pipeline.parameters, templates = pipeline.templates, knowledgeSources = pipeline.knowledgeSources, personas = pipeline.personas;
|
|
5653
5562
|
if (tools === undefined || tools.llm === undefined) {
|
|
5654
5563
|
throw new MissingToolsError('LLM tools are required for preparing the pipeline');
|
|
@@ -5706,7 +5615,9 @@ function preparePipeline(pipeline, tools, options) {
|
|
|
5706
5615
|
// ----- /Templates preparation -----
|
|
5707
5616
|
// Note: Count total usage
|
|
5708
5617
|
currentPreparation.usage = llmToolsWithUsage.getTotalUsage();
|
|
5709
|
-
return [2 /*return*/, $asDeeplyFrozenSerializableJson('Prepared PipelineJson', __assign(__assign({}, clonePipeline(pipeline)), { templates:
|
|
5618
|
+
return [2 /*return*/, $asDeeplyFrozenSerializableJson('Prepared PipelineJson', __assign(__assign({}, clonePipeline(pipeline)), { templates: __spreadArray([], __read(templatesPrepared), false),
|
|
5619
|
+
// <- TODO: [🪓] Here should be no need for spreading new array, just ` templates: templatesPrepared`
|
|
5620
|
+
knowledgeSources: knowledgeSourcesPrepared, knowledgePieces: knowledgePiecesPrepared, personas: preparedPersonas, preparations: __spreadArray([], __read(preparations), false) }))];
|
|
5710
5621
|
}
|
|
5711
5622
|
});
|
|
5712
5623
|
});
|
|
@@ -8630,7 +8541,7 @@ function $registeredLlmToolsMessage() {
|
|
|
8630
8541
|
*/
|
|
8631
8542
|
function createLlmToolsFromConfiguration(configuration, options) {
|
|
8632
8543
|
if (options === void 0) { options = {}; }
|
|
8633
|
-
var _a = options.isVerbose, isVerbose = _a === void 0 ?
|
|
8544
|
+
var _a = options.isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a, userId = options.userId;
|
|
8634
8545
|
var llmTools = configuration.map(function (llmConfiguration) {
|
|
8635
8546
|
var registeredItem = $llmToolsRegister
|
|
8636
8547
|
.list()
|
|
@@ -8691,6 +8602,115 @@ function $provideLlmToolsFromEnv(options) {
|
|
|
8691
8602
|
* TODO: [®] DRY Register logic
|
|
8692
8603
|
*/
|
|
8693
8604
|
|
|
8605
|
+
/**
|
|
8606
|
+
* @@@
|
|
8607
|
+
*
|
|
8608
|
+
* @public exported from `@promptbook/node`
|
|
8609
|
+
*/
|
|
8610
|
+
function $provideExecutablesForNode(options) {
|
|
8611
|
+
return __awaiter(this, void 0, void 0, function () {
|
|
8612
|
+
var _a, _b, isAutoInstalled, _c, isVerbose;
|
|
8613
|
+
return __generator(this, function (_d) {
|
|
8614
|
+
if (!$isRunningInNode()) {
|
|
8615
|
+
throw new EnvironmentMismatchError('Function `$getScrapersForNode` works only in Node.js environment');
|
|
8616
|
+
}
|
|
8617
|
+
_a = options || {}, _b = _a.isAutoInstalled, isAutoInstalled = _b === void 0 ? DEFAULT_IS_AUTO_INSTALLED : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? DEFAULT_IS_VERBOSE : _c;
|
|
8618
|
+
TODO_USE(isAutoInstalled);
|
|
8619
|
+
TODO_USE(isVerbose);
|
|
8620
|
+
return [2 /*return*/, {
|
|
8621
|
+
// TODO: !!!!!! use `locate-app` library here
|
|
8622
|
+
pandocPath: 'C:/Users/me/AppData/Local/Pandoc/pandoc.exe',
|
|
8623
|
+
libreOfficePath: 'C:/Program Files/LibreOffice/program/swriter.exe',
|
|
8624
|
+
}];
|
|
8625
|
+
});
|
|
8626
|
+
});
|
|
8627
|
+
}
|
|
8628
|
+
/**
|
|
8629
|
+
* TODO: [🧠] THis should be maybe in different folder
|
|
8630
|
+
* Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
|
|
8631
|
+
*/
|
|
8632
|
+
|
|
8633
|
+
/**
|
|
8634
|
+
* @@@
|
|
8635
|
+
*
|
|
8636
|
+
* @public exported from `@promptbook/node`
|
|
8637
|
+
*/
|
|
8638
|
+
function $provideFilesystemForNode(options) {
|
|
8639
|
+
if (!$isRunningInNode()) {
|
|
8640
|
+
throw new EnvironmentMismatchError('Function `$provideFilesystemForNode` works only in Node.js environment');
|
|
8641
|
+
}
|
|
8642
|
+
var _a = (options || {}).isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a;
|
|
8643
|
+
TODO_USE(isVerbose);
|
|
8644
|
+
return {
|
|
8645
|
+
stat: stat,
|
|
8646
|
+
access: access,
|
|
8647
|
+
constants: constants,
|
|
8648
|
+
readFile: readFile,
|
|
8649
|
+
readdir: readdir,
|
|
8650
|
+
};
|
|
8651
|
+
}
|
|
8652
|
+
/**
|
|
8653
|
+
* Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
|
|
8654
|
+
*/
|
|
8655
|
+
|
|
8656
|
+
/**
|
|
8657
|
+
* @@@
|
|
8658
|
+
*
|
|
8659
|
+
* 1) @@@
|
|
8660
|
+
* 2) @@@
|
|
8661
|
+
*
|
|
8662
|
+
* @public exported from `@promptbook/node`
|
|
8663
|
+
*/
|
|
8664
|
+
function $provideScrapersForNode(tools, options) {
|
|
8665
|
+
return __awaiter(this, void 0, void 0, function () {
|
|
8666
|
+
var _a, _b, isAutoInstalled, _c, isVerbose, scrapers, _d, _e, scraperFactory, scraper, e_1_1;
|
|
8667
|
+
var e_1, _f;
|
|
8668
|
+
return __generator(this, function (_g) {
|
|
8669
|
+
switch (_g.label) {
|
|
8670
|
+
case 0:
|
|
8671
|
+
if (!$isRunningInNode()) {
|
|
8672
|
+
throw new EnvironmentMismatchError('Function `$getScrapersForNode` works only in Node.js environment');
|
|
8673
|
+
}
|
|
8674
|
+
_a = options || {}, _b = _a.isAutoInstalled, isAutoInstalled = _b === void 0 ? DEFAULT_IS_AUTO_INSTALLED : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? DEFAULT_IS_VERBOSE : _c;
|
|
8675
|
+
TODO_USE(isAutoInstalled);
|
|
8676
|
+
TODO_USE(isVerbose);
|
|
8677
|
+
scrapers = [];
|
|
8678
|
+
_g.label = 1;
|
|
8679
|
+
case 1:
|
|
8680
|
+
_g.trys.push([1, 6, 7, 8]);
|
|
8681
|
+
_d = __values($scrapersRegister.list()), _e = _d.next();
|
|
8682
|
+
_g.label = 2;
|
|
8683
|
+
case 2:
|
|
8684
|
+
if (!!_e.done) return [3 /*break*/, 5];
|
|
8685
|
+
scraperFactory = _e.value;
|
|
8686
|
+
return [4 /*yield*/, scraperFactory(tools, options || {})];
|
|
8687
|
+
case 3:
|
|
8688
|
+
scraper = _g.sent();
|
|
8689
|
+
scrapers.push(scraper);
|
|
8690
|
+
_g.label = 4;
|
|
8691
|
+
case 4:
|
|
8692
|
+
_e = _d.next();
|
|
8693
|
+
return [3 /*break*/, 2];
|
|
8694
|
+
case 5: return [3 /*break*/, 8];
|
|
8695
|
+
case 6:
|
|
8696
|
+
e_1_1 = _g.sent();
|
|
8697
|
+
e_1 = { error: e_1_1 };
|
|
8698
|
+
return [3 /*break*/, 8];
|
|
8699
|
+
case 7:
|
|
8700
|
+
try {
|
|
8701
|
+
if (_e && !_e.done && (_f = _d.return)) _f.call(_d);
|
|
8702
|
+
}
|
|
8703
|
+
finally { if (e_1) throw e_1.error; }
|
|
8704
|
+
return [7 /*endfinally*/];
|
|
8705
|
+
case 8: return [2 /*return*/, scrapers];
|
|
8706
|
+
}
|
|
8707
|
+
});
|
|
8708
|
+
});
|
|
8709
|
+
}
|
|
8710
|
+
/**
|
|
8711
|
+
* Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
|
|
8712
|
+
*/
|
|
8713
|
+
|
|
8694
8714
|
/**
|
|
8695
8715
|
* Extracts code block from markdown.
|
|
8696
8716
|
*
|
|
@@ -9145,7 +9165,7 @@ var JavascriptExecutionTools = JavascriptEvalExecutionTools;
|
|
|
9145
9165
|
*/
|
|
9146
9166
|
function $provideExecutionToolsForNode(options) {
|
|
9147
9167
|
return __awaiter(this, void 0, void 0, function () {
|
|
9148
|
-
var fs, llm, tools;
|
|
9168
|
+
var fs, llm, executables, tools;
|
|
9149
9169
|
var _a;
|
|
9150
9170
|
return __generator(this, function (_b) {
|
|
9151
9171
|
switch (_b.label) {
|
|
@@ -9155,12 +9175,16 @@ function $provideExecutionToolsForNode(options) {
|
|
|
9155
9175
|
}
|
|
9156
9176
|
fs = $provideFilesystemForNode();
|
|
9157
9177
|
llm = $provideLlmToolsFromEnv(options);
|
|
9178
|
+
return [4 /*yield*/, $provideExecutablesForNode(options)];
|
|
9179
|
+
case 1:
|
|
9180
|
+
executables = _b.sent();
|
|
9158
9181
|
_a = {
|
|
9159
9182
|
llm: llm,
|
|
9160
|
-
fs: fs
|
|
9183
|
+
fs: fs,
|
|
9184
|
+
executables: executables
|
|
9161
9185
|
};
|
|
9162
|
-
return [4 /*yield*/, $provideScrapersForNode({ fs: fs, llm: llm }, options)];
|
|
9163
|
-
case
|
|
9186
|
+
return [4 /*yield*/, $provideScrapersForNode({ fs: fs, llm: llm, executables: executables }, options)];
|
|
9187
|
+
case 2:
|
|
9164
9188
|
tools = (_a.scrapers = _b.sent(),
|
|
9165
9189
|
_a.script = [new JavascriptExecutionTools(options)],
|
|
9166
9190
|
_a);
|
|
@@ -9398,7 +9422,9 @@ function createCollectionFromDirectory(path, tools, options) {
|
|
|
9398
9422
|
throw new EnvironmentMismatchError('Can not create collection without filesystem tools');
|
|
9399
9423
|
// <- TODO: [🧠] What is the best error type here`
|
|
9400
9424
|
}
|
|
9401
|
-
makedLibraryFilePath = join(path, "".concat(
|
|
9425
|
+
makedLibraryFilePath = join(path, "".concat(DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME
|
|
9426
|
+
// <- TODO: [🦒] Allow to override (pass different value into the function)
|
|
9427
|
+
, ".json"));
|
|
9402
9428
|
return [4 /*yield*/, isFileExisting(makedLibraryFilePath, tools.fs)];
|
|
9403
9429
|
case 3:
|
|
9404
9430
|
if (!(_f.sent())) {
|
|
@@ -9409,7 +9435,7 @@ function createCollectionFromDirectory(path, tools, options) {
|
|
|
9409
9435
|
// TODO: !! Implement;
|
|
9410
9436
|
// TODO: [🌗]
|
|
9411
9437
|
}
|
|
9412
|
-
_a = options || {}, _b = _a.isRecursive, isRecursive = _b === void 0 ? true : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ?
|
|
9438
|
+
_a = options || {}, _b = _a.isRecursive, isRecursive = _b === void 0 ? true : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? DEFAULT_IS_VERBOSE : _c, _d = _a.isLazyLoaded, isLazyLoaded = _d === void 0 ? false : _d, _e = _a.isCrashedOnError, isCrashedOnError = _e === void 0 ? true : _e;
|
|
9413
9439
|
collection = createCollectionFromPromise(function () { return __awaiter(_this, void 0, void 0, function () {
|
|
9414
9440
|
var fileNames, collection, _loop_1, fileNames_1, fileNames_1_1, fileName, e_1_1;
|
|
9415
9441
|
var e_1, _a;
|
|
@@ -9881,7 +9907,7 @@ function $currentDate() {
|
|
|
9881
9907
|
function cacheLlmTools(llmTools, options) {
|
|
9882
9908
|
var _this = this;
|
|
9883
9909
|
if (options === void 0) { options = {}; }
|
|
9884
|
-
var _a = options.storage, storage = _a === void 0 ? new MemoryStorage() : _a, _b = options.
|
|
9910
|
+
var _a = options.storage, storage = _a === void 0 ? new MemoryStorage() : _a, _b = options.isCacheReloaded, isCacheReloaded = _b === void 0 ? false : _b;
|
|
9885
9911
|
var proxyTools = __assign(__assign({}, llmTools), {
|
|
9886
9912
|
// <- Note: [🥫]
|
|
9887
9913
|
get title() {
|
|
@@ -9903,7 +9929,7 @@ function cacheLlmTools(llmTools, options) {
|
|
|
9903
9929
|
key = titleToName(prompt.title.substring(0, MAX_FILENAME_LENGTH - 10) +
|
|
9904
9930
|
'-' +
|
|
9905
9931
|
sha256(hexEncoder.parse(JSON.stringify(prompt.parameters))).toString( /* hex */));
|
|
9906
|
-
if (!!
|
|
9932
|
+
if (!!isCacheReloaded) return [3 /*break*/, 2];
|
|
9907
9933
|
return [4 /*yield*/, storage.getItem(key)];
|
|
9908
9934
|
case 1:
|
|
9909
9935
|
_a = _c.sent();
|
|
@@ -9989,12 +10015,14 @@ function $provideLlmToolsForCli(options) {
|
|
|
9989
10015
|
if (!$isRunningInNode()) {
|
|
9990
10016
|
throw new EnvironmentMismatchError('Function `$provideLlmToolsForTestingAndScriptsAndPlayground` works only in Node.js environment');
|
|
9991
10017
|
}
|
|
9992
|
-
var
|
|
10018
|
+
var isCacheReloaded = (options !== null && options !== void 0 ? options : {}).isCacheReloaded;
|
|
9993
10019
|
return cacheLlmTools(countTotalUsage(
|
|
9994
10020
|
// <- Note: for example here we don`t want the [🌯]
|
|
9995
10021
|
$provideLlmToolsFromEnv()), {
|
|
9996
|
-
storage: new FileCacheStorage({ fs: $provideFilesystemForNode() }, {
|
|
9997
|
-
|
|
10022
|
+
storage: new FileCacheStorage({ fs: $provideFilesystemForNode() }, {
|
|
10023
|
+
rootFolderPath: join(process.cwd(), DEFAULT_EXECUTIONS_CACHE_DIRNAME),
|
|
10024
|
+
}),
|
|
10025
|
+
isCacheReloaded: isCacheReloaded,
|
|
9998
10026
|
});
|
|
9999
10027
|
}
|
|
10000
10028
|
/**
|
|
@@ -10021,13 +10049,13 @@ function initializeMakeCommand(program) {
|
|
|
10021
10049
|
makeCommand.option('-f, --format <format>', spaceTrim$1("\n Output format of builded collection \"javascript\", \"typescript\" or \"json\"\n\n Note: You can use multiple formats separated by comma\n "), 'javascript' /* <- Note: [🏳🌈] */);
|
|
10022
10050
|
makeCommand.option('--no-validation', "Do not validate logic of pipelines in collection", true);
|
|
10023
10051
|
makeCommand.option('--validation', "Types of validations separated by comma (options \"logic\",\"imports\")", 'logic,imports');
|
|
10024
|
-
makeCommand.option('--reload
|
|
10052
|
+
makeCommand.option('--reload', "Call LLM models even if same prompt with result is in the cache", false);
|
|
10025
10053
|
makeCommand.option('--verbose', "Is output verbose", false);
|
|
10026
|
-
makeCommand.option('-o, --out-file <path>', spaceTrim$1("\n Where to save the builded collection\n\n Note: If you keep it \"".concat(
|
|
10054
|
+
makeCommand.option('-o, --out-file <path>', spaceTrim$1("\n Where to save the builded collection\n\n Note: If you keep it \"".concat(DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME, "\" it will be saved in the root of the promptbook directory\n If you set it to a path, it will be saved in that path\n BUT you can use only one format and set correct extension\n ")), DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME);
|
|
10027
10055
|
makeCommand.action(function (path, _a) {
|
|
10028
|
-
var projectName = _a.projectName, format = _a.format, validation = _a.validation,
|
|
10056
|
+
var projectName = _a.projectName, format = _a.format, validation = _a.validation, isCacheReloaded = _a.reloadCache, isVerbose = _a.verbose, outFile = _a.outFile;
|
|
10029
10057
|
return __awaiter(_this, void 0, void 0, function () {
|
|
10030
|
-
var formats, validations, options, fs, llm, tools, collection, validations_1, validations_1_1, validation_1, _b, _c, pipelineUrl, pipeline, e_1_1, e_2_1, collectionJson, collectionJsonString, collectionJsonItems, saveFile;
|
|
10058
|
+
var formats, validations, options, fs, llm, executables, tools, collection, validations_1, validations_1_1, validation_1, _b, _c, pipelineUrl, pipeline, e_1_1, e_2_1, collectionJson, collectionJsonString, collectionJsonItems, saveFile;
|
|
10031
10059
|
var _d, e_2, _e, e_1, _f;
|
|
10032
10060
|
var _this = this;
|
|
10033
10061
|
return __generator(this, function (_g) {
|
|
@@ -10041,22 +10069,25 @@ function initializeMakeCommand(program) {
|
|
|
10041
10069
|
.split(',')
|
|
10042
10070
|
.map(function (_) { return _.trim(); })
|
|
10043
10071
|
.filter(function (_) { return _ !== ''; });
|
|
10044
|
-
if (outFile !==
|
|
10072
|
+
if (outFile !== DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME && formats.length !== 1) {
|
|
10045
10073
|
console.error(colors.red("You can only use one format if you specify --out-file"));
|
|
10046
10074
|
process.exit(1);
|
|
10047
10075
|
}
|
|
10048
10076
|
options = {
|
|
10049
10077
|
isVerbose: isVerbose,
|
|
10050
|
-
|
|
10078
|
+
isCacheReloaded: isCacheReloaded,
|
|
10051
10079
|
};
|
|
10052
10080
|
fs = $provideFilesystemForNode(options);
|
|
10053
10081
|
llm = $provideLlmToolsForCli(options);
|
|
10082
|
+
return [4 /*yield*/, $provideExecutablesForNode(options)];
|
|
10083
|
+
case 1:
|
|
10084
|
+
executables = _g.sent();
|
|
10054
10085
|
_d = {
|
|
10055
10086
|
llm: llm,
|
|
10056
10087
|
fs: fs
|
|
10057
10088
|
};
|
|
10058
|
-
return [4 /*yield*/, $provideScrapersForNode({ fs: fs, llm: llm }, options)];
|
|
10059
|
-
case
|
|
10089
|
+
return [4 /*yield*/, $provideScrapersForNode({ fs: fs, llm: llm, executables: executables }, options)];
|
|
10090
|
+
case 2:
|
|
10060
10091
|
tools = (_d.scrapers = _g.sent(),
|
|
10061
10092
|
_d.script = [
|
|
10062
10093
|
/*new JavascriptExecutionTools(options)*/
|
|
@@ -10065,31 +10096,31 @@ function initializeMakeCommand(program) {
|
|
|
10065
10096
|
return [4 /*yield*/, createCollectionFromDirectory(path, tools, {
|
|
10066
10097
|
isVerbose: isVerbose,
|
|
10067
10098
|
isRecursive: true,
|
|
10068
|
-
// <- TODO: [🍖]
|
|
10099
|
+
// <- TODO: [🍖] Add `intermediateFilesStrategy`
|
|
10069
10100
|
})];
|
|
10070
|
-
case 2:
|
|
10071
|
-
collection = _g.sent();
|
|
10072
|
-
_g.label = 3;
|
|
10073
10101
|
case 3:
|
|
10074
|
-
_g.
|
|
10075
|
-
validations_1 = __values(validations), validations_1_1 = validations_1.next();
|
|
10102
|
+
collection = _g.sent();
|
|
10076
10103
|
_g.label = 4;
|
|
10077
10104
|
case 4:
|
|
10078
|
-
|
|
10079
|
-
|
|
10105
|
+
_g.trys.push([4, 16, 17, 18]);
|
|
10106
|
+
validations_1 = __values(validations), validations_1_1 = validations_1.next();
|
|
10080
10107
|
_g.label = 5;
|
|
10081
10108
|
case 5:
|
|
10082
|
-
|
|
10109
|
+
if (!!validations_1_1.done) return [3 /*break*/, 15];
|
|
10110
|
+
validation_1 = validations_1_1.value;
|
|
10111
|
+
_g.label = 6;
|
|
10112
|
+
case 6:
|
|
10113
|
+
_g.trys.push([6, 12, 13, 14]);
|
|
10083
10114
|
e_1 = void 0;
|
|
10084
10115
|
return [4 /*yield*/, collection.listPipelines()];
|
|
10085
|
-
case 6:
|
|
10086
|
-
_b = (__values.apply(void 0, [_g.sent()])), _c = _b.next();
|
|
10087
|
-
_g.label = 7;
|
|
10088
10116
|
case 7:
|
|
10089
|
-
|
|
10117
|
+
_b = (__values.apply(void 0, [_g.sent()])), _c = _b.next();
|
|
10118
|
+
_g.label = 8;
|
|
10119
|
+
case 8:
|
|
10120
|
+
if (!!_c.done) return [3 /*break*/, 11];
|
|
10090
10121
|
pipelineUrl = _c.value;
|
|
10091
10122
|
return [4 /*yield*/, collection.getPipelineByUrl(pipelineUrl)];
|
|
10092
|
-
case
|
|
10123
|
+
case 9:
|
|
10093
10124
|
pipeline = _g.sent();
|
|
10094
10125
|
if (validation_1 === 'logic') {
|
|
10095
10126
|
validatePipeline(pipeline);
|
|
@@ -10097,37 +10128,37 @@ function initializeMakeCommand(program) {
|
|
|
10097
10128
|
console.info(colors.cyan("Validated logic of ".concat(pipeline.pipelineUrl)));
|
|
10098
10129
|
}
|
|
10099
10130
|
}
|
|
10100
|
-
_g.label =
|
|
10101
|
-
case
|
|
10131
|
+
_g.label = 10;
|
|
10132
|
+
case 10:
|
|
10102
10133
|
_c = _b.next();
|
|
10103
|
-
return [3 /*break*/,
|
|
10104
|
-
case
|
|
10105
|
-
case
|
|
10134
|
+
return [3 /*break*/, 8];
|
|
10135
|
+
case 11: return [3 /*break*/, 14];
|
|
10136
|
+
case 12:
|
|
10106
10137
|
e_1_1 = _g.sent();
|
|
10107
10138
|
e_1 = { error: e_1_1 };
|
|
10108
|
-
return [3 /*break*/,
|
|
10109
|
-
case
|
|
10139
|
+
return [3 /*break*/, 14];
|
|
10140
|
+
case 13:
|
|
10110
10141
|
try {
|
|
10111
10142
|
if (_c && !_c.done && (_f = _b.return)) _f.call(_b);
|
|
10112
10143
|
}
|
|
10113
10144
|
finally { if (e_1) throw e_1.error; }
|
|
10114
10145
|
return [7 /*endfinally*/];
|
|
10115
|
-
case
|
|
10146
|
+
case 14:
|
|
10116
10147
|
validations_1_1 = validations_1.next();
|
|
10117
|
-
return [3 /*break*/,
|
|
10118
|
-
case
|
|
10119
|
-
case
|
|
10148
|
+
return [3 /*break*/, 5];
|
|
10149
|
+
case 15: return [3 /*break*/, 18];
|
|
10150
|
+
case 16:
|
|
10120
10151
|
e_2_1 = _g.sent();
|
|
10121
10152
|
e_2 = { error: e_2_1 };
|
|
10122
|
-
return [3 /*break*/,
|
|
10123
|
-
case
|
|
10153
|
+
return [3 /*break*/, 18];
|
|
10154
|
+
case 17:
|
|
10124
10155
|
try {
|
|
10125
10156
|
if (validations_1_1 && !validations_1_1.done && (_e = validations_1.return)) _e.call(validations_1);
|
|
10126
10157
|
}
|
|
10127
10158
|
finally { if (e_2) throw e_2.error; }
|
|
10128
10159
|
return [7 /*endfinally*/];
|
|
10129
|
-
case
|
|
10130
|
-
case
|
|
10160
|
+
case 18: return [4 /*yield*/, collectionToJson(collection)];
|
|
10161
|
+
case 19:
|
|
10131
10162
|
collectionJson = _g.sent();
|
|
10132
10163
|
collectionJsonString = stringifyPipelineJson(collectionJson).trim();
|
|
10133
10164
|
collectionJsonItems = (function () {
|
|
@@ -10146,9 +10177,9 @@ function initializeMakeCommand(program) {
|
|
|
10146
10177
|
return __generator(this, function (_a) {
|
|
10147
10178
|
switch (_a.label) {
|
|
10148
10179
|
case 0:
|
|
10149
|
-
filename = outFile !==
|
|
10180
|
+
filename = outFile !== DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME
|
|
10150
10181
|
? outFile
|
|
10151
|
-
: join(path, "".concat(
|
|
10182
|
+
: join(path, "".concat(DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME, ".").concat(extension));
|
|
10152
10183
|
if (!outFile.endsWith(".".concat(extension))) {
|
|
10153
10184
|
console.warn(colors.yellow("Warning: Extension of output file should be \"".concat(extension, "\"")));
|
|
10154
10185
|
}
|
|
@@ -10164,27 +10195,27 @@ function initializeMakeCommand(program) {
|
|
|
10164
10195
|
}
|
|
10165
10196
|
});
|
|
10166
10197
|
}); };
|
|
10167
|
-
if (!formats.includes('json')) return [3 /*break*/,
|
|
10198
|
+
if (!formats.includes('json')) return [3 /*break*/, 21];
|
|
10168
10199
|
formats = formats.filter(function (format) { return format !== 'json'; });
|
|
10169
10200
|
return [4 /*yield*/, saveFile('json', collectionJsonString)];
|
|
10170
|
-
case 19:
|
|
10171
|
-
_g.sent();
|
|
10172
|
-
_g.label = 20;
|
|
10173
10201
|
case 20:
|
|
10174
|
-
|
|
10202
|
+
_g.sent();
|
|
10203
|
+
_g.label = 21;
|
|
10204
|
+
case 21:
|
|
10205
|
+
if (!(formats.includes('javascript') || formats.includes('js'))) return [3 /*break*/, 23];
|
|
10175
10206
|
formats = formats.filter(function (format) { return format !== 'javascript' && format !== 'js'; });
|
|
10176
10207
|
return [4 /*yield*/, saveFile('js', spaceTrim$1(function (block) { return "\n // ".concat(block(GENERATOR_WARNING_BY_PROMPTBOOK_CLI), "\n\n import { createCollectionFromJson } from '@promptbook/core';\n\n /**\n * Pipeline collection for ").concat(projectName, "\n *\n * ").concat(block(GENERATOR_WARNING_BY_PROMPTBOOK_CLI), "\n *\n * @private internal cache for `getPipelineCollection`\n */\n let pipelineCollection = null;\n\n\n /**\n * Get pipeline collection for ").concat(projectName, "\n *\n * ").concat(block(GENERATOR_WARNING_BY_PROMPTBOOK_CLI), "\n *\n * @returns {PipelineCollection} Library of promptbooks for ").concat(projectName, "\n */\n export function getPipelineCollection(){\n if(pipelineCollection===null){\n pipelineCollection = createCollectionFromJson(\n ").concat(block(collectionJsonItems), "\n );\n }\n\n return pipelineCollection;\n }\n "); }))];
|
|
10177
|
-
case 21:
|
|
10178
|
-
(_g.sent()) + '\n';
|
|
10179
|
-
_g.label = 22;
|
|
10180
10208
|
case 22:
|
|
10181
|
-
|
|
10209
|
+
(_g.sent()) + '\n';
|
|
10210
|
+
_g.label = 23;
|
|
10211
|
+
case 23:
|
|
10212
|
+
if (!(formats.includes('typescript') || formats.includes('ts'))) return [3 /*break*/, 25];
|
|
10182
10213
|
formats = formats.filter(function (format) { return format !== 'typescript' && format !== 'ts'; });
|
|
10183
10214
|
return [4 /*yield*/, saveFile('ts', spaceTrim$1(function (block) { return "\n // ".concat(block(GENERATOR_WARNING_BY_PROMPTBOOK_CLI), "\n\n import { createCollectionFromJson } from '@promptbook/core';\n import type { PipelineCollection } from '@promptbook/types';\n\n /**\n * Pipeline collection for ").concat(projectName, "\n *\n * ").concat(block(GENERATOR_WARNING_BY_PROMPTBOOK_CLI), "\n *\n * @private internal cache for `getPipelineCollection`\n */\n let pipelineCollection: null | PipelineCollection = null;\n\n\n /**\n * Get pipeline collection for ").concat(projectName, "\n *\n * ").concat(block(GENERATOR_WARNING_BY_PROMPTBOOK_CLI), "\n *\n * @returns {PipelineCollection} Library of promptbooks for ").concat(projectName, "\n */\n export function getPipelineCollection(): PipelineCollection{\n if(pipelineCollection===null){\n pipelineCollection = createCollectionFromJson(\n ").concat(block(collectionJsonItems), "\n );\n }\n\n return pipelineCollection;\n }\n "); }) + '\n')];
|
|
10184
|
-
case 23:
|
|
10185
|
-
_g.sent();
|
|
10186
|
-
_g.label = 24;
|
|
10187
10215
|
case 24:
|
|
10216
|
+
_g.sent();
|
|
10217
|
+
_g.label = 25;
|
|
10218
|
+
case 25:
|
|
10188
10219
|
if (formats.length > 0) {
|
|
10189
10220
|
console.warn(colors.yellow("Format ".concat(formats.join(' and '), " is not supported")));
|
|
10190
10221
|
}
|
|
@@ -10446,76 +10477,79 @@ function initializeTestCommand(program) {
|
|
|
10446
10477
|
// <- TODO: [🧟♂️] Unite path to promptbook collection argument
|
|
10447
10478
|
'Pipelines to test as glob pattern');
|
|
10448
10479
|
testCommand.option('-i, --ignore <glob>', "Ignore as glob pattern");
|
|
10449
|
-
testCommand.option('--reload
|
|
10480
|
+
testCommand.option('--reload', "Call LLM models even if same prompt with result is in the cache ", false);
|
|
10450
10481
|
testCommand.option('-v, --verbose', "Is output verbose", false);
|
|
10451
10482
|
testCommand.action(function (filesGlob, _a) {
|
|
10452
|
-
var ignore = _a.ignore,
|
|
10483
|
+
var ignore = _a.ignore, isCacheReloaded = _a.reloadCache, isVerbose = _a.verbose;
|
|
10453
10484
|
return __awaiter(_this, void 0, void 0, function () {
|
|
10454
|
-
var options, fs, llm, tools, filenames, filenames_1, filenames_1_1, filename, pipeline, pipelineMarkdown, _b, _c, error_1, e_1_1;
|
|
10485
|
+
var options, fs, llm, executables, tools, filenames, filenames_1, filenames_1_1, filename, pipeline, pipelineMarkdown, _b, _c, error_1, e_1_1;
|
|
10455
10486
|
var _d, e_1, _e;
|
|
10456
10487
|
return __generator(this, function (_f) {
|
|
10457
10488
|
switch (_f.label) {
|
|
10458
10489
|
case 0:
|
|
10459
10490
|
options = {
|
|
10460
10491
|
isVerbose: isVerbose,
|
|
10461
|
-
|
|
10492
|
+
isCacheReloaded: isCacheReloaded,
|
|
10462
10493
|
};
|
|
10463
10494
|
fs = $provideFilesystemForNode(options);
|
|
10464
10495
|
llm = $provideLlmToolsForCli(options);
|
|
10496
|
+
return [4 /*yield*/, $provideExecutablesForNode(options)];
|
|
10497
|
+
case 1:
|
|
10498
|
+
executables = _f.sent();
|
|
10465
10499
|
_d = {
|
|
10466
10500
|
llm: llm,
|
|
10467
10501
|
fs: fs
|
|
10468
10502
|
};
|
|
10469
|
-
return [4 /*yield*/, $provideScrapersForNode({ fs: fs, llm: llm }, options)];
|
|
10470
|
-
case
|
|
10503
|
+
return [4 /*yield*/, $provideScrapersForNode({ fs: fs, llm: llm, executables: executables }, options)];
|
|
10504
|
+
case 2:
|
|
10471
10505
|
tools = (_d.scrapers = _f.sent(),
|
|
10472
10506
|
_d.script = [
|
|
10473
10507
|
/*new JavascriptExecutionTools(options)*/
|
|
10474
10508
|
],
|
|
10475
10509
|
_d);
|
|
10476
10510
|
return [4 /*yield*/, glob(filesGlob, { ignore: ignore })];
|
|
10477
|
-
case 2:
|
|
10478
|
-
filenames = _f.sent();
|
|
10479
|
-
_f.label = 3;
|
|
10480
10511
|
case 3:
|
|
10481
|
-
_f.
|
|
10482
|
-
filenames_1 = __values(filenames), filenames_1_1 = filenames_1.next();
|
|
10512
|
+
filenames = _f.sent();
|
|
10483
10513
|
_f.label = 4;
|
|
10484
10514
|
case 4:
|
|
10485
|
-
|
|
10486
|
-
|
|
10515
|
+
_f.trys.push([4, 16, 17, 18]);
|
|
10516
|
+
filenames_1 = __values(filenames), filenames_1_1 = filenames_1.next();
|
|
10487
10517
|
_f.label = 5;
|
|
10488
10518
|
case 5:
|
|
10489
|
-
|
|
10519
|
+
if (!!filenames_1_1.done) return [3 /*break*/, 15];
|
|
10520
|
+
filename = filenames_1_1.value;
|
|
10521
|
+
_f.label = 6;
|
|
10522
|
+
case 6:
|
|
10523
|
+
_f.trys.push([6, 13, , 14]);
|
|
10490
10524
|
pipeline = void 0;
|
|
10491
|
-
if (!filename.endsWith('.ptbk.md')) return [3 /*break*/,
|
|
10525
|
+
if (!filename.endsWith('.ptbk.md')) return [3 /*break*/, 9];
|
|
10492
10526
|
return [4 /*yield*/, readFile(filename, 'utf-8')];
|
|
10493
|
-
case
|
|
10527
|
+
case 7:
|
|
10494
10528
|
pipelineMarkdown = (_f.sent());
|
|
10495
10529
|
return [4 /*yield*/, pipelineStringToJson(pipelineMarkdown, tools)];
|
|
10496
|
-
case
|
|
10530
|
+
case 8:
|
|
10497
10531
|
pipeline = _f.sent();
|
|
10498
10532
|
if (isVerbose) {
|
|
10499
10533
|
console.info(colors.green("Parsed ".concat(filename)));
|
|
10500
10534
|
}
|
|
10501
|
-
_f.label =
|
|
10502
|
-
case
|
|
10503
|
-
if (!filename.endsWith('.ptbk.json')) return [3 /*break*/,
|
|
10535
|
+
_f.label = 9;
|
|
10536
|
+
case 9:
|
|
10537
|
+
if (!filename.endsWith('.ptbk.json')) return [3 /*break*/, 11];
|
|
10504
10538
|
_c = (_b = JSON).parse;
|
|
10505
10539
|
return [4 /*yield*/, readFile(filename, 'utf-8')];
|
|
10506
|
-
case 9:
|
|
10507
|
-
pipeline = _c.apply(_b, [_f.sent()]);
|
|
10508
|
-
return [3 /*break*/, 11];
|
|
10509
10540
|
case 10:
|
|
10541
|
+
pipeline = _c.apply(_b, [_f.sent()]);
|
|
10542
|
+
return [3 /*break*/, 12];
|
|
10543
|
+
case 11:
|
|
10510
10544
|
if (isVerbose) {
|
|
10511
10545
|
console.info(colors.gray("Skipping ".concat(filename)));
|
|
10512
10546
|
}
|
|
10513
|
-
return [3 /*break*/,
|
|
10514
|
-
case
|
|
10547
|
+
return [3 /*break*/, 14];
|
|
10548
|
+
case 12:
|
|
10515
10549
|
validatePipeline(pipeline);
|
|
10516
10550
|
console.info(colors.green("Validated ".concat(filename)));
|
|
10517
|
-
return [3 /*break*/,
|
|
10518
|
-
case
|
|
10551
|
+
return [3 /*break*/, 14];
|
|
10552
|
+
case 13:
|
|
10519
10553
|
error_1 = _f.sent();
|
|
10520
10554
|
if (!(error_1 instanceof Error)) {
|
|
10521
10555
|
throw error_1;
|
|
@@ -10524,22 +10558,22 @@ function initializeTestCommand(program) {
|
|
|
10524
10558
|
console.error(colors.bgRed(error_1.name /* <- 11:11 */));
|
|
10525
10559
|
console.error(colors.red(error_1.stack || error_1.message));
|
|
10526
10560
|
process.exit(1);
|
|
10527
|
-
return [3 /*break*/,
|
|
10528
|
-
case
|
|
10561
|
+
return [3 /*break*/, 14];
|
|
10562
|
+
case 14:
|
|
10529
10563
|
filenames_1_1 = filenames_1.next();
|
|
10530
|
-
return [3 /*break*/,
|
|
10531
|
-
case
|
|
10532
|
-
case
|
|
10564
|
+
return [3 /*break*/, 5];
|
|
10565
|
+
case 15: return [3 /*break*/, 18];
|
|
10566
|
+
case 16:
|
|
10533
10567
|
e_1_1 = _f.sent();
|
|
10534
10568
|
e_1 = { error: e_1_1 };
|
|
10535
|
-
return [3 /*break*/,
|
|
10536
|
-
case
|
|
10569
|
+
return [3 /*break*/, 18];
|
|
10570
|
+
case 17:
|
|
10537
10571
|
try {
|
|
10538
10572
|
if (filenames_1_1 && !filenames_1_1.done && (_e = filenames_1.return)) _e.call(filenames_1);
|
|
10539
10573
|
}
|
|
10540
10574
|
finally { if (e_1) throw e_1.error; }
|
|
10541
10575
|
return [7 /*endfinally*/];
|
|
10542
|
-
case
|
|
10576
|
+
case 18:
|
|
10543
10577
|
console.info(colors.green("All pipelines are valid"));
|
|
10544
10578
|
process.exit(0);
|
|
10545
10579
|
return [2 /*return*/];
|
|
@@ -13010,12 +13044,12 @@ function $execCommand(options) {
|
|
|
13010
13044
|
*/
|
|
13011
13045
|
function getScraperIntermediateSource(source, options) {
|
|
13012
13046
|
return __awaiter(this, void 0, void 0, function () {
|
|
13013
|
-
var sourceFilename, url, rootDirname, cacheDirname,
|
|
13047
|
+
var sourceFilename, url, rootDirname, cacheDirname, intermediateFilesStrategy, extension, isVerbose, hash, semanticName, pieces, name, cacheFilename, isDestroyed, fileHandler;
|
|
13014
13048
|
return __generator(this, function (_a) {
|
|
13015
13049
|
switch (_a.label) {
|
|
13016
13050
|
case 0:
|
|
13017
13051
|
sourceFilename = source.filename, url = source.url;
|
|
13018
|
-
rootDirname = options.rootDirname, cacheDirname = options.cacheDirname,
|
|
13052
|
+
rootDirname = options.rootDirname, cacheDirname = options.cacheDirname, intermediateFilesStrategy = options.intermediateFilesStrategy, extension = options.extension, isVerbose = options.isVerbose;
|
|
13019
13053
|
hash = SHA256(
|
|
13020
13054
|
// <- TODO: [🥬] Encapsulate sha256 to some private utility function
|
|
13021
13055
|
hexEncoder.parse(sourceFilename || url || 'untitled'))
|
|
@@ -13025,7 +13059,7 @@ function getScraperIntermediateSource(source, options) {
|
|
|
13025
13059
|
pieces = ['intermediate', semanticName, hash].filter(function (piece) { return piece !== ''; });
|
|
13026
13060
|
name = pieces.join('-').split('--').join('-');
|
|
13027
13061
|
// <- TODO: Use MAX_FILENAME_LENGTH
|
|
13028
|
-
TODO_USE(rootDirname); // <- TODO:
|
|
13062
|
+
TODO_USE(rootDirname); // <- TODO: [😡]
|
|
13029
13063
|
cacheFilename = join.apply(void 0, __spreadArray(__spreadArray([process.cwd(),
|
|
13030
13064
|
cacheDirname], __read(nameToSubfolderPath(hash /* <- TODO: [🎎] Maybe add some SHA256 prefix */)), false), [name], false)).split('\\')
|
|
13031
13065
|
.join('/') +
|
|
@@ -13045,7 +13079,7 @@ function getScraperIntermediateSource(source, options) {
|
|
|
13045
13079
|
return __generator(this, function (_a) {
|
|
13046
13080
|
switch (_a.label) {
|
|
13047
13081
|
case 0:
|
|
13048
|
-
if (!
|
|
13082
|
+
if (!(intermediateFilesStrategy === 'HIDE_AND_CLEAN')) return [3 /*break*/, 2];
|
|
13049
13083
|
if (isVerbose) {
|
|
13050
13084
|
console.info('legacyDocumentScraper: Clening cache');
|
|
13051
13085
|
}
|
|
@@ -13069,7 +13103,7 @@ function getScraperIntermediateSource(source, options) {
|
|
|
13069
13103
|
/**
|
|
13070
13104
|
* Note: Not using `FileCacheStorage` for two reasons:
|
|
13071
13105
|
* 1) Need to store more than serialized JSONs
|
|
13072
|
-
* 2) Need to switch between a `rootDirname` and `cacheDirname` <- TODO:
|
|
13106
|
+
* 2) Need to switch between a `rootDirname` and `cacheDirname` <- TODO: [😡]
|
|
13073
13107
|
* TODO: [🐱🐉][🧠] Make some smart crop
|
|
13074
13108
|
* Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
|
|
13075
13109
|
*/
|
|
@@ -13086,7 +13120,7 @@ var markdownScraperMetadata = $deepFreeze({
|
|
|
13086
13120
|
mimeTypes: ['text/markdown', 'text/plain'],
|
|
13087
13121
|
documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
|
|
13088
13122
|
isAvilableInBrowser: true,
|
|
13089
|
-
requiredExecutables: [
|
|
13123
|
+
requiredExecutables: [],
|
|
13090
13124
|
}); /* <- TODO: [🤛] */
|
|
13091
13125
|
/**
|
|
13092
13126
|
* Registration of known scraper metadata
|
|
@@ -13130,7 +13164,7 @@ var MarkdownScraper = /** @class */ (function () {
|
|
|
13130
13164
|
return __generator(this, function (_k) {
|
|
13131
13165
|
switch (_k.label) {
|
|
13132
13166
|
case 0:
|
|
13133
|
-
_a = this.options, _b = _a.maxParallelCount, maxParallelCount = _b === void 0 ?
|
|
13167
|
+
_a = this.options, _b = _a.maxParallelCount, maxParallelCount = _b === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? DEFAULT_IS_VERBOSE : _c;
|
|
13134
13168
|
llm = this.tools.llm;
|
|
13135
13169
|
if (llm === undefined) {
|
|
13136
13170
|
throw new MissingToolsError('LLM tools are required for scraping external files');
|
|
@@ -13229,7 +13263,8 @@ var MarkdownScraper = /** @class */ (function () {
|
|
|
13229
13263
|
embeddingResult = _c.sent();
|
|
13230
13264
|
index.push({
|
|
13231
13265
|
modelName: embeddingResult.modelName,
|
|
13232
|
-
position: embeddingResult.content,
|
|
13266
|
+
position: __spreadArray([], __read(embeddingResult.content), false),
|
|
13267
|
+
// <- TODO: [🪓] Here should be no need for spreading new array, just `position: embeddingResult.content`
|
|
13233
13268
|
});
|
|
13234
13269
|
_c.label = 6;
|
|
13235
13270
|
case 6: return [3 /*break*/, 8];
|
|
@@ -13280,7 +13315,7 @@ var documentScraperMetadata = $deepFreeze({
|
|
|
13280
13315
|
mimeTypes: ['application/vnd.openxmlformats-officedocument.wordprocessingml.document'],
|
|
13281
13316
|
documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
|
|
13282
13317
|
isAvilableInBrowser: false,
|
|
13283
|
-
requiredExecutables: ['
|
|
13318
|
+
requiredExecutables: ['Pandoc'],
|
|
13284
13319
|
}); /* <- TODO: [🤛] */
|
|
13285
13320
|
/**
|
|
13286
13321
|
* Registration of known scraper metadata
|
|
@@ -13320,12 +13355,13 @@ var DocumentScraper = /** @class */ (function () {
|
|
|
13320
13355
|
* Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object
|
|
13321
13356
|
*/
|
|
13322
13357
|
DocumentScraper.prototype.$convert = function (source) {
|
|
13358
|
+
var _a;
|
|
13323
13359
|
return __awaiter(this, void 0, void 0, function () {
|
|
13324
|
-
var
|
|
13360
|
+
var _b, _c, rootDirname, _d, cacheDirname, _e, intermediateFilesStrategy, _f, isVerbose, extension, cacheFilehandler, command_1;
|
|
13325
13361
|
return __generator(this, function (_g) {
|
|
13326
13362
|
switch (_g.label) {
|
|
13327
13363
|
case 0:
|
|
13328
|
-
|
|
13364
|
+
_b = this.options, _c = _b.rootDirname, rootDirname = _c === void 0 ? process.cwd() : _c, _d = _b.cacheDirname, cacheDirname = _d === void 0 ? DEFAULT_SCRAPE_CACHE_DIRNAME : _d, _e = _b.intermediateFilesStrategy, intermediateFilesStrategy = _e === void 0 ? DEFAULT_INTERMEDIATE_FILES_STRATEGY : _e, _f = _b.isVerbose, isVerbose = _f === void 0 ? DEFAULT_IS_VERBOSE : _f;
|
|
13329
13365
|
if (!$isRunningInNode()) {
|
|
13330
13366
|
throw new KnowledgeScrapeError('Scraping .docx files is only supported in Node environment');
|
|
13331
13367
|
}
|
|
@@ -13333,7 +13369,7 @@ var DocumentScraper = /** @class */ (function () {
|
|
|
13333
13369
|
throw new EnvironmentMismatchError('Can not scrape documents without filesystem tools');
|
|
13334
13370
|
// <- TODO: [🧠] What is the best error type here`
|
|
13335
13371
|
}
|
|
13336
|
-
if (
|
|
13372
|
+
if (((_a = this.tools.executables) === null || _a === void 0 ? void 0 : _a.pandocPath) === undefined) {
|
|
13337
13373
|
throw new MissingToolsError('Pandoc is required for scraping .docx files');
|
|
13338
13374
|
}
|
|
13339
13375
|
if (source.filename === null) {
|
|
@@ -13344,7 +13380,7 @@ var DocumentScraper = /** @class */ (function () {
|
|
|
13344
13380
|
return [4 /*yield*/, getScraperIntermediateSource(source, {
|
|
13345
13381
|
rootDirname: rootDirname,
|
|
13346
13382
|
cacheDirname: cacheDirname,
|
|
13347
|
-
|
|
13383
|
+
intermediateFilesStrategy: intermediateFilesStrategy,
|
|
13348
13384
|
extension: 'md',
|
|
13349
13385
|
isVerbose: isVerbose,
|
|
13350
13386
|
})];
|
|
@@ -13353,11 +13389,9 @@ var DocumentScraper = /** @class */ (function () {
|
|
|
13353
13389
|
return [4 /*yield*/, isFileExisting(cacheFilehandler.filename, this.tools.fs)];
|
|
13354
13390
|
case 2:
|
|
13355
13391
|
if (!!(_g.sent())) return [3 /*break*/, 5];
|
|
13356
|
-
command_1 = "\"".concat(
|
|
13357
|
-
// TODO: !!!!!! [🕊] Make execCommand standard (?node-)util of the promptbook
|
|
13392
|
+
command_1 = "\"".concat(this.tools.executables.pandocPath, "\" -f ").concat(extension, " -t markdown \"").concat(source.filename, "\" -o \"").concat(cacheFilehandler.filename, "\"");
|
|
13358
13393
|
return [4 /*yield*/, $execCommand(command_1)];
|
|
13359
13394
|
case 3:
|
|
13360
|
-
// TODO: !!!!!! [🕊] Make execCommand standard (?node-)util of the promptbook
|
|
13361
13395
|
_g.sent();
|
|
13362
13396
|
return [4 /*yield*/, isFileExisting(cacheFilehandler.filename, this.tools.fs)];
|
|
13363
13397
|
case 4:
|
|
@@ -13436,7 +13470,11 @@ var legacyDocumentScraperMetadata = $deepFreeze({
|
|
|
13436
13470
|
mimeTypes: ['application/msword', 'text/rtf'],
|
|
13437
13471
|
documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
|
|
13438
13472
|
isAvilableInBrowser: false,
|
|
13439
|
-
requiredExecutables: [
|
|
13473
|
+
requiredExecutables: [
|
|
13474
|
+
'Pandoc',
|
|
13475
|
+
'LibreOffice',
|
|
13476
|
+
// <- TODO: [🧠] Should be 'LibreOffice' here, its dependency of dependency
|
|
13477
|
+
],
|
|
13440
13478
|
}); /* <- TODO: [🤛] */
|
|
13441
13479
|
/**
|
|
13442
13480
|
* Registration of known scraper metadata
|
|
@@ -13449,7 +13487,7 @@ var legacyDocumentScraperMetadata = $deepFreeze({
|
|
|
13449
13487
|
var _LegacyDocumentScraperMetadataRegistration = $scrapersMetadataRegister.register(legacyDocumentScraperMetadata);
|
|
13450
13488
|
|
|
13451
13489
|
/**
|
|
13452
|
-
* Scraper for .
|
|
13490
|
+
* Scraper for old document files (like .doc and .rtf)
|
|
13453
13491
|
*
|
|
13454
13492
|
* @see `documentationUrl` for more details
|
|
13455
13493
|
* @public exported from `@promptbook/legacy-documents`
|
|
@@ -13476,12 +13514,13 @@ var LegacyDocumentScraper = /** @class */ (function () {
|
|
|
13476
13514
|
* Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object
|
|
13477
13515
|
*/
|
|
13478
13516
|
LegacyDocumentScraper.prototype.$convert = function (source) {
|
|
13517
|
+
var _a;
|
|
13479
13518
|
return __awaiter(this, void 0, void 0, function () {
|
|
13480
|
-
var
|
|
13519
|
+
var _b, _c, rootDirname, _d, cacheDirname, _e, intermediateFilesStrategy, _f, isVerbose, extension, cacheFilehandler, documentSourceOutdirPathForLibreOffice_1, command_1, files_1, file;
|
|
13481
13520
|
return __generator(this, function (_g) {
|
|
13482
13521
|
switch (_g.label) {
|
|
13483
13522
|
case 0:
|
|
13484
|
-
|
|
13523
|
+
_b = this.options, _c = _b.rootDirname, rootDirname = _c === void 0 ? process.cwd() : _c, _d = _b.cacheDirname, cacheDirname = _d === void 0 ? DEFAULT_SCRAPE_CACHE_DIRNAME : _d, _e = _b.intermediateFilesStrategy, intermediateFilesStrategy = _e === void 0 ? DEFAULT_INTERMEDIATE_FILES_STRATEGY : _e, _f = _b.isVerbose, isVerbose = _f === void 0 ? DEFAULT_IS_VERBOSE : _f;
|
|
13485
13524
|
if (!$isRunningInNode()) {
|
|
13486
13525
|
throw new KnowledgeScrapeError('Scraping .doc files is only supported in Node environment');
|
|
13487
13526
|
}
|
|
@@ -13489,7 +13528,7 @@ var LegacyDocumentScraper = /** @class */ (function () {
|
|
|
13489
13528
|
throw new EnvironmentMismatchError('Can not scrape (legacy) documents without filesystem tools');
|
|
13490
13529
|
// <- TODO: [🧠] What is the best error type here`
|
|
13491
13530
|
}
|
|
13492
|
-
if (
|
|
13531
|
+
if (((_a = this.tools.executables) === null || _a === void 0 ? void 0 : _a.libreOfficePath) === undefined) {
|
|
13493
13532
|
throw new MissingToolsError('LibreOffice is required for scraping .doc and .rtf files');
|
|
13494
13533
|
}
|
|
13495
13534
|
if (source.filename === null) {
|
|
@@ -13500,7 +13539,7 @@ var LegacyDocumentScraper = /** @class */ (function () {
|
|
|
13500
13539
|
return [4 /*yield*/, getScraperIntermediateSource(source, {
|
|
13501
13540
|
rootDirname: rootDirname,
|
|
13502
13541
|
cacheDirname: cacheDirname,
|
|
13503
|
-
|
|
13542
|
+
intermediateFilesStrategy: intermediateFilesStrategy,
|
|
13504
13543
|
extension: 'docx',
|
|
13505
13544
|
isVerbose: isVerbose,
|
|
13506
13545
|
})];
|
|
@@ -13515,11 +13554,9 @@ var LegacyDocumentScraper = /** @class */ (function () {
|
|
|
13515
13554
|
documentSourceOutdirPathForLibreOffice_1 = join(dirname(cacheFilehandler.filename), 'libreoffice')
|
|
13516
13555
|
.split('\\')
|
|
13517
13556
|
.join('/');
|
|
13518
|
-
command_1 = "\"".concat(
|
|
13519
|
-
// TODO: !!!!!! [🕊] Make execCommand standard (?node-)util of the promptbook - this should trigger build polution error
|
|
13557
|
+
command_1 = "\"".concat(this.tools.executables.libreOfficePath, "\" --headless --convert-to docx \"").concat(source.filename, "\" --outdir \"").concat(documentSourceOutdirPathForLibreOffice_1, "\"");
|
|
13520
13558
|
return [4 /*yield*/, $execCommand(command_1)];
|
|
13521
13559
|
case 3:
|
|
13522
|
-
// TODO: !!!!!! [🕊] Make execCommand standard (?node-)util of the promptbook - this should trigger build polution error
|
|
13523
13560
|
_g.sent();
|
|
13524
13561
|
return [4 /*yield*/, readdir(documentSourceOutdirPathForLibreOffice_1)];
|
|
13525
13562
|
case 4:
|
|
@@ -13676,7 +13713,7 @@ var pdfScraperMetadata = $deepFreeze({
|
|
|
13676
13713
|
mimeTypes: ['application/pdf'],
|
|
13677
13714
|
documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
|
|
13678
13715
|
isAvilableInBrowser: true,
|
|
13679
|
-
requiredExecutables: [
|
|
13716
|
+
requiredExecutables: [],
|
|
13680
13717
|
}); /* <- TODO: [🤛] */
|
|
13681
13718
|
/**
|
|
13682
13719
|
* Registration of known scraper metadata
|
|
@@ -13689,7 +13726,7 @@ var pdfScraperMetadata = $deepFreeze({
|
|
|
13689
13726
|
var _PdfScraperMetadataRegistration = $scrapersMetadataRegister.register(pdfScraperMetadata);
|
|
13690
13727
|
|
|
13691
13728
|
/**
|
|
13692
|
-
* Scraper for .
|
|
13729
|
+
* Scraper for .pdf files
|
|
13693
13730
|
*
|
|
13694
13731
|
* @see `documentationUrl` for more details
|
|
13695
13732
|
* @public exported from `@promptbook/pdf`
|
|
@@ -13732,9 +13769,8 @@ var PdfScraper = /** @class */ (function () {
|
|
|
13732
13769
|
TODO_USE(this.options);
|
|
13733
13770
|
/*
|
|
13734
13771
|
const {
|
|
13735
|
-
externalProgramsPaths = {},
|
|
13736
13772
|
cacheDirname = SCRAPE_CACHE_DIRNAME,
|
|
13737
|
-
|
|
13773
|
+
intermediateFilesStrategy = DEFAULT_INTERMEDIATE_FILES_STRATEGY,
|
|
13738
13774
|
isVerbose = IS_VERBOSE,
|
|
13739
13775
|
} = options;
|
|
13740
13776
|
*/
|
|
@@ -13787,7 +13823,7 @@ var websiteScraperMetadata = $deepFreeze({
|
|
|
13787
13823
|
mimeTypes: ['text/html'],
|
|
13788
13824
|
documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
|
|
13789
13825
|
isAvilableInBrowser: false,
|
|
13790
|
-
requiredExecutables: [
|
|
13826
|
+
requiredExecutables: [],
|
|
13791
13827
|
}); /* <- TODO: [🤛] */
|
|
13792
13828
|
/**
|
|
13793
13829
|
* Registration of known scraper metadata
|
|
@@ -13800,32 +13836,29 @@ var websiteScraperMetadata = $deepFreeze({
|
|
|
13800
13836
|
var _WebsiteScraperMetadataRegistration = $scrapersMetadataRegister.register(websiteScraperMetadata);
|
|
13801
13837
|
|
|
13802
13838
|
/**
|
|
13803
|
-
*
|
|
13839
|
+
* Create a new showdown converter instance
|
|
13804
13840
|
*
|
|
13805
|
-
* @
|
|
13806
|
-
* @private for markdown and html knowledge scrapers
|
|
13807
|
-
*/
|
|
13808
|
-
var markdownConverter = new Converter({
|
|
13809
|
-
flavor: 'github', // <- TODO: !!!!!! Explicitly specify the flavor of promptbook markdown
|
|
13810
|
-
/*
|
|
13811
|
-
> import showdownHighlight from 'showdown-highlight';
|
|
13812
|
-
> extensions: [
|
|
13813
|
-
> showdownHighlight({
|
|
13814
|
-
> // Whether to add the classes to the <pre> tag, default is false
|
|
13815
|
-
> pre: true,
|
|
13816
|
-
> // Whether to use hljs' auto language detection, default is true
|
|
13817
|
-
> auto_detection: true,
|
|
13818
|
-
> }),
|
|
13819
|
-
> ],
|
|
13820
|
-
*/
|
|
13821
|
-
});
|
|
13822
|
-
/**
|
|
13823
|
-
* TODO: !!!!!! Figure out better name not to confuse with `Converter`
|
|
13824
|
-
* TODO: !!!!!! Lazy-make converter
|
|
13841
|
+
* @private utility of `WebsiteScraper`
|
|
13825
13842
|
*/
|
|
13843
|
+
function createShowdownConverter() {
|
|
13844
|
+
return new Converter({
|
|
13845
|
+
flavor: 'github',
|
|
13846
|
+
/*
|
|
13847
|
+
> import showdownHighlight from 'showdown-highlight';
|
|
13848
|
+
> extensions: [
|
|
13849
|
+
> showdownHighlight({
|
|
13850
|
+
> // Whether to add the classes to the <pre> tag, default is false
|
|
13851
|
+
> pre: true,
|
|
13852
|
+
> // Whether to use hljs' auto language detection, default is true
|
|
13853
|
+
> auto_detection: true,
|
|
13854
|
+
> }),
|
|
13855
|
+
> ],
|
|
13856
|
+
*/
|
|
13857
|
+
});
|
|
13858
|
+
}
|
|
13826
13859
|
|
|
13827
13860
|
/**
|
|
13828
|
-
* Scraper for
|
|
13861
|
+
* Scraper for websites
|
|
13829
13862
|
*
|
|
13830
13863
|
* @see `documentationUrl` for more details
|
|
13831
13864
|
* @public exported from `@promptbook/website-crawler`
|
|
@@ -13835,6 +13868,7 @@ var WebsiteScraper = /** @class */ (function () {
|
|
|
13835
13868
|
this.tools = tools;
|
|
13836
13869
|
this.options = options;
|
|
13837
13870
|
this.markdownScraper = new MarkdownScraper(tools, options);
|
|
13871
|
+
this.showdownConverter = createShowdownConverter();
|
|
13838
13872
|
}
|
|
13839
13873
|
Object.defineProperty(WebsiteScraper.prototype, "metadata", {
|
|
13840
13874
|
/**
|
|
@@ -13855,12 +13889,11 @@ var WebsiteScraper = /** @class */ (function () {
|
|
|
13855
13889
|
return __awaiter(this, void 0, void 0, function () {
|
|
13856
13890
|
var _a, _b,
|
|
13857
13891
|
// TODO: [🧠] Maybe in node use headless browser not just JSDOM
|
|
13858
|
-
|
|
13859
|
-
rootDirname, _c, cacheDirname, _d, isCacheCleaned, _e, isVerbose, jsdom, _f, reader, article, html, i, cacheFilehandler, markdown;
|
|
13892
|
+
rootDirname, _c, cacheDirname, _d, intermediateFilesStrategy, _e, isVerbose, jsdom, _f, reader, article, html, i, cacheFilehandler, markdown;
|
|
13860
13893
|
return __generator(this, function (_g) {
|
|
13861
13894
|
switch (_g.label) {
|
|
13862
13895
|
case 0:
|
|
13863
|
-
_a = this.options, _b = _a.rootDirname, rootDirname = _b === void 0 ? process.cwd() : _b, _c = _a.cacheDirname, cacheDirname = _c === void 0 ?
|
|
13896
|
+
_a = this.options, _b = _a.rootDirname, rootDirname = _b === void 0 ? process.cwd() : _b, _c = _a.cacheDirname, cacheDirname = _c === void 0 ? DEFAULT_SCRAPE_CACHE_DIRNAME : _c, _d = _a.intermediateFilesStrategy, intermediateFilesStrategy = _d === void 0 ? DEFAULT_INTERMEDIATE_FILES_STRATEGY : _d, _e = _a.isVerbose, isVerbose = _e === void 0 ? DEFAULT_IS_VERBOSE : _e;
|
|
13864
13897
|
// TODO: !!!!!! Does this work in browser? Make it work.
|
|
13865
13898
|
if (source.url === null) {
|
|
13866
13899
|
throw new KnowledgeScrapeError('Website scraper requires URL');
|
|
@@ -13873,10 +13906,6 @@ var WebsiteScraper = /** @class */ (function () {
|
|
|
13873
13906
|
}]))();
|
|
13874
13907
|
reader = new Readability(jsdom.window.document);
|
|
13875
13908
|
article = reader.parse();
|
|
13876
|
-
console.log(article);
|
|
13877
|
-
return [4 /*yield*/, forTime(10000)];
|
|
13878
|
-
case 2:
|
|
13879
|
-
_g.sent();
|
|
13880
13909
|
html = (article === null || article === void 0 ? void 0 : article.content) || (article === null || article === void 0 ? void 0 : article.textContent) || jsdom.window.document.body.innerHTML;
|
|
13881
13910
|
// Note: Unwrap html such as it is convertable by `markdownConverter`
|
|
13882
13911
|
for (i = 0; i < 2; i++) {
|
|
@@ -13888,16 +13917,16 @@ var WebsiteScraper = /** @class */ (function () {
|
|
|
13888
13917
|
return [4 /*yield*/, getScraperIntermediateSource(source, {
|
|
13889
13918
|
rootDirname: rootDirname,
|
|
13890
13919
|
cacheDirname: cacheDirname,
|
|
13891
|
-
|
|
13920
|
+
intermediateFilesStrategy: intermediateFilesStrategy,
|
|
13892
13921
|
extension: 'html',
|
|
13893
13922
|
isVerbose: isVerbose,
|
|
13894
13923
|
})];
|
|
13895
|
-
case
|
|
13924
|
+
case 2:
|
|
13896
13925
|
cacheFilehandler = _g.sent();
|
|
13897
13926
|
return [4 /*yield*/, writeFile(cacheFilehandler.filename, html, 'utf-8')];
|
|
13898
|
-
case
|
|
13927
|
+
case 3:
|
|
13899
13928
|
_g.sent();
|
|
13900
|
-
markdown =
|
|
13929
|
+
markdown = this.showdownConverter.makeMarkdown(html, jsdom.window.document);
|
|
13901
13930
|
return [2 /*return*/, __assign(__assign({}, cacheFilehandler), { markdown: markdown })];
|
|
13902
13931
|
}
|
|
13903
13932
|
});
|
|
@@ -13941,7 +13970,6 @@ var WebsiteScraper = /** @class */ (function () {
|
|
|
13941
13970
|
return WebsiteScraper;
|
|
13942
13971
|
}());
|
|
13943
13972
|
/**
|
|
13944
|
-
* TODO: !!!!!! Put into separate package
|
|
13945
13973
|
* TODO: [👣] Scraped website in .md can act as cache item - there is no need to run conversion each time
|
|
13946
13974
|
* TODO: [🪂] Do it in parallel 11:11
|
|
13947
13975
|
* Note: No need to aggregate usage here, it is done by intercepting the llmTools
|