@promptbook/cli 0.72.0-14 → 0.72.0-15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/esm/index.es.js +544 -516
- package/esm/index.es.js.map +1 -1
- package/esm/typings/src/_packages/core.index.d.ts +22 -18
- package/esm/typings/src/_packages/node.index.d.ts +2 -0
- package/esm/typings/src/_packages/types.index.d.ts +10 -10
- package/esm/typings/src/_packages/utils.index.d.ts +2 -2
- package/esm/typings/src/collection/PipelineCollection.d.ts +1 -1
- package/esm/typings/src/collection/SimplePipelineCollection.d.ts +2 -2
- package/esm/typings/src/collection/collectionToJson.d.ts +1 -1
- package/esm/typings/src/collection/constructors/createCollectionFromJson.d.ts +1 -1
- package/esm/typings/src/collection/constructors/createCollectionFromPromise.d.ts +1 -1
- package/esm/typings/src/commands/_common/types/CommandParser.d.ts +5 -5
- package/esm/typings/src/config.d.ts +21 -14
- package/esm/typings/src/execution/EmbeddingVector.d.ts +1 -1
- package/esm/typings/src/execution/Executables.d.ts +18 -0
- package/esm/typings/src/execution/ExecutionTools.d.ts +9 -3
- package/esm/typings/src/execution/LlmExecutionTools.d.ts +1 -1
- package/esm/typings/src/execution/PipelineExecutorResult.d.ts +2 -2
- package/esm/typings/src/execution/assertsExecutionSuccessful.d.ts +1 -0
- package/esm/typings/src/execution/createPipelineExecutor/00-CreatePipelineExecutorOptions.d.ts +29 -6
- package/esm/typings/src/execution/createPipelineExecutor/10-executePipeline.d.ts +2 -11
- package/esm/typings/src/execution/createPipelineExecutor/20-executeTemplate.d.ts +4 -13
- package/esm/typings/src/execution/createPipelineExecutor/40-executeAttempts.d.ts +9 -14
- package/esm/typings/src/execution/translation/automatic-translate/automatic-translators/LindatAutomaticTranslator.d.ts +11 -3
- package/esm/typings/src/execution/utils/addUsage.d.ts +1 -1
- package/esm/typings/src/execution/utils/forEachAsync.d.ts +1 -1
- package/esm/typings/src/formats/_common/FormatDefinition.d.ts +2 -2
- package/esm/typings/src/formats/_common/FormatSubvalueDefinition.d.ts +2 -2
- package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsForCli.d.ts +2 -2
- package/esm/typings/src/llm-providers/_common/register/LlmToolsConfiguration.d.ts +1 -1
- package/esm/typings/src/llm-providers/_common/utils/cache/CacheLlmToolsOptions.d.ts +4 -1
- package/esm/typings/src/llm-providers/anthropic-claude/AnthropicClaudeExecutionTools.d.ts +1 -1
- package/esm/typings/src/llm-providers/anthropic-claude/anthropic-claude-models.d.ts +1 -1
- package/esm/typings/src/llm-providers/azure-openai/AzureOpenAiExecutionTools.d.ts +1 -1
- package/esm/typings/src/llm-providers/mocked/$fakeTextToExpectations.d.ts +1 -1
- package/esm/typings/src/llm-providers/mocked/MockedEchoLlmExecutionTools.d.ts +1 -1
- package/esm/typings/src/llm-providers/mocked/MockedFackedLlmExecutionTools.d.ts +1 -1
- package/esm/typings/src/llm-providers/multiple/MultipleLlmExecutionTools.d.ts +4 -5
- package/esm/typings/src/llm-providers/multiple/joinLlmExecutionTools.d.ts +1 -1
- package/esm/typings/src/llm-providers/openai/OpenAiExecutionTools.d.ts +1 -1
- package/esm/typings/src/llm-providers/openai/openai-models.d.ts +1 -1
- package/esm/typings/src/llm-providers/remote/RemoteLlmExecutionTools.d.ts +1 -1
- package/esm/typings/src/llm-providers/remote/interfaces/PromptbookServer_ListModels_Response.d.ts +3 -3
- package/esm/typings/src/llm-providers/remote/interfaces/PromptbookServer_Prompt_Response.d.ts +2 -2
- package/esm/typings/src/prepare/PrepareAndScrapeOptions.d.ts +4 -23
- package/esm/typings/src/prepare/prepareTemplates.d.ts +1 -1
- package/esm/typings/src/scrapers/_common/Scraper.d.ts +1 -5
- package/esm/typings/src/scrapers/_common/prepareKnowledgePieces.d.ts +1 -1
- package/esm/typings/src/scrapers/_common/register/$provideExecutablesForNode.d.ts +12 -0
- package/esm/typings/src/scrapers/_common/register/$provideScrapersForBrowser.d.ts +2 -2
- package/esm/typings/src/scrapers/_common/register/$provideScrapersForNode.d.ts +2 -2
- package/esm/typings/src/scrapers/_common/register/ScraperAndConverterMetadata.d.ts +2 -2
- package/esm/typings/src/scrapers/_common/utils/getScraperIntermediateSource.d.ts +2 -2
- package/esm/typings/src/scrapers/document/DocumentScraper.d.ts +2 -2
- package/esm/typings/src/scrapers/document/createDocumentScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/document/register-metadata.d.ts +1 -1
- package/esm/typings/src/scrapers/document-legacy/LegacyDocumentScraper.d.ts +3 -3
- package/esm/typings/src/scrapers/document-legacy/createLegacyDocumentScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/document-legacy/register-metadata.d.ts +1 -1
- package/esm/typings/src/scrapers/markdown/MarkdownScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/markdown/createMarkdownScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/markdown/register-metadata.d.ts +1 -1
- package/esm/typings/src/scrapers/pdf/PdfScraper.d.ts +2 -2
- package/esm/typings/src/scrapers/pdf/createPdfScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/pdf/register-metadata.d.ts +1 -1
- package/esm/typings/src/scrapers/website/WebsiteScraper.d.ts +6 -3
- package/esm/typings/src/scrapers/website/createWebsiteScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/website/register-metadata.d.ts +1 -1
- package/esm/typings/src/scrapers/website/utils/createShowdownConverter.d.ts +7 -0
- package/esm/typings/src/scripting/javascript/utils/preserve.d.ts +1 -1
- package/esm/typings/src/storage/file-cache-storage/utils/nameToSubfolderPath.d.ts +1 -1
- package/esm/typings/src/types/Arrayable.d.ts +1 -1
- package/esm/typings/src/types/IntermediateFilesStrategy.d.ts +7 -0
- package/esm/typings/src/types/PipelineJson/KnowledgePieceJson.d.ts +4 -4
- package/esm/typings/src/types/PipelineJson/KnowledgeSourceJson.d.ts +1 -1
- package/esm/typings/src/types/PipelineJson/PersonaJson.d.ts +1 -1
- package/esm/typings/src/types/PipelineJson/TemplateJsonCommon.d.ts +2 -2
- package/esm/typings/src/types/Prompt.d.ts +1 -1
- package/esm/typings/src/types/execution-report/ExecutionReportJson.d.ts +1 -1
- package/esm/typings/src/utils/$Register.d.ts +1 -1
- package/esm/typings/src/utils/FromtoItems.d.ts +1 -1
- package/esm/typings/src/utils/arrayableToArray.d.ts +1 -1
- package/esm/typings/src/utils/emojis.d.ts +1 -1
- package/esm/typings/src/utils/execCommand/$execCommand.d.ts +2 -2
- package/esm/typings/src/utils/execCommand/{IExecCommandOptions.d.ts → ExecCommandOptions.d.ts} +2 -6
- package/esm/typings/src/utils/execCommand/execCommandNormalizeOptions.d.ts +3 -3
- package/esm/typings/src/utils/expectation-counters/countSentences.d.ts +1 -1
- package/esm/typings/src/utils/markdown/extractAllBlocksFromMarkdown.d.ts +1 -1
- package/esm/typings/src/utils/markdown/splitMarkdownIntoSections.d.ts +1 -1
- package/esm/typings/src/utils/normalization/IKeywords.d.ts +2 -2
- package/esm/typings/src/utils/normalization/parseKeywords.d.ts +2 -2
- package/esm/typings/src/utils/normalization/parseKeywordsFromString.d.ts +2 -2
- package/esm/typings/src/utils/normalization/searchKeywords.d.ts +2 -2
- package/esm/typings/src/utils/organization/TODO_USE.d.ts +1 -1
- package/esm/typings/src/utils/organization/keepUnused.d.ts +1 -1
- package/esm/typings/src/utils/random/$randomSeed.d.ts +1 -1
- package/esm/typings/src/utils/sets/intersection.d.ts +1 -1
- package/esm/typings/src/utils/sets/union.d.ts +1 -1
- package/esm/typings/src/utils/unwrapResult.d.ts +4 -4
- package/package.json +3 -2
- package/umd/index.umd.js +544 -516
- package/umd/index.umd.js.map +1 -1
- package/esm/typings/src/execution/createPipelineExecutor/00-CreatePipelineExecutorSettings.d.ts +0 -29
- package/esm/typings/src/scrapers/website/utils/markdownConverter.d.ts +0 -12
- /package/esm/typings/src/scrapers/website/utils/{markdownConverter.test.d.ts → createShowdownConverter.test.d.ts} +0 -0
package/umd/index.umd.js
CHANGED
|
@@ -39,7 +39,7 @@
|
|
|
39
39
|
/**
|
|
40
40
|
* The version of the Promptbook library
|
|
41
41
|
*/
|
|
42
|
-
var PROMPTBOOK_VERSION = '0.72.0-
|
|
42
|
+
var PROMPTBOOK_VERSION = '0.72.0-14';
|
|
43
43
|
// TODO: [main] !!!! List here all the versions and annotate + put into script
|
|
44
44
|
|
|
45
45
|
/*! *****************************************************************************
|
|
@@ -412,7 +412,8 @@
|
|
|
412
412
|
*
|
|
413
413
|
* @public exported from `@promptbook/core`
|
|
414
414
|
*/
|
|
415
|
-
var CLAIM = "
|
|
415
|
+
var CLAIM = "Build responsible, controlled and transparent applications on top of LLM models!";
|
|
416
|
+
// <- TODO: [🐊] Pick the best claim
|
|
416
417
|
/**
|
|
417
418
|
* Warning message for the generated sections and files files
|
|
418
419
|
*
|
|
@@ -445,23 +446,30 @@
|
|
|
445
446
|
*/
|
|
446
447
|
var IMMEDIATE_TIME = 10;
|
|
447
448
|
/**
|
|
448
|
-
* The maximum
|
|
449
|
+
* The maximum length of the (generated) filename
|
|
449
450
|
*
|
|
450
451
|
* @public exported from `@promptbook/core`
|
|
451
452
|
*/
|
|
452
|
-
var
|
|
453
|
+
var MAX_FILENAME_LENGTH = 30;
|
|
453
454
|
/**
|
|
454
|
-
*
|
|
455
|
+
* Strategy for caching the intermediate results for knowledge sources
|
|
455
456
|
*
|
|
456
457
|
* @public exported from `@promptbook/core`
|
|
457
458
|
*/
|
|
458
|
-
var
|
|
459
|
+
var DEFAULT_INTERMEDIATE_FILES_STRATEGY = 'HIDE_AND_KEEP';
|
|
460
|
+
// <- TODO: [😡] Change to 'VISIBLE'
|
|
459
461
|
/**
|
|
460
|
-
* The maximum
|
|
462
|
+
* The maximum number of (LLM) tasks running in parallel
|
|
461
463
|
*
|
|
462
464
|
* @public exported from `@promptbook/core`
|
|
463
465
|
*/
|
|
464
|
-
var
|
|
466
|
+
var DEFAULT_MAX_PARALLEL_COUNT = 5; // <- TODO: [🤹♂️]
|
|
467
|
+
/**
|
|
468
|
+
* The maximum number of attempts to execute LLM task before giving up
|
|
469
|
+
*
|
|
470
|
+
* @public exported from `@promptbook/core`
|
|
471
|
+
*/
|
|
472
|
+
var DEFAULT_MAX_EXECUTION_ATTEMPTS = 3; // <- TODO: [🤹♂️]
|
|
465
473
|
/**
|
|
466
474
|
* Where to store the cache of executions for promptbook CLI
|
|
467
475
|
*
|
|
@@ -469,7 +477,7 @@
|
|
|
469
477
|
*
|
|
470
478
|
* @public exported from `@promptbook/core`
|
|
471
479
|
*/
|
|
472
|
-
var
|
|
480
|
+
var DEFAULT_EXECUTIONS_CACHE_DIRNAME = '/.promptbook/executions-cache';
|
|
473
481
|
/**
|
|
474
482
|
* Where to store the scrape cache
|
|
475
483
|
*
|
|
@@ -477,13 +485,13 @@
|
|
|
477
485
|
*
|
|
478
486
|
* @public exported from `@promptbook/core`
|
|
479
487
|
*/
|
|
480
|
-
var
|
|
488
|
+
var DEFAULT_SCRAPE_CACHE_DIRNAME = '/.promptbook/scrape-cache';
|
|
481
489
|
/**
|
|
482
490
|
* The name of the builded pipeline collection made by CLI `ptbk make` and for lookup in `createCollectionFromDirectory`
|
|
483
491
|
*
|
|
484
492
|
* @public exported from `@promptbook/core`
|
|
485
493
|
*/
|
|
486
|
-
var
|
|
494
|
+
var DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME = "index";
|
|
487
495
|
/**
|
|
488
496
|
* Nonce which is used for replacing things in strings
|
|
489
497
|
*
|
|
@@ -547,13 +555,13 @@
|
|
|
547
555
|
*
|
|
548
556
|
* @public exported from `@promptbook/core`
|
|
549
557
|
*/
|
|
550
|
-
var
|
|
558
|
+
var DEFAULT_IS_VERBOSE = false;
|
|
551
559
|
/**
|
|
552
560
|
* @@@
|
|
553
561
|
*
|
|
554
562
|
* @public exported from `@promptbook/core`
|
|
555
563
|
*/
|
|
556
|
-
var
|
|
564
|
+
var DEFAULT_IS_AUTO_INSTALLED = false;
|
|
557
565
|
/**
|
|
558
566
|
* @@@
|
|
559
567
|
*
|
|
@@ -626,271 +634,6 @@
|
|
|
626
634
|
* Note: [🟡] Code in this file should never be published outside of `@promptbook/cli`
|
|
627
635
|
*/
|
|
628
636
|
|
|
629
|
-
/**
|
|
630
|
-
* Just marks a place of place where should be something implemented
|
|
631
|
-
* No side effects.
|
|
632
|
-
*
|
|
633
|
-
* Note: It can be usefull suppressing eslint errors of unused variables
|
|
634
|
-
*
|
|
635
|
-
* @param value any values
|
|
636
|
-
* @returns void
|
|
637
|
-
* @private within the repository
|
|
638
|
-
*/
|
|
639
|
-
function TODO_USE() {
|
|
640
|
-
var value = [];
|
|
641
|
-
for (var _i = 0; _i < arguments.length; _i++) {
|
|
642
|
-
value[_i] = arguments[_i];
|
|
643
|
-
}
|
|
644
|
-
}
|
|
645
|
-
|
|
646
|
-
/**
|
|
647
|
-
* @@@
|
|
648
|
-
*
|
|
649
|
-
* @public exported from `@promptbook/node`
|
|
650
|
-
*/
|
|
651
|
-
function $provideFilesystemForNode(options) {
|
|
652
|
-
if (!$isRunningInNode()) {
|
|
653
|
-
throw new EnvironmentMismatchError('Function `$provideFilesystemForNode` works only in Node.js environment');
|
|
654
|
-
}
|
|
655
|
-
var _a = (options || {}).isVerbose, isVerbose = _a === void 0 ? IS_VERBOSE : _a;
|
|
656
|
-
TODO_USE(isVerbose);
|
|
657
|
-
return {
|
|
658
|
-
stat: promises.stat,
|
|
659
|
-
access: promises.access,
|
|
660
|
-
constants: promises.constants,
|
|
661
|
-
readFile: promises.readFile,
|
|
662
|
-
readdir: promises.readdir,
|
|
663
|
-
};
|
|
664
|
-
}
|
|
665
|
-
/**
|
|
666
|
-
* Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
|
|
667
|
-
*/
|
|
668
|
-
|
|
669
|
-
/**
|
|
670
|
-
* This error type indicates that some part of the code is not implemented yet
|
|
671
|
-
*
|
|
672
|
-
* @public exported from `@promptbook/core`
|
|
673
|
-
*/
|
|
674
|
-
var NotYetImplementedError = /** @class */ (function (_super) {
|
|
675
|
-
__extends(NotYetImplementedError, _super);
|
|
676
|
-
function NotYetImplementedError(message) {
|
|
677
|
-
var _this = _super.call(this, spaceTrim.spaceTrim(function (block) { return "\n ".concat(block(message), "\n\n Note: This feature is not implemented yet but it will be soon.\n\n If you want speed up the implementation or just read more, look here:\n https://github.com/webgptorg/promptbook\n\n Or contact us on me@pavolhejny.com\n\n "); })) || this;
|
|
678
|
-
_this.name = 'NotYetImplementedError';
|
|
679
|
-
Object.setPrototypeOf(_this, NotYetImplementedError.prototype);
|
|
680
|
-
return _this;
|
|
681
|
-
}
|
|
682
|
-
return NotYetImplementedError;
|
|
683
|
-
}(Error));
|
|
684
|
-
|
|
685
|
-
/**
|
|
686
|
-
* @@@
|
|
687
|
-
*
|
|
688
|
-
* Note: `$` is used to indicate that this function is not a pure function - it access global scope
|
|
689
|
-
*
|
|
690
|
-
* @private internal function of `$Register`
|
|
691
|
-
*/
|
|
692
|
-
function $getGlobalScope() {
|
|
693
|
-
return Function('return this')();
|
|
694
|
-
}
|
|
695
|
-
|
|
696
|
-
/**
|
|
697
|
-
* @@@
|
|
698
|
-
*
|
|
699
|
-
* @param text @@@
|
|
700
|
-
* @returns @@@
|
|
701
|
-
* @example 'HELLO_WORLD'
|
|
702
|
-
* @example 'I_LOVE_PROMPTBOOK'
|
|
703
|
-
* @public exported from `@promptbook/utils`
|
|
704
|
-
*/
|
|
705
|
-
function normalizeTo_SCREAMING_CASE(text) {
|
|
706
|
-
var e_1, _a;
|
|
707
|
-
var charType;
|
|
708
|
-
var lastCharType = 'OTHER';
|
|
709
|
-
var normalizedName = '';
|
|
710
|
-
try {
|
|
711
|
-
for (var text_1 = __values(text), text_1_1 = text_1.next(); !text_1_1.done; text_1_1 = text_1.next()) {
|
|
712
|
-
var char = text_1_1.value;
|
|
713
|
-
var normalizedChar = void 0;
|
|
714
|
-
if (/^[a-z]$/.test(char)) {
|
|
715
|
-
charType = 'LOWERCASE';
|
|
716
|
-
normalizedChar = char.toUpperCase();
|
|
717
|
-
}
|
|
718
|
-
else if (/^[A-Z]$/.test(char)) {
|
|
719
|
-
charType = 'UPPERCASE';
|
|
720
|
-
normalizedChar = char;
|
|
721
|
-
}
|
|
722
|
-
else if (/^[0-9]$/.test(char)) {
|
|
723
|
-
charType = 'NUMBER';
|
|
724
|
-
normalizedChar = char;
|
|
725
|
-
}
|
|
726
|
-
else {
|
|
727
|
-
charType = 'OTHER';
|
|
728
|
-
normalizedChar = '_';
|
|
729
|
-
}
|
|
730
|
-
if (charType !== lastCharType &&
|
|
731
|
-
!(lastCharType === 'UPPERCASE' && charType === 'LOWERCASE') &&
|
|
732
|
-
!(lastCharType === 'NUMBER') &&
|
|
733
|
-
!(charType === 'NUMBER')) {
|
|
734
|
-
normalizedName += '_';
|
|
735
|
-
}
|
|
736
|
-
normalizedName += normalizedChar;
|
|
737
|
-
lastCharType = charType;
|
|
738
|
-
}
|
|
739
|
-
}
|
|
740
|
-
catch (e_1_1) { e_1 = { error: e_1_1 }; }
|
|
741
|
-
finally {
|
|
742
|
-
try {
|
|
743
|
-
if (text_1_1 && !text_1_1.done && (_a = text_1.return)) _a.call(text_1);
|
|
744
|
-
}
|
|
745
|
-
finally { if (e_1) throw e_1.error; }
|
|
746
|
-
}
|
|
747
|
-
normalizedName = normalizedName.replace(/_+/g, '_');
|
|
748
|
-
normalizedName = normalizedName.replace(/_?\/_?/g, '/');
|
|
749
|
-
normalizedName = normalizedName.replace(/^_/, '');
|
|
750
|
-
normalizedName = normalizedName.replace(/_$/, '');
|
|
751
|
-
return normalizedName;
|
|
752
|
-
}
|
|
753
|
-
/**
|
|
754
|
-
* TODO: Tests
|
|
755
|
-
* > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: 'Moje tabule' })).toEqual('/VtG7sR9rRJqwNEdM2/Moje tabule');
|
|
756
|
-
* > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: 'ěščřžžýáíúů' })).toEqual('/VtG7sR9rRJqwNEdM2/escrzyaieuu');
|
|
757
|
-
* > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: ' ahoj ' })).toEqual('/VtG7sR9rRJqwNEdM2/ahoj');
|
|
758
|
-
* > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: ' ahoj_ahojAhoj ahoj ' })).toEqual('/VtG7sR9rRJqwNEdM2/ahoj-ahoj-ahoj-ahoj');
|
|
759
|
-
* TODO: [🌺] Use some intermediate util splitWords
|
|
760
|
-
*/
|
|
761
|
-
|
|
762
|
-
/**
|
|
763
|
-
* @@@
|
|
764
|
-
*
|
|
765
|
-
* @param text @@@
|
|
766
|
-
* @returns @@@
|
|
767
|
-
* @example 'hello_world'
|
|
768
|
-
* @example 'i_love_promptbook'
|
|
769
|
-
* @public exported from `@promptbook/utils`
|
|
770
|
-
*/
|
|
771
|
-
function normalizeTo_snake_case(text) {
|
|
772
|
-
return normalizeTo_SCREAMING_CASE(text).toLowerCase();
|
|
773
|
-
}
|
|
774
|
-
|
|
775
|
-
/**
|
|
776
|
-
* Register is @@@
|
|
777
|
-
*
|
|
778
|
-
* Note: `$` is used to indicate that this function is not a pure function - it accesses and adds variables in global scope.
|
|
779
|
-
*
|
|
780
|
-
* @private internal utility, exported are only signleton instances of this class
|
|
781
|
-
*/
|
|
782
|
-
var $Register = /** @class */ (function () {
|
|
783
|
-
function $Register(registerName) {
|
|
784
|
-
this.registerName = registerName;
|
|
785
|
-
var storageName = "_promptbook_".concat(normalizeTo_snake_case(registerName));
|
|
786
|
-
var globalScope = $getGlobalScope();
|
|
787
|
-
if (globalScope[storageName] === undefined) {
|
|
788
|
-
globalScope[storageName] = [];
|
|
789
|
-
}
|
|
790
|
-
else if (!Array.isArray(globalScope[storageName])) {
|
|
791
|
-
throw new UnexpectedError("Expected (global) ".concat(storageName, " to be an array, but got ").concat(typeof globalScope[storageName]));
|
|
792
|
-
}
|
|
793
|
-
this.storage = globalScope[storageName];
|
|
794
|
-
}
|
|
795
|
-
$Register.prototype.list = function () {
|
|
796
|
-
// <- TODO: ReadonlyDeep<Array<TRegistered>>
|
|
797
|
-
return this.storage;
|
|
798
|
-
};
|
|
799
|
-
$Register.prototype.register = function (registered) {
|
|
800
|
-
var packageName = registered.packageName, className = registered.className;
|
|
801
|
-
var existingRegistrationIndex = this.storage.findIndex(function (item) { return item.packageName === packageName && item.className === className; });
|
|
802
|
-
var existingRegistration = this.storage[existingRegistrationIndex];
|
|
803
|
-
if (!existingRegistration) {
|
|
804
|
-
this.storage.push(registered);
|
|
805
|
-
}
|
|
806
|
-
else {
|
|
807
|
-
this.storage[existingRegistrationIndex] = registered;
|
|
808
|
-
}
|
|
809
|
-
return {
|
|
810
|
-
registerName: this.registerName,
|
|
811
|
-
packageName: packageName,
|
|
812
|
-
className: className,
|
|
813
|
-
get isDestroyed() {
|
|
814
|
-
return false;
|
|
815
|
-
},
|
|
816
|
-
destroy: function () {
|
|
817
|
-
throw new NotYetImplementedError("Registration to ".concat(this.registerName, " is permanent in this version of Promptbook"));
|
|
818
|
-
},
|
|
819
|
-
};
|
|
820
|
-
};
|
|
821
|
-
return $Register;
|
|
822
|
-
}());
|
|
823
|
-
|
|
824
|
-
/**
|
|
825
|
-
* @@@
|
|
826
|
-
*
|
|
827
|
-
* Note: `$` is used to indicate that this interacts with the global scope
|
|
828
|
-
* @singleton Only one instance of each register is created per build, but thare can be more @@@
|
|
829
|
-
* @public exported from `@promptbook/core`
|
|
830
|
-
*/
|
|
831
|
-
var $scrapersRegister = new $Register('scraper_constructors');
|
|
832
|
-
/**
|
|
833
|
-
* TODO: [®] DRY Register logic
|
|
834
|
-
*/
|
|
835
|
-
|
|
836
|
-
/**
|
|
837
|
-
* !!!!!!
|
|
838
|
-
*
|
|
839
|
-
* 1) @@@
|
|
840
|
-
* 2) @@@
|
|
841
|
-
*
|
|
842
|
-
* @public exported from `@promptbook/node`
|
|
843
|
-
*/
|
|
844
|
-
function $provideScrapersForNode(tools, options) {
|
|
845
|
-
return __awaiter(this, void 0, void 0, function () {
|
|
846
|
-
var _a, _b, isAutoInstalled, _c, isVerbose, scrapers, _d, _e, scraperFactory, scraper, e_1_1;
|
|
847
|
-
var e_1, _f;
|
|
848
|
-
return __generator(this, function (_g) {
|
|
849
|
-
switch (_g.label) {
|
|
850
|
-
case 0:
|
|
851
|
-
if (!$isRunningInNode()) {
|
|
852
|
-
throw new EnvironmentMismatchError('Function `$getScrapersForNode` works only in Node.js environment');
|
|
853
|
-
}
|
|
854
|
-
_a = options || {}, _b = _a.isAutoInstalled, isAutoInstalled = _b === void 0 ? IS_AUTO_INSTALLED : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? IS_VERBOSE : _c;
|
|
855
|
-
TODO_USE(isAutoInstalled);
|
|
856
|
-
TODO_USE(isVerbose);
|
|
857
|
-
scrapers = [];
|
|
858
|
-
_g.label = 1;
|
|
859
|
-
case 1:
|
|
860
|
-
_g.trys.push([1, 6, 7, 8]);
|
|
861
|
-
_d = __values($scrapersRegister.list()), _e = _d.next();
|
|
862
|
-
_g.label = 2;
|
|
863
|
-
case 2:
|
|
864
|
-
if (!!_e.done) return [3 /*break*/, 5];
|
|
865
|
-
scraperFactory = _e.value;
|
|
866
|
-
return [4 /*yield*/, scraperFactory(tools, options || {})];
|
|
867
|
-
case 3:
|
|
868
|
-
scraper = _g.sent();
|
|
869
|
-
scrapers.push(scraper);
|
|
870
|
-
_g.label = 4;
|
|
871
|
-
case 4:
|
|
872
|
-
_e = _d.next();
|
|
873
|
-
return [3 /*break*/, 2];
|
|
874
|
-
case 5: return [3 /*break*/, 8];
|
|
875
|
-
case 6:
|
|
876
|
-
e_1_1 = _g.sent();
|
|
877
|
-
e_1 = { error: e_1_1 };
|
|
878
|
-
return [3 /*break*/, 8];
|
|
879
|
-
case 7:
|
|
880
|
-
try {
|
|
881
|
-
if (_e && !_e.done && (_f = _d.return)) _f.call(_d);
|
|
882
|
-
}
|
|
883
|
-
finally { if (e_1) throw e_1.error; }
|
|
884
|
-
return [7 /*endfinally*/];
|
|
885
|
-
case 8: return [2 /*return*/, scrapers];
|
|
886
|
-
}
|
|
887
|
-
});
|
|
888
|
-
});
|
|
889
|
-
}
|
|
890
|
-
/**
|
|
891
|
-
* Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
|
|
892
|
-
*/
|
|
893
|
-
|
|
894
637
|
/**
|
|
895
638
|
* Converts PipelineCollection to serialized JSON
|
|
896
639
|
*
|
|
@@ -976,7 +719,6 @@
|
|
|
976
719
|
pipelineString += '\n\n';
|
|
977
720
|
pipelineString += description;
|
|
978
721
|
}
|
|
979
|
-
// TODO:> const commands: Array<Command>
|
|
980
722
|
var commands = [];
|
|
981
723
|
if (pipelineUrl) {
|
|
982
724
|
commands.push("PIPELINE URL ".concat(pipelineUrl));
|
|
@@ -1032,7 +774,6 @@
|
|
|
1032
774
|
pipelineString += '\n\n';
|
|
1033
775
|
pipelineString += description_1;
|
|
1034
776
|
}
|
|
1035
|
-
// TODO:> const commands: Array<Command>
|
|
1036
777
|
var commands_1 = [];
|
|
1037
778
|
var contentLanguage = 'text';
|
|
1038
779
|
if (templateType === 'PROMPT_TEMPLATE') {
|
|
@@ -1331,8 +1072,7 @@
|
|
|
1331
1072
|
* Multiple LLM Execution Tools is a proxy server that uses multiple execution tools internally and exposes the executor interface externally.
|
|
1332
1073
|
*
|
|
1333
1074
|
* Note: Internal utility of `joinLlmExecutionTools` but exposed type
|
|
1334
|
-
* @public exported from `@promptbook/
|
|
1335
|
-
* TODO: !!!!!! Export as runtime class not just type
|
|
1075
|
+
* @public exported from `@promptbook/core`
|
|
1336
1076
|
*/
|
|
1337
1077
|
var MultipleLlmExecutionTools = /** @class */ (function () {
|
|
1338
1078
|
/**
|
|
@@ -2491,6 +2231,22 @@
|
|
|
2491
2231
|
return LimitReachedError;
|
|
2492
2232
|
}(Error));
|
|
2493
2233
|
|
|
2234
|
+
/**
|
|
2235
|
+
* This error type indicates that some part of the code is not implemented yet
|
|
2236
|
+
*
|
|
2237
|
+
* @public exported from `@promptbook/core`
|
|
2238
|
+
*/
|
|
2239
|
+
var NotYetImplementedError = /** @class */ (function (_super) {
|
|
2240
|
+
__extends(NotYetImplementedError, _super);
|
|
2241
|
+
function NotYetImplementedError(message) {
|
|
2242
|
+
var _this = _super.call(this, spaceTrim.spaceTrim(function (block) { return "\n ".concat(block(message), "\n\n Note: This feature is not implemented yet but it will be soon.\n\n If you want speed up the implementation or just read more, look here:\n https://github.com/webgptorg/promptbook\n\n Or contact us on me@pavolhejny.com\n\n "); })) || this;
|
|
2243
|
+
_this.name = 'NotYetImplementedError';
|
|
2244
|
+
Object.setPrototypeOf(_this, NotYetImplementedError.prototype);
|
|
2245
|
+
return _this;
|
|
2246
|
+
}
|
|
2247
|
+
return NotYetImplementedError;
|
|
2248
|
+
}(Error));
|
|
2249
|
+
|
|
2494
2250
|
/**
|
|
2495
2251
|
* Index of all custom errors
|
|
2496
2252
|
*
|
|
@@ -2552,6 +2308,7 @@
|
|
|
2552
2308
|
}
|
|
2553
2309
|
}
|
|
2554
2310
|
/**
|
|
2311
|
+
* TODO: [🐚] This function should be removed OR changed OR be completely rewritten
|
|
2555
2312
|
* TODO: [🧠] Can this return type be better typed than void
|
|
2556
2313
|
*/
|
|
2557
2314
|
|
|
@@ -2809,6 +2566,23 @@
|
|
|
2809
2566
|
return union;
|
|
2810
2567
|
}
|
|
2811
2568
|
|
|
2569
|
+
/**
|
|
2570
|
+
* Just marks a place of place where should be something implemented
|
|
2571
|
+
* No side effects.
|
|
2572
|
+
*
|
|
2573
|
+
* Note: It can be usefull suppressing eslint errors of unused variables
|
|
2574
|
+
*
|
|
2575
|
+
* @param value any values
|
|
2576
|
+
* @returns void
|
|
2577
|
+
* @private within the repository
|
|
2578
|
+
*/
|
|
2579
|
+
function TODO_USE() {
|
|
2580
|
+
var value = [];
|
|
2581
|
+
for (var _i = 0; _i < arguments.length; _i++) {
|
|
2582
|
+
value[_i] = arguments[_i];
|
|
2583
|
+
}
|
|
2584
|
+
}
|
|
2585
|
+
|
|
2812
2586
|
/**
|
|
2813
2587
|
* This error indicates problems parsing the format value
|
|
2814
2588
|
*
|
|
@@ -3841,12 +3615,11 @@
|
|
|
3841
3615
|
*/
|
|
3842
3616
|
function executeAttempts(options) {
|
|
3843
3617
|
return __awaiter(this, void 0, void 0, function () {
|
|
3844
|
-
var jokerParameterNames, priority, maxAttempts, preparedContent, parameters, template, preparedPipeline, tools,
|
|
3618
|
+
var jokerParameterNames, priority, maxAttempts, preparedContent, parameters, template, preparedPipeline, tools, $executionReport, pipelineIdentification, maxExecutionAttempts, $ongoingTemplateResult, _llms, llmTools, _loop_1, attempt, state_1;
|
|
3845
3619
|
return __generator(this, function (_a) {
|
|
3846
3620
|
switch (_a.label) {
|
|
3847
3621
|
case 0:
|
|
3848
|
-
jokerParameterNames = options.jokerParameterNames, priority = options.priority, maxAttempts = options.maxAttempts, preparedContent = options.preparedContent, parameters = options.parameters, template = options.template, preparedPipeline = options.preparedPipeline, tools = options.tools,
|
|
3849
|
-
maxExecutionAttempts = settings.maxExecutionAttempts;
|
|
3622
|
+
jokerParameterNames = options.jokerParameterNames, priority = options.priority, maxAttempts = options.maxAttempts, preparedContent = options.preparedContent, parameters = options.parameters, template = options.template, preparedPipeline = options.preparedPipeline, tools = options.tools, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification, maxExecutionAttempts = options.maxExecutionAttempts;
|
|
3850
3623
|
$ongoingTemplateResult = {
|
|
3851
3624
|
$result: null,
|
|
3852
3625
|
$resultString: null,
|
|
@@ -4212,12 +3985,12 @@
|
|
|
4212
3985
|
*/
|
|
4213
3986
|
function executeFormatSubvalues(options) {
|
|
4214
3987
|
return __awaiter(this, void 0, void 0, function () {
|
|
4215
|
-
var template, jokerParameterNames, parameters, priority,
|
|
3988
|
+
var template, jokerParameterNames, parameters, priority, csvSettings, pipelineIdentification, parameterValue, formatDefinition, subvalueDefinition, formatSettings, resultString;
|
|
4216
3989
|
var _this = this;
|
|
4217
3990
|
return __generator(this, function (_a) {
|
|
4218
3991
|
switch (_a.label) {
|
|
4219
3992
|
case 0:
|
|
4220
|
-
template = options.template, jokerParameterNames = options.jokerParameterNames, parameters = options.parameters, priority = options.priority,
|
|
3993
|
+
template = options.template, jokerParameterNames = options.jokerParameterNames, parameters = options.parameters, priority = options.priority, csvSettings = options.csvSettings, pipelineIdentification = options.pipelineIdentification;
|
|
4221
3994
|
if (template.foreach === undefined) {
|
|
4222
3995
|
return [2 /*return*/, /* not await */ executeAttempts(options)];
|
|
4223
3996
|
}
|
|
@@ -4247,7 +4020,7 @@
|
|
|
4247
4020
|
.join('\n')), "\n\n [\u26F7] This should never happen because subformat name should be validated during parsing\n\n ").concat(block(pipelineIdentification), "\n "); }));
|
|
4248
4021
|
}
|
|
4249
4022
|
if (formatDefinition.formatName === 'CSV') {
|
|
4250
|
-
formatSettings =
|
|
4023
|
+
formatSettings = csvSettings;
|
|
4251
4024
|
// <- TODO: [🤹♂️] More universal, make simmilar pattern for other formats for example \n vs \r\n in text
|
|
4252
4025
|
}
|
|
4253
4026
|
return [4 /*yield*/, subvalueDefinition.mapValues(parameterValue, template.foreach.outputSubparameterName, formatSettings, function (subparameters, index) { return __awaiter(_this, void 0, void 0, function () {
|
|
@@ -4400,13 +4173,12 @@
|
|
|
4400
4173
|
*/
|
|
4401
4174
|
function executeTemplate(options) {
|
|
4402
4175
|
return __awaiter(this, void 0, void 0, function () {
|
|
4403
|
-
var currentTemplate, preparedPipeline, parametersToPass, tools, onProgress,
|
|
4404
|
-
var e_1,
|
|
4405
|
-
return __generator(this, function (
|
|
4406
|
-
switch (
|
|
4176
|
+
var currentTemplate, preparedPipeline, parametersToPass, tools, onProgress, $executionReport, pipelineIdentification, _a, maxExecutionAttempts, name, title, priority, usedParameterNames, dependentParameterNames, definedParameters, _b, _c, _d, definedParameterNames, parameters, _loop_1, _e, _f, parameterName, maxAttempts, jokerParameterNames, preparedContent, resultString;
|
|
4177
|
+
var e_1, _g, _h;
|
|
4178
|
+
return __generator(this, function (_j) {
|
|
4179
|
+
switch (_j.label) {
|
|
4407
4180
|
case 0:
|
|
4408
|
-
currentTemplate = options.currentTemplate, preparedPipeline = options.preparedPipeline, parametersToPass = options.parametersToPass, tools = options.tools, onProgress = options.onProgress,
|
|
4409
|
-
maxExecutionAttempts = settings.maxExecutionAttempts;
|
|
4181
|
+
currentTemplate = options.currentTemplate, preparedPipeline = options.preparedPipeline, parametersToPass = options.parametersToPass, tools = options.tools, onProgress = options.onProgress, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification, _a = options.maxExecutionAttempts, maxExecutionAttempts = _a === void 0 ? DEFAULT_MAX_EXECUTION_ATTEMPTS : _a;
|
|
4410
4182
|
name = "pipeline-executor-frame-".concat(currentTemplate.name);
|
|
4411
4183
|
title = currentTemplate.title;
|
|
4412
4184
|
priority = preparedPipeline.templates.length - preparedPipeline.templates.indexOf(currentTemplate);
|
|
@@ -4421,7 +4193,7 @@
|
|
|
4421
4193
|
// <- [🍸]
|
|
4422
4194
|
})];
|
|
4423
4195
|
case 1:
|
|
4424
|
-
|
|
4196
|
+
_j.sent();
|
|
4425
4197
|
usedParameterNames = extractParameterNamesFromTemplate(currentTemplate);
|
|
4426
4198
|
dependentParameterNames = new Set(currentTemplate.dependentParameterNames);
|
|
4427
4199
|
// TODO: [👩🏾🤝👩🏻] Use here `mapAvailableToExpectedParameters`
|
|
@@ -4432,15 +4204,15 @@
|
|
|
4432
4204
|
.map(function (name) { return "{".concat(name, "}"); })
|
|
4433
4205
|
.join(', '), "\n\n ").concat(block(pipelineIdentification), "\n\n "); }));
|
|
4434
4206
|
}
|
|
4435
|
-
|
|
4436
|
-
|
|
4207
|
+
_c = (_b = Object).freeze;
|
|
4208
|
+
_d = [{}];
|
|
4437
4209
|
return [4 /*yield*/, getReservedParametersForTemplate({
|
|
4438
4210
|
preparedPipeline: preparedPipeline,
|
|
4439
4211
|
template: currentTemplate,
|
|
4440
4212
|
pipelineIdentification: pipelineIdentification,
|
|
4441
4213
|
})];
|
|
4442
4214
|
case 2:
|
|
4443
|
-
definedParameters =
|
|
4215
|
+
definedParameters = _c.apply(_b, [__assign.apply(void 0, [__assign.apply(void 0, _d.concat([(_j.sent())])), parametersToPass])]);
|
|
4444
4216
|
definedParameterNames = new Set(Object.keys(definedParameters));
|
|
4445
4217
|
parameters = {};
|
|
4446
4218
|
_loop_1 = function (parameterName) {
|
|
@@ -4460,15 +4232,15 @@
|
|
|
4460
4232
|
try {
|
|
4461
4233
|
// Note: [2] Check that all used parameters are defined and removing unused parameters for this template
|
|
4462
4234
|
// TODO: [👩🏾🤝👩🏻] Use here `mapAvailableToExpectedParameters`
|
|
4463
|
-
for (
|
|
4464
|
-
parameterName =
|
|
4235
|
+
for (_e = __values(Array.from(union(definedParameterNames, usedParameterNames, dependentParameterNames))), _f = _e.next(); !_f.done; _f = _e.next()) {
|
|
4236
|
+
parameterName = _f.value;
|
|
4465
4237
|
_loop_1(parameterName);
|
|
4466
4238
|
}
|
|
4467
4239
|
}
|
|
4468
4240
|
catch (e_1_1) { e_1 = { error: e_1_1 }; }
|
|
4469
4241
|
finally {
|
|
4470
4242
|
try {
|
|
4471
|
-
if (
|
|
4243
|
+
if (_f && !_f.done && (_g = _e.return)) _g.call(_e);
|
|
4472
4244
|
}
|
|
4473
4245
|
finally { if (e_1) throw e_1.error; }
|
|
4474
4246
|
}
|
|
@@ -4488,12 +4260,11 @@
|
|
|
4488
4260
|
template: currentTemplate,
|
|
4489
4261
|
preparedPipeline: preparedPipeline,
|
|
4490
4262
|
tools: tools,
|
|
4491
|
-
settings: settings,
|
|
4492
4263
|
$executionReport: $executionReport,
|
|
4493
4264
|
pipelineIdentification: pipelineIdentification,
|
|
4494
4265
|
})];
|
|
4495
4266
|
case 3:
|
|
4496
|
-
resultString =
|
|
4267
|
+
resultString = _j.sent();
|
|
4497
4268
|
return [4 /*yield*/, onProgress({
|
|
4498
4269
|
name: name,
|
|
4499
4270
|
title: title,
|
|
@@ -4505,12 +4276,12 @@
|
|
|
4505
4276
|
// <- [🍸]
|
|
4506
4277
|
})];
|
|
4507
4278
|
case 4:
|
|
4508
|
-
|
|
4509
|
-
return [2 /*return*/, Object.freeze((
|
|
4510
|
-
|
|
4279
|
+
_j.sent();
|
|
4280
|
+
return [2 /*return*/, Object.freeze((_h = {},
|
|
4281
|
+
_h[currentTemplate.resultingParameterName] =
|
|
4511
4282
|
// <- Note: [👩👩👧] No need to detect parameter collision here because pipeline checks logic consistency during construction
|
|
4512
4283
|
resultString,
|
|
4513
|
-
|
|
4284
|
+
_h))];
|
|
4514
4285
|
}
|
|
4515
4286
|
});
|
|
4516
4287
|
});
|
|
@@ -4569,13 +4340,12 @@
|
|
|
4569
4340
|
*/
|
|
4570
4341
|
function executePipeline(options) {
|
|
4571
4342
|
return __awaiter(this, void 0, void 0, function () {
|
|
4572
|
-
var inputParameters, tools, onProgress, pipeline, setPreparedPipeline, pipelineIdentification,
|
|
4343
|
+
var inputParameters, tools, onProgress, pipeline, setPreparedPipeline, pipelineIdentification, maxParallelCount, rootDirname, _a, isVerbose, preparedPipeline, errors, warnings, executionReport, isReturned, _b, _c, parameter, e_1_1, _loop_1, _d, _e, parameterName, state_1, e_2_1, parametersToPass, resovedParameterNames_1, unresovedTemplates_1, resolving_1, loopLimit, _loop_2, error_1, usage_1, outputParameters_1, usage, outputParameters;
|
|
4573
4344
|
var e_1, _f, e_2, _g;
|
|
4574
4345
|
return __generator(this, function (_h) {
|
|
4575
4346
|
switch (_h.label) {
|
|
4576
4347
|
case 0:
|
|
4577
|
-
inputParameters = options.inputParameters, tools = options.tools, onProgress = options.onProgress, pipeline = options.pipeline, setPreparedPipeline = options.setPreparedPipeline, pipelineIdentification = options.pipelineIdentification,
|
|
4578
|
-
maxParallelCount = settings.maxParallelCount, rootDirname = settings.rootDirname, _a = settings.isVerbose, isVerbose = _a === void 0 ? IS_VERBOSE : _a;
|
|
4348
|
+
inputParameters = options.inputParameters, tools = options.tools, onProgress = options.onProgress, pipeline = options.pipeline, setPreparedPipeline = options.setPreparedPipeline, pipelineIdentification = options.pipelineIdentification, maxParallelCount = options.maxParallelCount, rootDirname = options.rootDirname, _a = options.isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a;
|
|
4579
4349
|
preparedPipeline = options.preparedPipeline;
|
|
4580
4350
|
if (!(preparedPipeline === undefined)) return [3 /*break*/, 2];
|
|
4581
4351
|
return [4 /*yield*/, preparePipeline(pipeline, tools, {
|
|
@@ -4760,12 +4530,7 @@
|
|
|
4760
4530
|
return [3 /*break*/, 4];
|
|
4761
4531
|
case 3:
|
|
4762
4532
|
unresovedTemplates_1 = unresovedTemplates_1.filter(function (template) { return template !== currentTemplate; });
|
|
4763
|
-
work_1 = executeTemplate({
|
|
4764
|
-
currentTemplate: currentTemplate,
|
|
4765
|
-
preparedPipeline: preparedPipeline,
|
|
4766
|
-
parametersToPass: parametersToPass,
|
|
4767
|
-
tools: tools,
|
|
4768
|
-
onProgress: function (progress) {
|
|
4533
|
+
work_1 = executeTemplate(__assign(__assign({}, options), { currentTemplate: currentTemplate, preparedPipeline: preparedPipeline, parametersToPass: parametersToPass, tools: tools, onProgress: function (progress) {
|
|
4769
4534
|
if (isReturned) {
|
|
4770
4535
|
throw new UnexpectedError(spaceTrim.spaceTrim(function (block) { return "\n Can not call `onProgress` after pipeline execution is finished\n\n ".concat(block(pipelineIdentification), "\n\n ").concat(block(JSON.stringify(progress, null, 4)
|
|
4771
4536
|
.split('\n')
|
|
@@ -4775,11 +4540,7 @@
|
|
|
4775
4540
|
if (onProgress) {
|
|
4776
4541
|
onProgress(progress);
|
|
4777
4542
|
}
|
|
4778
|
-
},
|
|
4779
|
-
settings: settings,
|
|
4780
|
-
$executionReport: executionReport,
|
|
4781
|
-
pipelineIdentification: spaceTrim.spaceTrim(function (block) { return "\n ".concat(block(pipelineIdentification), "\n Template name: ").concat(currentTemplate.name, "\n Template title: ").concat(currentTemplate.title, "\n "); }),
|
|
4782
|
-
})
|
|
4543
|
+
}, $executionReport: executionReport, pipelineIdentification: spaceTrim.spaceTrim(function (block) { return "\n ".concat(block(pipelineIdentification), "\n Template name: ").concat(currentTemplate.name, "\n Template title: ").concat(currentTemplate.title, "\n "); }) }))
|
|
4783
4544
|
.then(function (newParametersToPass) {
|
|
4784
4545
|
parametersToPass = __assign(__assign({}, newParametersToPass), parametersToPass);
|
|
4785
4546
|
resovedParameterNames_1 = __spreadArray(__spreadArray([], __read(resovedParameterNames_1), false), [currentTemplate.resultingParameterName], false);
|
|
@@ -4883,8 +4644,7 @@
|
|
|
4883
4644
|
*/
|
|
4884
4645
|
function createPipelineExecutor(options) {
|
|
4885
4646
|
var _this = this;
|
|
4886
|
-
var pipeline = options.pipeline, tools = options.tools, _a = options.
|
|
4887
|
-
var _b = settings.maxExecutionAttempts, maxExecutionAttempts = _b === void 0 ? MAX_EXECUTION_ATTEMPTS : _b, _c = settings.maxParallelCount, maxParallelCount = _c === void 0 ? MAX_PARALLEL_COUNT : _c, _d = settings.csvSettings, csvSettings = _d === void 0 ? DEFAULT_CSV_SETTINGS : _d, _e = settings.isVerbose, isVerbose = _e === void 0 ? IS_VERBOSE : _e, _f = settings.isNotPreparedWarningSupressed, isNotPreparedWarningSupressed = _f === void 0 ? false : _f, _g = settings.rootDirname, rootDirname = _g === void 0 ? null : _g;
|
|
4647
|
+
var pipeline = options.pipeline, tools = options.tools, _a = options.maxExecutionAttempts, maxExecutionAttempts = _a === void 0 ? DEFAULT_MAX_EXECUTION_ATTEMPTS : _a, _b = options.maxParallelCount, maxParallelCount = _b === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _b, _c = options.csvSettings, csvSettings = _c === void 0 ? DEFAULT_CSV_SETTINGS : _c, _d = options.isVerbose, isVerbose = _d === void 0 ? DEFAULT_IS_VERBOSE : _d, _e = options.isNotPreparedWarningSupressed, isNotPreparedWarningSupressed = _e === void 0 ? false : _e, _f = options.rootDirname, rootDirname = _f === void 0 ? null : _f;
|
|
4888
4648
|
validatePipeline(pipeline);
|
|
4889
4649
|
var pipelineIdentification = (function () {
|
|
4890
4650
|
// Note: This is a 😐 implementation of [🚞]
|
|
@@ -4918,14 +4678,12 @@
|
|
|
4918
4678
|
tools: tools,
|
|
4919
4679
|
onProgress: onProgress,
|
|
4920
4680
|
pipelineIdentification: spaceTrim.spaceTrim(function (block) { return "\n ".concat(block(pipelineIdentification), "\n ").concat(runCount === 1 ? '' : "Run #".concat(runCount), "\n "); }),
|
|
4921
|
-
|
|
4922
|
-
|
|
4923
|
-
|
|
4924
|
-
|
|
4925
|
-
|
|
4926
|
-
|
|
4927
|
-
rootDirname: rootDirname,
|
|
4928
|
-
},
|
|
4681
|
+
maxExecutionAttempts: maxExecutionAttempts,
|
|
4682
|
+
maxParallelCount: maxParallelCount,
|
|
4683
|
+
csvSettings: csvSettings,
|
|
4684
|
+
isVerbose: isVerbose,
|
|
4685
|
+
isNotPreparedWarningSupressed: isNotPreparedWarningSupressed,
|
|
4686
|
+
rootDirname: rootDirname,
|
|
4929
4687
|
})];
|
|
4930
4688
|
});
|
|
4931
4689
|
}); };
|
|
@@ -4948,7 +4706,7 @@
|
|
|
4948
4706
|
return __generator(this, function (_d) {
|
|
4949
4707
|
switch (_d.label) {
|
|
4950
4708
|
case 0:
|
|
4951
|
-
_a = options.isVerbose, isVerbose = _a === void 0 ?
|
|
4709
|
+
_a = options.isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a;
|
|
4952
4710
|
if (tools === undefined || tools.llm === undefined) {
|
|
4953
4711
|
throw new MissingToolsError('LLM tools are required for preparing persona');
|
|
4954
4712
|
}
|
|
@@ -5004,20 +4762,171 @@
|
|
|
5004
4762
|
*/
|
|
5005
4763
|
|
|
5006
4764
|
/**
|
|
5007
|
-
* This error indicates that the promptbook can not retrieve knowledge from external sources
|
|
4765
|
+
* This error indicates that the promptbook can not retrieve knowledge from external sources
|
|
4766
|
+
*
|
|
4767
|
+
* @public exported from `@promptbook/core`
|
|
4768
|
+
*/
|
|
4769
|
+
var KnowledgeScrapeError = /** @class */ (function (_super) {
|
|
4770
|
+
__extends(KnowledgeScrapeError, _super);
|
|
4771
|
+
function KnowledgeScrapeError(message) {
|
|
4772
|
+
var _this = _super.call(this, message) || this;
|
|
4773
|
+
_this.name = 'KnowledgeScrapeError';
|
|
4774
|
+
Object.setPrototypeOf(_this, KnowledgeScrapeError.prototype);
|
|
4775
|
+
return _this;
|
|
4776
|
+
}
|
|
4777
|
+
return KnowledgeScrapeError;
|
|
4778
|
+
}(Error));
|
|
4779
|
+
|
|
4780
|
+
/**
|
|
4781
|
+
* @@@
|
|
4782
|
+
*
|
|
4783
|
+
* Note: `$` is used to indicate that this function is not a pure function - it access global scope
|
|
4784
|
+
*
|
|
4785
|
+
* @private internal function of `$Register`
|
|
4786
|
+
*/
|
|
4787
|
+
function $getGlobalScope() {
|
|
4788
|
+
return Function('return this')();
|
|
4789
|
+
}
|
|
4790
|
+
|
|
4791
|
+
/**
|
|
4792
|
+
* @@@
|
|
4793
|
+
*
|
|
4794
|
+
* @param text @@@
|
|
4795
|
+
* @returns @@@
|
|
4796
|
+
* @example 'HELLO_WORLD'
|
|
4797
|
+
* @example 'I_LOVE_PROMPTBOOK'
|
|
4798
|
+
* @public exported from `@promptbook/utils`
|
|
4799
|
+
*/
|
|
4800
|
+
function normalizeTo_SCREAMING_CASE(text) {
|
|
4801
|
+
var e_1, _a;
|
|
4802
|
+
var charType;
|
|
4803
|
+
var lastCharType = 'OTHER';
|
|
4804
|
+
var normalizedName = '';
|
|
4805
|
+
try {
|
|
4806
|
+
for (var text_1 = __values(text), text_1_1 = text_1.next(); !text_1_1.done; text_1_1 = text_1.next()) {
|
|
4807
|
+
var char = text_1_1.value;
|
|
4808
|
+
var normalizedChar = void 0;
|
|
4809
|
+
if (/^[a-z]$/.test(char)) {
|
|
4810
|
+
charType = 'LOWERCASE';
|
|
4811
|
+
normalizedChar = char.toUpperCase();
|
|
4812
|
+
}
|
|
4813
|
+
else if (/^[A-Z]$/.test(char)) {
|
|
4814
|
+
charType = 'UPPERCASE';
|
|
4815
|
+
normalizedChar = char;
|
|
4816
|
+
}
|
|
4817
|
+
else if (/^[0-9]$/.test(char)) {
|
|
4818
|
+
charType = 'NUMBER';
|
|
4819
|
+
normalizedChar = char;
|
|
4820
|
+
}
|
|
4821
|
+
else {
|
|
4822
|
+
charType = 'OTHER';
|
|
4823
|
+
normalizedChar = '_';
|
|
4824
|
+
}
|
|
4825
|
+
if (charType !== lastCharType &&
|
|
4826
|
+
!(lastCharType === 'UPPERCASE' && charType === 'LOWERCASE') &&
|
|
4827
|
+
!(lastCharType === 'NUMBER') &&
|
|
4828
|
+
!(charType === 'NUMBER')) {
|
|
4829
|
+
normalizedName += '_';
|
|
4830
|
+
}
|
|
4831
|
+
normalizedName += normalizedChar;
|
|
4832
|
+
lastCharType = charType;
|
|
4833
|
+
}
|
|
4834
|
+
}
|
|
4835
|
+
catch (e_1_1) { e_1 = { error: e_1_1 }; }
|
|
4836
|
+
finally {
|
|
4837
|
+
try {
|
|
4838
|
+
if (text_1_1 && !text_1_1.done && (_a = text_1.return)) _a.call(text_1);
|
|
4839
|
+
}
|
|
4840
|
+
finally { if (e_1) throw e_1.error; }
|
|
4841
|
+
}
|
|
4842
|
+
normalizedName = normalizedName.replace(/_+/g, '_');
|
|
4843
|
+
normalizedName = normalizedName.replace(/_?\/_?/g, '/');
|
|
4844
|
+
normalizedName = normalizedName.replace(/^_/, '');
|
|
4845
|
+
normalizedName = normalizedName.replace(/_$/, '');
|
|
4846
|
+
return normalizedName;
|
|
4847
|
+
}
|
|
4848
|
+
/**
|
|
4849
|
+
* TODO: Tests
|
|
4850
|
+
* > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: 'Moje tabule' })).toEqual('/VtG7sR9rRJqwNEdM2/Moje tabule');
|
|
4851
|
+
* > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: 'ěščřžžýáíúů' })).toEqual('/VtG7sR9rRJqwNEdM2/escrzyaieuu');
|
|
4852
|
+
* > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: ' ahoj ' })).toEqual('/VtG7sR9rRJqwNEdM2/ahoj');
|
|
4853
|
+
* > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: ' ahoj_ahojAhoj ahoj ' })).toEqual('/VtG7sR9rRJqwNEdM2/ahoj-ahoj-ahoj-ahoj');
|
|
4854
|
+
* TODO: [🌺] Use some intermediate util splitWords
|
|
4855
|
+
*/
|
|
4856
|
+
|
|
4857
|
+
/**
|
|
4858
|
+
* @@@
|
|
4859
|
+
*
|
|
4860
|
+
* @param text @@@
|
|
4861
|
+
* @returns @@@
|
|
4862
|
+
* @example 'hello_world'
|
|
4863
|
+
* @example 'i_love_promptbook'
|
|
4864
|
+
* @public exported from `@promptbook/utils`
|
|
4865
|
+
*/
|
|
4866
|
+
function normalizeTo_snake_case(text) {
|
|
4867
|
+
return normalizeTo_SCREAMING_CASE(text).toLowerCase();
|
|
4868
|
+
}
|
|
4869
|
+
|
|
4870
|
+
/**
|
|
4871
|
+
* Register is @@@
|
|
4872
|
+
*
|
|
4873
|
+
* Note: `$` is used to indicate that this function is not a pure function - it accesses and adds variables in global scope.
|
|
4874
|
+
*
|
|
4875
|
+
* @private internal utility, exported are only signleton instances of this class
|
|
4876
|
+
*/
|
|
4877
|
+
var $Register = /** @class */ (function () {
|
|
4878
|
+
function $Register(registerName) {
|
|
4879
|
+
this.registerName = registerName;
|
|
4880
|
+
var storageName = "_promptbook_".concat(normalizeTo_snake_case(registerName));
|
|
4881
|
+
var globalScope = $getGlobalScope();
|
|
4882
|
+
if (globalScope[storageName] === undefined) {
|
|
4883
|
+
globalScope[storageName] = [];
|
|
4884
|
+
}
|
|
4885
|
+
else if (!Array.isArray(globalScope[storageName])) {
|
|
4886
|
+
throw new UnexpectedError("Expected (global) ".concat(storageName, " to be an array, but got ").concat(typeof globalScope[storageName]));
|
|
4887
|
+
}
|
|
4888
|
+
this.storage = globalScope[storageName];
|
|
4889
|
+
}
|
|
4890
|
+
$Register.prototype.list = function () {
|
|
4891
|
+
// <- TODO: ReadonlyDeep<ReadonlyArray<TRegistered>>
|
|
4892
|
+
return this.storage;
|
|
4893
|
+
};
|
|
4894
|
+
$Register.prototype.register = function (registered) {
|
|
4895
|
+
var packageName = registered.packageName, className = registered.className;
|
|
4896
|
+
var existingRegistrationIndex = this.storage.findIndex(function (item) { return item.packageName === packageName && item.className === className; });
|
|
4897
|
+
var existingRegistration = this.storage[existingRegistrationIndex];
|
|
4898
|
+
if (!existingRegistration) {
|
|
4899
|
+
this.storage.push(registered);
|
|
4900
|
+
}
|
|
4901
|
+
else {
|
|
4902
|
+
this.storage[existingRegistrationIndex] = registered;
|
|
4903
|
+
}
|
|
4904
|
+
return {
|
|
4905
|
+
registerName: this.registerName,
|
|
4906
|
+
packageName: packageName,
|
|
4907
|
+
className: className,
|
|
4908
|
+
get isDestroyed() {
|
|
4909
|
+
return false;
|
|
4910
|
+
},
|
|
4911
|
+
destroy: function () {
|
|
4912
|
+
throw new NotYetImplementedError("Registration to ".concat(this.registerName, " is permanent in this version of Promptbook"));
|
|
4913
|
+
},
|
|
4914
|
+
};
|
|
4915
|
+
};
|
|
4916
|
+
return $Register;
|
|
4917
|
+
}());
|
|
4918
|
+
|
|
4919
|
+
/**
|
|
4920
|
+
* @@@
|
|
5008
4921
|
*
|
|
4922
|
+
* Note: `$` is used to indicate that this interacts with the global scope
|
|
4923
|
+
* @singleton Only one instance of each register is created per build, but thare can be more @@@
|
|
5009
4924
|
* @public exported from `@promptbook/core`
|
|
5010
4925
|
*/
|
|
5011
|
-
var
|
|
5012
|
-
|
|
5013
|
-
|
|
5014
|
-
|
|
5015
|
-
_this.name = 'KnowledgeScrapeError';
|
|
5016
|
-
Object.setPrototypeOf(_this, KnowledgeScrapeError.prototype);
|
|
5017
|
-
return _this;
|
|
5018
|
-
}
|
|
5019
|
-
return KnowledgeScrapeError;
|
|
5020
|
-
}(Error));
|
|
4926
|
+
var $scrapersMetadataRegister = new $Register('scrapers_metadata');
|
|
4927
|
+
/**
|
|
4928
|
+
* TODO: [®] DRY Register logic
|
|
4929
|
+
*/
|
|
5021
4930
|
|
|
5022
4931
|
/**
|
|
5023
4932
|
* @@@
|
|
@@ -5026,12 +4935,12 @@
|
|
|
5026
4935
|
* @singleton Only one instance of each register is created per build, but thare can be more @@@
|
|
5027
4936
|
* @public exported from `@promptbook/core`
|
|
5028
4937
|
*/
|
|
5029
|
-
var $
|
|
4938
|
+
var $scrapersRegister = new $Register('scraper_constructors');
|
|
5030
4939
|
/**
|
|
5031
4940
|
* TODO: [®] DRY Register logic
|
|
5032
4941
|
*/
|
|
5033
4942
|
|
|
5034
|
-
// TODO: !!!!!! Maybe delete this function
|
|
4943
|
+
// TODO: !!!!!!last - Maybe delete this function
|
|
5035
4944
|
/**
|
|
5036
4945
|
* Creates a message with all registered scrapers
|
|
5037
4946
|
*
|
|
@@ -5198,7 +5107,6 @@
|
|
|
5198
5107
|
* @private within the repository
|
|
5199
5108
|
*/
|
|
5200
5109
|
function sourceContentToName(sourceContent) {
|
|
5201
|
-
// TODO: !!!!!! Better name for source than gibberish hash
|
|
5202
5110
|
var hash = cryptoJs.SHA256(hexEncoder__default["default"].parse(JSON.stringify(sourceContent)))
|
|
5203
5111
|
// <- TODO: [🥬] Encapsulate sha256 to some private utility function
|
|
5204
5112
|
.toString( /* hex */)
|
|
@@ -5311,7 +5219,7 @@
|
|
|
5311
5219
|
case 0:
|
|
5312
5220
|
sourceContent = knowledgeSource.sourceContent;
|
|
5313
5221
|
name = knowledgeSource.name;
|
|
5314
|
-
_b = options || {}, _c = _b.rootDirname, rootDirname = _c === void 0 ? null : _c, _d = _b.isVerbose, isVerbose = _d === void 0 ?
|
|
5222
|
+
_b = options || {}, _c = _b.rootDirname, rootDirname = _c === void 0 ? null : _c, _d = _b.isVerbose, isVerbose = _d === void 0 ? DEFAULT_IS_VERBOSE : _d;
|
|
5315
5223
|
TODO_USE(isVerbose);
|
|
5316
5224
|
if (!name) {
|
|
5317
5225
|
name = sourceContentToName(sourceContent);
|
|
@@ -5401,7 +5309,7 @@
|
|
|
5401
5309
|
content = _a.sent();
|
|
5402
5310
|
return [2 /*return*/, new Blob([
|
|
5403
5311
|
content,
|
|
5404
|
-
// <- TODO: !!!!!!
|
|
5312
|
+
// <- TODO: !!!!!! Test that this is working
|
|
5405
5313
|
], { type: mimeType_1 })];
|
|
5406
5314
|
}
|
|
5407
5315
|
});
|
|
@@ -5464,7 +5372,7 @@
|
|
|
5464
5372
|
return __generator(this, function (_c) {
|
|
5465
5373
|
switch (_c.label) {
|
|
5466
5374
|
case 0:
|
|
5467
|
-
_a = options.maxParallelCount, maxParallelCount = _a === void 0 ?
|
|
5375
|
+
_a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a, rootDirname = options.rootDirname, _b = options.isVerbose, isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b;
|
|
5468
5376
|
knowledgePreparedUnflatten = new Array(knowledgeSources.length);
|
|
5469
5377
|
return [4 /*yield*/, forEachAsync(knowledgeSources, { maxParallelCount: maxParallelCount }, function (knowledgeSource, index) { return __awaiter(_this, void 0, void 0, function () {
|
|
5470
5378
|
var partialPieces, sourceHandler, _a, _b, scraper, partialPiecesUnchecked, e_1_1, pieces;
|
|
@@ -5493,7 +5401,8 @@
|
|
|
5493
5401
|
case 4:
|
|
5494
5402
|
partialPiecesUnchecked = _d.sent();
|
|
5495
5403
|
if (partialPiecesUnchecked !== null) {
|
|
5496
|
-
partialPieces = partialPiecesUnchecked;
|
|
5404
|
+
partialPieces = __spreadArray([], __read(partialPiecesUnchecked), false);
|
|
5405
|
+
// <- TODO: [🪓] Here should be no need for spreading new array, just `partialPieces = partialPiecesUnchecked`
|
|
5497
5406
|
return [3 /*break*/, 6];
|
|
5498
5407
|
}
|
|
5499
5408
|
_d.label = 5;
|
|
@@ -5541,7 +5450,7 @@
|
|
|
5541
5450
|
> /**
|
|
5542
5451
|
> * Unprepared knowledge
|
|
5543
5452
|
> * /
|
|
5544
|
-
> readonly knowledgeSources:
|
|
5453
|
+
> readonly knowledgeSources: ReadonlyArray<KnowledgeSourceJson>;
|
|
5545
5454
|
> };
|
|
5546
5455
|
>
|
|
5547
5456
|
> export async function prepareKnowledgePieces(
|
|
@@ -5599,7 +5508,7 @@
|
|
|
5599
5508
|
return __generator(this, function (_b) {
|
|
5600
5509
|
switch (_b.label) {
|
|
5601
5510
|
case 0:
|
|
5602
|
-
_a = options.maxParallelCount, maxParallelCount = _a === void 0 ?
|
|
5511
|
+
_a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a;
|
|
5603
5512
|
templates = pipeline.templates, parameters = pipeline.parameters, knowledgePiecesCount = pipeline.knowledgePiecesCount;
|
|
5604
5513
|
// TODO: [main] !!!!! Apply samples to each template (if missing and is for the template defined)
|
|
5605
5514
|
TODO_USE(parameters);
|
|
@@ -5661,7 +5570,7 @@
|
|
|
5661
5570
|
if (isPipelinePrepared(pipeline)) {
|
|
5662
5571
|
return [2 /*return*/, pipeline];
|
|
5663
5572
|
}
|
|
5664
|
-
rootDirname = options.rootDirname, _a = options.maxParallelCount, maxParallelCount = _a === void 0 ?
|
|
5573
|
+
rootDirname = options.rootDirname, _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a, _b = options.isVerbose, isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b;
|
|
5665
5574
|
parameters = pipeline.parameters, templates = pipeline.templates, knowledgeSources = pipeline.knowledgeSources, personas = pipeline.personas;
|
|
5666
5575
|
if (tools === undefined || tools.llm === undefined) {
|
|
5667
5576
|
throw new MissingToolsError('LLM tools are required for preparing the pipeline');
|
|
@@ -5719,7 +5628,9 @@
|
|
|
5719
5628
|
// ----- /Templates preparation -----
|
|
5720
5629
|
// Note: Count total usage
|
|
5721
5630
|
currentPreparation.usage = llmToolsWithUsage.getTotalUsage();
|
|
5722
|
-
return [2 /*return*/, $asDeeplyFrozenSerializableJson('Prepared PipelineJson', __assign(__assign({}, clonePipeline(pipeline)), { templates:
|
|
5631
|
+
return [2 /*return*/, $asDeeplyFrozenSerializableJson('Prepared PipelineJson', __assign(__assign({}, clonePipeline(pipeline)), { templates: __spreadArray([], __read(templatesPrepared), false),
|
|
5632
|
+
// <- TODO: [🪓] Here should be no need for spreading new array, just ` templates: templatesPrepared`
|
|
5633
|
+
knowledgeSources: knowledgeSourcesPrepared, knowledgePieces: knowledgePiecesPrepared, personas: preparedPersonas, preparations: __spreadArray([], __read(preparations), false) }))];
|
|
5723
5634
|
}
|
|
5724
5635
|
});
|
|
5725
5636
|
});
|
|
@@ -8643,7 +8554,7 @@
|
|
|
8643
8554
|
*/
|
|
8644
8555
|
function createLlmToolsFromConfiguration(configuration, options) {
|
|
8645
8556
|
if (options === void 0) { options = {}; }
|
|
8646
|
-
var _a = options.isVerbose, isVerbose = _a === void 0 ?
|
|
8557
|
+
var _a = options.isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a, userId = options.userId;
|
|
8647
8558
|
var llmTools = configuration.map(function (llmConfiguration) {
|
|
8648
8559
|
var registeredItem = $llmToolsRegister
|
|
8649
8560
|
.list()
|
|
@@ -8704,6 +8615,115 @@
|
|
|
8704
8615
|
* TODO: [®] DRY Register logic
|
|
8705
8616
|
*/
|
|
8706
8617
|
|
|
8618
|
+
/**
|
|
8619
|
+
* @@@
|
|
8620
|
+
*
|
|
8621
|
+
* @public exported from `@promptbook/node`
|
|
8622
|
+
*/
|
|
8623
|
+
function $provideExecutablesForNode(options) {
|
|
8624
|
+
return __awaiter(this, void 0, void 0, function () {
|
|
8625
|
+
var _a, _b, isAutoInstalled, _c, isVerbose;
|
|
8626
|
+
return __generator(this, function (_d) {
|
|
8627
|
+
if (!$isRunningInNode()) {
|
|
8628
|
+
throw new EnvironmentMismatchError('Function `$getScrapersForNode` works only in Node.js environment');
|
|
8629
|
+
}
|
|
8630
|
+
_a = options || {}, _b = _a.isAutoInstalled, isAutoInstalled = _b === void 0 ? DEFAULT_IS_AUTO_INSTALLED : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? DEFAULT_IS_VERBOSE : _c;
|
|
8631
|
+
TODO_USE(isAutoInstalled);
|
|
8632
|
+
TODO_USE(isVerbose);
|
|
8633
|
+
return [2 /*return*/, {
|
|
8634
|
+
// TODO: !!!!!! use `locate-app` library here
|
|
8635
|
+
pandocPath: 'C:/Users/me/AppData/Local/Pandoc/pandoc.exe',
|
|
8636
|
+
libreOfficePath: 'C:/Program Files/LibreOffice/program/swriter.exe',
|
|
8637
|
+
}];
|
|
8638
|
+
});
|
|
8639
|
+
});
|
|
8640
|
+
}
|
|
8641
|
+
/**
|
|
8642
|
+
* TODO: [🧠] THis should be maybe in different folder
|
|
8643
|
+
* Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
|
|
8644
|
+
*/
|
|
8645
|
+
|
|
8646
|
+
/**
|
|
8647
|
+
* @@@
|
|
8648
|
+
*
|
|
8649
|
+
* @public exported from `@promptbook/node`
|
|
8650
|
+
*/
|
|
8651
|
+
function $provideFilesystemForNode(options) {
|
|
8652
|
+
if (!$isRunningInNode()) {
|
|
8653
|
+
throw new EnvironmentMismatchError('Function `$provideFilesystemForNode` works only in Node.js environment');
|
|
8654
|
+
}
|
|
8655
|
+
var _a = (options || {}).isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a;
|
|
8656
|
+
TODO_USE(isVerbose);
|
|
8657
|
+
return {
|
|
8658
|
+
stat: promises.stat,
|
|
8659
|
+
access: promises.access,
|
|
8660
|
+
constants: promises.constants,
|
|
8661
|
+
readFile: promises.readFile,
|
|
8662
|
+
readdir: promises.readdir,
|
|
8663
|
+
};
|
|
8664
|
+
}
|
|
8665
|
+
/**
|
|
8666
|
+
* Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
|
|
8667
|
+
*/
|
|
8668
|
+
|
|
8669
|
+
/**
|
|
8670
|
+
* @@@
|
|
8671
|
+
*
|
|
8672
|
+
* 1) @@@
|
|
8673
|
+
* 2) @@@
|
|
8674
|
+
*
|
|
8675
|
+
* @public exported from `@promptbook/node`
|
|
8676
|
+
*/
|
|
8677
|
+
function $provideScrapersForNode(tools, options) {
|
|
8678
|
+
return __awaiter(this, void 0, void 0, function () {
|
|
8679
|
+
var _a, _b, isAutoInstalled, _c, isVerbose, scrapers, _d, _e, scraperFactory, scraper, e_1_1;
|
|
8680
|
+
var e_1, _f;
|
|
8681
|
+
return __generator(this, function (_g) {
|
|
8682
|
+
switch (_g.label) {
|
|
8683
|
+
case 0:
|
|
8684
|
+
if (!$isRunningInNode()) {
|
|
8685
|
+
throw new EnvironmentMismatchError('Function `$getScrapersForNode` works only in Node.js environment');
|
|
8686
|
+
}
|
|
8687
|
+
_a = options || {}, _b = _a.isAutoInstalled, isAutoInstalled = _b === void 0 ? DEFAULT_IS_AUTO_INSTALLED : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? DEFAULT_IS_VERBOSE : _c;
|
|
8688
|
+
TODO_USE(isAutoInstalled);
|
|
8689
|
+
TODO_USE(isVerbose);
|
|
8690
|
+
scrapers = [];
|
|
8691
|
+
_g.label = 1;
|
|
8692
|
+
case 1:
|
|
8693
|
+
_g.trys.push([1, 6, 7, 8]);
|
|
8694
|
+
_d = __values($scrapersRegister.list()), _e = _d.next();
|
|
8695
|
+
_g.label = 2;
|
|
8696
|
+
case 2:
|
|
8697
|
+
if (!!_e.done) return [3 /*break*/, 5];
|
|
8698
|
+
scraperFactory = _e.value;
|
|
8699
|
+
return [4 /*yield*/, scraperFactory(tools, options || {})];
|
|
8700
|
+
case 3:
|
|
8701
|
+
scraper = _g.sent();
|
|
8702
|
+
scrapers.push(scraper);
|
|
8703
|
+
_g.label = 4;
|
|
8704
|
+
case 4:
|
|
8705
|
+
_e = _d.next();
|
|
8706
|
+
return [3 /*break*/, 2];
|
|
8707
|
+
case 5: return [3 /*break*/, 8];
|
|
8708
|
+
case 6:
|
|
8709
|
+
e_1_1 = _g.sent();
|
|
8710
|
+
e_1 = { error: e_1_1 };
|
|
8711
|
+
return [3 /*break*/, 8];
|
|
8712
|
+
case 7:
|
|
8713
|
+
try {
|
|
8714
|
+
if (_e && !_e.done && (_f = _d.return)) _f.call(_d);
|
|
8715
|
+
}
|
|
8716
|
+
finally { if (e_1) throw e_1.error; }
|
|
8717
|
+
return [7 /*endfinally*/];
|
|
8718
|
+
case 8: return [2 /*return*/, scrapers];
|
|
8719
|
+
}
|
|
8720
|
+
});
|
|
8721
|
+
});
|
|
8722
|
+
}
|
|
8723
|
+
/**
|
|
8724
|
+
* Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
|
|
8725
|
+
*/
|
|
8726
|
+
|
|
8707
8727
|
/**
|
|
8708
8728
|
* Extracts code block from markdown.
|
|
8709
8729
|
*
|
|
@@ -9158,7 +9178,7 @@
|
|
|
9158
9178
|
*/
|
|
9159
9179
|
function $provideExecutionToolsForNode(options) {
|
|
9160
9180
|
return __awaiter(this, void 0, void 0, function () {
|
|
9161
|
-
var fs, llm, tools;
|
|
9181
|
+
var fs, llm, executables, tools;
|
|
9162
9182
|
var _a;
|
|
9163
9183
|
return __generator(this, function (_b) {
|
|
9164
9184
|
switch (_b.label) {
|
|
@@ -9168,12 +9188,16 @@
|
|
|
9168
9188
|
}
|
|
9169
9189
|
fs = $provideFilesystemForNode();
|
|
9170
9190
|
llm = $provideLlmToolsFromEnv(options);
|
|
9191
|
+
return [4 /*yield*/, $provideExecutablesForNode(options)];
|
|
9192
|
+
case 1:
|
|
9193
|
+
executables = _b.sent();
|
|
9171
9194
|
_a = {
|
|
9172
9195
|
llm: llm,
|
|
9173
|
-
fs: fs
|
|
9196
|
+
fs: fs,
|
|
9197
|
+
executables: executables
|
|
9174
9198
|
};
|
|
9175
|
-
return [4 /*yield*/, $provideScrapersForNode({ fs: fs, llm: llm }, options)];
|
|
9176
|
-
case
|
|
9199
|
+
return [4 /*yield*/, $provideScrapersForNode({ fs: fs, llm: llm, executables: executables }, options)];
|
|
9200
|
+
case 2:
|
|
9177
9201
|
tools = (_a.scrapers = _b.sent(),
|
|
9178
9202
|
_a.script = [new JavascriptExecutionTools(options)],
|
|
9179
9203
|
_a);
|
|
@@ -9411,7 +9435,9 @@
|
|
|
9411
9435
|
throw new EnvironmentMismatchError('Can not create collection without filesystem tools');
|
|
9412
9436
|
// <- TODO: [🧠] What is the best error type here`
|
|
9413
9437
|
}
|
|
9414
|
-
makedLibraryFilePath = path.join(path$1, "".concat(
|
|
9438
|
+
makedLibraryFilePath = path.join(path$1, "".concat(DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME
|
|
9439
|
+
// <- TODO: [🦒] Allow to override (pass different value into the function)
|
|
9440
|
+
, ".json"));
|
|
9415
9441
|
return [4 /*yield*/, isFileExisting(makedLibraryFilePath, tools.fs)];
|
|
9416
9442
|
case 3:
|
|
9417
9443
|
if (!(_f.sent())) {
|
|
@@ -9422,7 +9448,7 @@
|
|
|
9422
9448
|
// TODO: !! Implement;
|
|
9423
9449
|
// TODO: [🌗]
|
|
9424
9450
|
}
|
|
9425
|
-
_a = options || {}, _b = _a.isRecursive, isRecursive = _b === void 0 ? true : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ?
|
|
9451
|
+
_a = options || {}, _b = _a.isRecursive, isRecursive = _b === void 0 ? true : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? DEFAULT_IS_VERBOSE : _c, _d = _a.isLazyLoaded, isLazyLoaded = _d === void 0 ? false : _d, _e = _a.isCrashedOnError, isCrashedOnError = _e === void 0 ? true : _e;
|
|
9426
9452
|
collection = createCollectionFromPromise(function () { return __awaiter(_this, void 0, void 0, function () {
|
|
9427
9453
|
var fileNames, collection, _loop_1, fileNames_1, fileNames_1_1, fileName, e_1_1;
|
|
9428
9454
|
var e_1, _a;
|
|
@@ -9894,7 +9920,7 @@
|
|
|
9894
9920
|
function cacheLlmTools(llmTools, options) {
|
|
9895
9921
|
var _this = this;
|
|
9896
9922
|
if (options === void 0) { options = {}; }
|
|
9897
|
-
var _a = options.storage, storage = _a === void 0 ? new MemoryStorage() : _a, _b = options.
|
|
9923
|
+
var _a = options.storage, storage = _a === void 0 ? new MemoryStorage() : _a, _b = options.isCacheReloaded, isCacheReloaded = _b === void 0 ? false : _b;
|
|
9898
9924
|
var proxyTools = __assign(__assign({}, llmTools), {
|
|
9899
9925
|
// <- Note: [🥫]
|
|
9900
9926
|
get title() {
|
|
@@ -9916,7 +9942,7 @@
|
|
|
9916
9942
|
key = titleToName(prompt.title.substring(0, MAX_FILENAME_LENGTH - 10) +
|
|
9917
9943
|
'-' +
|
|
9918
9944
|
sha256__default["default"](hexEncoder__default["default"].parse(JSON.stringify(prompt.parameters))).toString( /* hex */));
|
|
9919
|
-
if (!!
|
|
9945
|
+
if (!!isCacheReloaded) return [3 /*break*/, 2];
|
|
9920
9946
|
return [4 /*yield*/, storage.getItem(key)];
|
|
9921
9947
|
case 1:
|
|
9922
9948
|
_a = _c.sent();
|
|
@@ -10002,12 +10028,14 @@
|
|
|
10002
10028
|
if (!$isRunningInNode()) {
|
|
10003
10029
|
throw new EnvironmentMismatchError('Function `$provideLlmToolsForTestingAndScriptsAndPlayground` works only in Node.js environment');
|
|
10004
10030
|
}
|
|
10005
|
-
var
|
|
10031
|
+
var isCacheReloaded = (options !== null && options !== void 0 ? options : {}).isCacheReloaded;
|
|
10006
10032
|
return cacheLlmTools(countTotalUsage(
|
|
10007
10033
|
// <- Note: for example here we don`t want the [🌯]
|
|
10008
10034
|
$provideLlmToolsFromEnv()), {
|
|
10009
|
-
storage: new FileCacheStorage({ fs: $provideFilesystemForNode() }, {
|
|
10010
|
-
|
|
10035
|
+
storage: new FileCacheStorage({ fs: $provideFilesystemForNode() }, {
|
|
10036
|
+
rootFolderPath: path.join(process.cwd(), DEFAULT_EXECUTIONS_CACHE_DIRNAME),
|
|
10037
|
+
}),
|
|
10038
|
+
isCacheReloaded: isCacheReloaded,
|
|
10011
10039
|
});
|
|
10012
10040
|
}
|
|
10013
10041
|
/**
|
|
@@ -10034,13 +10062,13 @@
|
|
|
10034
10062
|
makeCommand.option('-f, --format <format>', spaceTrim__default["default"]("\n Output format of builded collection \"javascript\", \"typescript\" or \"json\"\n\n Note: You can use multiple formats separated by comma\n "), 'javascript' /* <- Note: [🏳🌈] */);
|
|
10035
10063
|
makeCommand.option('--no-validation', "Do not validate logic of pipelines in collection", true);
|
|
10036
10064
|
makeCommand.option('--validation', "Types of validations separated by comma (options \"logic\",\"imports\")", 'logic,imports');
|
|
10037
|
-
makeCommand.option('--reload
|
|
10065
|
+
makeCommand.option('--reload', "Call LLM models even if same prompt with result is in the cache", false);
|
|
10038
10066
|
makeCommand.option('--verbose', "Is output verbose", false);
|
|
10039
|
-
makeCommand.option('-o, --out-file <path>', spaceTrim__default["default"]("\n Where to save the builded collection\n\n Note: If you keep it \"".concat(
|
|
10067
|
+
makeCommand.option('-o, --out-file <path>', spaceTrim__default["default"]("\n Where to save the builded collection\n\n Note: If you keep it \"".concat(DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME, "\" it will be saved in the root of the promptbook directory\n If you set it to a path, it will be saved in that path\n BUT you can use only one format and set correct extension\n ")), DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME);
|
|
10040
10068
|
makeCommand.action(function (path$1, _a) {
|
|
10041
|
-
var projectName = _a.projectName, format = _a.format, validation = _a.validation,
|
|
10069
|
+
var projectName = _a.projectName, format = _a.format, validation = _a.validation, isCacheReloaded = _a.reloadCache, isVerbose = _a.verbose, outFile = _a.outFile;
|
|
10042
10070
|
return __awaiter(_this, void 0, void 0, function () {
|
|
10043
|
-
var formats, validations, options, fs, llm, tools, collection, validations_1, validations_1_1, validation_1, _b, _c, pipelineUrl, pipeline, e_1_1, e_2_1, collectionJson, collectionJsonString, collectionJsonItems, saveFile;
|
|
10071
|
+
var formats, validations, options, fs, llm, executables, tools, collection, validations_1, validations_1_1, validation_1, _b, _c, pipelineUrl, pipeline, e_1_1, e_2_1, collectionJson, collectionJsonString, collectionJsonItems, saveFile;
|
|
10044
10072
|
var _d, e_2, _e, e_1, _f;
|
|
10045
10073
|
var _this = this;
|
|
10046
10074
|
return __generator(this, function (_g) {
|
|
@@ -10054,22 +10082,25 @@
|
|
|
10054
10082
|
.split(',')
|
|
10055
10083
|
.map(function (_) { return _.trim(); })
|
|
10056
10084
|
.filter(function (_) { return _ !== ''; });
|
|
10057
|
-
if (outFile !==
|
|
10085
|
+
if (outFile !== DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME && formats.length !== 1) {
|
|
10058
10086
|
console.error(colors__default["default"].red("You can only use one format if you specify --out-file"));
|
|
10059
10087
|
process.exit(1);
|
|
10060
10088
|
}
|
|
10061
10089
|
options = {
|
|
10062
10090
|
isVerbose: isVerbose,
|
|
10063
|
-
|
|
10091
|
+
isCacheReloaded: isCacheReloaded,
|
|
10064
10092
|
};
|
|
10065
10093
|
fs = $provideFilesystemForNode(options);
|
|
10066
10094
|
llm = $provideLlmToolsForCli(options);
|
|
10095
|
+
return [4 /*yield*/, $provideExecutablesForNode(options)];
|
|
10096
|
+
case 1:
|
|
10097
|
+
executables = _g.sent();
|
|
10067
10098
|
_d = {
|
|
10068
10099
|
llm: llm,
|
|
10069
10100
|
fs: fs
|
|
10070
10101
|
};
|
|
10071
|
-
return [4 /*yield*/, $provideScrapersForNode({ fs: fs, llm: llm }, options)];
|
|
10072
|
-
case
|
|
10102
|
+
return [4 /*yield*/, $provideScrapersForNode({ fs: fs, llm: llm, executables: executables }, options)];
|
|
10103
|
+
case 2:
|
|
10073
10104
|
tools = (_d.scrapers = _g.sent(),
|
|
10074
10105
|
_d.script = [
|
|
10075
10106
|
/*new JavascriptExecutionTools(options)*/
|
|
@@ -10078,31 +10109,31 @@
|
|
|
10078
10109
|
return [4 /*yield*/, createCollectionFromDirectory(path$1, tools, {
|
|
10079
10110
|
isVerbose: isVerbose,
|
|
10080
10111
|
isRecursive: true,
|
|
10081
|
-
// <- TODO: [🍖]
|
|
10112
|
+
// <- TODO: [🍖] Add `intermediateFilesStrategy`
|
|
10082
10113
|
})];
|
|
10083
|
-
case 2:
|
|
10084
|
-
collection = _g.sent();
|
|
10085
|
-
_g.label = 3;
|
|
10086
10114
|
case 3:
|
|
10087
|
-
_g.
|
|
10088
|
-
validations_1 = __values(validations), validations_1_1 = validations_1.next();
|
|
10115
|
+
collection = _g.sent();
|
|
10089
10116
|
_g.label = 4;
|
|
10090
10117
|
case 4:
|
|
10091
|
-
|
|
10092
|
-
|
|
10118
|
+
_g.trys.push([4, 16, 17, 18]);
|
|
10119
|
+
validations_1 = __values(validations), validations_1_1 = validations_1.next();
|
|
10093
10120
|
_g.label = 5;
|
|
10094
10121
|
case 5:
|
|
10095
|
-
|
|
10122
|
+
if (!!validations_1_1.done) return [3 /*break*/, 15];
|
|
10123
|
+
validation_1 = validations_1_1.value;
|
|
10124
|
+
_g.label = 6;
|
|
10125
|
+
case 6:
|
|
10126
|
+
_g.trys.push([6, 12, 13, 14]);
|
|
10096
10127
|
e_1 = void 0;
|
|
10097
10128
|
return [4 /*yield*/, collection.listPipelines()];
|
|
10098
|
-
case 6:
|
|
10099
|
-
_b = (__values.apply(void 0, [_g.sent()])), _c = _b.next();
|
|
10100
|
-
_g.label = 7;
|
|
10101
10129
|
case 7:
|
|
10102
|
-
|
|
10130
|
+
_b = (__values.apply(void 0, [_g.sent()])), _c = _b.next();
|
|
10131
|
+
_g.label = 8;
|
|
10132
|
+
case 8:
|
|
10133
|
+
if (!!_c.done) return [3 /*break*/, 11];
|
|
10103
10134
|
pipelineUrl = _c.value;
|
|
10104
10135
|
return [4 /*yield*/, collection.getPipelineByUrl(pipelineUrl)];
|
|
10105
|
-
case
|
|
10136
|
+
case 9:
|
|
10106
10137
|
pipeline = _g.sent();
|
|
10107
10138
|
if (validation_1 === 'logic') {
|
|
10108
10139
|
validatePipeline(pipeline);
|
|
@@ -10110,37 +10141,37 @@
|
|
|
10110
10141
|
console.info(colors__default["default"].cyan("Validated logic of ".concat(pipeline.pipelineUrl)));
|
|
10111
10142
|
}
|
|
10112
10143
|
}
|
|
10113
|
-
_g.label =
|
|
10114
|
-
case
|
|
10144
|
+
_g.label = 10;
|
|
10145
|
+
case 10:
|
|
10115
10146
|
_c = _b.next();
|
|
10116
|
-
return [3 /*break*/,
|
|
10117
|
-
case
|
|
10118
|
-
case
|
|
10147
|
+
return [3 /*break*/, 8];
|
|
10148
|
+
case 11: return [3 /*break*/, 14];
|
|
10149
|
+
case 12:
|
|
10119
10150
|
e_1_1 = _g.sent();
|
|
10120
10151
|
e_1 = { error: e_1_1 };
|
|
10121
|
-
return [3 /*break*/,
|
|
10122
|
-
case
|
|
10152
|
+
return [3 /*break*/, 14];
|
|
10153
|
+
case 13:
|
|
10123
10154
|
try {
|
|
10124
10155
|
if (_c && !_c.done && (_f = _b.return)) _f.call(_b);
|
|
10125
10156
|
}
|
|
10126
10157
|
finally { if (e_1) throw e_1.error; }
|
|
10127
10158
|
return [7 /*endfinally*/];
|
|
10128
|
-
case
|
|
10159
|
+
case 14:
|
|
10129
10160
|
validations_1_1 = validations_1.next();
|
|
10130
|
-
return [3 /*break*/,
|
|
10131
|
-
case
|
|
10132
|
-
case
|
|
10161
|
+
return [3 /*break*/, 5];
|
|
10162
|
+
case 15: return [3 /*break*/, 18];
|
|
10163
|
+
case 16:
|
|
10133
10164
|
e_2_1 = _g.sent();
|
|
10134
10165
|
e_2 = { error: e_2_1 };
|
|
10135
|
-
return [3 /*break*/,
|
|
10136
|
-
case
|
|
10166
|
+
return [3 /*break*/, 18];
|
|
10167
|
+
case 17:
|
|
10137
10168
|
try {
|
|
10138
10169
|
if (validations_1_1 && !validations_1_1.done && (_e = validations_1.return)) _e.call(validations_1);
|
|
10139
10170
|
}
|
|
10140
10171
|
finally { if (e_2) throw e_2.error; }
|
|
10141
10172
|
return [7 /*endfinally*/];
|
|
10142
|
-
case
|
|
10143
|
-
case
|
|
10173
|
+
case 18: return [4 /*yield*/, collectionToJson(collection)];
|
|
10174
|
+
case 19:
|
|
10144
10175
|
collectionJson = _g.sent();
|
|
10145
10176
|
collectionJsonString = stringifyPipelineJson(collectionJson).trim();
|
|
10146
10177
|
collectionJsonItems = (function () {
|
|
@@ -10159,9 +10190,9 @@
|
|
|
10159
10190
|
return __generator(this, function (_a) {
|
|
10160
10191
|
switch (_a.label) {
|
|
10161
10192
|
case 0:
|
|
10162
|
-
filename = outFile !==
|
|
10193
|
+
filename = outFile !== DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME
|
|
10163
10194
|
? outFile
|
|
10164
|
-
: path.join(path$1, "".concat(
|
|
10195
|
+
: path.join(path$1, "".concat(DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME, ".").concat(extension));
|
|
10165
10196
|
if (!outFile.endsWith(".".concat(extension))) {
|
|
10166
10197
|
console.warn(colors__default["default"].yellow("Warning: Extension of output file should be \"".concat(extension, "\"")));
|
|
10167
10198
|
}
|
|
@@ -10177,27 +10208,27 @@
|
|
|
10177
10208
|
}
|
|
10178
10209
|
});
|
|
10179
10210
|
}); };
|
|
10180
|
-
if (!formats.includes('json')) return [3 /*break*/,
|
|
10211
|
+
if (!formats.includes('json')) return [3 /*break*/, 21];
|
|
10181
10212
|
formats = formats.filter(function (format) { return format !== 'json'; });
|
|
10182
10213
|
return [4 /*yield*/, saveFile('json', collectionJsonString)];
|
|
10183
|
-
case 19:
|
|
10184
|
-
_g.sent();
|
|
10185
|
-
_g.label = 20;
|
|
10186
10214
|
case 20:
|
|
10187
|
-
|
|
10215
|
+
_g.sent();
|
|
10216
|
+
_g.label = 21;
|
|
10217
|
+
case 21:
|
|
10218
|
+
if (!(formats.includes('javascript') || formats.includes('js'))) return [3 /*break*/, 23];
|
|
10188
10219
|
formats = formats.filter(function (format) { return format !== 'javascript' && format !== 'js'; });
|
|
10189
10220
|
return [4 /*yield*/, saveFile('js', spaceTrim__default["default"](function (block) { return "\n // ".concat(block(GENERATOR_WARNING_BY_PROMPTBOOK_CLI), "\n\n import { createCollectionFromJson } from '@promptbook/core';\n\n /**\n * Pipeline collection for ").concat(projectName, "\n *\n * ").concat(block(GENERATOR_WARNING_BY_PROMPTBOOK_CLI), "\n *\n * @private internal cache for `getPipelineCollection`\n */\n let pipelineCollection = null;\n\n\n /**\n * Get pipeline collection for ").concat(projectName, "\n *\n * ").concat(block(GENERATOR_WARNING_BY_PROMPTBOOK_CLI), "\n *\n * @returns {PipelineCollection} Library of promptbooks for ").concat(projectName, "\n */\n export function getPipelineCollection(){\n if(pipelineCollection===null){\n pipelineCollection = createCollectionFromJson(\n ").concat(block(collectionJsonItems), "\n );\n }\n\n return pipelineCollection;\n }\n "); }))];
|
|
10190
|
-
case 21:
|
|
10191
|
-
(_g.sent()) + '\n';
|
|
10192
|
-
_g.label = 22;
|
|
10193
10221
|
case 22:
|
|
10194
|
-
|
|
10222
|
+
(_g.sent()) + '\n';
|
|
10223
|
+
_g.label = 23;
|
|
10224
|
+
case 23:
|
|
10225
|
+
if (!(formats.includes('typescript') || formats.includes('ts'))) return [3 /*break*/, 25];
|
|
10195
10226
|
formats = formats.filter(function (format) { return format !== 'typescript' && format !== 'ts'; });
|
|
10196
10227
|
return [4 /*yield*/, saveFile('ts', spaceTrim__default["default"](function (block) { return "\n // ".concat(block(GENERATOR_WARNING_BY_PROMPTBOOK_CLI), "\n\n import { createCollectionFromJson } from '@promptbook/core';\n import type { PipelineCollection } from '@promptbook/types';\n\n /**\n * Pipeline collection for ").concat(projectName, "\n *\n * ").concat(block(GENERATOR_WARNING_BY_PROMPTBOOK_CLI), "\n *\n * @private internal cache for `getPipelineCollection`\n */\n let pipelineCollection: null | PipelineCollection = null;\n\n\n /**\n * Get pipeline collection for ").concat(projectName, "\n *\n * ").concat(block(GENERATOR_WARNING_BY_PROMPTBOOK_CLI), "\n *\n * @returns {PipelineCollection} Library of promptbooks for ").concat(projectName, "\n */\n export function getPipelineCollection(): PipelineCollection{\n if(pipelineCollection===null){\n pipelineCollection = createCollectionFromJson(\n ").concat(block(collectionJsonItems), "\n );\n }\n\n return pipelineCollection;\n }\n "); }) + '\n')];
|
|
10197
|
-
case 23:
|
|
10198
|
-
_g.sent();
|
|
10199
|
-
_g.label = 24;
|
|
10200
10228
|
case 24:
|
|
10229
|
+
_g.sent();
|
|
10230
|
+
_g.label = 25;
|
|
10231
|
+
case 25:
|
|
10201
10232
|
if (formats.length > 0) {
|
|
10202
10233
|
console.warn(colors__default["default"].yellow("Format ".concat(formats.join(' and '), " is not supported")));
|
|
10203
10234
|
}
|
|
@@ -10459,76 +10490,79 @@
|
|
|
10459
10490
|
// <- TODO: [🧟♂️] Unite path to promptbook collection argument
|
|
10460
10491
|
'Pipelines to test as glob pattern');
|
|
10461
10492
|
testCommand.option('-i, --ignore <glob>', "Ignore as glob pattern");
|
|
10462
|
-
testCommand.option('--reload
|
|
10493
|
+
testCommand.option('--reload', "Call LLM models even if same prompt with result is in the cache ", false);
|
|
10463
10494
|
testCommand.option('-v, --verbose', "Is output verbose", false);
|
|
10464
10495
|
testCommand.action(function (filesGlob, _a) {
|
|
10465
|
-
var ignore = _a.ignore,
|
|
10496
|
+
var ignore = _a.ignore, isCacheReloaded = _a.reloadCache, isVerbose = _a.verbose;
|
|
10466
10497
|
return __awaiter(_this, void 0, void 0, function () {
|
|
10467
|
-
var options, fs, llm, tools, filenames, filenames_1, filenames_1_1, filename, pipeline, pipelineMarkdown, _b, _c, error_1, e_1_1;
|
|
10498
|
+
var options, fs, llm, executables, tools, filenames, filenames_1, filenames_1_1, filename, pipeline, pipelineMarkdown, _b, _c, error_1, e_1_1;
|
|
10468
10499
|
var _d, e_1, _e;
|
|
10469
10500
|
return __generator(this, function (_f) {
|
|
10470
10501
|
switch (_f.label) {
|
|
10471
10502
|
case 0:
|
|
10472
10503
|
options = {
|
|
10473
10504
|
isVerbose: isVerbose,
|
|
10474
|
-
|
|
10505
|
+
isCacheReloaded: isCacheReloaded,
|
|
10475
10506
|
};
|
|
10476
10507
|
fs = $provideFilesystemForNode(options);
|
|
10477
10508
|
llm = $provideLlmToolsForCli(options);
|
|
10509
|
+
return [4 /*yield*/, $provideExecutablesForNode(options)];
|
|
10510
|
+
case 1:
|
|
10511
|
+
executables = _f.sent();
|
|
10478
10512
|
_d = {
|
|
10479
10513
|
llm: llm,
|
|
10480
10514
|
fs: fs
|
|
10481
10515
|
};
|
|
10482
|
-
return [4 /*yield*/, $provideScrapersForNode({ fs: fs, llm: llm }, options)];
|
|
10483
|
-
case
|
|
10516
|
+
return [4 /*yield*/, $provideScrapersForNode({ fs: fs, llm: llm, executables: executables }, options)];
|
|
10517
|
+
case 2:
|
|
10484
10518
|
tools = (_d.scrapers = _f.sent(),
|
|
10485
10519
|
_d.script = [
|
|
10486
10520
|
/*new JavascriptExecutionTools(options)*/
|
|
10487
10521
|
],
|
|
10488
10522
|
_d);
|
|
10489
10523
|
return [4 /*yield*/, glob__default["default"](filesGlob, { ignore: ignore })];
|
|
10490
|
-
case 2:
|
|
10491
|
-
filenames = _f.sent();
|
|
10492
|
-
_f.label = 3;
|
|
10493
10524
|
case 3:
|
|
10494
|
-
_f.
|
|
10495
|
-
filenames_1 = __values(filenames), filenames_1_1 = filenames_1.next();
|
|
10525
|
+
filenames = _f.sent();
|
|
10496
10526
|
_f.label = 4;
|
|
10497
10527
|
case 4:
|
|
10498
|
-
|
|
10499
|
-
|
|
10528
|
+
_f.trys.push([4, 16, 17, 18]);
|
|
10529
|
+
filenames_1 = __values(filenames), filenames_1_1 = filenames_1.next();
|
|
10500
10530
|
_f.label = 5;
|
|
10501
10531
|
case 5:
|
|
10502
|
-
|
|
10532
|
+
if (!!filenames_1_1.done) return [3 /*break*/, 15];
|
|
10533
|
+
filename = filenames_1_1.value;
|
|
10534
|
+
_f.label = 6;
|
|
10535
|
+
case 6:
|
|
10536
|
+
_f.trys.push([6, 13, , 14]);
|
|
10503
10537
|
pipeline = void 0;
|
|
10504
|
-
if (!filename.endsWith('.ptbk.md')) return [3 /*break*/,
|
|
10538
|
+
if (!filename.endsWith('.ptbk.md')) return [3 /*break*/, 9];
|
|
10505
10539
|
return [4 /*yield*/, promises.readFile(filename, 'utf-8')];
|
|
10506
|
-
case
|
|
10540
|
+
case 7:
|
|
10507
10541
|
pipelineMarkdown = (_f.sent());
|
|
10508
10542
|
return [4 /*yield*/, pipelineStringToJson(pipelineMarkdown, tools)];
|
|
10509
|
-
case
|
|
10543
|
+
case 8:
|
|
10510
10544
|
pipeline = _f.sent();
|
|
10511
10545
|
if (isVerbose) {
|
|
10512
10546
|
console.info(colors__default["default"].green("Parsed ".concat(filename)));
|
|
10513
10547
|
}
|
|
10514
|
-
_f.label =
|
|
10515
|
-
case
|
|
10516
|
-
if (!filename.endsWith('.ptbk.json')) return [3 /*break*/,
|
|
10548
|
+
_f.label = 9;
|
|
10549
|
+
case 9:
|
|
10550
|
+
if (!filename.endsWith('.ptbk.json')) return [3 /*break*/, 11];
|
|
10517
10551
|
_c = (_b = JSON).parse;
|
|
10518
10552
|
return [4 /*yield*/, promises.readFile(filename, 'utf-8')];
|
|
10519
|
-
case 9:
|
|
10520
|
-
pipeline = _c.apply(_b, [_f.sent()]);
|
|
10521
|
-
return [3 /*break*/, 11];
|
|
10522
10553
|
case 10:
|
|
10554
|
+
pipeline = _c.apply(_b, [_f.sent()]);
|
|
10555
|
+
return [3 /*break*/, 12];
|
|
10556
|
+
case 11:
|
|
10523
10557
|
if (isVerbose) {
|
|
10524
10558
|
console.info(colors__default["default"].gray("Skipping ".concat(filename)));
|
|
10525
10559
|
}
|
|
10526
|
-
return [3 /*break*/,
|
|
10527
|
-
case
|
|
10560
|
+
return [3 /*break*/, 14];
|
|
10561
|
+
case 12:
|
|
10528
10562
|
validatePipeline(pipeline);
|
|
10529
10563
|
console.info(colors__default["default"].green("Validated ".concat(filename)));
|
|
10530
|
-
return [3 /*break*/,
|
|
10531
|
-
case
|
|
10564
|
+
return [3 /*break*/, 14];
|
|
10565
|
+
case 13:
|
|
10532
10566
|
error_1 = _f.sent();
|
|
10533
10567
|
if (!(error_1 instanceof Error)) {
|
|
10534
10568
|
throw error_1;
|
|
@@ -10537,22 +10571,22 @@
|
|
|
10537
10571
|
console.error(colors__default["default"].bgRed(error_1.name /* <- 11:11 */));
|
|
10538
10572
|
console.error(colors__default["default"].red(error_1.stack || error_1.message));
|
|
10539
10573
|
process.exit(1);
|
|
10540
|
-
return [3 /*break*/,
|
|
10541
|
-
case
|
|
10574
|
+
return [3 /*break*/, 14];
|
|
10575
|
+
case 14:
|
|
10542
10576
|
filenames_1_1 = filenames_1.next();
|
|
10543
|
-
return [3 /*break*/,
|
|
10544
|
-
case
|
|
10545
|
-
case
|
|
10577
|
+
return [3 /*break*/, 5];
|
|
10578
|
+
case 15: return [3 /*break*/, 18];
|
|
10579
|
+
case 16:
|
|
10546
10580
|
e_1_1 = _f.sent();
|
|
10547
10581
|
e_1 = { error: e_1_1 };
|
|
10548
|
-
return [3 /*break*/,
|
|
10549
|
-
case
|
|
10582
|
+
return [3 /*break*/, 18];
|
|
10583
|
+
case 17:
|
|
10550
10584
|
try {
|
|
10551
10585
|
if (filenames_1_1 && !filenames_1_1.done && (_e = filenames_1.return)) _e.call(filenames_1);
|
|
10552
10586
|
}
|
|
10553
10587
|
finally { if (e_1) throw e_1.error; }
|
|
10554
10588
|
return [7 /*endfinally*/];
|
|
10555
|
-
case
|
|
10589
|
+
case 18:
|
|
10556
10590
|
console.info(colors__default["default"].green("All pipelines are valid"));
|
|
10557
10591
|
process.exit(0);
|
|
10558
10592
|
return [2 /*return*/];
|
|
@@ -13023,12 +13057,12 @@
|
|
|
13023
13057
|
*/
|
|
13024
13058
|
function getScraperIntermediateSource(source, options) {
|
|
13025
13059
|
return __awaiter(this, void 0, void 0, function () {
|
|
13026
|
-
var sourceFilename, url, rootDirname, cacheDirname,
|
|
13060
|
+
var sourceFilename, url, rootDirname, cacheDirname, intermediateFilesStrategy, extension, isVerbose, hash, semanticName, pieces, name, cacheFilename, isDestroyed, fileHandler;
|
|
13027
13061
|
return __generator(this, function (_a) {
|
|
13028
13062
|
switch (_a.label) {
|
|
13029
13063
|
case 0:
|
|
13030
13064
|
sourceFilename = source.filename, url = source.url;
|
|
13031
|
-
rootDirname = options.rootDirname, cacheDirname = options.cacheDirname,
|
|
13065
|
+
rootDirname = options.rootDirname, cacheDirname = options.cacheDirname, intermediateFilesStrategy = options.intermediateFilesStrategy, extension = options.extension, isVerbose = options.isVerbose;
|
|
13032
13066
|
hash = cryptoJs.SHA256(
|
|
13033
13067
|
// <- TODO: [🥬] Encapsulate sha256 to some private utility function
|
|
13034
13068
|
hexEncoder__default["default"].parse(sourceFilename || url || 'untitled'))
|
|
@@ -13038,7 +13072,7 @@
|
|
|
13038
13072
|
pieces = ['intermediate', semanticName, hash].filter(function (piece) { return piece !== ''; });
|
|
13039
13073
|
name = pieces.join('-').split('--').join('-');
|
|
13040
13074
|
// <- TODO: Use MAX_FILENAME_LENGTH
|
|
13041
|
-
TODO_USE(rootDirname); // <- TODO:
|
|
13075
|
+
TODO_USE(rootDirname); // <- TODO: [😡]
|
|
13042
13076
|
cacheFilename = path.join.apply(void 0, __spreadArray(__spreadArray([process.cwd(),
|
|
13043
13077
|
cacheDirname], __read(nameToSubfolderPath(hash /* <- TODO: [🎎] Maybe add some SHA256 prefix */)), false), [name], false)).split('\\')
|
|
13044
13078
|
.join('/') +
|
|
@@ -13058,7 +13092,7 @@
|
|
|
13058
13092
|
return __generator(this, function (_a) {
|
|
13059
13093
|
switch (_a.label) {
|
|
13060
13094
|
case 0:
|
|
13061
|
-
if (!
|
|
13095
|
+
if (!(intermediateFilesStrategy === 'HIDE_AND_CLEAN')) return [3 /*break*/, 2];
|
|
13062
13096
|
if (isVerbose) {
|
|
13063
13097
|
console.info('legacyDocumentScraper: Clening cache');
|
|
13064
13098
|
}
|
|
@@ -13082,7 +13116,7 @@
|
|
|
13082
13116
|
/**
|
|
13083
13117
|
* Note: Not using `FileCacheStorage` for two reasons:
|
|
13084
13118
|
* 1) Need to store more than serialized JSONs
|
|
13085
|
-
* 2) Need to switch between a `rootDirname` and `cacheDirname` <- TODO:
|
|
13119
|
+
* 2) Need to switch between a `rootDirname` and `cacheDirname` <- TODO: [😡]
|
|
13086
13120
|
* TODO: [🐱🐉][🧠] Make some smart crop
|
|
13087
13121
|
* Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
|
|
13088
13122
|
*/
|
|
@@ -13099,7 +13133,7 @@
|
|
|
13099
13133
|
mimeTypes: ['text/markdown', 'text/plain'],
|
|
13100
13134
|
documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
|
|
13101
13135
|
isAvilableInBrowser: true,
|
|
13102
|
-
requiredExecutables: [
|
|
13136
|
+
requiredExecutables: [],
|
|
13103
13137
|
}); /* <- TODO: [🤛] */
|
|
13104
13138
|
/**
|
|
13105
13139
|
* Registration of known scraper metadata
|
|
@@ -13143,7 +13177,7 @@
|
|
|
13143
13177
|
return __generator(this, function (_k) {
|
|
13144
13178
|
switch (_k.label) {
|
|
13145
13179
|
case 0:
|
|
13146
|
-
_a = this.options, _b = _a.maxParallelCount, maxParallelCount = _b === void 0 ?
|
|
13180
|
+
_a = this.options, _b = _a.maxParallelCount, maxParallelCount = _b === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? DEFAULT_IS_VERBOSE : _c;
|
|
13147
13181
|
llm = this.tools.llm;
|
|
13148
13182
|
if (llm === undefined) {
|
|
13149
13183
|
throw new MissingToolsError('LLM tools are required for scraping external files');
|
|
@@ -13242,7 +13276,8 @@
|
|
|
13242
13276
|
embeddingResult = _c.sent();
|
|
13243
13277
|
index.push({
|
|
13244
13278
|
modelName: embeddingResult.modelName,
|
|
13245
|
-
position: embeddingResult.content,
|
|
13279
|
+
position: __spreadArray([], __read(embeddingResult.content), false),
|
|
13280
|
+
// <- TODO: [🪓] Here should be no need for spreading new array, just `position: embeddingResult.content`
|
|
13246
13281
|
});
|
|
13247
13282
|
_c.label = 6;
|
|
13248
13283
|
case 6: return [3 /*break*/, 8];
|
|
@@ -13293,7 +13328,7 @@
|
|
|
13293
13328
|
mimeTypes: ['application/vnd.openxmlformats-officedocument.wordprocessingml.document'],
|
|
13294
13329
|
documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
|
|
13295
13330
|
isAvilableInBrowser: false,
|
|
13296
|
-
requiredExecutables: ['
|
|
13331
|
+
requiredExecutables: ['Pandoc'],
|
|
13297
13332
|
}); /* <- TODO: [🤛] */
|
|
13298
13333
|
/**
|
|
13299
13334
|
* Registration of known scraper metadata
|
|
@@ -13333,12 +13368,13 @@
|
|
|
13333
13368
|
* Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object
|
|
13334
13369
|
*/
|
|
13335
13370
|
DocumentScraper.prototype.$convert = function (source) {
|
|
13371
|
+
var _a;
|
|
13336
13372
|
return __awaiter(this, void 0, void 0, function () {
|
|
13337
|
-
var
|
|
13373
|
+
var _b, _c, rootDirname, _d, cacheDirname, _e, intermediateFilesStrategy, _f, isVerbose, extension, cacheFilehandler, command_1;
|
|
13338
13374
|
return __generator(this, function (_g) {
|
|
13339
13375
|
switch (_g.label) {
|
|
13340
13376
|
case 0:
|
|
13341
|
-
|
|
13377
|
+
_b = this.options, _c = _b.rootDirname, rootDirname = _c === void 0 ? process.cwd() : _c, _d = _b.cacheDirname, cacheDirname = _d === void 0 ? DEFAULT_SCRAPE_CACHE_DIRNAME : _d, _e = _b.intermediateFilesStrategy, intermediateFilesStrategy = _e === void 0 ? DEFAULT_INTERMEDIATE_FILES_STRATEGY : _e, _f = _b.isVerbose, isVerbose = _f === void 0 ? DEFAULT_IS_VERBOSE : _f;
|
|
13342
13378
|
if (!$isRunningInNode()) {
|
|
13343
13379
|
throw new KnowledgeScrapeError('Scraping .docx files is only supported in Node environment');
|
|
13344
13380
|
}
|
|
@@ -13346,7 +13382,7 @@
|
|
|
13346
13382
|
throw new EnvironmentMismatchError('Can not scrape documents without filesystem tools');
|
|
13347
13383
|
// <- TODO: [🧠] What is the best error type here`
|
|
13348
13384
|
}
|
|
13349
|
-
if (
|
|
13385
|
+
if (((_a = this.tools.executables) === null || _a === void 0 ? void 0 : _a.pandocPath) === undefined) {
|
|
13350
13386
|
throw new MissingToolsError('Pandoc is required for scraping .docx files');
|
|
13351
13387
|
}
|
|
13352
13388
|
if (source.filename === null) {
|
|
@@ -13357,7 +13393,7 @@
|
|
|
13357
13393
|
return [4 /*yield*/, getScraperIntermediateSource(source, {
|
|
13358
13394
|
rootDirname: rootDirname,
|
|
13359
13395
|
cacheDirname: cacheDirname,
|
|
13360
|
-
|
|
13396
|
+
intermediateFilesStrategy: intermediateFilesStrategy,
|
|
13361
13397
|
extension: 'md',
|
|
13362
13398
|
isVerbose: isVerbose,
|
|
13363
13399
|
})];
|
|
@@ -13366,11 +13402,9 @@
|
|
|
13366
13402
|
return [4 /*yield*/, isFileExisting(cacheFilehandler.filename, this.tools.fs)];
|
|
13367
13403
|
case 2:
|
|
13368
13404
|
if (!!(_g.sent())) return [3 /*break*/, 5];
|
|
13369
|
-
command_1 = "\"".concat(
|
|
13370
|
-
// TODO: !!!!!! [🕊] Make execCommand standard (?node-)util of the promptbook
|
|
13405
|
+
command_1 = "\"".concat(this.tools.executables.pandocPath, "\" -f ").concat(extension, " -t markdown \"").concat(source.filename, "\" -o \"").concat(cacheFilehandler.filename, "\"");
|
|
13371
13406
|
return [4 /*yield*/, $execCommand(command_1)];
|
|
13372
13407
|
case 3:
|
|
13373
|
-
// TODO: !!!!!! [🕊] Make execCommand standard (?node-)util of the promptbook
|
|
13374
13408
|
_g.sent();
|
|
13375
13409
|
return [4 /*yield*/, isFileExisting(cacheFilehandler.filename, this.tools.fs)];
|
|
13376
13410
|
case 4:
|
|
@@ -13449,7 +13483,11 @@
|
|
|
13449
13483
|
mimeTypes: ['application/msword', 'text/rtf'],
|
|
13450
13484
|
documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
|
|
13451
13485
|
isAvilableInBrowser: false,
|
|
13452
|
-
requiredExecutables: [
|
|
13486
|
+
requiredExecutables: [
|
|
13487
|
+
'Pandoc',
|
|
13488
|
+
'LibreOffice',
|
|
13489
|
+
// <- TODO: [🧠] Should be 'LibreOffice' here, its dependency of dependency
|
|
13490
|
+
],
|
|
13453
13491
|
}); /* <- TODO: [🤛] */
|
|
13454
13492
|
/**
|
|
13455
13493
|
* Registration of known scraper metadata
|
|
@@ -13462,7 +13500,7 @@
|
|
|
13462
13500
|
var _LegacyDocumentScraperMetadataRegistration = $scrapersMetadataRegister.register(legacyDocumentScraperMetadata);
|
|
13463
13501
|
|
|
13464
13502
|
/**
|
|
13465
|
-
* Scraper for .
|
|
13503
|
+
* Scraper for old document files (like .doc and .rtf)
|
|
13466
13504
|
*
|
|
13467
13505
|
* @see `documentationUrl` for more details
|
|
13468
13506
|
* @public exported from `@promptbook/legacy-documents`
|
|
@@ -13489,12 +13527,13 @@
|
|
|
13489
13527
|
* Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object
|
|
13490
13528
|
*/
|
|
13491
13529
|
LegacyDocumentScraper.prototype.$convert = function (source) {
|
|
13530
|
+
var _a;
|
|
13492
13531
|
return __awaiter(this, void 0, void 0, function () {
|
|
13493
|
-
var
|
|
13532
|
+
var _b, _c, rootDirname, _d, cacheDirname, _e, intermediateFilesStrategy, _f, isVerbose, extension, cacheFilehandler, documentSourceOutdirPathForLibreOffice_1, command_1, files_1, file;
|
|
13494
13533
|
return __generator(this, function (_g) {
|
|
13495
13534
|
switch (_g.label) {
|
|
13496
13535
|
case 0:
|
|
13497
|
-
|
|
13536
|
+
_b = this.options, _c = _b.rootDirname, rootDirname = _c === void 0 ? process.cwd() : _c, _d = _b.cacheDirname, cacheDirname = _d === void 0 ? DEFAULT_SCRAPE_CACHE_DIRNAME : _d, _e = _b.intermediateFilesStrategy, intermediateFilesStrategy = _e === void 0 ? DEFAULT_INTERMEDIATE_FILES_STRATEGY : _e, _f = _b.isVerbose, isVerbose = _f === void 0 ? DEFAULT_IS_VERBOSE : _f;
|
|
13498
13537
|
if (!$isRunningInNode()) {
|
|
13499
13538
|
throw new KnowledgeScrapeError('Scraping .doc files is only supported in Node environment');
|
|
13500
13539
|
}
|
|
@@ -13502,7 +13541,7 @@
|
|
|
13502
13541
|
throw new EnvironmentMismatchError('Can not scrape (legacy) documents without filesystem tools');
|
|
13503
13542
|
// <- TODO: [🧠] What is the best error type here`
|
|
13504
13543
|
}
|
|
13505
|
-
if (
|
|
13544
|
+
if (((_a = this.tools.executables) === null || _a === void 0 ? void 0 : _a.libreOfficePath) === undefined) {
|
|
13506
13545
|
throw new MissingToolsError('LibreOffice is required for scraping .doc and .rtf files');
|
|
13507
13546
|
}
|
|
13508
13547
|
if (source.filename === null) {
|
|
@@ -13513,7 +13552,7 @@
|
|
|
13513
13552
|
return [4 /*yield*/, getScraperIntermediateSource(source, {
|
|
13514
13553
|
rootDirname: rootDirname,
|
|
13515
13554
|
cacheDirname: cacheDirname,
|
|
13516
|
-
|
|
13555
|
+
intermediateFilesStrategy: intermediateFilesStrategy,
|
|
13517
13556
|
extension: 'docx',
|
|
13518
13557
|
isVerbose: isVerbose,
|
|
13519
13558
|
})];
|
|
@@ -13528,11 +13567,9 @@
|
|
|
13528
13567
|
documentSourceOutdirPathForLibreOffice_1 = path.join(path.dirname(cacheFilehandler.filename), 'libreoffice')
|
|
13529
13568
|
.split('\\')
|
|
13530
13569
|
.join('/');
|
|
13531
|
-
command_1 = "\"".concat(
|
|
13532
|
-
// TODO: !!!!!! [🕊] Make execCommand standard (?node-)util of the promptbook - this should trigger build polution error
|
|
13570
|
+
command_1 = "\"".concat(this.tools.executables.libreOfficePath, "\" --headless --convert-to docx \"").concat(source.filename, "\" --outdir \"").concat(documentSourceOutdirPathForLibreOffice_1, "\"");
|
|
13533
13571
|
return [4 /*yield*/, $execCommand(command_1)];
|
|
13534
13572
|
case 3:
|
|
13535
|
-
// TODO: !!!!!! [🕊] Make execCommand standard (?node-)util of the promptbook - this should trigger build polution error
|
|
13536
13573
|
_g.sent();
|
|
13537
13574
|
return [4 /*yield*/, promises.readdir(documentSourceOutdirPathForLibreOffice_1)];
|
|
13538
13575
|
case 4:
|
|
@@ -13689,7 +13726,7 @@
|
|
|
13689
13726
|
mimeTypes: ['application/pdf'],
|
|
13690
13727
|
documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
|
|
13691
13728
|
isAvilableInBrowser: true,
|
|
13692
|
-
requiredExecutables: [
|
|
13729
|
+
requiredExecutables: [],
|
|
13693
13730
|
}); /* <- TODO: [🤛] */
|
|
13694
13731
|
/**
|
|
13695
13732
|
* Registration of known scraper metadata
|
|
@@ -13702,7 +13739,7 @@
|
|
|
13702
13739
|
var _PdfScraperMetadataRegistration = $scrapersMetadataRegister.register(pdfScraperMetadata);
|
|
13703
13740
|
|
|
13704
13741
|
/**
|
|
13705
|
-
* Scraper for .
|
|
13742
|
+
* Scraper for .pdf files
|
|
13706
13743
|
*
|
|
13707
13744
|
* @see `documentationUrl` for more details
|
|
13708
13745
|
* @public exported from `@promptbook/pdf`
|
|
@@ -13745,9 +13782,8 @@
|
|
|
13745
13782
|
TODO_USE(this.options);
|
|
13746
13783
|
/*
|
|
13747
13784
|
const {
|
|
13748
|
-
externalProgramsPaths = {},
|
|
13749
13785
|
cacheDirname = SCRAPE_CACHE_DIRNAME,
|
|
13750
|
-
|
|
13786
|
+
intermediateFilesStrategy = DEFAULT_INTERMEDIATE_FILES_STRATEGY,
|
|
13751
13787
|
isVerbose = IS_VERBOSE,
|
|
13752
13788
|
} = options;
|
|
13753
13789
|
*/
|
|
@@ -13800,7 +13836,7 @@
|
|
|
13800
13836
|
mimeTypes: ['text/html'],
|
|
13801
13837
|
documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
|
|
13802
13838
|
isAvilableInBrowser: false,
|
|
13803
|
-
requiredExecutables: [
|
|
13839
|
+
requiredExecutables: [],
|
|
13804
13840
|
}); /* <- TODO: [🤛] */
|
|
13805
13841
|
/**
|
|
13806
13842
|
* Registration of known scraper metadata
|
|
@@ -13813,32 +13849,29 @@
|
|
|
13813
13849
|
var _WebsiteScraperMetadataRegistration = $scrapersMetadataRegister.register(websiteScraperMetadata);
|
|
13814
13850
|
|
|
13815
13851
|
/**
|
|
13816
|
-
*
|
|
13852
|
+
* Create a new showdown converter instance
|
|
13817
13853
|
*
|
|
13818
|
-
* @
|
|
13819
|
-
* @private for markdown and html knowledge scrapers
|
|
13820
|
-
*/
|
|
13821
|
-
var markdownConverter = new showdown.Converter({
|
|
13822
|
-
flavor: 'github', // <- TODO: !!!!!! Explicitly specify the flavor of promptbook markdown
|
|
13823
|
-
/*
|
|
13824
|
-
> import showdownHighlight from 'showdown-highlight';
|
|
13825
|
-
> extensions: [
|
|
13826
|
-
> showdownHighlight({
|
|
13827
|
-
> // Whether to add the classes to the <pre> tag, default is false
|
|
13828
|
-
> pre: true,
|
|
13829
|
-
> // Whether to use hljs' auto language detection, default is true
|
|
13830
|
-
> auto_detection: true,
|
|
13831
|
-
> }),
|
|
13832
|
-
> ],
|
|
13833
|
-
*/
|
|
13834
|
-
});
|
|
13835
|
-
/**
|
|
13836
|
-
* TODO: !!!!!! Figure out better name not to confuse with `Converter`
|
|
13837
|
-
* TODO: !!!!!! Lazy-make converter
|
|
13854
|
+
* @private utility of `WebsiteScraper`
|
|
13838
13855
|
*/
|
|
13856
|
+
function createShowdownConverter() {
|
|
13857
|
+
return new showdown.Converter({
|
|
13858
|
+
flavor: 'github',
|
|
13859
|
+
/*
|
|
13860
|
+
> import showdownHighlight from 'showdown-highlight';
|
|
13861
|
+
> extensions: [
|
|
13862
|
+
> showdownHighlight({
|
|
13863
|
+
> // Whether to add the classes to the <pre> tag, default is false
|
|
13864
|
+
> pre: true,
|
|
13865
|
+
> // Whether to use hljs' auto language detection, default is true
|
|
13866
|
+
> auto_detection: true,
|
|
13867
|
+
> }),
|
|
13868
|
+
> ],
|
|
13869
|
+
*/
|
|
13870
|
+
});
|
|
13871
|
+
}
|
|
13839
13872
|
|
|
13840
13873
|
/**
|
|
13841
|
-
* Scraper for
|
|
13874
|
+
* Scraper for websites
|
|
13842
13875
|
*
|
|
13843
13876
|
* @see `documentationUrl` for more details
|
|
13844
13877
|
* @public exported from `@promptbook/website-crawler`
|
|
@@ -13848,6 +13881,7 @@
|
|
|
13848
13881
|
this.tools = tools;
|
|
13849
13882
|
this.options = options;
|
|
13850
13883
|
this.markdownScraper = new MarkdownScraper(tools, options);
|
|
13884
|
+
this.showdownConverter = createShowdownConverter();
|
|
13851
13885
|
}
|
|
13852
13886
|
Object.defineProperty(WebsiteScraper.prototype, "metadata", {
|
|
13853
13887
|
/**
|
|
@@ -13868,12 +13902,11 @@
|
|
|
13868
13902
|
return __awaiter(this, void 0, void 0, function () {
|
|
13869
13903
|
var _a, _b,
|
|
13870
13904
|
// TODO: [🧠] Maybe in node use headless browser not just JSDOM
|
|
13871
|
-
|
|
13872
|
-
rootDirname, _c, cacheDirname, _d, isCacheCleaned, _e, isVerbose, jsdom$1, _f, reader, article, html, i, cacheFilehandler, markdown;
|
|
13905
|
+
rootDirname, _c, cacheDirname, _d, intermediateFilesStrategy, _e, isVerbose, jsdom$1, _f, reader, article, html, i, cacheFilehandler, markdown;
|
|
13873
13906
|
return __generator(this, function (_g) {
|
|
13874
13907
|
switch (_g.label) {
|
|
13875
13908
|
case 0:
|
|
13876
|
-
_a = this.options, _b = _a.rootDirname, rootDirname = _b === void 0 ? process.cwd() : _b, _c = _a.cacheDirname, cacheDirname = _c === void 0 ?
|
|
13909
|
+
_a = this.options, _b = _a.rootDirname, rootDirname = _b === void 0 ? process.cwd() : _b, _c = _a.cacheDirname, cacheDirname = _c === void 0 ? DEFAULT_SCRAPE_CACHE_DIRNAME : _c, _d = _a.intermediateFilesStrategy, intermediateFilesStrategy = _d === void 0 ? DEFAULT_INTERMEDIATE_FILES_STRATEGY : _d, _e = _a.isVerbose, isVerbose = _e === void 0 ? DEFAULT_IS_VERBOSE : _e;
|
|
13877
13910
|
// TODO: !!!!!! Does this work in browser? Make it work.
|
|
13878
13911
|
if (source.url === null) {
|
|
13879
13912
|
throw new KnowledgeScrapeError('Website scraper requires URL');
|
|
@@ -13886,10 +13919,6 @@
|
|
|
13886
13919
|
}]))();
|
|
13887
13920
|
reader = new readability.Readability(jsdom$1.window.document);
|
|
13888
13921
|
article = reader.parse();
|
|
13889
|
-
console.log(article);
|
|
13890
|
-
return [4 /*yield*/, waitasecond.forTime(10000)];
|
|
13891
|
-
case 2:
|
|
13892
|
-
_g.sent();
|
|
13893
13922
|
html = (article === null || article === void 0 ? void 0 : article.content) || (article === null || article === void 0 ? void 0 : article.textContent) || jsdom$1.window.document.body.innerHTML;
|
|
13894
13923
|
// Note: Unwrap html such as it is convertable by `markdownConverter`
|
|
13895
13924
|
for (i = 0; i < 2; i++) {
|
|
@@ -13901,16 +13930,16 @@
|
|
|
13901
13930
|
return [4 /*yield*/, getScraperIntermediateSource(source, {
|
|
13902
13931
|
rootDirname: rootDirname,
|
|
13903
13932
|
cacheDirname: cacheDirname,
|
|
13904
|
-
|
|
13933
|
+
intermediateFilesStrategy: intermediateFilesStrategy,
|
|
13905
13934
|
extension: 'html',
|
|
13906
13935
|
isVerbose: isVerbose,
|
|
13907
13936
|
})];
|
|
13908
|
-
case
|
|
13937
|
+
case 2:
|
|
13909
13938
|
cacheFilehandler = _g.sent();
|
|
13910
13939
|
return [4 /*yield*/, promises.writeFile(cacheFilehandler.filename, html, 'utf-8')];
|
|
13911
|
-
case
|
|
13940
|
+
case 3:
|
|
13912
13941
|
_g.sent();
|
|
13913
|
-
markdown =
|
|
13942
|
+
markdown = this.showdownConverter.makeMarkdown(html, jsdom$1.window.document);
|
|
13914
13943
|
return [2 /*return*/, __assign(__assign({}, cacheFilehandler), { markdown: markdown })];
|
|
13915
13944
|
}
|
|
13916
13945
|
});
|
|
@@ -13954,7 +13983,6 @@
|
|
|
13954
13983
|
return WebsiteScraper;
|
|
13955
13984
|
}());
|
|
13956
13985
|
/**
|
|
13957
|
-
* TODO: !!!!!! Put into separate package
|
|
13958
13986
|
* TODO: [👣] Scraped website in .md can act as cache item - there is no need to run conversion each time
|
|
13959
13987
|
* TODO: [🪂] Do it in parallel 11:11
|
|
13960
13988
|
* Note: No need to aggregate usage here, it is done by intercepting the llmTools
|