@promptbook/cli 0.72.0-9 → 0.72.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -8
- package/esm/index.es.js +987 -649
- package/esm/index.es.js.map +1 -1
- package/esm/typings/src/_packages/core.index.d.ts +24 -18
- package/esm/typings/src/_packages/node.index.d.ts +2 -0
- package/esm/typings/src/_packages/types.index.d.ts +18 -12
- package/esm/typings/src/_packages/utils.index.d.ts +2 -2
- package/esm/typings/src/collection/PipelineCollection.d.ts +1 -1
- package/esm/typings/src/collection/SimplePipelineCollection.d.ts +2 -2
- package/esm/typings/src/collection/collectionToJson.d.ts +1 -1
- package/esm/typings/src/collection/constructors/createCollectionFromJson.d.ts +1 -1
- package/esm/typings/src/collection/constructors/createCollectionFromPromise.d.ts +1 -1
- package/esm/typings/src/commands/_common/types/CommandParser.d.ts +5 -5
- package/esm/typings/src/config.d.ts +21 -14
- package/esm/typings/src/executables/$provideExecutablesForNode.d.ts +12 -0
- package/esm/typings/src/executables/apps/locateLibreoffice.d.ts +11 -0
- package/esm/typings/src/executables/apps/locatePandoc.d.ts +11 -0
- package/esm/typings/src/executables/locateApp.d.ts +33 -0
- package/esm/typings/src/executables/locateApp.test.d.ts +1 -0
- package/esm/typings/src/executables/platforms/locateAppOnLinux.d.ts +12 -0
- package/esm/typings/src/executables/platforms/locateAppOnMacOs.d.ts +12 -0
- package/esm/typings/src/executables/platforms/locateAppOnWindows.d.ts +12 -0
- package/esm/typings/src/execution/EmbeddingVector.d.ts +1 -1
- package/esm/typings/src/execution/Executables.d.ts +18 -0
- package/esm/typings/src/execution/ExecutionTools.d.ts +9 -3
- package/esm/typings/src/execution/FilesystemTools.d.ts +1 -1
- package/esm/typings/src/execution/LlmExecutionTools.d.ts +2 -2
- package/esm/typings/src/execution/PipelineExecutorResult.d.ts +2 -2
- package/esm/typings/src/execution/assertsExecutionSuccessful.d.ts +3 -2
- package/esm/typings/src/execution/createPipelineExecutor/00-CreatePipelineExecutorOptions.d.ts +29 -6
- package/esm/typings/src/execution/createPipelineExecutor/10-executePipeline.d.ts +2 -11
- package/esm/typings/src/execution/createPipelineExecutor/20-executeTemplate.d.ts +4 -13
- package/esm/typings/src/execution/createPipelineExecutor/40-executeAttempts.d.ts +9 -14
- package/esm/typings/src/execution/translation/automatic-translate/automatic-translators/LindatAutomaticTranslator.d.ts +11 -3
- package/esm/typings/src/execution/utils/addUsage.d.ts +1 -1
- package/esm/typings/src/execution/utils/forEachAsync.d.ts +1 -1
- package/esm/typings/src/formats/_common/FormatDefinition.d.ts +2 -2
- package/esm/typings/src/formats/_common/FormatSubvalueDefinition.d.ts +2 -2
- package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsForCli.d.ts +2 -2
- package/esm/typings/src/llm-providers/_common/register/LlmToolsConfiguration.d.ts +1 -1
- package/esm/typings/src/llm-providers/_common/register/createLlmToolsFromConfiguration.d.ts +7 -0
- package/esm/typings/src/llm-providers/_common/utils/cache/CacheLlmToolsOptions.d.ts +4 -1
- package/esm/typings/src/llm-providers/anthropic-claude/AnthropicClaudeExecutionTools.d.ts +1 -1
- package/esm/typings/src/llm-providers/anthropic-claude/AnthropicClaudeExecutionToolsOptions.d.ts +2 -2
- package/esm/typings/src/llm-providers/anthropic-claude/anthropic-claude-models.d.ts +1 -1
- package/esm/typings/src/llm-providers/azure-openai/AzureOpenAiExecutionTools.d.ts +1 -1
- package/esm/typings/src/llm-providers/azure-openai/AzureOpenAiExecutionToolsOptions.d.ts +2 -1
- package/esm/typings/src/llm-providers/langtail/LangtailExecutionTools.d.ts +1 -1
- package/esm/typings/src/llm-providers/mocked/$fakeTextToExpectations.d.ts +1 -1
- package/esm/typings/src/llm-providers/mocked/MockedEchoLlmExecutionTools.d.ts +1 -1
- package/esm/typings/src/llm-providers/mocked/MockedFackedLlmExecutionTools.d.ts +1 -1
- package/esm/typings/src/llm-providers/multiple/MultipleLlmExecutionTools.d.ts +4 -5
- package/esm/typings/src/llm-providers/multiple/joinLlmExecutionTools.d.ts +1 -1
- package/esm/typings/src/llm-providers/openai/OpenAiAssistantExecutionTools.d.ts +0 -1
- package/esm/typings/src/llm-providers/openai/OpenAiExecutionTools.d.ts +2 -2
- package/esm/typings/src/llm-providers/openai/OpenAiExecutionToolsOptions.d.ts +2 -2
- package/esm/typings/src/llm-providers/openai/openai-models.d.ts +1 -1
- package/esm/typings/src/llm-providers/remote/RemoteLlmExecutionTools.d.ts +4 -4
- package/esm/typings/src/llm-providers/remote/interfaces/PromptbookServer_ListModels_Request.d.ts +15 -6
- package/esm/typings/src/llm-providers/remote/interfaces/PromptbookServer_ListModels_Response.d.ts +4 -4
- package/esm/typings/src/llm-providers/remote/interfaces/PromptbookServer_Prompt_Request.d.ts +6 -12
- package/esm/typings/src/llm-providers/remote/interfaces/PromptbookServer_Prompt_Response.d.ts +3 -3
- package/esm/typings/src/llm-providers/remote/interfaces/RemoteLlmExecutionToolsOptions.d.ts +9 -14
- package/esm/typings/src/llm-providers/remote/interfaces/RemoteServerOptions.d.ts +23 -8
- package/esm/typings/src/llm-providers/remote/startRemoteServer.d.ts +1 -1
- package/esm/typings/src/prepare/PrepareAndScrapeOptions.d.ts +4 -23
- package/esm/typings/src/prepare/prepareTemplates.d.ts +1 -1
- package/esm/typings/src/scrapers/_common/Scraper.d.ts +2 -9
- package/esm/typings/src/scrapers/_common/prepareKnowledgePieces.d.ts +1 -1
- package/esm/typings/src/scrapers/_common/register/$provideScrapersForBrowser.d.ts +2 -2
- package/esm/typings/src/scrapers/_common/register/$provideScrapersForNode.d.ts +2 -2
- package/esm/typings/src/scrapers/_common/register/ScraperAndConverterMetadata.d.ts +2 -2
- package/esm/typings/src/scrapers/_common/utils/getScraperIntermediateSource.d.ts +2 -2
- package/esm/typings/src/scrapers/_common/utils/makeKnowledgeSourceHandler.d.ts +1 -1
- package/esm/typings/src/scrapers/document/DocumentScraper.d.ts +2 -2
- package/esm/typings/src/scrapers/document/createDocumentScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/document/register-metadata.d.ts +1 -1
- package/esm/typings/src/scrapers/document-legacy/LegacyDocumentScraper.d.ts +3 -3
- package/esm/typings/src/scrapers/document-legacy/createLegacyDocumentScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/document-legacy/register-metadata.d.ts +1 -1
- package/esm/typings/src/scrapers/markdown/MarkdownScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/markdown/createMarkdownScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/markdown/register-metadata.d.ts +1 -1
- package/esm/typings/src/scrapers/pdf/PdfScraper.d.ts +2 -2
- package/esm/typings/src/scrapers/pdf/createPdfScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/pdf/register-metadata.d.ts +1 -1
- package/esm/typings/src/scrapers/website/WebsiteScraper.d.ts +6 -3
- package/esm/typings/src/scrapers/website/createWebsiteScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/website/register-metadata.d.ts +1 -1
- package/esm/typings/src/scrapers/website/utils/createShowdownConverter.d.ts +7 -0
- package/esm/typings/src/scrapers/website/utils/createShowdownConverter.test.d.ts +1 -0
- package/esm/typings/src/scripting/javascript/utils/preserve.d.ts +1 -1
- package/esm/typings/src/storage/file-cache-storage/utils/nameToSubfolderPath.d.ts +1 -1
- package/esm/typings/src/types/Arrayable.d.ts +1 -1
- package/esm/typings/src/types/IntermediateFilesStrategy.d.ts +7 -0
- package/esm/typings/src/types/PipelineJson/KnowledgePieceJson.d.ts +4 -4
- package/esm/typings/src/types/PipelineJson/KnowledgeSourceJson.d.ts +1 -1
- package/esm/typings/src/types/PipelineJson/PersonaJson.d.ts +1 -1
- package/esm/typings/src/types/PipelineJson/TemplateJsonCommon.d.ts +2 -2
- package/esm/typings/src/types/Prompt.d.ts +2 -1
- package/esm/typings/src/types/execution-report/ExecutionReportJson.d.ts +1 -1
- package/esm/typings/src/types/typeAliases.d.ts +11 -8
- package/esm/typings/src/utils/$Register.d.ts +1 -1
- package/esm/typings/src/utils/FromtoItems.d.ts +1 -1
- package/esm/typings/src/utils/arrayableToArray.d.ts +1 -1
- package/esm/typings/src/utils/emojis.d.ts +1 -1
- package/esm/typings/src/utils/execCommand/$execCommand.d.ts +4 -2
- package/esm/typings/src/utils/execCommand/$execCommandNormalizeOptions.d.ts +12 -0
- package/esm/typings/src/utils/execCommand/$execCommands.d.ts +1 -0
- package/esm/typings/src/utils/execCommand/ExecCommandOptions.d.ts +45 -0
- package/esm/typings/src/utils/expectation-counters/countSentences.d.ts +1 -1
- package/esm/typings/src/utils/files/isExecutable.d.ts +11 -0
- package/esm/typings/src/utils/markdown/extractAllBlocksFromMarkdown.d.ts +1 -1
- package/esm/typings/src/utils/markdown/splitMarkdownIntoSections.d.ts +1 -1
- package/esm/typings/src/utils/normalization/IKeywords.d.ts +2 -2
- package/esm/typings/src/utils/normalization/parseKeywords.d.ts +2 -2
- package/esm/typings/src/utils/normalization/parseKeywordsFromString.d.ts +2 -2
- package/esm/typings/src/utils/normalization/searchKeywords.d.ts +2 -2
- package/esm/typings/src/utils/organization/TODO_USE.d.ts +1 -1
- package/esm/typings/src/utils/organization/keepUnused.d.ts +1 -1
- package/esm/typings/src/utils/random/$randomSeed.d.ts +1 -1
- package/esm/typings/src/utils/sets/intersection.d.ts +1 -1
- package/esm/typings/src/utils/sets/union.d.ts +1 -1
- package/esm/typings/src/utils/unwrapResult.d.ts +4 -4
- package/package.json +3 -2
- package/umd/index.umd.js +988 -651
- package/umd/index.umd.js.map +1 -1
- package/esm/typings/src/execution/createPipelineExecutor/00-CreatePipelineExecutorSettings.d.ts +0 -29
- package/esm/typings/src/scrapers/website/utils/markdownConverter.d.ts +0 -12
- package/esm/typings/src/utils/execCommand/IExecCommandOptions.d.ts +0 -23
- package/esm/typings/src/utils/execCommand/execCommandNormalizeOptions.d.ts +0 -10
- /package/esm/typings/src/{scrapers/website/WebsiteScraper.test.d.ts → executables/apps/locateLibreoffice.test.d.ts} +0 -0
- /package/esm/typings/src/{scrapers/website/utils/markdownConverter.test.d.ts → executables/apps/locatePandoc.test.d.ts} +0 -0
package/umd/index.umd.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
(function (global, factory) {
|
|
2
|
-
typeof exports === 'object' && typeof module !== 'undefined' ? factory(exports, require('commander'), require('spacetrim'), require('colors'), require('waitasecond'), require('fs/promises'), require('path'), require('prettier'), require('prettier/parser-html'), require('papaparse'), require('crypto-js'), require('crypto-js/enc-hex'), require('mime-types'), require('dotenv'), require('crypto-js/sha256'), require('glob-promise'), require('socket.io-client'), require('@anthropic-ai/sdk'), require('@azure/openai'), require('openai'), require('
|
|
3
|
-
typeof define === 'function' && define.amd ? define(['exports', 'commander', 'spacetrim', 'colors', 'waitasecond', 'fs/promises', 'path', 'prettier', 'prettier/parser-html', 'papaparse', 'crypto-js', 'crypto-js/enc-hex', 'mime-types', 'dotenv', 'crypto-js/sha256', 'glob-promise', 'socket.io-client', '@anthropic-ai/sdk', '@azure/openai', 'openai', '
|
|
4
|
-
(global = typeof globalThis !== 'undefined' ? globalThis : global || self, factory(global["promptbook-cli"] = {}, global.commander, global.spaceTrim, global.colors, global.waitasecond, global.promises, global.path, global.prettier, global.parserHtml, global.papaparse, global.cryptoJs, global.hexEncoder, global.mimeTypes, global.dotenv, global.sha256, global.glob, global.socket_ioClient, global.Anthropic, global.openai, global.OpenAI, global.
|
|
5
|
-
})(this, (function (exports, commander, spaceTrim, colors, waitasecond, promises, path, prettier, parserHtml, papaparse, cryptoJs, hexEncoder, mimeTypes, dotenv, sha256, glob, socket_ioClient, Anthropic, openai, OpenAI,
|
|
2
|
+
typeof exports === 'object' && typeof module !== 'undefined' ? factory(exports, require('commander'), require('spacetrim'), require('colors'), require('waitasecond'), require('fs/promises'), require('path'), require('prettier'), require('prettier/parser-html'), require('papaparse'), require('crypto-js'), require('crypto-js/enc-hex'), require('mime-types'), require('child_process'), require('util'), require('dotenv'), require('crypto-js/sha256'), require('glob-promise'), require('socket.io-client'), require('@anthropic-ai/sdk'), require('@azure/openai'), require('openai'), require('@mozilla/readability'), require('jsdom'), require('showdown')) :
|
|
3
|
+
typeof define === 'function' && define.amd ? define(['exports', 'commander', 'spacetrim', 'colors', 'waitasecond', 'fs/promises', 'path', 'prettier', 'prettier/parser-html', 'papaparse', 'crypto-js', 'crypto-js/enc-hex', 'mime-types', 'child_process', 'util', 'dotenv', 'crypto-js/sha256', 'glob-promise', 'socket.io-client', '@anthropic-ai/sdk', '@azure/openai', 'openai', '@mozilla/readability', 'jsdom', 'showdown'], factory) :
|
|
4
|
+
(global = typeof globalThis !== 'undefined' ? globalThis : global || self, factory(global["promptbook-cli"] = {}, global.commander, global.spaceTrim, global.colors, global.waitasecond, global.promises, global.path, global.prettier, global.parserHtml, global.papaparse, global.cryptoJs, global.hexEncoder, global.mimeTypes, global.child_process, global.util, global.dotenv, global.sha256, global.glob, global.socket_ioClient, global.Anthropic, global.openai, global.OpenAI, global.readability, global.jsdom, global.showdown));
|
|
5
|
+
})(this, (function (exports, commander, spaceTrim, colors, waitasecond, promises, path, prettier, parserHtml, papaparse, cryptoJs, hexEncoder, mimeTypes, child_process, util, dotenv, sha256, glob, socket_ioClient, Anthropic, openai, OpenAI, readability, jsdom, showdown) { 'use strict';
|
|
6
6
|
|
|
7
7
|
function _interopDefaultLegacy (e) { return e && typeof e === 'object' && 'default' in e ? e : { 'default': e }; }
|
|
8
8
|
|
|
@@ -39,7 +39,7 @@
|
|
|
39
39
|
/**
|
|
40
40
|
* The version of the Promptbook library
|
|
41
41
|
*/
|
|
42
|
-
var PROMPTBOOK_VERSION = '0.72.0-
|
|
42
|
+
var PROMPTBOOK_VERSION = '0.72.0-34';
|
|
43
43
|
// TODO: [main] !!!! List here all the versions and annotate + put into script
|
|
44
44
|
|
|
45
45
|
/*! *****************************************************************************
|
|
@@ -412,7 +412,8 @@
|
|
|
412
412
|
*
|
|
413
413
|
* @public exported from `@promptbook/core`
|
|
414
414
|
*/
|
|
415
|
-
var CLAIM = "
|
|
415
|
+
var CLAIM = "Build responsible, controlled and transparent applications on top of LLM models!";
|
|
416
|
+
// <- TODO: [🐊] Pick the best claim
|
|
416
417
|
/**
|
|
417
418
|
* Warning message for the generated sections and files files
|
|
418
419
|
*
|
|
@@ -432,6 +433,7 @@
|
|
|
432
433
|
* @private within the repository - too low-level in comparison with other `MAX_...`
|
|
433
434
|
*/
|
|
434
435
|
var CONNECTION_TIMEOUT_MS = 7 * 1000;
|
|
436
|
+
// <- TODO: [⏳] Standartize timeouts, Make DEFAULT_TIMEOUT_MS as global constant
|
|
435
437
|
/**
|
|
436
438
|
* How many times to retry the connections
|
|
437
439
|
*
|
|
@@ -445,23 +447,30 @@
|
|
|
445
447
|
*/
|
|
446
448
|
var IMMEDIATE_TIME = 10;
|
|
447
449
|
/**
|
|
448
|
-
* The maximum
|
|
450
|
+
* The maximum length of the (generated) filename
|
|
449
451
|
*
|
|
450
452
|
* @public exported from `@promptbook/core`
|
|
451
453
|
*/
|
|
452
|
-
var
|
|
454
|
+
var MAX_FILENAME_LENGTH = 30;
|
|
453
455
|
/**
|
|
454
|
-
*
|
|
456
|
+
* Strategy for caching the intermediate results for knowledge sources
|
|
455
457
|
*
|
|
456
458
|
* @public exported from `@promptbook/core`
|
|
457
459
|
*/
|
|
458
|
-
var
|
|
460
|
+
var DEFAULT_INTERMEDIATE_FILES_STRATEGY = 'HIDE_AND_KEEP';
|
|
461
|
+
// <- TODO: [😡] Change to 'VISIBLE'
|
|
459
462
|
/**
|
|
460
|
-
* The maximum
|
|
463
|
+
* The maximum number of (LLM) tasks running in parallel
|
|
461
464
|
*
|
|
462
465
|
* @public exported from `@promptbook/core`
|
|
463
466
|
*/
|
|
464
|
-
var
|
|
467
|
+
var DEFAULT_MAX_PARALLEL_COUNT = 5; // <- TODO: [🤹♂️]
|
|
468
|
+
/**
|
|
469
|
+
* The maximum number of attempts to execute LLM task before giving up
|
|
470
|
+
*
|
|
471
|
+
* @public exported from `@promptbook/core`
|
|
472
|
+
*/
|
|
473
|
+
var DEFAULT_MAX_EXECUTION_ATTEMPTS = 3; // <- TODO: [🤹♂️]
|
|
465
474
|
/**
|
|
466
475
|
* Where to store the cache of executions for promptbook CLI
|
|
467
476
|
*
|
|
@@ -469,7 +478,7 @@
|
|
|
469
478
|
*
|
|
470
479
|
* @public exported from `@promptbook/core`
|
|
471
480
|
*/
|
|
472
|
-
var
|
|
481
|
+
var DEFAULT_EXECUTIONS_CACHE_DIRNAME = '/.promptbook/executions-cache';
|
|
473
482
|
/**
|
|
474
483
|
* Where to store the scrape cache
|
|
475
484
|
*
|
|
@@ -477,13 +486,13 @@
|
|
|
477
486
|
*
|
|
478
487
|
* @public exported from `@promptbook/core`
|
|
479
488
|
*/
|
|
480
|
-
var
|
|
489
|
+
var DEFAULT_SCRAPE_CACHE_DIRNAME = '/.promptbook/scrape-cache';
|
|
481
490
|
/**
|
|
482
491
|
* The name of the builded pipeline collection made by CLI `ptbk make` and for lookup in `createCollectionFromDirectory`
|
|
483
492
|
*
|
|
484
493
|
* @public exported from `@promptbook/core`
|
|
485
494
|
*/
|
|
486
|
-
var
|
|
495
|
+
var DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME = "index";
|
|
487
496
|
/**
|
|
488
497
|
* Nonce which is used for replacing things in strings
|
|
489
498
|
*
|
|
@@ -547,13 +556,13 @@
|
|
|
547
556
|
*
|
|
548
557
|
* @public exported from `@promptbook/core`
|
|
549
558
|
*/
|
|
550
|
-
var
|
|
559
|
+
var DEFAULT_IS_VERBOSE = false;
|
|
551
560
|
/**
|
|
552
561
|
* @@@
|
|
553
562
|
*
|
|
554
563
|
* @public exported from `@promptbook/core`
|
|
555
564
|
*/
|
|
556
|
-
var
|
|
565
|
+
var DEFAULT_IS_AUTO_INSTALLED = false;
|
|
557
566
|
/**
|
|
558
567
|
* @@@
|
|
559
568
|
*
|
|
@@ -626,271 +635,6 @@
|
|
|
626
635
|
* Note: [🟡] Code in this file should never be published outside of `@promptbook/cli`
|
|
627
636
|
*/
|
|
628
637
|
|
|
629
|
-
/**
|
|
630
|
-
* Just marks a place of place where should be something implemented
|
|
631
|
-
* No side effects.
|
|
632
|
-
*
|
|
633
|
-
* Note: It can be usefull suppressing eslint errors of unused variables
|
|
634
|
-
*
|
|
635
|
-
* @param value any values
|
|
636
|
-
* @returns void
|
|
637
|
-
* @private within the repository
|
|
638
|
-
*/
|
|
639
|
-
function TODO_USE() {
|
|
640
|
-
var value = [];
|
|
641
|
-
for (var _i = 0; _i < arguments.length; _i++) {
|
|
642
|
-
value[_i] = arguments[_i];
|
|
643
|
-
}
|
|
644
|
-
}
|
|
645
|
-
|
|
646
|
-
/**
|
|
647
|
-
* @@@
|
|
648
|
-
*
|
|
649
|
-
* @public exported from `@promptbook/node`
|
|
650
|
-
*/
|
|
651
|
-
function $provideFilesystemForNode(options) {
|
|
652
|
-
if (!$isRunningInNode()) {
|
|
653
|
-
throw new EnvironmentMismatchError('Function `$provideFilesystemForNode` works only in Node.js environment');
|
|
654
|
-
}
|
|
655
|
-
var _a = (options || {}).isVerbose, isVerbose = _a === void 0 ? IS_VERBOSE : _a;
|
|
656
|
-
TODO_USE(isVerbose);
|
|
657
|
-
return {
|
|
658
|
-
stat: promises.stat,
|
|
659
|
-
access: promises.access,
|
|
660
|
-
constants: promises.constants,
|
|
661
|
-
readFile: promises.readFile,
|
|
662
|
-
readdir: promises.readdir,
|
|
663
|
-
};
|
|
664
|
-
}
|
|
665
|
-
/**
|
|
666
|
-
* Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
|
|
667
|
-
*/
|
|
668
|
-
|
|
669
|
-
/**
|
|
670
|
-
* This error type indicates that some part of the code is not implemented yet
|
|
671
|
-
*
|
|
672
|
-
* @public exported from `@promptbook/core`
|
|
673
|
-
*/
|
|
674
|
-
var NotYetImplementedError = /** @class */ (function (_super) {
|
|
675
|
-
__extends(NotYetImplementedError, _super);
|
|
676
|
-
function NotYetImplementedError(message) {
|
|
677
|
-
var _this = _super.call(this, spaceTrim.spaceTrim(function (block) { return "\n ".concat(block(message), "\n\n Note: This feature is not implemented yet but it will be soon.\n\n If you want speed up the implementation or just read more, look here:\n https://github.com/webgptorg/promptbook\n\n Or contact us on me@pavolhejny.com\n\n "); })) || this;
|
|
678
|
-
_this.name = 'NotYetImplementedError';
|
|
679
|
-
Object.setPrototypeOf(_this, NotYetImplementedError.prototype);
|
|
680
|
-
return _this;
|
|
681
|
-
}
|
|
682
|
-
return NotYetImplementedError;
|
|
683
|
-
}(Error));
|
|
684
|
-
|
|
685
|
-
/**
|
|
686
|
-
* @@@
|
|
687
|
-
*
|
|
688
|
-
* Note: `$` is used to indicate that this function is not a pure function - it access global scope
|
|
689
|
-
*
|
|
690
|
-
* @private internal function of `$Register`
|
|
691
|
-
*/
|
|
692
|
-
function $getGlobalScope() {
|
|
693
|
-
return Function('return this')();
|
|
694
|
-
}
|
|
695
|
-
|
|
696
|
-
/**
|
|
697
|
-
* @@@
|
|
698
|
-
*
|
|
699
|
-
* @param text @@@
|
|
700
|
-
* @returns @@@
|
|
701
|
-
* @example 'HELLO_WORLD'
|
|
702
|
-
* @example 'I_LOVE_PROMPTBOOK'
|
|
703
|
-
* @public exported from `@promptbook/utils`
|
|
704
|
-
*/
|
|
705
|
-
function normalizeTo_SCREAMING_CASE(text) {
|
|
706
|
-
var e_1, _a;
|
|
707
|
-
var charType;
|
|
708
|
-
var lastCharType = 'OTHER';
|
|
709
|
-
var normalizedName = '';
|
|
710
|
-
try {
|
|
711
|
-
for (var text_1 = __values(text), text_1_1 = text_1.next(); !text_1_1.done; text_1_1 = text_1.next()) {
|
|
712
|
-
var char = text_1_1.value;
|
|
713
|
-
var normalizedChar = void 0;
|
|
714
|
-
if (/^[a-z]$/.test(char)) {
|
|
715
|
-
charType = 'LOWERCASE';
|
|
716
|
-
normalizedChar = char.toUpperCase();
|
|
717
|
-
}
|
|
718
|
-
else if (/^[A-Z]$/.test(char)) {
|
|
719
|
-
charType = 'UPPERCASE';
|
|
720
|
-
normalizedChar = char;
|
|
721
|
-
}
|
|
722
|
-
else if (/^[0-9]$/.test(char)) {
|
|
723
|
-
charType = 'NUMBER';
|
|
724
|
-
normalizedChar = char;
|
|
725
|
-
}
|
|
726
|
-
else {
|
|
727
|
-
charType = 'OTHER';
|
|
728
|
-
normalizedChar = '_';
|
|
729
|
-
}
|
|
730
|
-
if (charType !== lastCharType &&
|
|
731
|
-
!(lastCharType === 'UPPERCASE' && charType === 'LOWERCASE') &&
|
|
732
|
-
!(lastCharType === 'NUMBER') &&
|
|
733
|
-
!(charType === 'NUMBER')) {
|
|
734
|
-
normalizedName += '_';
|
|
735
|
-
}
|
|
736
|
-
normalizedName += normalizedChar;
|
|
737
|
-
lastCharType = charType;
|
|
738
|
-
}
|
|
739
|
-
}
|
|
740
|
-
catch (e_1_1) { e_1 = { error: e_1_1 }; }
|
|
741
|
-
finally {
|
|
742
|
-
try {
|
|
743
|
-
if (text_1_1 && !text_1_1.done && (_a = text_1.return)) _a.call(text_1);
|
|
744
|
-
}
|
|
745
|
-
finally { if (e_1) throw e_1.error; }
|
|
746
|
-
}
|
|
747
|
-
normalizedName = normalizedName.replace(/_+/g, '_');
|
|
748
|
-
normalizedName = normalizedName.replace(/_?\/_?/g, '/');
|
|
749
|
-
normalizedName = normalizedName.replace(/^_/, '');
|
|
750
|
-
normalizedName = normalizedName.replace(/_$/, '');
|
|
751
|
-
return normalizedName;
|
|
752
|
-
}
|
|
753
|
-
/**
|
|
754
|
-
* TODO: Tests
|
|
755
|
-
* > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: 'Moje tabule' })).toEqual('/VtG7sR9rRJqwNEdM2/Moje tabule');
|
|
756
|
-
* > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: 'ěščřžžýáíúů' })).toEqual('/VtG7sR9rRJqwNEdM2/escrzyaieuu');
|
|
757
|
-
* > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: ' ahoj ' })).toEqual('/VtG7sR9rRJqwNEdM2/ahoj');
|
|
758
|
-
* > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: ' ahoj_ahojAhoj ahoj ' })).toEqual('/VtG7sR9rRJqwNEdM2/ahoj-ahoj-ahoj-ahoj');
|
|
759
|
-
* TODO: [🌺] Use some intermediate util splitWords
|
|
760
|
-
*/
|
|
761
|
-
|
|
762
|
-
/**
|
|
763
|
-
* @@@
|
|
764
|
-
*
|
|
765
|
-
* @param text @@@
|
|
766
|
-
* @returns @@@
|
|
767
|
-
* @example 'hello_world'
|
|
768
|
-
* @example 'i_love_promptbook'
|
|
769
|
-
* @public exported from `@promptbook/utils`
|
|
770
|
-
*/
|
|
771
|
-
function normalizeTo_snake_case(text) {
|
|
772
|
-
return normalizeTo_SCREAMING_CASE(text).toLowerCase();
|
|
773
|
-
}
|
|
774
|
-
|
|
775
|
-
/**
|
|
776
|
-
* Register is @@@
|
|
777
|
-
*
|
|
778
|
-
* Note: `$` is used to indicate that this function is not a pure function - it accesses and adds variables in global scope.
|
|
779
|
-
*
|
|
780
|
-
* @private internal utility, exported are only signleton instances of this class
|
|
781
|
-
*/
|
|
782
|
-
var $Register = /** @class */ (function () {
|
|
783
|
-
function $Register(registerName) {
|
|
784
|
-
this.registerName = registerName;
|
|
785
|
-
var storageName = "_promptbook_".concat(normalizeTo_snake_case(registerName));
|
|
786
|
-
var globalScope = $getGlobalScope();
|
|
787
|
-
if (globalScope[storageName] === undefined) {
|
|
788
|
-
globalScope[storageName] = [];
|
|
789
|
-
}
|
|
790
|
-
else if (!Array.isArray(globalScope[storageName])) {
|
|
791
|
-
throw new UnexpectedError("Expected (global) ".concat(storageName, " to be an array, but got ").concat(typeof globalScope[storageName]));
|
|
792
|
-
}
|
|
793
|
-
this.storage = globalScope[storageName];
|
|
794
|
-
}
|
|
795
|
-
$Register.prototype.list = function () {
|
|
796
|
-
// <- TODO: ReadonlyDeep<Array<TRegistered>>
|
|
797
|
-
return this.storage;
|
|
798
|
-
};
|
|
799
|
-
$Register.prototype.register = function (registered) {
|
|
800
|
-
var packageName = registered.packageName, className = registered.className;
|
|
801
|
-
var existingRegistrationIndex = this.storage.findIndex(function (item) { return item.packageName === packageName && item.className === className; });
|
|
802
|
-
var existingRegistration = this.storage[existingRegistrationIndex];
|
|
803
|
-
if (!existingRegistration) {
|
|
804
|
-
this.storage.push(registered);
|
|
805
|
-
}
|
|
806
|
-
else {
|
|
807
|
-
this.storage[existingRegistrationIndex] = registered;
|
|
808
|
-
}
|
|
809
|
-
return {
|
|
810
|
-
registerName: this.registerName,
|
|
811
|
-
packageName: packageName,
|
|
812
|
-
className: className,
|
|
813
|
-
get isDestroyed() {
|
|
814
|
-
return false;
|
|
815
|
-
},
|
|
816
|
-
destroy: function () {
|
|
817
|
-
throw new NotYetImplementedError("Registration to ".concat(this.registerName, " is permanent in this version of Promptbook"));
|
|
818
|
-
},
|
|
819
|
-
};
|
|
820
|
-
};
|
|
821
|
-
return $Register;
|
|
822
|
-
}());
|
|
823
|
-
|
|
824
|
-
/**
|
|
825
|
-
* @@@
|
|
826
|
-
*
|
|
827
|
-
* Note: `$` is used to indicate that this interacts with the global scope
|
|
828
|
-
* @singleton Only one instance of each register is created per build, but thare can be more @@@
|
|
829
|
-
* @public exported from `@promptbook/core`
|
|
830
|
-
*/
|
|
831
|
-
var $scrapersRegister = new $Register('scraper_constructors');
|
|
832
|
-
/**
|
|
833
|
-
* TODO: [®] DRY Register logic
|
|
834
|
-
*/
|
|
835
|
-
|
|
836
|
-
/**
|
|
837
|
-
* !!!!!!
|
|
838
|
-
*
|
|
839
|
-
* 1) @@@
|
|
840
|
-
* 2) @@@
|
|
841
|
-
*
|
|
842
|
-
* @public exported from `@promptbook/node`
|
|
843
|
-
*/
|
|
844
|
-
function $provideScrapersForNode(tools, options) {
|
|
845
|
-
return __awaiter(this, void 0, void 0, function () {
|
|
846
|
-
var _a, _b, isAutoInstalled, _c, isVerbose, scrapers, _d, _e, scraperFactory, scraper, e_1_1;
|
|
847
|
-
var e_1, _f;
|
|
848
|
-
return __generator(this, function (_g) {
|
|
849
|
-
switch (_g.label) {
|
|
850
|
-
case 0:
|
|
851
|
-
if (!$isRunningInNode()) {
|
|
852
|
-
throw new EnvironmentMismatchError('Function `$getScrapersForNode` works only in Node.js environment');
|
|
853
|
-
}
|
|
854
|
-
_a = options || {}, _b = _a.isAutoInstalled, isAutoInstalled = _b === void 0 ? IS_AUTO_INSTALLED : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? IS_VERBOSE : _c;
|
|
855
|
-
TODO_USE(isAutoInstalled);
|
|
856
|
-
TODO_USE(isVerbose);
|
|
857
|
-
scrapers = [];
|
|
858
|
-
_g.label = 1;
|
|
859
|
-
case 1:
|
|
860
|
-
_g.trys.push([1, 6, 7, 8]);
|
|
861
|
-
_d = __values($scrapersRegister.list()), _e = _d.next();
|
|
862
|
-
_g.label = 2;
|
|
863
|
-
case 2:
|
|
864
|
-
if (!!_e.done) return [3 /*break*/, 5];
|
|
865
|
-
scraperFactory = _e.value;
|
|
866
|
-
return [4 /*yield*/, scraperFactory(tools, options || {})];
|
|
867
|
-
case 3:
|
|
868
|
-
scraper = _g.sent();
|
|
869
|
-
scrapers.push(scraper);
|
|
870
|
-
_g.label = 4;
|
|
871
|
-
case 4:
|
|
872
|
-
_e = _d.next();
|
|
873
|
-
return [3 /*break*/, 2];
|
|
874
|
-
case 5: return [3 /*break*/, 8];
|
|
875
|
-
case 6:
|
|
876
|
-
e_1_1 = _g.sent();
|
|
877
|
-
e_1 = { error: e_1_1 };
|
|
878
|
-
return [3 /*break*/, 8];
|
|
879
|
-
case 7:
|
|
880
|
-
try {
|
|
881
|
-
if (_e && !_e.done && (_f = _d.return)) _f.call(_d);
|
|
882
|
-
}
|
|
883
|
-
finally { if (e_1) throw e_1.error; }
|
|
884
|
-
return [7 /*endfinally*/];
|
|
885
|
-
case 8: return [2 /*return*/, scrapers];
|
|
886
|
-
}
|
|
887
|
-
});
|
|
888
|
-
});
|
|
889
|
-
}
|
|
890
|
-
/**
|
|
891
|
-
* Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
|
|
892
|
-
*/
|
|
893
|
-
|
|
894
638
|
/**
|
|
895
639
|
* Converts PipelineCollection to serialized JSON
|
|
896
640
|
*
|
|
@@ -976,7 +720,6 @@
|
|
|
976
720
|
pipelineString += '\n\n';
|
|
977
721
|
pipelineString += description;
|
|
978
722
|
}
|
|
979
|
-
// TODO:> const commands: Array<Command>
|
|
980
723
|
var commands = [];
|
|
981
724
|
if (pipelineUrl) {
|
|
982
725
|
commands.push("PIPELINE URL ".concat(pipelineUrl));
|
|
@@ -1032,7 +775,6 @@
|
|
|
1032
775
|
pipelineString += '\n\n';
|
|
1033
776
|
pipelineString += description_1;
|
|
1034
777
|
}
|
|
1035
|
-
// TODO:> const commands: Array<Command>
|
|
1036
778
|
var commands_1 = [];
|
|
1037
779
|
var contentLanguage = 'text';
|
|
1038
780
|
if (templateType === 'PROMPT_TEMPLATE') {
|
|
@@ -1331,8 +1073,7 @@
|
|
|
1331
1073
|
* Multiple LLM Execution Tools is a proxy server that uses multiple execution tools internally and exposes the executor interface externally.
|
|
1332
1074
|
*
|
|
1333
1075
|
* Note: Internal utility of `joinLlmExecutionTools` but exposed type
|
|
1334
|
-
* @public exported from `@promptbook/
|
|
1335
|
-
* TODO: !!!!!! Export as runtime class not just type
|
|
1076
|
+
* @public exported from `@promptbook/core`
|
|
1336
1077
|
*/
|
|
1337
1078
|
var MultipleLlmExecutionTools = /** @class */ (function () {
|
|
1338
1079
|
/**
|
|
@@ -2491,6 +2232,22 @@
|
|
|
2491
2232
|
return LimitReachedError;
|
|
2492
2233
|
}(Error));
|
|
2493
2234
|
|
|
2235
|
+
/**
|
|
2236
|
+
* This error type indicates that some part of the code is not implemented yet
|
|
2237
|
+
*
|
|
2238
|
+
* @public exported from `@promptbook/core`
|
|
2239
|
+
*/
|
|
2240
|
+
var NotYetImplementedError = /** @class */ (function (_super) {
|
|
2241
|
+
__extends(NotYetImplementedError, _super);
|
|
2242
|
+
function NotYetImplementedError(message) {
|
|
2243
|
+
var _this = _super.call(this, spaceTrim.spaceTrim(function (block) { return "\n ".concat(block(message), "\n\n Note: This feature is not implemented yet but it will be soon.\n\n If you want speed up the implementation or just read more, look here:\n https://github.com/webgptorg/promptbook\n\n Or contact us on me@pavolhejny.com\n\n "); })) || this;
|
|
2244
|
+
_this.name = 'NotYetImplementedError';
|
|
2245
|
+
Object.setPrototypeOf(_this, NotYetImplementedError.prototype);
|
|
2246
|
+
return _this;
|
|
2247
|
+
}
|
|
2248
|
+
return NotYetImplementedError;
|
|
2249
|
+
}(Error));
|
|
2250
|
+
|
|
2494
2251
|
/**
|
|
2495
2252
|
* Index of all custom errors
|
|
2496
2253
|
*
|
|
@@ -2552,6 +2309,7 @@
|
|
|
2552
2309
|
}
|
|
2553
2310
|
}
|
|
2554
2311
|
/**
|
|
2312
|
+
* TODO: [🐚] This function should be removed OR changed OR be completely rewritten
|
|
2555
2313
|
* TODO: [🧠] Can this return type be better typed than void
|
|
2556
2314
|
*/
|
|
2557
2315
|
|
|
@@ -2809,6 +2567,23 @@
|
|
|
2809
2567
|
return union;
|
|
2810
2568
|
}
|
|
2811
2569
|
|
|
2570
|
+
/**
|
|
2571
|
+
* Just marks a place of place where should be something implemented
|
|
2572
|
+
* No side effects.
|
|
2573
|
+
*
|
|
2574
|
+
* Note: It can be usefull suppressing eslint errors of unused variables
|
|
2575
|
+
*
|
|
2576
|
+
* @param value any values
|
|
2577
|
+
* @returns void
|
|
2578
|
+
* @private within the repository
|
|
2579
|
+
*/
|
|
2580
|
+
function TODO_USE() {
|
|
2581
|
+
var value = [];
|
|
2582
|
+
for (var _i = 0; _i < arguments.length; _i++) {
|
|
2583
|
+
value[_i] = arguments[_i];
|
|
2584
|
+
}
|
|
2585
|
+
}
|
|
2586
|
+
|
|
2812
2587
|
/**
|
|
2813
2588
|
* This error indicates problems parsing the format value
|
|
2814
2589
|
*
|
|
@@ -3841,12 +3616,11 @@
|
|
|
3841
3616
|
*/
|
|
3842
3617
|
function executeAttempts(options) {
|
|
3843
3618
|
return __awaiter(this, void 0, void 0, function () {
|
|
3844
|
-
var jokerParameterNames, priority, maxAttempts, preparedContent, parameters, template, preparedPipeline, tools,
|
|
3619
|
+
var jokerParameterNames, priority, maxAttempts, preparedContent, parameters, template, preparedPipeline, tools, $executionReport, pipelineIdentification, maxExecutionAttempts, $ongoingTemplateResult, _llms, llmTools, _loop_1, attempt, state_1;
|
|
3845
3620
|
return __generator(this, function (_a) {
|
|
3846
3621
|
switch (_a.label) {
|
|
3847
3622
|
case 0:
|
|
3848
|
-
jokerParameterNames = options.jokerParameterNames, priority = options.priority, maxAttempts = options.maxAttempts, preparedContent = options.preparedContent, parameters = options.parameters, template = options.template, preparedPipeline = options.preparedPipeline, tools = options.tools,
|
|
3849
|
-
maxExecutionAttempts = settings.maxExecutionAttempts;
|
|
3623
|
+
jokerParameterNames = options.jokerParameterNames, priority = options.priority, maxAttempts = options.maxAttempts, preparedContent = options.preparedContent, parameters = options.parameters, template = options.template, preparedPipeline = options.preparedPipeline, tools = options.tools, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification, maxExecutionAttempts = options.maxExecutionAttempts;
|
|
3850
3624
|
$ongoingTemplateResult = {
|
|
3851
3625
|
$result: null,
|
|
3852
3626
|
$resultString: null,
|
|
@@ -4212,12 +3986,12 @@
|
|
|
4212
3986
|
*/
|
|
4213
3987
|
function executeFormatSubvalues(options) {
|
|
4214
3988
|
return __awaiter(this, void 0, void 0, function () {
|
|
4215
|
-
var template, jokerParameterNames, parameters, priority,
|
|
3989
|
+
var template, jokerParameterNames, parameters, priority, csvSettings, pipelineIdentification, parameterValue, formatDefinition, subvalueDefinition, formatSettings, resultString;
|
|
4216
3990
|
var _this = this;
|
|
4217
3991
|
return __generator(this, function (_a) {
|
|
4218
3992
|
switch (_a.label) {
|
|
4219
3993
|
case 0:
|
|
4220
|
-
template = options.template, jokerParameterNames = options.jokerParameterNames, parameters = options.parameters, priority = options.priority,
|
|
3994
|
+
template = options.template, jokerParameterNames = options.jokerParameterNames, parameters = options.parameters, priority = options.priority, csvSettings = options.csvSettings, pipelineIdentification = options.pipelineIdentification;
|
|
4221
3995
|
if (template.foreach === undefined) {
|
|
4222
3996
|
return [2 /*return*/, /* not await */ executeAttempts(options)];
|
|
4223
3997
|
}
|
|
@@ -4247,7 +4021,7 @@
|
|
|
4247
4021
|
.join('\n')), "\n\n [\u26F7] This should never happen because subformat name should be validated during parsing\n\n ").concat(block(pipelineIdentification), "\n "); }));
|
|
4248
4022
|
}
|
|
4249
4023
|
if (formatDefinition.formatName === 'CSV') {
|
|
4250
|
-
formatSettings =
|
|
4024
|
+
formatSettings = csvSettings;
|
|
4251
4025
|
// <- TODO: [🤹♂️] More universal, make simmilar pattern for other formats for example \n vs \r\n in text
|
|
4252
4026
|
}
|
|
4253
4027
|
return [4 /*yield*/, subvalueDefinition.mapValues(parameterValue, template.foreach.outputSubparameterName, formatSettings, function (subparameters, index) { return __awaiter(_this, void 0, void 0, function () {
|
|
@@ -4400,13 +4174,12 @@
|
|
|
4400
4174
|
*/
|
|
4401
4175
|
function executeTemplate(options) {
|
|
4402
4176
|
return __awaiter(this, void 0, void 0, function () {
|
|
4403
|
-
var currentTemplate, preparedPipeline, parametersToPass, tools, onProgress,
|
|
4404
|
-
var e_1,
|
|
4405
|
-
return __generator(this, function (
|
|
4406
|
-
switch (
|
|
4177
|
+
var currentTemplate, preparedPipeline, parametersToPass, tools, onProgress, $executionReport, pipelineIdentification, _a, maxExecutionAttempts, name, title, priority, usedParameterNames, dependentParameterNames, definedParameters, _b, _c, _d, definedParameterNames, parameters, _loop_1, _e, _f, parameterName, maxAttempts, jokerParameterNames, preparedContent, resultString;
|
|
4178
|
+
var e_1, _g, _h;
|
|
4179
|
+
return __generator(this, function (_j) {
|
|
4180
|
+
switch (_j.label) {
|
|
4407
4181
|
case 0:
|
|
4408
|
-
currentTemplate = options.currentTemplate, preparedPipeline = options.preparedPipeline, parametersToPass = options.parametersToPass, tools = options.tools, onProgress = options.onProgress,
|
|
4409
|
-
maxExecutionAttempts = settings.maxExecutionAttempts;
|
|
4182
|
+
currentTemplate = options.currentTemplate, preparedPipeline = options.preparedPipeline, parametersToPass = options.parametersToPass, tools = options.tools, onProgress = options.onProgress, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification, _a = options.maxExecutionAttempts, maxExecutionAttempts = _a === void 0 ? DEFAULT_MAX_EXECUTION_ATTEMPTS : _a;
|
|
4410
4183
|
name = "pipeline-executor-frame-".concat(currentTemplate.name);
|
|
4411
4184
|
title = currentTemplate.title;
|
|
4412
4185
|
priority = preparedPipeline.templates.length - preparedPipeline.templates.indexOf(currentTemplate);
|
|
@@ -4421,7 +4194,7 @@
|
|
|
4421
4194
|
// <- [🍸]
|
|
4422
4195
|
})];
|
|
4423
4196
|
case 1:
|
|
4424
|
-
|
|
4197
|
+
_j.sent();
|
|
4425
4198
|
usedParameterNames = extractParameterNamesFromTemplate(currentTemplate);
|
|
4426
4199
|
dependentParameterNames = new Set(currentTemplate.dependentParameterNames);
|
|
4427
4200
|
// TODO: [👩🏾🤝👩🏻] Use here `mapAvailableToExpectedParameters`
|
|
@@ -4432,15 +4205,15 @@
|
|
|
4432
4205
|
.map(function (name) { return "{".concat(name, "}"); })
|
|
4433
4206
|
.join(', '), "\n\n ").concat(block(pipelineIdentification), "\n\n "); }));
|
|
4434
4207
|
}
|
|
4435
|
-
|
|
4436
|
-
|
|
4208
|
+
_c = (_b = Object).freeze;
|
|
4209
|
+
_d = [{}];
|
|
4437
4210
|
return [4 /*yield*/, getReservedParametersForTemplate({
|
|
4438
4211
|
preparedPipeline: preparedPipeline,
|
|
4439
4212
|
template: currentTemplate,
|
|
4440
4213
|
pipelineIdentification: pipelineIdentification,
|
|
4441
4214
|
})];
|
|
4442
4215
|
case 2:
|
|
4443
|
-
definedParameters =
|
|
4216
|
+
definedParameters = _c.apply(_b, [__assign.apply(void 0, [__assign.apply(void 0, _d.concat([(_j.sent())])), parametersToPass])]);
|
|
4444
4217
|
definedParameterNames = new Set(Object.keys(definedParameters));
|
|
4445
4218
|
parameters = {};
|
|
4446
4219
|
_loop_1 = function (parameterName) {
|
|
@@ -4460,15 +4233,15 @@
|
|
|
4460
4233
|
try {
|
|
4461
4234
|
// Note: [2] Check that all used parameters are defined and removing unused parameters for this template
|
|
4462
4235
|
// TODO: [👩🏾🤝👩🏻] Use here `mapAvailableToExpectedParameters`
|
|
4463
|
-
for (
|
|
4464
|
-
parameterName =
|
|
4236
|
+
for (_e = __values(Array.from(union(definedParameterNames, usedParameterNames, dependentParameterNames))), _f = _e.next(); !_f.done; _f = _e.next()) {
|
|
4237
|
+
parameterName = _f.value;
|
|
4465
4238
|
_loop_1(parameterName);
|
|
4466
4239
|
}
|
|
4467
4240
|
}
|
|
4468
4241
|
catch (e_1_1) { e_1 = { error: e_1_1 }; }
|
|
4469
4242
|
finally {
|
|
4470
4243
|
try {
|
|
4471
|
-
if (
|
|
4244
|
+
if (_f && !_f.done && (_g = _e.return)) _g.call(_e);
|
|
4472
4245
|
}
|
|
4473
4246
|
finally { if (e_1) throw e_1.error; }
|
|
4474
4247
|
}
|
|
@@ -4488,12 +4261,11 @@
|
|
|
4488
4261
|
template: currentTemplate,
|
|
4489
4262
|
preparedPipeline: preparedPipeline,
|
|
4490
4263
|
tools: tools,
|
|
4491
|
-
settings: settings,
|
|
4492
4264
|
$executionReport: $executionReport,
|
|
4493
4265
|
pipelineIdentification: pipelineIdentification,
|
|
4494
4266
|
})];
|
|
4495
4267
|
case 3:
|
|
4496
|
-
resultString =
|
|
4268
|
+
resultString = _j.sent();
|
|
4497
4269
|
return [4 /*yield*/, onProgress({
|
|
4498
4270
|
name: name,
|
|
4499
4271
|
title: title,
|
|
@@ -4505,12 +4277,12 @@
|
|
|
4505
4277
|
// <- [🍸]
|
|
4506
4278
|
})];
|
|
4507
4279
|
case 4:
|
|
4508
|
-
|
|
4509
|
-
return [2 /*return*/, Object.freeze((
|
|
4510
|
-
|
|
4280
|
+
_j.sent();
|
|
4281
|
+
return [2 /*return*/, Object.freeze((_h = {},
|
|
4282
|
+
_h[currentTemplate.resultingParameterName] =
|
|
4511
4283
|
// <- Note: [👩👩👧] No need to detect parameter collision here because pipeline checks logic consistency during construction
|
|
4512
4284
|
resultString,
|
|
4513
|
-
|
|
4285
|
+
_h))];
|
|
4514
4286
|
}
|
|
4515
4287
|
});
|
|
4516
4288
|
});
|
|
@@ -4569,13 +4341,12 @@
|
|
|
4569
4341
|
*/
|
|
4570
4342
|
function executePipeline(options) {
|
|
4571
4343
|
return __awaiter(this, void 0, void 0, function () {
|
|
4572
|
-
var inputParameters, tools, onProgress, pipeline, setPreparedPipeline, pipelineIdentification,
|
|
4344
|
+
var inputParameters, tools, onProgress, pipeline, setPreparedPipeline, pipelineIdentification, maxParallelCount, rootDirname, _a, isVerbose, preparedPipeline, errors, warnings, executionReport, isReturned, _b, _c, parameter, e_1_1, _loop_1, _d, _e, parameterName, state_1, e_2_1, parametersToPass, resovedParameterNames_1, unresovedTemplates_1, resolving_1, loopLimit, _loop_2, error_1, usage_1, outputParameters_1, usage, outputParameters;
|
|
4573
4345
|
var e_1, _f, e_2, _g;
|
|
4574
4346
|
return __generator(this, function (_h) {
|
|
4575
4347
|
switch (_h.label) {
|
|
4576
4348
|
case 0:
|
|
4577
|
-
inputParameters = options.inputParameters, tools = options.tools, onProgress = options.onProgress, pipeline = options.pipeline, setPreparedPipeline = options.setPreparedPipeline, pipelineIdentification = options.pipelineIdentification,
|
|
4578
|
-
maxParallelCount = settings.maxParallelCount, rootDirname = settings.rootDirname, _a = settings.isVerbose, isVerbose = _a === void 0 ? IS_VERBOSE : _a;
|
|
4349
|
+
inputParameters = options.inputParameters, tools = options.tools, onProgress = options.onProgress, pipeline = options.pipeline, setPreparedPipeline = options.setPreparedPipeline, pipelineIdentification = options.pipelineIdentification, maxParallelCount = options.maxParallelCount, rootDirname = options.rootDirname, _a = options.isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a;
|
|
4579
4350
|
preparedPipeline = options.preparedPipeline;
|
|
4580
4351
|
if (!(preparedPipeline === undefined)) return [3 /*break*/, 2];
|
|
4581
4352
|
return [4 /*yield*/, preparePipeline(pipeline, tools, {
|
|
@@ -4760,12 +4531,7 @@
|
|
|
4760
4531
|
return [3 /*break*/, 4];
|
|
4761
4532
|
case 3:
|
|
4762
4533
|
unresovedTemplates_1 = unresovedTemplates_1.filter(function (template) { return template !== currentTemplate; });
|
|
4763
|
-
work_1 = executeTemplate({
|
|
4764
|
-
currentTemplate: currentTemplate,
|
|
4765
|
-
preparedPipeline: preparedPipeline,
|
|
4766
|
-
parametersToPass: parametersToPass,
|
|
4767
|
-
tools: tools,
|
|
4768
|
-
onProgress: function (progress) {
|
|
4534
|
+
work_1 = executeTemplate(__assign(__assign({}, options), { currentTemplate: currentTemplate, preparedPipeline: preparedPipeline, parametersToPass: parametersToPass, tools: tools, onProgress: function (progress) {
|
|
4769
4535
|
if (isReturned) {
|
|
4770
4536
|
throw new UnexpectedError(spaceTrim.spaceTrim(function (block) { return "\n Can not call `onProgress` after pipeline execution is finished\n\n ".concat(block(pipelineIdentification), "\n\n ").concat(block(JSON.stringify(progress, null, 4)
|
|
4771
4537
|
.split('\n')
|
|
@@ -4775,11 +4541,7 @@
|
|
|
4775
4541
|
if (onProgress) {
|
|
4776
4542
|
onProgress(progress);
|
|
4777
4543
|
}
|
|
4778
|
-
},
|
|
4779
|
-
settings: settings,
|
|
4780
|
-
$executionReport: executionReport,
|
|
4781
|
-
pipelineIdentification: spaceTrim.spaceTrim(function (block) { return "\n ".concat(block(pipelineIdentification), "\n Template name: ").concat(currentTemplate.name, "\n Template title: ").concat(currentTemplate.title, "\n "); }),
|
|
4782
|
-
})
|
|
4544
|
+
}, $executionReport: executionReport, pipelineIdentification: spaceTrim.spaceTrim(function (block) { return "\n ".concat(block(pipelineIdentification), "\n Template name: ").concat(currentTemplate.name, "\n Template title: ").concat(currentTemplate.title, "\n "); }) }))
|
|
4783
4545
|
.then(function (newParametersToPass) {
|
|
4784
4546
|
parametersToPass = __assign(__assign({}, newParametersToPass), parametersToPass);
|
|
4785
4547
|
resovedParameterNames_1 = __spreadArray(__spreadArray([], __read(resovedParameterNames_1), false), [currentTemplate.resultingParameterName], false);
|
|
@@ -4883,8 +4645,7 @@
|
|
|
4883
4645
|
*/
|
|
4884
4646
|
function createPipelineExecutor(options) {
|
|
4885
4647
|
var _this = this;
|
|
4886
|
-
var pipeline = options.pipeline, tools = options.tools, _a = options.
|
|
4887
|
-
var _b = settings.maxExecutionAttempts, maxExecutionAttempts = _b === void 0 ? MAX_EXECUTION_ATTEMPTS : _b, _c = settings.maxParallelCount, maxParallelCount = _c === void 0 ? MAX_PARALLEL_COUNT : _c, _d = settings.csvSettings, csvSettings = _d === void 0 ? DEFAULT_CSV_SETTINGS : _d, _e = settings.isVerbose, isVerbose = _e === void 0 ? IS_VERBOSE : _e, _f = settings.isNotPreparedWarningSupressed, isNotPreparedWarningSupressed = _f === void 0 ? false : _f, _g = settings.rootDirname, rootDirname = _g === void 0 ? null : _g;
|
|
4648
|
+
var pipeline = options.pipeline, tools = options.tools, _a = options.maxExecutionAttempts, maxExecutionAttempts = _a === void 0 ? DEFAULT_MAX_EXECUTION_ATTEMPTS : _a, _b = options.maxParallelCount, maxParallelCount = _b === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _b, _c = options.csvSettings, csvSettings = _c === void 0 ? DEFAULT_CSV_SETTINGS : _c, _d = options.isVerbose, isVerbose = _d === void 0 ? DEFAULT_IS_VERBOSE : _d, _e = options.isNotPreparedWarningSupressed, isNotPreparedWarningSupressed = _e === void 0 ? false : _e, _f = options.rootDirname, rootDirname = _f === void 0 ? null : _f;
|
|
4888
4649
|
validatePipeline(pipeline);
|
|
4889
4650
|
var pipelineIdentification = (function () {
|
|
4890
4651
|
// Note: This is a 😐 implementation of [🚞]
|
|
@@ -4918,14 +4679,12 @@
|
|
|
4918
4679
|
tools: tools,
|
|
4919
4680
|
onProgress: onProgress,
|
|
4920
4681
|
pipelineIdentification: spaceTrim.spaceTrim(function (block) { return "\n ".concat(block(pipelineIdentification), "\n ").concat(runCount === 1 ? '' : "Run #".concat(runCount), "\n "); }),
|
|
4921
|
-
|
|
4922
|
-
|
|
4923
|
-
|
|
4924
|
-
|
|
4925
|
-
|
|
4926
|
-
|
|
4927
|
-
rootDirname: rootDirname,
|
|
4928
|
-
},
|
|
4682
|
+
maxExecutionAttempts: maxExecutionAttempts,
|
|
4683
|
+
maxParallelCount: maxParallelCount,
|
|
4684
|
+
csvSettings: csvSettings,
|
|
4685
|
+
isVerbose: isVerbose,
|
|
4686
|
+
isNotPreparedWarningSupressed: isNotPreparedWarningSupressed,
|
|
4687
|
+
rootDirname: rootDirname,
|
|
4929
4688
|
})];
|
|
4930
4689
|
});
|
|
4931
4690
|
}); };
|
|
@@ -4948,7 +4707,7 @@
|
|
|
4948
4707
|
return __generator(this, function (_d) {
|
|
4949
4708
|
switch (_d.label) {
|
|
4950
4709
|
case 0:
|
|
4951
|
-
_a = options.isVerbose, isVerbose = _a === void 0 ?
|
|
4710
|
+
_a = options.isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a;
|
|
4952
4711
|
if (tools === undefined || tools.llm === undefined) {
|
|
4953
4712
|
throw new MissingToolsError('LLM tools are required for preparing persona');
|
|
4954
4713
|
}
|
|
@@ -5004,20 +4763,171 @@
|
|
|
5004
4763
|
*/
|
|
5005
4764
|
|
|
5006
4765
|
/**
|
|
5007
|
-
* This error indicates that the promptbook can not retrieve knowledge from external sources
|
|
4766
|
+
* This error indicates that the promptbook can not retrieve knowledge from external sources
|
|
4767
|
+
*
|
|
4768
|
+
* @public exported from `@promptbook/core`
|
|
4769
|
+
*/
|
|
4770
|
+
var KnowledgeScrapeError = /** @class */ (function (_super) {
|
|
4771
|
+
__extends(KnowledgeScrapeError, _super);
|
|
4772
|
+
function KnowledgeScrapeError(message) {
|
|
4773
|
+
var _this = _super.call(this, message) || this;
|
|
4774
|
+
_this.name = 'KnowledgeScrapeError';
|
|
4775
|
+
Object.setPrototypeOf(_this, KnowledgeScrapeError.prototype);
|
|
4776
|
+
return _this;
|
|
4777
|
+
}
|
|
4778
|
+
return KnowledgeScrapeError;
|
|
4779
|
+
}(Error));
|
|
4780
|
+
|
|
4781
|
+
/**
|
|
4782
|
+
* @@@
|
|
4783
|
+
*
|
|
4784
|
+
* Note: `$` is used to indicate that this function is not a pure function - it access global scope
|
|
4785
|
+
*
|
|
4786
|
+
* @private internal function of `$Register`
|
|
4787
|
+
*/
|
|
4788
|
+
function $getGlobalScope() {
|
|
4789
|
+
return Function('return this')();
|
|
4790
|
+
}
|
|
4791
|
+
|
|
4792
|
+
/**
|
|
4793
|
+
* @@@
|
|
4794
|
+
*
|
|
4795
|
+
* @param text @@@
|
|
4796
|
+
* @returns @@@
|
|
4797
|
+
* @example 'HELLO_WORLD'
|
|
4798
|
+
* @example 'I_LOVE_PROMPTBOOK'
|
|
4799
|
+
* @public exported from `@promptbook/utils`
|
|
4800
|
+
*/
|
|
4801
|
+
function normalizeTo_SCREAMING_CASE(text) {
|
|
4802
|
+
var e_1, _a;
|
|
4803
|
+
var charType;
|
|
4804
|
+
var lastCharType = 'OTHER';
|
|
4805
|
+
var normalizedName = '';
|
|
4806
|
+
try {
|
|
4807
|
+
for (var text_1 = __values(text), text_1_1 = text_1.next(); !text_1_1.done; text_1_1 = text_1.next()) {
|
|
4808
|
+
var char = text_1_1.value;
|
|
4809
|
+
var normalizedChar = void 0;
|
|
4810
|
+
if (/^[a-z]$/.test(char)) {
|
|
4811
|
+
charType = 'LOWERCASE';
|
|
4812
|
+
normalizedChar = char.toUpperCase();
|
|
4813
|
+
}
|
|
4814
|
+
else if (/^[A-Z]$/.test(char)) {
|
|
4815
|
+
charType = 'UPPERCASE';
|
|
4816
|
+
normalizedChar = char;
|
|
4817
|
+
}
|
|
4818
|
+
else if (/^[0-9]$/.test(char)) {
|
|
4819
|
+
charType = 'NUMBER';
|
|
4820
|
+
normalizedChar = char;
|
|
4821
|
+
}
|
|
4822
|
+
else {
|
|
4823
|
+
charType = 'OTHER';
|
|
4824
|
+
normalizedChar = '_';
|
|
4825
|
+
}
|
|
4826
|
+
if (charType !== lastCharType &&
|
|
4827
|
+
!(lastCharType === 'UPPERCASE' && charType === 'LOWERCASE') &&
|
|
4828
|
+
!(lastCharType === 'NUMBER') &&
|
|
4829
|
+
!(charType === 'NUMBER')) {
|
|
4830
|
+
normalizedName += '_';
|
|
4831
|
+
}
|
|
4832
|
+
normalizedName += normalizedChar;
|
|
4833
|
+
lastCharType = charType;
|
|
4834
|
+
}
|
|
4835
|
+
}
|
|
4836
|
+
catch (e_1_1) { e_1 = { error: e_1_1 }; }
|
|
4837
|
+
finally {
|
|
4838
|
+
try {
|
|
4839
|
+
if (text_1_1 && !text_1_1.done && (_a = text_1.return)) _a.call(text_1);
|
|
4840
|
+
}
|
|
4841
|
+
finally { if (e_1) throw e_1.error; }
|
|
4842
|
+
}
|
|
4843
|
+
normalizedName = normalizedName.replace(/_+/g, '_');
|
|
4844
|
+
normalizedName = normalizedName.replace(/_?\/_?/g, '/');
|
|
4845
|
+
normalizedName = normalizedName.replace(/^_/, '');
|
|
4846
|
+
normalizedName = normalizedName.replace(/_$/, '');
|
|
4847
|
+
return normalizedName;
|
|
4848
|
+
}
|
|
4849
|
+
/**
|
|
4850
|
+
* TODO: Tests
|
|
4851
|
+
* > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: 'Moje tabule' })).toEqual('/VtG7sR9rRJqwNEdM2/Moje tabule');
|
|
4852
|
+
* > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: 'ěščřžžýáíúů' })).toEqual('/VtG7sR9rRJqwNEdM2/escrzyaieuu');
|
|
4853
|
+
* > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: ' ahoj ' })).toEqual('/VtG7sR9rRJqwNEdM2/ahoj');
|
|
4854
|
+
* > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: ' ahoj_ahojAhoj ahoj ' })).toEqual('/VtG7sR9rRJqwNEdM2/ahoj-ahoj-ahoj-ahoj');
|
|
4855
|
+
* TODO: [🌺] Use some intermediate util splitWords
|
|
4856
|
+
*/
|
|
4857
|
+
|
|
4858
|
+
/**
|
|
4859
|
+
* @@@
|
|
4860
|
+
*
|
|
4861
|
+
* @param text @@@
|
|
4862
|
+
* @returns @@@
|
|
4863
|
+
* @example 'hello_world'
|
|
4864
|
+
* @example 'i_love_promptbook'
|
|
4865
|
+
* @public exported from `@promptbook/utils`
|
|
4866
|
+
*/
|
|
4867
|
+
function normalizeTo_snake_case(text) {
|
|
4868
|
+
return normalizeTo_SCREAMING_CASE(text).toLowerCase();
|
|
4869
|
+
}
|
|
4870
|
+
|
|
4871
|
+
/**
|
|
4872
|
+
* Register is @@@
|
|
4873
|
+
*
|
|
4874
|
+
* Note: `$` is used to indicate that this function is not a pure function - it accesses and adds variables in global scope.
|
|
4875
|
+
*
|
|
4876
|
+
* @private internal utility, exported are only signleton instances of this class
|
|
4877
|
+
*/
|
|
4878
|
+
var $Register = /** @class */ (function () {
|
|
4879
|
+
function $Register(registerName) {
|
|
4880
|
+
this.registerName = registerName;
|
|
4881
|
+
var storageName = "_promptbook_".concat(normalizeTo_snake_case(registerName));
|
|
4882
|
+
var globalScope = $getGlobalScope();
|
|
4883
|
+
if (globalScope[storageName] === undefined) {
|
|
4884
|
+
globalScope[storageName] = [];
|
|
4885
|
+
}
|
|
4886
|
+
else if (!Array.isArray(globalScope[storageName])) {
|
|
4887
|
+
throw new UnexpectedError("Expected (global) ".concat(storageName, " to be an array, but got ").concat(typeof globalScope[storageName]));
|
|
4888
|
+
}
|
|
4889
|
+
this.storage = globalScope[storageName];
|
|
4890
|
+
}
|
|
4891
|
+
$Register.prototype.list = function () {
|
|
4892
|
+
// <- TODO: ReadonlyDeep<ReadonlyArray<TRegistered>>
|
|
4893
|
+
return this.storage;
|
|
4894
|
+
};
|
|
4895
|
+
$Register.prototype.register = function (registered) {
|
|
4896
|
+
var packageName = registered.packageName, className = registered.className;
|
|
4897
|
+
var existingRegistrationIndex = this.storage.findIndex(function (item) { return item.packageName === packageName && item.className === className; });
|
|
4898
|
+
var existingRegistration = this.storage[existingRegistrationIndex];
|
|
4899
|
+
if (!existingRegistration) {
|
|
4900
|
+
this.storage.push(registered);
|
|
4901
|
+
}
|
|
4902
|
+
else {
|
|
4903
|
+
this.storage[existingRegistrationIndex] = registered;
|
|
4904
|
+
}
|
|
4905
|
+
return {
|
|
4906
|
+
registerName: this.registerName,
|
|
4907
|
+
packageName: packageName,
|
|
4908
|
+
className: className,
|
|
4909
|
+
get isDestroyed() {
|
|
4910
|
+
return false;
|
|
4911
|
+
},
|
|
4912
|
+
destroy: function () {
|
|
4913
|
+
throw new NotYetImplementedError("Registration to ".concat(this.registerName, " is permanent in this version of Promptbook"));
|
|
4914
|
+
},
|
|
4915
|
+
};
|
|
4916
|
+
};
|
|
4917
|
+
return $Register;
|
|
4918
|
+
}());
|
|
4919
|
+
|
|
4920
|
+
/**
|
|
4921
|
+
* @@@
|
|
5008
4922
|
*
|
|
4923
|
+
* Note: `$` is used to indicate that this interacts with the global scope
|
|
4924
|
+
* @singleton Only one instance of each register is created per build, but thare can be more @@@
|
|
5009
4925
|
* @public exported from `@promptbook/core`
|
|
5010
4926
|
*/
|
|
5011
|
-
var
|
|
5012
|
-
|
|
5013
|
-
|
|
5014
|
-
|
|
5015
|
-
_this.name = 'KnowledgeScrapeError';
|
|
5016
|
-
Object.setPrototypeOf(_this, KnowledgeScrapeError.prototype);
|
|
5017
|
-
return _this;
|
|
5018
|
-
}
|
|
5019
|
-
return KnowledgeScrapeError;
|
|
5020
|
-
}(Error));
|
|
4927
|
+
var $scrapersMetadataRegister = new $Register('scrapers_metadata');
|
|
4928
|
+
/**
|
|
4929
|
+
* TODO: [®] DRY Register logic
|
|
4930
|
+
*/
|
|
5021
4931
|
|
|
5022
4932
|
/**
|
|
5023
4933
|
* @@@
|
|
@@ -5026,12 +4936,11 @@
|
|
|
5026
4936
|
* @singleton Only one instance of each register is created per build, but thare can be more @@@
|
|
5027
4937
|
* @public exported from `@promptbook/core`
|
|
5028
4938
|
*/
|
|
5029
|
-
var $
|
|
4939
|
+
var $scrapersRegister = new $Register('scraper_constructors');
|
|
5030
4940
|
/**
|
|
5031
4941
|
* TODO: [®] DRY Register logic
|
|
5032
4942
|
*/
|
|
5033
4943
|
|
|
5034
|
-
// TODO: !!!!!! Maybe delete this function
|
|
5035
4944
|
/**
|
|
5036
4945
|
* Creates a message with all registered scrapers
|
|
5037
4946
|
*
|
|
@@ -5198,7 +5107,6 @@
|
|
|
5198
5107
|
* @private within the repository
|
|
5199
5108
|
*/
|
|
5200
5109
|
function sourceContentToName(sourceContent) {
|
|
5201
|
-
// TODO: !!!!!! Better name for source than gibberish hash
|
|
5202
5110
|
var hash = cryptoJs.SHA256(hexEncoder__default["default"].parse(JSON.stringify(sourceContent)))
|
|
5203
5111
|
// <- TODO: [🥬] Encapsulate sha256 to some private utility function
|
|
5204
5112
|
.toString( /* hex */)
|
|
@@ -5298,20 +5206,20 @@
|
|
|
5298
5206
|
/**
|
|
5299
5207
|
* @@@
|
|
5300
5208
|
*
|
|
5301
|
-
* @
|
|
5209
|
+
* @public exported from `@promptbook/core`
|
|
5302
5210
|
*/
|
|
5303
5211
|
function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
|
|
5304
5212
|
var _a;
|
|
5305
5213
|
return __awaiter(this, void 0, void 0, function () {
|
|
5306
5214
|
var sourceContent, name, _b, _c, rootDirname, _d,
|
|
5307
5215
|
// <- TODO: process.cwd() if running in Node.js
|
|
5308
|
-
isVerbose, url, response_1, mimeType, filename_1, fileExtension,
|
|
5216
|
+
isVerbose, url, response_1, mimeType, filename_1, fileExtension, mimeType;
|
|
5309
5217
|
return __generator(this, function (_e) {
|
|
5310
5218
|
switch (_e.label) {
|
|
5311
5219
|
case 0:
|
|
5312
5220
|
sourceContent = knowledgeSource.sourceContent;
|
|
5313
5221
|
name = knowledgeSource.name;
|
|
5314
|
-
_b = options || {}, _c = _b.rootDirname, rootDirname = _c === void 0 ? null : _c, _d = _b.isVerbose, isVerbose = _d === void 0 ?
|
|
5222
|
+
_b = options || {}, _c = _b.rootDirname, rootDirname = _c === void 0 ? null : _c, _d = _b.isVerbose, isVerbose = _d === void 0 ? DEFAULT_IS_VERBOSE : _d;
|
|
5315
5223
|
TODO_USE(isVerbose);
|
|
5316
5224
|
if (!name) {
|
|
5317
5225
|
name = sourceContentToName(sourceContent);
|
|
@@ -5327,19 +5235,14 @@
|
|
|
5327
5235
|
filename: null,
|
|
5328
5236
|
url: url,
|
|
5329
5237
|
mimeType: mimeType,
|
|
5330
|
-
|
|
5331
|
-
|
|
5332
|
-
|
|
5333
|
-
|
|
5334
|
-
|
|
5335
|
-
|
|
5336
|
-
|
|
5337
|
-
|
|
5338
|
-
return [2 /*return*/, content];
|
|
5339
|
-
}
|
|
5340
|
-
});
|
|
5341
|
-
});
|
|
5342
|
-
},
|
|
5238
|
+
/*
|
|
5239
|
+
TODO: [🥽]
|
|
5240
|
+
> async asBlob() {
|
|
5241
|
+
> // TODO: [👨🏻🤝👨🏻] This can be called multiple times BUT when called second time, response in already consumed
|
|
5242
|
+
> const content = await response.blob();
|
|
5243
|
+
> return content;
|
|
5244
|
+
> },
|
|
5245
|
+
*/
|
|
5343
5246
|
asJson: function () {
|
|
5344
5247
|
return __awaiter(this, void 0, void 0, function () {
|
|
5345
5248
|
var content;
|
|
@@ -5379,34 +5282,31 @@
|
|
|
5379
5282
|
}
|
|
5380
5283
|
filename_1 = path.join(rootDirname, sourceContent).split('\\').join('/');
|
|
5381
5284
|
fileExtension = getFileExtension(filename_1);
|
|
5382
|
-
|
|
5285
|
+
mimeType = extensionToMimeType(fileExtension || '');
|
|
5383
5286
|
return [4 /*yield*/, isFileExisting(filename_1, tools.fs)];
|
|
5384
5287
|
case 3:
|
|
5385
5288
|
if (!(_e.sent())) {
|
|
5386
5289
|
throw new NotFoundError(spaceTrim__default["default"](function (block) { return "\n Can not make source handler for file which does not exist:\n\n File:\n ".concat(block(filename_1), "\n "); }));
|
|
5387
5290
|
}
|
|
5388
|
-
// TODO:
|
|
5291
|
+
// TODO: [🧠][😿] Test security file - file is scoped to the project (BUT maybe do this in `filesystemTools`)
|
|
5389
5292
|
return [2 /*return*/, {
|
|
5390
5293
|
source: name,
|
|
5391
5294
|
filename: filename_1,
|
|
5392
5295
|
url: null,
|
|
5393
|
-
mimeType:
|
|
5394
|
-
|
|
5395
|
-
|
|
5396
|
-
|
|
5397
|
-
|
|
5398
|
-
|
|
5399
|
-
|
|
5400
|
-
|
|
5401
|
-
|
|
5402
|
-
|
|
5403
|
-
|
|
5404
|
-
|
|
5405
|
-
|
|
5406
|
-
|
|
5407
|
-
});
|
|
5408
|
-
});
|
|
5409
|
-
},
|
|
5296
|
+
mimeType: mimeType,
|
|
5297
|
+
/*
|
|
5298
|
+
TODO: [🥽]
|
|
5299
|
+
> async asBlob() {
|
|
5300
|
+
> const content = await tools.fs!.readFile(filename);
|
|
5301
|
+
> return new Blob(
|
|
5302
|
+
> [
|
|
5303
|
+
> content,
|
|
5304
|
+
> // <- TODO: [🥽] This is NOT tested, test it
|
|
5305
|
+
> ],
|
|
5306
|
+
> { type: mimeType },
|
|
5307
|
+
> );
|
|
5308
|
+
> },
|
|
5309
|
+
*/
|
|
5410
5310
|
asJson: function () {
|
|
5411
5311
|
return __awaiter(this, void 0, void 0, function () {
|
|
5412
5312
|
var _a, _b;
|
|
@@ -5442,9 +5342,14 @@
|
|
|
5442
5342
|
asJson: function () {
|
|
5443
5343
|
throw new UnexpectedError('Did not expect that `markdownScraper` would need to get the content `asJson`');
|
|
5444
5344
|
},
|
|
5445
|
-
|
|
5446
|
-
|
|
5447
|
-
|
|
5345
|
+
/*
|
|
5346
|
+
TODO: [🥽]
|
|
5347
|
+
> asBlob() {
|
|
5348
|
+
> throw new UnexpectedError(
|
|
5349
|
+
> 'Did not expect that `markdownScraper` would need to get the content `asBlob`',
|
|
5350
|
+
> );
|
|
5351
|
+
> },
|
|
5352
|
+
*/
|
|
5448
5353
|
}];
|
|
5449
5354
|
}
|
|
5450
5355
|
});
|
|
@@ -5464,7 +5369,7 @@
|
|
|
5464
5369
|
return __generator(this, function (_c) {
|
|
5465
5370
|
switch (_c.label) {
|
|
5466
5371
|
case 0:
|
|
5467
|
-
_a = options.maxParallelCount, maxParallelCount = _a === void 0 ?
|
|
5372
|
+
_a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a, rootDirname = options.rootDirname, _b = options.isVerbose, isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b;
|
|
5468
5373
|
knowledgePreparedUnflatten = new Array(knowledgeSources.length);
|
|
5469
5374
|
return [4 /*yield*/, forEachAsync(knowledgeSources, { maxParallelCount: maxParallelCount }, function (knowledgeSource, index) { return __awaiter(_this, void 0, void 0, function () {
|
|
5470
5375
|
var partialPieces, sourceHandler, _a, _b, scraper, partialPiecesUnchecked, e_1_1, pieces;
|
|
@@ -5493,7 +5398,8 @@
|
|
|
5493
5398
|
case 4:
|
|
5494
5399
|
partialPiecesUnchecked = _d.sent();
|
|
5495
5400
|
if (partialPiecesUnchecked !== null) {
|
|
5496
|
-
partialPieces = partialPiecesUnchecked;
|
|
5401
|
+
partialPieces = __spreadArray([], __read(partialPiecesUnchecked), false);
|
|
5402
|
+
// <- TODO: [🪓] Here should be no need for spreading new array, just `partialPieces = partialPiecesUnchecked`
|
|
5497
5403
|
return [3 /*break*/, 6];
|
|
5498
5404
|
}
|
|
5499
5405
|
_d.label = 5;
|
|
@@ -5541,7 +5447,7 @@
|
|
|
5541
5447
|
> /**
|
|
5542
5448
|
> * Unprepared knowledge
|
|
5543
5449
|
> * /
|
|
5544
|
-
> readonly knowledgeSources:
|
|
5450
|
+
> readonly knowledgeSources: ReadonlyArray<KnowledgeSourceJson>;
|
|
5545
5451
|
> };
|
|
5546
5452
|
>
|
|
5547
5453
|
> export async function prepareKnowledgePieces(
|
|
@@ -5599,7 +5505,7 @@
|
|
|
5599
5505
|
return __generator(this, function (_b) {
|
|
5600
5506
|
switch (_b.label) {
|
|
5601
5507
|
case 0:
|
|
5602
|
-
_a = options.maxParallelCount, maxParallelCount = _a === void 0 ?
|
|
5508
|
+
_a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a;
|
|
5603
5509
|
templates = pipeline.templates, parameters = pipeline.parameters, knowledgePiecesCount = pipeline.knowledgePiecesCount;
|
|
5604
5510
|
// TODO: [main] !!!!! Apply samples to each template (if missing and is for the template defined)
|
|
5605
5511
|
TODO_USE(parameters);
|
|
@@ -5661,7 +5567,7 @@
|
|
|
5661
5567
|
if (isPipelinePrepared(pipeline)) {
|
|
5662
5568
|
return [2 /*return*/, pipeline];
|
|
5663
5569
|
}
|
|
5664
|
-
rootDirname = options.rootDirname, _a = options.maxParallelCount, maxParallelCount = _a === void 0 ?
|
|
5570
|
+
rootDirname = options.rootDirname, _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a, _b = options.isVerbose, isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b;
|
|
5665
5571
|
parameters = pipeline.parameters, templates = pipeline.templates, knowledgeSources = pipeline.knowledgeSources, personas = pipeline.personas;
|
|
5666
5572
|
if (tools === undefined || tools.llm === undefined) {
|
|
5667
5573
|
throw new MissingToolsError('LLM tools are required for preparing the pipeline');
|
|
@@ -5719,7 +5625,9 @@
|
|
|
5719
5625
|
// ----- /Templates preparation -----
|
|
5720
5626
|
// Note: Count total usage
|
|
5721
5627
|
currentPreparation.usage = llmToolsWithUsage.getTotalUsage();
|
|
5722
|
-
return [2 /*return*/, $asDeeplyFrozenSerializableJson('Prepared PipelineJson', __assign(__assign({}, clonePipeline(pipeline)), { templates:
|
|
5628
|
+
return [2 /*return*/, $asDeeplyFrozenSerializableJson('Prepared PipelineJson', __assign(__assign({}, clonePipeline(pipeline)), { templates: __spreadArray([], __read(templatesPrepared), false),
|
|
5629
|
+
// <- TODO: [🪓] Here should be no need for spreading new array, just ` templates: templatesPrepared`
|
|
5630
|
+
knowledgeSources: knowledgeSourcesPrepared, knowledgePieces: knowledgePiecesPrepared, personas: preparedPersonas, preparations: __spreadArray([], __read(preparations), false) }))];
|
|
5723
5631
|
}
|
|
5724
5632
|
});
|
|
5725
5633
|
});
|
|
@@ -5766,6 +5674,7 @@
|
|
|
5766
5674
|
'KNOWLEDGE ./hejny-cv.md',
|
|
5767
5675
|
'KNOWLEDGE ./hejny-cv.pdf',
|
|
5768
5676
|
'KNOWLEDGE ./hejny-cv.docx',
|
|
5677
|
+
// <- TODO: [😿] Allow ONLY files scoped in the (sub)directory NOT ../ and test it
|
|
5769
5678
|
],
|
|
5770
5679
|
/**
|
|
5771
5680
|
* Parses the KNOWLEDGE command
|
|
@@ -8440,38 +8349,359 @@
|
|
|
8440
8349
|
* Note: This function does not validate logic of the pipeline only the parsing
|
|
8441
8350
|
* Note: This function acts as compilation process
|
|
8442
8351
|
*
|
|
8443
|
-
* @param pipelineString {Promptbook} in string markdown format (.ptbk.md)
|
|
8444
|
-
* @param tools - Tools for the preparation and scraping - if not provided together with `llm`, the preparation will be skipped
|
|
8445
|
-
* @param options - Options and tools for the compilation
|
|
8446
|
-
* @returns {Promptbook} compiled in JSON format (.ptbk.json)
|
|
8447
|
-
* @throws {ParseError} if the promptbook string is not valid
|
|
8448
|
-
* @public exported from `@promptbook/core`
|
|
8352
|
+
* @param pipelineString {Promptbook} in string markdown format (.ptbk.md)
|
|
8353
|
+
* @param tools - Tools for the preparation and scraping - if not provided together with `llm`, the preparation will be skipped
|
|
8354
|
+
* @param options - Options and tools for the compilation
|
|
8355
|
+
* @returns {Promptbook} compiled in JSON format (.ptbk.json)
|
|
8356
|
+
* @throws {ParseError} if the promptbook string is not valid
|
|
8357
|
+
* @public exported from `@promptbook/core`
|
|
8358
|
+
*/
|
|
8359
|
+
function pipelineStringToJson(pipelineString, tools, options) {
|
|
8360
|
+
return __awaiter(this, void 0, void 0, function () {
|
|
8361
|
+
var pipelineJson;
|
|
8362
|
+
return __generator(this, function (_a) {
|
|
8363
|
+
switch (_a.label) {
|
|
8364
|
+
case 0:
|
|
8365
|
+
pipelineJson = pipelineStringToJsonSync(pipelineString);
|
|
8366
|
+
if (!(tools !== undefined && tools.llm !== undefined)) return [3 /*break*/, 2];
|
|
8367
|
+
return [4 /*yield*/, preparePipeline(pipelineJson, tools, options || {
|
|
8368
|
+
rootDirname: null,
|
|
8369
|
+
})];
|
|
8370
|
+
case 1:
|
|
8371
|
+
pipelineJson = _a.sent();
|
|
8372
|
+
_a.label = 2;
|
|
8373
|
+
case 2:
|
|
8374
|
+
// Note: No need to use `$asDeeplyFrozenSerializableJson` because `pipelineStringToJsonSync` and `preparePipeline` already do that
|
|
8375
|
+
return [2 /*return*/, pipelineJson];
|
|
8376
|
+
}
|
|
8377
|
+
});
|
|
8378
|
+
});
|
|
8379
|
+
}
|
|
8380
|
+
/**
|
|
8381
|
+
* TODO: [🏏] Leverage the batch API and build queues @see https://platform.openai.com/docs/guides/batch
|
|
8382
|
+
* TODO: [🛠] Actions, instruments (and maybe knowledge) => Functions and tools
|
|
8383
|
+
* TODO: [🧠] Should be in generated JSON file GENERATOR_WARNING
|
|
8384
|
+
*/
|
|
8385
|
+
|
|
8386
|
+
// Note: We want to use the `exec` as async function
|
|
8387
|
+
var exec$1 = util.promisify(child_process.exec);
|
|
8388
|
+
/**
|
|
8389
|
+
* @@@
|
|
8390
|
+
*
|
|
8391
|
+
* @private within the repository
|
|
8392
|
+
*/
|
|
8393
|
+
function locateAppOnLinux(_a) {
|
|
8394
|
+
var appName = _a.appName, linuxWhich = _a.linuxWhich;
|
|
8395
|
+
return __awaiter(this, void 0, void 0, function () {
|
|
8396
|
+
var _b, stderr, stdout, error_1;
|
|
8397
|
+
return __generator(this, function (_c) {
|
|
8398
|
+
switch (_c.label) {
|
|
8399
|
+
case 0:
|
|
8400
|
+
_c.trys.push([0, 2, , 3]);
|
|
8401
|
+
return [4 /*yield*/, exec$1("which ".concat(linuxWhich))];
|
|
8402
|
+
case 1:
|
|
8403
|
+
_b = _c.sent(), stderr = _b.stderr, stdout = _b.stdout;
|
|
8404
|
+
if (!stderr && stdout) {
|
|
8405
|
+
return [2 /*return*/, stdout.trim()];
|
|
8406
|
+
}
|
|
8407
|
+
throw new Error("Can not locate app ".concat(appName, " on Linux.\n ").concat(stderr));
|
|
8408
|
+
case 2:
|
|
8409
|
+
error_1 = _c.sent();
|
|
8410
|
+
if (!(error_1 instanceof Error)) {
|
|
8411
|
+
throw error_1;
|
|
8412
|
+
}
|
|
8413
|
+
return [2 /*return*/, null];
|
|
8414
|
+
case 3: return [2 /*return*/];
|
|
8415
|
+
}
|
|
8416
|
+
});
|
|
8417
|
+
});
|
|
8418
|
+
}
|
|
8419
|
+
/**
|
|
8420
|
+
* TODO: [🧠][♿] Maybe export through `@promptbook/node`
|
|
8421
|
+
* Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
|
|
8422
|
+
*/
|
|
8423
|
+
|
|
8424
|
+
/**
|
|
8425
|
+
* @@@
|
|
8426
|
+
*
|
|
8427
|
+
* @public exported from `@promptbook/node`
|
|
8428
|
+
*/
|
|
8429
|
+
function $provideFilesystemForNode(options) {
|
|
8430
|
+
if (!$isRunningInNode()) {
|
|
8431
|
+
throw new EnvironmentMismatchError('Function `$provideFilesystemForNode` works only in Node.js environment');
|
|
8432
|
+
}
|
|
8433
|
+
var _a = (options || {}).isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a;
|
|
8434
|
+
TODO_USE(isVerbose);
|
|
8435
|
+
return {
|
|
8436
|
+
stat: promises.stat,
|
|
8437
|
+
access: promises.access,
|
|
8438
|
+
constants: promises.constants,
|
|
8439
|
+
readFile: promises.readFile,
|
|
8440
|
+
writeFile: promises.writeFile,
|
|
8441
|
+
readdir: promises.readdir,
|
|
8442
|
+
};
|
|
8443
|
+
}
|
|
8444
|
+
/**
|
|
8445
|
+
* Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
|
|
8446
|
+
*/
|
|
8447
|
+
|
|
8448
|
+
/**
|
|
8449
|
+
* Checks if the file is executable
|
|
8450
|
+
*
|
|
8451
|
+
* @private within the repository
|
|
8452
|
+
*/
|
|
8453
|
+
function isExecutable(path, fs) {
|
|
8454
|
+
return __awaiter(this, void 0, void 0, function () {
|
|
8455
|
+
return __generator(this, function (_a) {
|
|
8456
|
+
switch (_a.label) {
|
|
8457
|
+
case 0:
|
|
8458
|
+
_a.trys.push([0, 2, , 3]);
|
|
8459
|
+
return [4 /*yield*/, fs.access(path, fs.constants.X_OK)];
|
|
8460
|
+
case 1:
|
|
8461
|
+
_a.sent();
|
|
8462
|
+
return [2 /*return*/, true];
|
|
8463
|
+
case 2:
|
|
8464
|
+
_a.sent();
|
|
8465
|
+
return [2 /*return*/, false];
|
|
8466
|
+
case 3: return [2 /*return*/];
|
|
8467
|
+
}
|
|
8468
|
+
});
|
|
8469
|
+
});
|
|
8470
|
+
}
|
|
8471
|
+
/**
|
|
8472
|
+
* Note: Not [~🟢~] because it is not directly dependent on `fs
|
|
8473
|
+
* TODO: [🖇] What about symlinks?
|
|
8474
|
+
*/
|
|
8475
|
+
|
|
8476
|
+
// Note: Module `userhome` has no types available, so it is imported using `require`
|
|
8477
|
+
// @see https://stackoverflow.com/questions/37000981/how-to-import-node-module-in-typescript-without-type-definitions
|
|
8478
|
+
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
|
8479
|
+
var userhome = require('userhome');
|
|
8480
|
+
// Note: We want to use the `exec` as async function
|
|
8481
|
+
var exec = util.promisify(child_process.exec);
|
|
8482
|
+
/**
|
|
8483
|
+
* @@@
|
|
8484
|
+
*
|
|
8485
|
+
* @private within the repository
|
|
8486
|
+
*/
|
|
8487
|
+
function locateAppOnMacOs(_a) {
|
|
8488
|
+
var appName = _a.appName, macOsName = _a.macOsName;
|
|
8489
|
+
return __awaiter(this, void 0, void 0, function () {
|
|
8490
|
+
var toExec, regPath, altPath, _b, stderr, stdout, error_1;
|
|
8491
|
+
return __generator(this, function (_c) {
|
|
8492
|
+
switch (_c.label) {
|
|
8493
|
+
case 0:
|
|
8494
|
+
_c.trys.push([0, 6, , 7]);
|
|
8495
|
+
toExec = "/Contents/MacOS/".concat(macOsName);
|
|
8496
|
+
regPath = "/Applications/".concat(macOsName, ".app") + toExec;
|
|
8497
|
+
altPath = userhome(regPath.slice(1));
|
|
8498
|
+
return [4 /*yield*/, isExecutable(regPath, $provideFilesystemForNode())];
|
|
8499
|
+
case 1:
|
|
8500
|
+
if (!_c.sent()) return [3 /*break*/, 2];
|
|
8501
|
+
return [2 /*return*/, regPath];
|
|
8502
|
+
case 2: return [4 /*yield*/, isExecutable(altPath, $provideFilesystemForNode())];
|
|
8503
|
+
case 3:
|
|
8504
|
+
if (_c.sent()) {
|
|
8505
|
+
return [2 /*return*/, altPath];
|
|
8506
|
+
}
|
|
8507
|
+
_c.label = 4;
|
|
8508
|
+
case 4: return [4 /*yield*/, exec("mdfind 'kMDItemDisplayName == \"".concat(macOsName, "\" && kMDItemKind == Application'"))];
|
|
8509
|
+
case 5:
|
|
8510
|
+
_b = _c.sent(), stderr = _b.stderr, stdout = _b.stdout;
|
|
8511
|
+
if (!stderr && stdout) {
|
|
8512
|
+
return [2 /*return*/, stdout.trim() + toExec];
|
|
8513
|
+
}
|
|
8514
|
+
throw new Error("Can not locate app ".concat(appName, " on macOS.\n ").concat(stderr));
|
|
8515
|
+
case 6:
|
|
8516
|
+
error_1 = _c.sent();
|
|
8517
|
+
if (!(error_1 instanceof Error)) {
|
|
8518
|
+
throw error_1;
|
|
8519
|
+
}
|
|
8520
|
+
return [2 /*return*/, null];
|
|
8521
|
+
case 7: return [2 /*return*/];
|
|
8522
|
+
}
|
|
8523
|
+
});
|
|
8524
|
+
});
|
|
8525
|
+
}
|
|
8526
|
+
/**
|
|
8527
|
+
* TODO: [🧠][♿] Maybe export through `@promptbook/node`
|
|
8528
|
+
* Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
|
|
8529
|
+
*/
|
|
8530
|
+
|
|
8531
|
+
/**
|
|
8532
|
+
* @@@
|
|
8533
|
+
*
|
|
8534
|
+
* @private within the repository
|
|
8535
|
+
*/
|
|
8536
|
+
function locateAppOnWindows(_a) {
|
|
8537
|
+
var appName = _a.appName, windowsSuffix = _a.windowsSuffix;
|
|
8538
|
+
return __awaiter(this, void 0, void 0, function () {
|
|
8539
|
+
var prefixes, prefixes_1, prefixes_1_1, prefix, path$1, e_1_1, error_1;
|
|
8540
|
+
var e_1, _b;
|
|
8541
|
+
return __generator(this, function (_c) {
|
|
8542
|
+
switch (_c.label) {
|
|
8543
|
+
case 0:
|
|
8544
|
+
_c.trys.push([0, 9, , 10]);
|
|
8545
|
+
prefixes = [
|
|
8546
|
+
process.env.LOCALAPPDATA,
|
|
8547
|
+
path.join(process.env.LOCALAPPDATA || '', 'Programs'),
|
|
8548
|
+
process.env.PROGRAMFILES,
|
|
8549
|
+
process.env['PROGRAMFILES(X86)'],
|
|
8550
|
+
];
|
|
8551
|
+
_c.label = 1;
|
|
8552
|
+
case 1:
|
|
8553
|
+
_c.trys.push([1, 6, 7, 8]);
|
|
8554
|
+
prefixes_1 = __values(prefixes), prefixes_1_1 = prefixes_1.next();
|
|
8555
|
+
_c.label = 2;
|
|
8556
|
+
case 2:
|
|
8557
|
+
if (!!prefixes_1_1.done) return [3 /*break*/, 5];
|
|
8558
|
+
prefix = prefixes_1_1.value;
|
|
8559
|
+
path$1 = prefix + windowsSuffix;
|
|
8560
|
+
return [4 /*yield*/, isExecutable(path$1, $provideFilesystemForNode())];
|
|
8561
|
+
case 3:
|
|
8562
|
+
if (_c.sent()) {
|
|
8563
|
+
return [2 /*return*/, path$1];
|
|
8564
|
+
}
|
|
8565
|
+
_c.label = 4;
|
|
8566
|
+
case 4:
|
|
8567
|
+
prefixes_1_1 = prefixes_1.next();
|
|
8568
|
+
return [3 /*break*/, 2];
|
|
8569
|
+
case 5: return [3 /*break*/, 8];
|
|
8570
|
+
case 6:
|
|
8571
|
+
e_1_1 = _c.sent();
|
|
8572
|
+
e_1 = { error: e_1_1 };
|
|
8573
|
+
return [3 /*break*/, 8];
|
|
8574
|
+
case 7:
|
|
8575
|
+
try {
|
|
8576
|
+
if (prefixes_1_1 && !prefixes_1_1.done && (_b = prefixes_1.return)) _b.call(prefixes_1);
|
|
8577
|
+
}
|
|
8578
|
+
finally { if (e_1) throw e_1.error; }
|
|
8579
|
+
return [7 /*endfinally*/];
|
|
8580
|
+
case 8: throw new Error("Can not locate app ".concat(appName, " on Windows."));
|
|
8581
|
+
case 9:
|
|
8582
|
+
error_1 = _c.sent();
|
|
8583
|
+
if (!(error_1 instanceof Error)) {
|
|
8584
|
+
throw error_1;
|
|
8585
|
+
}
|
|
8586
|
+
return [2 /*return*/, null];
|
|
8587
|
+
case 10: return [2 /*return*/];
|
|
8588
|
+
}
|
|
8589
|
+
});
|
|
8590
|
+
});
|
|
8591
|
+
}
|
|
8592
|
+
/**
|
|
8593
|
+
* TODO: [🧠][♿] Maybe export through `@promptbook/node`
|
|
8594
|
+
* Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
|
|
8595
|
+
*/
|
|
8596
|
+
|
|
8597
|
+
/**
|
|
8598
|
+
* Locates an application on the system
|
|
8599
|
+
*
|
|
8600
|
+
* @private within the repository
|
|
8601
|
+
*/
|
|
8602
|
+
function locateApp(options) {
|
|
8603
|
+
if (!$isRunningInNode()) {
|
|
8604
|
+
throw new EnvironmentMismatchError('Locating apps works only in Node.js environment');
|
|
8605
|
+
}
|
|
8606
|
+
var appName = options.appName, linuxWhich = options.linuxWhich, windowsSuffix = options.windowsSuffix, macOsName = options.macOsName;
|
|
8607
|
+
if (process.platform === 'win32') {
|
|
8608
|
+
if (windowsSuffix) {
|
|
8609
|
+
return locateAppOnWindows({ appName: appName, windowsSuffix: windowsSuffix });
|
|
8610
|
+
}
|
|
8611
|
+
else {
|
|
8612
|
+
throw new Error("".concat(appName, " is not available on Windows."));
|
|
8613
|
+
}
|
|
8614
|
+
}
|
|
8615
|
+
else if (process.platform === 'darwin') {
|
|
8616
|
+
if (macOsName) {
|
|
8617
|
+
return locateAppOnMacOs({ appName: appName, macOsName: macOsName });
|
|
8618
|
+
}
|
|
8619
|
+
else {
|
|
8620
|
+
throw new Error("".concat(appName, " is not available on macOS."));
|
|
8621
|
+
}
|
|
8622
|
+
}
|
|
8623
|
+
else {
|
|
8624
|
+
if (linuxWhich) {
|
|
8625
|
+
return locateAppOnLinux({ appName: appName, linuxWhich: linuxWhich });
|
|
8626
|
+
}
|
|
8627
|
+
else {
|
|
8628
|
+
throw new Error("".concat(appName, " is not available on Linux."));
|
|
8629
|
+
}
|
|
8630
|
+
}
|
|
8631
|
+
}
|
|
8632
|
+
/**
|
|
8633
|
+
* TODO: [🧠][♿] Maybe export through `@promptbook/node`
|
|
8634
|
+
* Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
|
|
8635
|
+
*/
|
|
8636
|
+
|
|
8637
|
+
/**
|
|
8638
|
+
* @@@
|
|
8639
|
+
*
|
|
8640
|
+
* @private within the repository
|
|
8641
|
+
*/
|
|
8642
|
+
function locateLibreoffice() {
|
|
8643
|
+
return locateApp({
|
|
8644
|
+
appName: 'Libreoffice',
|
|
8645
|
+
linuxWhich: 'libreoffice',
|
|
8646
|
+
windowsSuffix: '\\LibreOffice\\program\\soffice.exe',
|
|
8647
|
+
macOsName: 'LibreOffice',
|
|
8648
|
+
});
|
|
8649
|
+
}
|
|
8650
|
+
/**
|
|
8651
|
+
* TODO: [🧠][♿] Maybe export through `@promptbook/node` OR `@promptbook/legacy-documents`
|
|
8652
|
+
* Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
|
|
8653
|
+
*/
|
|
8654
|
+
|
|
8655
|
+
/**
|
|
8656
|
+
* @@@
|
|
8657
|
+
*
|
|
8658
|
+
* @private within the repository
|
|
8659
|
+
*/
|
|
8660
|
+
function locatePandoc() {
|
|
8661
|
+
return locateApp({
|
|
8662
|
+
appName: 'Pandoc',
|
|
8663
|
+
linuxWhich: 'pandoc',
|
|
8664
|
+
windowsSuffix: '\\Pandoc\\pandoc.exe',
|
|
8665
|
+
macOsName: 'Pandoc',
|
|
8666
|
+
});
|
|
8667
|
+
}
|
|
8668
|
+
/**
|
|
8669
|
+
* TODO: [🧠][♿] Maybe export through `@promptbook/node` OR `@promptbook/documents`
|
|
8670
|
+
* Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
|
|
8671
|
+
*/
|
|
8672
|
+
|
|
8673
|
+
/**
|
|
8674
|
+
* @@@
|
|
8675
|
+
*
|
|
8676
|
+
* @public exported from `@promptbook/node`
|
|
8449
8677
|
*/
|
|
8450
|
-
function
|
|
8678
|
+
function $provideExecutablesForNode(options) {
|
|
8451
8679
|
return __awaiter(this, void 0, void 0, function () {
|
|
8452
|
-
var
|
|
8453
|
-
|
|
8454
|
-
|
|
8680
|
+
var _a, _b, isAutoInstalled, _c, isVerbose;
|
|
8681
|
+
var _d;
|
|
8682
|
+
return __generator(this, function (_e) {
|
|
8683
|
+
switch (_e.label) {
|
|
8455
8684
|
case 0:
|
|
8456
|
-
|
|
8457
|
-
|
|
8458
|
-
|
|
8459
|
-
|
|
8460
|
-
|
|
8685
|
+
if (!$isRunningInNode()) {
|
|
8686
|
+
throw new EnvironmentMismatchError('Function `$getScrapersForNode` works only in Node.js environment');
|
|
8687
|
+
}
|
|
8688
|
+
_a = options || {}, _b = _a.isAutoInstalled, isAutoInstalled = _b === void 0 ? DEFAULT_IS_AUTO_INSTALLED : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? DEFAULT_IS_VERBOSE : _c;
|
|
8689
|
+
TODO_USE(isAutoInstalled); // <- TODO: [🔱][🧠] Auto-install the executables
|
|
8690
|
+
TODO_USE(isVerbose);
|
|
8691
|
+
_d = {};
|
|
8692
|
+
return [4 /*yield*/, locatePandoc()];
|
|
8461
8693
|
case 1:
|
|
8462
|
-
|
|
8463
|
-
|
|
8464
|
-
case 2:
|
|
8465
|
-
|
|
8466
|
-
return [2 /*return*/, pipelineJson];
|
|
8694
|
+
_d.pandocPath = (_e.sent()) || undefined;
|
|
8695
|
+
return [4 /*yield*/, locateLibreoffice()];
|
|
8696
|
+
case 2: return [2 /*return*/, (_d.libreOfficePath = (_e.sent()) || undefined,
|
|
8697
|
+
_d)];
|
|
8467
8698
|
}
|
|
8468
8699
|
});
|
|
8469
8700
|
});
|
|
8470
8701
|
}
|
|
8471
8702
|
/**
|
|
8472
|
-
* TODO: [
|
|
8473
|
-
*
|
|
8474
|
-
* TODO: [🧠] Should be in generated JSON file GENERATOR_WARNING
|
|
8703
|
+
* TODO: [🧠] Allow to override the executables without need to call `locatePandoc` / `locateLibreoffice` in case of provided
|
|
8704
|
+
* Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
|
|
8475
8705
|
*/
|
|
8476
8706
|
|
|
8477
8707
|
/**
|
|
@@ -8643,7 +8873,7 @@
|
|
|
8643
8873
|
*/
|
|
8644
8874
|
function createLlmToolsFromConfiguration(configuration, options) {
|
|
8645
8875
|
if (options === void 0) { options = {}; }
|
|
8646
|
-
var _a = options.isVerbose, isVerbose = _a === void 0 ?
|
|
8876
|
+
var _a = options.isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a, userId = options.userId;
|
|
8647
8877
|
var llmTools = configuration.map(function (llmConfiguration) {
|
|
8648
8878
|
var registeredItem = $llmToolsRegister
|
|
8649
8879
|
.list()
|
|
@@ -8654,7 +8884,7 @@
|
|
|
8654
8884
|
if (registeredItem === undefined) {
|
|
8655
8885
|
throw new Error(spaceTrim__default["default"](function (block) { return "\n There is no constructor for LLM provider `".concat(llmConfiguration.className, "` from `").concat(llmConfiguration.packageName, "`\n\n You have probably forgotten install and import the provider package.\n To fix this issue, you can:\n\n Install:\n\n > npm install ").concat(llmConfiguration.packageName, "\n\n And import:\n\n > import '").concat(llmConfiguration.packageName, "';\n\n\n ").concat(block($registeredLlmToolsMessage()), "\n "); }));
|
|
8656
8886
|
}
|
|
8657
|
-
return registeredItem(__assign({ isVerbose: isVerbose }, llmConfiguration.options));
|
|
8887
|
+
return registeredItem(__assign({ isVerbose: isVerbose, userId: userId }, llmConfiguration.options));
|
|
8658
8888
|
});
|
|
8659
8889
|
return joinLlmExecutionTools.apply(void 0, __spreadArray([], __read(llmTools), false));
|
|
8660
8890
|
}
|
|
@@ -8704,6 +8934,64 @@
|
|
|
8704
8934
|
* TODO: [®] DRY Register logic
|
|
8705
8935
|
*/
|
|
8706
8936
|
|
|
8937
|
+
/**
|
|
8938
|
+
* @@@
|
|
8939
|
+
*
|
|
8940
|
+
* 1) @@@
|
|
8941
|
+
* 2) @@@
|
|
8942
|
+
*
|
|
8943
|
+
* @public exported from `@promptbook/node`
|
|
8944
|
+
*/
|
|
8945
|
+
function $provideScrapersForNode(tools, options) {
|
|
8946
|
+
return __awaiter(this, void 0, void 0, function () {
|
|
8947
|
+
var _a, _b, isAutoInstalled, _c, isVerbose, scrapers, _d, _e, scraperFactory, scraper, e_1_1;
|
|
8948
|
+
var e_1, _f;
|
|
8949
|
+
return __generator(this, function (_g) {
|
|
8950
|
+
switch (_g.label) {
|
|
8951
|
+
case 0:
|
|
8952
|
+
if (!$isRunningInNode()) {
|
|
8953
|
+
throw new EnvironmentMismatchError('Function `$getScrapersForNode` works only in Node.js environment');
|
|
8954
|
+
}
|
|
8955
|
+
_a = options || {}, _b = _a.isAutoInstalled, isAutoInstalled = _b === void 0 ? DEFAULT_IS_AUTO_INSTALLED : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? DEFAULT_IS_VERBOSE : _c;
|
|
8956
|
+
TODO_USE(isAutoInstalled);
|
|
8957
|
+
TODO_USE(isVerbose);
|
|
8958
|
+
scrapers = [];
|
|
8959
|
+
_g.label = 1;
|
|
8960
|
+
case 1:
|
|
8961
|
+
_g.trys.push([1, 6, 7, 8]);
|
|
8962
|
+
_d = __values($scrapersRegister.list()), _e = _d.next();
|
|
8963
|
+
_g.label = 2;
|
|
8964
|
+
case 2:
|
|
8965
|
+
if (!!_e.done) return [3 /*break*/, 5];
|
|
8966
|
+
scraperFactory = _e.value;
|
|
8967
|
+
return [4 /*yield*/, scraperFactory(tools, options || {})];
|
|
8968
|
+
case 3:
|
|
8969
|
+
scraper = _g.sent();
|
|
8970
|
+
scrapers.push(scraper);
|
|
8971
|
+
_g.label = 4;
|
|
8972
|
+
case 4:
|
|
8973
|
+
_e = _d.next();
|
|
8974
|
+
return [3 /*break*/, 2];
|
|
8975
|
+
case 5: return [3 /*break*/, 8];
|
|
8976
|
+
case 6:
|
|
8977
|
+
e_1_1 = _g.sent();
|
|
8978
|
+
e_1 = { error: e_1_1 };
|
|
8979
|
+
return [3 /*break*/, 8];
|
|
8980
|
+
case 7:
|
|
8981
|
+
try {
|
|
8982
|
+
if (_e && !_e.done && (_f = _d.return)) _f.call(_d);
|
|
8983
|
+
}
|
|
8984
|
+
finally { if (e_1) throw e_1.error; }
|
|
8985
|
+
return [7 /*endfinally*/];
|
|
8986
|
+
case 8: return [2 /*return*/, scrapers];
|
|
8987
|
+
}
|
|
8988
|
+
});
|
|
8989
|
+
});
|
|
8990
|
+
}
|
|
8991
|
+
/**
|
|
8992
|
+
* Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
|
|
8993
|
+
*/
|
|
8994
|
+
|
|
8707
8995
|
/**
|
|
8708
8996
|
* Extracts code block from markdown.
|
|
8709
8997
|
*
|
|
@@ -9158,7 +9446,7 @@
|
|
|
9158
9446
|
*/
|
|
9159
9447
|
function $provideExecutionToolsForNode(options) {
|
|
9160
9448
|
return __awaiter(this, void 0, void 0, function () {
|
|
9161
|
-
var fs, llm, tools;
|
|
9449
|
+
var fs, llm, executables, tools;
|
|
9162
9450
|
var _a;
|
|
9163
9451
|
return __generator(this, function (_b) {
|
|
9164
9452
|
switch (_b.label) {
|
|
@@ -9168,12 +9456,16 @@
|
|
|
9168
9456
|
}
|
|
9169
9457
|
fs = $provideFilesystemForNode();
|
|
9170
9458
|
llm = $provideLlmToolsFromEnv(options);
|
|
9459
|
+
return [4 /*yield*/, $provideExecutablesForNode(options)];
|
|
9460
|
+
case 1:
|
|
9461
|
+
executables = _b.sent();
|
|
9171
9462
|
_a = {
|
|
9172
9463
|
llm: llm,
|
|
9173
|
-
fs: fs
|
|
9464
|
+
fs: fs,
|
|
9465
|
+
executables: executables
|
|
9174
9466
|
};
|
|
9175
|
-
return [4 /*yield*/, $provideScrapersForNode({ fs: fs, llm: llm }, options)];
|
|
9176
|
-
case
|
|
9467
|
+
return [4 /*yield*/, $provideScrapersForNode({ fs: fs, llm: llm, executables: executables }, options)];
|
|
9468
|
+
case 2:
|
|
9177
9469
|
tools = (_a.scrapers = _b.sent(),
|
|
9178
9470
|
_a.script = [new JavascriptExecutionTools(options)],
|
|
9179
9471
|
_a);
|
|
@@ -9411,7 +9703,9 @@
|
|
|
9411
9703
|
throw new EnvironmentMismatchError('Can not create collection without filesystem tools');
|
|
9412
9704
|
// <- TODO: [🧠] What is the best error type here`
|
|
9413
9705
|
}
|
|
9414
|
-
makedLibraryFilePath = path.join(path$1, "".concat(
|
|
9706
|
+
makedLibraryFilePath = path.join(path$1, "".concat(DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME
|
|
9707
|
+
// <- TODO: [🦒] Allow to override (pass different value into the function)
|
|
9708
|
+
, ".json"));
|
|
9415
9709
|
return [4 /*yield*/, isFileExisting(makedLibraryFilePath, tools.fs)];
|
|
9416
9710
|
case 3:
|
|
9417
9711
|
if (!(_f.sent())) {
|
|
@@ -9422,7 +9716,7 @@
|
|
|
9422
9716
|
// TODO: !! Implement;
|
|
9423
9717
|
// TODO: [🌗]
|
|
9424
9718
|
}
|
|
9425
|
-
_a = options || {}, _b = _a.isRecursive, isRecursive = _b === void 0 ? true : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ?
|
|
9719
|
+
_a = options || {}, _b = _a.isRecursive, isRecursive = _b === void 0 ? true : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? DEFAULT_IS_VERBOSE : _c, _d = _a.isLazyLoaded, isLazyLoaded = _d === void 0 ? false : _d, _e = _a.isCrashedOnError, isCrashedOnError = _e === void 0 ? true : _e;
|
|
9426
9720
|
collection = createCollectionFromPromise(function () { return __awaiter(_this, void 0, void 0, function () {
|
|
9427
9721
|
var fileNames, collection, _loop_1, fileNames_1, fileNames_1_1, fileName, e_1_1;
|
|
9428
9722
|
var e_1, _a;
|
|
@@ -9894,7 +10188,7 @@
|
|
|
9894
10188
|
function cacheLlmTools(llmTools, options) {
|
|
9895
10189
|
var _this = this;
|
|
9896
10190
|
if (options === void 0) { options = {}; }
|
|
9897
|
-
var _a = options.storage, storage = _a === void 0 ? new MemoryStorage() : _a, _b = options.
|
|
10191
|
+
var _a = options.storage, storage = _a === void 0 ? new MemoryStorage() : _a, _b = options.isCacheReloaded, isCacheReloaded = _b === void 0 ? false : _b;
|
|
9898
10192
|
var proxyTools = __assign(__assign({}, llmTools), {
|
|
9899
10193
|
// <- Note: [🥫]
|
|
9900
10194
|
get title() {
|
|
@@ -9916,7 +10210,7 @@
|
|
|
9916
10210
|
key = titleToName(prompt.title.substring(0, MAX_FILENAME_LENGTH - 10) +
|
|
9917
10211
|
'-' +
|
|
9918
10212
|
sha256__default["default"](hexEncoder__default["default"].parse(JSON.stringify(prompt.parameters))).toString( /* hex */));
|
|
9919
|
-
if (!!
|
|
10213
|
+
if (!!isCacheReloaded) return [3 /*break*/, 2];
|
|
9920
10214
|
return [4 /*yield*/, storage.getItem(key)];
|
|
9921
10215
|
case 1:
|
|
9922
10216
|
_a = _c.sent();
|
|
@@ -10002,12 +10296,14 @@
|
|
|
10002
10296
|
if (!$isRunningInNode()) {
|
|
10003
10297
|
throw new EnvironmentMismatchError('Function `$provideLlmToolsForTestingAndScriptsAndPlayground` works only in Node.js environment');
|
|
10004
10298
|
}
|
|
10005
|
-
var
|
|
10299
|
+
var isCacheReloaded = (options !== null && options !== void 0 ? options : {}).isCacheReloaded;
|
|
10006
10300
|
return cacheLlmTools(countTotalUsage(
|
|
10007
10301
|
// <- Note: for example here we don`t want the [🌯]
|
|
10008
10302
|
$provideLlmToolsFromEnv()), {
|
|
10009
|
-
storage: new FileCacheStorage({ fs: $provideFilesystemForNode() }, {
|
|
10010
|
-
|
|
10303
|
+
storage: new FileCacheStorage({ fs: $provideFilesystemForNode() }, {
|
|
10304
|
+
rootFolderPath: path.join(process.cwd(), DEFAULT_EXECUTIONS_CACHE_DIRNAME),
|
|
10305
|
+
}),
|
|
10306
|
+
isCacheReloaded: isCacheReloaded,
|
|
10011
10307
|
});
|
|
10012
10308
|
}
|
|
10013
10309
|
/**
|
|
@@ -10034,13 +10330,13 @@
|
|
|
10034
10330
|
makeCommand.option('-f, --format <format>', spaceTrim__default["default"]("\n Output format of builded collection \"javascript\", \"typescript\" or \"json\"\n\n Note: You can use multiple formats separated by comma\n "), 'javascript' /* <- Note: [🏳🌈] */);
|
|
10035
10331
|
makeCommand.option('--no-validation', "Do not validate logic of pipelines in collection", true);
|
|
10036
10332
|
makeCommand.option('--validation', "Types of validations separated by comma (options \"logic\",\"imports\")", 'logic,imports');
|
|
10037
|
-
makeCommand.option('--reload
|
|
10333
|
+
makeCommand.option('--reload', "Call LLM models even if same prompt with result is in the cache", false);
|
|
10038
10334
|
makeCommand.option('--verbose', "Is output verbose", false);
|
|
10039
|
-
makeCommand.option('-o, --out-file <path>', spaceTrim__default["default"]("\n Where to save the builded collection\n\n Note: If you keep it \"".concat(
|
|
10335
|
+
makeCommand.option('-o, --out-file <path>', spaceTrim__default["default"]("\n Where to save the builded collection\n\n Note: If you keep it \"".concat(DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME, "\" it will be saved in the root of the promptbook directory\n If you set it to a path, it will be saved in that path\n BUT you can use only one format and set correct extension\n ")), DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME);
|
|
10040
10336
|
makeCommand.action(function (path$1, _a) {
|
|
10041
|
-
var projectName = _a.projectName, format = _a.format, validation = _a.validation,
|
|
10337
|
+
var projectName = _a.projectName, format = _a.format, validation = _a.validation, isCacheReloaded = _a.reloadCache, isVerbose = _a.verbose, outFile = _a.outFile;
|
|
10042
10338
|
return __awaiter(_this, void 0, void 0, function () {
|
|
10043
|
-
var formats, validations, options, fs, llm, tools, collection, validations_1, validations_1_1, validation_1, _b, _c, pipelineUrl, pipeline, e_1_1, e_2_1, collectionJson, collectionJsonString, collectionJsonItems, saveFile;
|
|
10339
|
+
var formats, validations, options, fs, llm, executables, tools, collection, validations_1, validations_1_1, validation_1, _b, _c, pipelineUrl, pipeline, e_1_1, e_2_1, collectionJson, collectionJsonString, collectionJsonItems, saveFile;
|
|
10044
10340
|
var _d, e_2, _e, e_1, _f;
|
|
10045
10341
|
var _this = this;
|
|
10046
10342
|
return __generator(this, function (_g) {
|
|
@@ -10054,22 +10350,25 @@
|
|
|
10054
10350
|
.split(',')
|
|
10055
10351
|
.map(function (_) { return _.trim(); })
|
|
10056
10352
|
.filter(function (_) { return _ !== ''; });
|
|
10057
|
-
if (outFile !==
|
|
10353
|
+
if (outFile !== DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME && formats.length !== 1) {
|
|
10058
10354
|
console.error(colors__default["default"].red("You can only use one format if you specify --out-file"));
|
|
10059
10355
|
process.exit(1);
|
|
10060
10356
|
}
|
|
10061
10357
|
options = {
|
|
10062
10358
|
isVerbose: isVerbose,
|
|
10063
|
-
|
|
10359
|
+
isCacheReloaded: isCacheReloaded,
|
|
10064
10360
|
};
|
|
10065
10361
|
fs = $provideFilesystemForNode(options);
|
|
10066
10362
|
llm = $provideLlmToolsForCli(options);
|
|
10363
|
+
return [4 /*yield*/, $provideExecutablesForNode(options)];
|
|
10364
|
+
case 1:
|
|
10365
|
+
executables = _g.sent();
|
|
10067
10366
|
_d = {
|
|
10068
10367
|
llm: llm,
|
|
10069
10368
|
fs: fs
|
|
10070
10369
|
};
|
|
10071
|
-
return [4 /*yield*/, $provideScrapersForNode({ fs: fs, llm: llm }, options)];
|
|
10072
|
-
case
|
|
10370
|
+
return [4 /*yield*/, $provideScrapersForNode({ fs: fs, llm: llm, executables: executables }, options)];
|
|
10371
|
+
case 2:
|
|
10073
10372
|
tools = (_d.scrapers = _g.sent(),
|
|
10074
10373
|
_d.script = [
|
|
10075
10374
|
/*new JavascriptExecutionTools(options)*/
|
|
@@ -10078,31 +10377,31 @@
|
|
|
10078
10377
|
return [4 /*yield*/, createCollectionFromDirectory(path$1, tools, {
|
|
10079
10378
|
isVerbose: isVerbose,
|
|
10080
10379
|
isRecursive: true,
|
|
10081
|
-
// <- TODO: [🍖]
|
|
10380
|
+
// <- TODO: [🍖] Add `intermediateFilesStrategy`
|
|
10082
10381
|
})];
|
|
10083
|
-
case 2:
|
|
10084
|
-
collection = _g.sent();
|
|
10085
|
-
_g.label = 3;
|
|
10086
10382
|
case 3:
|
|
10087
|
-
_g.
|
|
10088
|
-
validations_1 = __values(validations), validations_1_1 = validations_1.next();
|
|
10383
|
+
collection = _g.sent();
|
|
10089
10384
|
_g.label = 4;
|
|
10090
10385
|
case 4:
|
|
10091
|
-
|
|
10092
|
-
|
|
10386
|
+
_g.trys.push([4, 16, 17, 18]);
|
|
10387
|
+
validations_1 = __values(validations), validations_1_1 = validations_1.next();
|
|
10093
10388
|
_g.label = 5;
|
|
10094
10389
|
case 5:
|
|
10095
|
-
|
|
10390
|
+
if (!!validations_1_1.done) return [3 /*break*/, 15];
|
|
10391
|
+
validation_1 = validations_1_1.value;
|
|
10392
|
+
_g.label = 6;
|
|
10393
|
+
case 6:
|
|
10394
|
+
_g.trys.push([6, 12, 13, 14]);
|
|
10096
10395
|
e_1 = void 0;
|
|
10097
10396
|
return [4 /*yield*/, collection.listPipelines()];
|
|
10098
|
-
case 6:
|
|
10099
|
-
_b = (__values.apply(void 0, [_g.sent()])), _c = _b.next();
|
|
10100
|
-
_g.label = 7;
|
|
10101
10397
|
case 7:
|
|
10102
|
-
|
|
10398
|
+
_b = (__values.apply(void 0, [_g.sent()])), _c = _b.next();
|
|
10399
|
+
_g.label = 8;
|
|
10400
|
+
case 8:
|
|
10401
|
+
if (!!_c.done) return [3 /*break*/, 11];
|
|
10103
10402
|
pipelineUrl = _c.value;
|
|
10104
10403
|
return [4 /*yield*/, collection.getPipelineByUrl(pipelineUrl)];
|
|
10105
|
-
case
|
|
10404
|
+
case 9:
|
|
10106
10405
|
pipeline = _g.sent();
|
|
10107
10406
|
if (validation_1 === 'logic') {
|
|
10108
10407
|
validatePipeline(pipeline);
|
|
@@ -10110,37 +10409,37 @@
|
|
|
10110
10409
|
console.info(colors__default["default"].cyan("Validated logic of ".concat(pipeline.pipelineUrl)));
|
|
10111
10410
|
}
|
|
10112
10411
|
}
|
|
10113
|
-
_g.label =
|
|
10114
|
-
case
|
|
10412
|
+
_g.label = 10;
|
|
10413
|
+
case 10:
|
|
10115
10414
|
_c = _b.next();
|
|
10116
|
-
return [3 /*break*/,
|
|
10117
|
-
case
|
|
10118
|
-
case
|
|
10415
|
+
return [3 /*break*/, 8];
|
|
10416
|
+
case 11: return [3 /*break*/, 14];
|
|
10417
|
+
case 12:
|
|
10119
10418
|
e_1_1 = _g.sent();
|
|
10120
10419
|
e_1 = { error: e_1_1 };
|
|
10121
|
-
return [3 /*break*/,
|
|
10122
|
-
case
|
|
10420
|
+
return [3 /*break*/, 14];
|
|
10421
|
+
case 13:
|
|
10123
10422
|
try {
|
|
10124
10423
|
if (_c && !_c.done && (_f = _b.return)) _f.call(_b);
|
|
10125
10424
|
}
|
|
10126
10425
|
finally { if (e_1) throw e_1.error; }
|
|
10127
10426
|
return [7 /*endfinally*/];
|
|
10128
|
-
case
|
|
10427
|
+
case 14:
|
|
10129
10428
|
validations_1_1 = validations_1.next();
|
|
10130
|
-
return [3 /*break*/,
|
|
10131
|
-
case
|
|
10132
|
-
case
|
|
10429
|
+
return [3 /*break*/, 5];
|
|
10430
|
+
case 15: return [3 /*break*/, 18];
|
|
10431
|
+
case 16:
|
|
10133
10432
|
e_2_1 = _g.sent();
|
|
10134
10433
|
e_2 = { error: e_2_1 };
|
|
10135
|
-
return [3 /*break*/,
|
|
10136
|
-
case
|
|
10434
|
+
return [3 /*break*/, 18];
|
|
10435
|
+
case 17:
|
|
10137
10436
|
try {
|
|
10138
10437
|
if (validations_1_1 && !validations_1_1.done && (_e = validations_1.return)) _e.call(validations_1);
|
|
10139
10438
|
}
|
|
10140
10439
|
finally { if (e_2) throw e_2.error; }
|
|
10141
10440
|
return [7 /*endfinally*/];
|
|
10142
|
-
case
|
|
10143
|
-
case
|
|
10441
|
+
case 18: return [4 /*yield*/, collectionToJson(collection)];
|
|
10442
|
+
case 19:
|
|
10144
10443
|
collectionJson = _g.sent();
|
|
10145
10444
|
collectionJsonString = stringifyPipelineJson(collectionJson).trim();
|
|
10146
10445
|
collectionJsonItems = (function () {
|
|
@@ -10159,9 +10458,9 @@
|
|
|
10159
10458
|
return __generator(this, function (_a) {
|
|
10160
10459
|
switch (_a.label) {
|
|
10161
10460
|
case 0:
|
|
10162
|
-
filename = outFile !==
|
|
10461
|
+
filename = outFile !== DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME
|
|
10163
10462
|
? outFile
|
|
10164
|
-
: path.join(path$1, "".concat(
|
|
10463
|
+
: path.join(path$1, "".concat(DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME, ".").concat(extension));
|
|
10165
10464
|
if (!outFile.endsWith(".".concat(extension))) {
|
|
10166
10465
|
console.warn(colors__default["default"].yellow("Warning: Extension of output file should be \"".concat(extension, "\"")));
|
|
10167
10466
|
}
|
|
@@ -10177,27 +10476,27 @@
|
|
|
10177
10476
|
}
|
|
10178
10477
|
});
|
|
10179
10478
|
}); };
|
|
10180
|
-
if (!formats.includes('json')) return [3 /*break*/,
|
|
10479
|
+
if (!formats.includes('json')) return [3 /*break*/, 21];
|
|
10181
10480
|
formats = formats.filter(function (format) { return format !== 'json'; });
|
|
10182
10481
|
return [4 /*yield*/, saveFile('json', collectionJsonString)];
|
|
10183
|
-
case 19:
|
|
10184
|
-
_g.sent();
|
|
10185
|
-
_g.label = 20;
|
|
10186
10482
|
case 20:
|
|
10187
|
-
|
|
10483
|
+
_g.sent();
|
|
10484
|
+
_g.label = 21;
|
|
10485
|
+
case 21:
|
|
10486
|
+
if (!(formats.includes('javascript') || formats.includes('js'))) return [3 /*break*/, 23];
|
|
10188
10487
|
formats = formats.filter(function (format) { return format !== 'javascript' && format !== 'js'; });
|
|
10189
10488
|
return [4 /*yield*/, saveFile('js', spaceTrim__default["default"](function (block) { return "\n // ".concat(block(GENERATOR_WARNING_BY_PROMPTBOOK_CLI), "\n\n import { createCollectionFromJson } from '@promptbook/core';\n\n /**\n * Pipeline collection for ").concat(projectName, "\n *\n * ").concat(block(GENERATOR_WARNING_BY_PROMPTBOOK_CLI), "\n *\n * @private internal cache for `getPipelineCollection`\n */\n let pipelineCollection = null;\n\n\n /**\n * Get pipeline collection for ").concat(projectName, "\n *\n * ").concat(block(GENERATOR_WARNING_BY_PROMPTBOOK_CLI), "\n *\n * @returns {PipelineCollection} Library of promptbooks for ").concat(projectName, "\n */\n export function getPipelineCollection(){\n if(pipelineCollection===null){\n pipelineCollection = createCollectionFromJson(\n ").concat(block(collectionJsonItems), "\n );\n }\n\n return pipelineCollection;\n }\n "); }))];
|
|
10190
|
-
case 21:
|
|
10191
|
-
(_g.sent()) + '\n';
|
|
10192
|
-
_g.label = 22;
|
|
10193
10489
|
case 22:
|
|
10194
|
-
|
|
10490
|
+
(_g.sent()) + '\n';
|
|
10491
|
+
_g.label = 23;
|
|
10492
|
+
case 23:
|
|
10493
|
+
if (!(formats.includes('typescript') || formats.includes('ts'))) return [3 /*break*/, 25];
|
|
10195
10494
|
formats = formats.filter(function (format) { return format !== 'typescript' && format !== 'ts'; });
|
|
10196
10495
|
return [4 /*yield*/, saveFile('ts', spaceTrim__default["default"](function (block) { return "\n // ".concat(block(GENERATOR_WARNING_BY_PROMPTBOOK_CLI), "\n\n import { createCollectionFromJson } from '@promptbook/core';\n import type { PipelineCollection } from '@promptbook/types';\n\n /**\n * Pipeline collection for ").concat(projectName, "\n *\n * ").concat(block(GENERATOR_WARNING_BY_PROMPTBOOK_CLI), "\n *\n * @private internal cache for `getPipelineCollection`\n */\n let pipelineCollection: null | PipelineCollection = null;\n\n\n /**\n * Get pipeline collection for ").concat(projectName, "\n *\n * ").concat(block(GENERATOR_WARNING_BY_PROMPTBOOK_CLI), "\n *\n * @returns {PipelineCollection} Library of promptbooks for ").concat(projectName, "\n */\n export function getPipelineCollection(): PipelineCollection{\n if(pipelineCollection===null){\n pipelineCollection = createCollectionFromJson(\n ").concat(block(collectionJsonItems), "\n );\n }\n\n return pipelineCollection;\n }\n "); }) + '\n')];
|
|
10197
|
-
case 23:
|
|
10198
|
-
_g.sent();
|
|
10199
|
-
_g.label = 24;
|
|
10200
10496
|
case 24:
|
|
10497
|
+
_g.sent();
|
|
10498
|
+
_g.label = 25;
|
|
10499
|
+
case 25:
|
|
10201
10500
|
if (formats.length > 0) {
|
|
10202
10501
|
console.warn(colors__default["default"].yellow("Format ".concat(formats.join(' and '), " is not supported")));
|
|
10203
10502
|
}
|
|
@@ -10459,76 +10758,79 @@
|
|
|
10459
10758
|
// <- TODO: [🧟♂️] Unite path to promptbook collection argument
|
|
10460
10759
|
'Pipelines to test as glob pattern');
|
|
10461
10760
|
testCommand.option('-i, --ignore <glob>', "Ignore as glob pattern");
|
|
10462
|
-
testCommand.option('--reload
|
|
10761
|
+
testCommand.option('--reload', "Call LLM models even if same prompt with result is in the cache ", false);
|
|
10463
10762
|
testCommand.option('-v, --verbose', "Is output verbose", false);
|
|
10464
10763
|
testCommand.action(function (filesGlob, _a) {
|
|
10465
|
-
var ignore = _a.ignore,
|
|
10764
|
+
var ignore = _a.ignore, isCacheReloaded = _a.reloadCache, isVerbose = _a.verbose;
|
|
10466
10765
|
return __awaiter(_this, void 0, void 0, function () {
|
|
10467
|
-
var options, fs, llm, tools, filenames, filenames_1, filenames_1_1, filename, pipeline, pipelineMarkdown, _b, _c, error_1, e_1_1;
|
|
10766
|
+
var options, fs, llm, executables, tools, filenames, filenames_1, filenames_1_1, filename, pipeline, pipelineMarkdown, _b, _c, error_1, e_1_1;
|
|
10468
10767
|
var _d, e_1, _e;
|
|
10469
10768
|
return __generator(this, function (_f) {
|
|
10470
10769
|
switch (_f.label) {
|
|
10471
10770
|
case 0:
|
|
10472
10771
|
options = {
|
|
10473
10772
|
isVerbose: isVerbose,
|
|
10474
|
-
|
|
10773
|
+
isCacheReloaded: isCacheReloaded,
|
|
10475
10774
|
};
|
|
10476
10775
|
fs = $provideFilesystemForNode(options);
|
|
10477
10776
|
llm = $provideLlmToolsForCli(options);
|
|
10777
|
+
return [4 /*yield*/, $provideExecutablesForNode(options)];
|
|
10778
|
+
case 1:
|
|
10779
|
+
executables = _f.sent();
|
|
10478
10780
|
_d = {
|
|
10479
10781
|
llm: llm,
|
|
10480
10782
|
fs: fs
|
|
10481
10783
|
};
|
|
10482
|
-
return [4 /*yield*/, $provideScrapersForNode({ fs: fs, llm: llm }, options)];
|
|
10483
|
-
case
|
|
10784
|
+
return [4 /*yield*/, $provideScrapersForNode({ fs: fs, llm: llm, executables: executables }, options)];
|
|
10785
|
+
case 2:
|
|
10484
10786
|
tools = (_d.scrapers = _f.sent(),
|
|
10485
10787
|
_d.script = [
|
|
10486
10788
|
/*new JavascriptExecutionTools(options)*/
|
|
10487
10789
|
],
|
|
10488
10790
|
_d);
|
|
10489
10791
|
return [4 /*yield*/, glob__default["default"](filesGlob, { ignore: ignore })];
|
|
10490
|
-
case 2:
|
|
10491
|
-
filenames = _f.sent();
|
|
10492
|
-
_f.label = 3;
|
|
10493
10792
|
case 3:
|
|
10494
|
-
_f.
|
|
10495
|
-
filenames_1 = __values(filenames), filenames_1_1 = filenames_1.next();
|
|
10793
|
+
filenames = _f.sent();
|
|
10496
10794
|
_f.label = 4;
|
|
10497
10795
|
case 4:
|
|
10498
|
-
|
|
10499
|
-
|
|
10796
|
+
_f.trys.push([4, 16, 17, 18]);
|
|
10797
|
+
filenames_1 = __values(filenames), filenames_1_1 = filenames_1.next();
|
|
10500
10798
|
_f.label = 5;
|
|
10501
10799
|
case 5:
|
|
10502
|
-
|
|
10800
|
+
if (!!filenames_1_1.done) return [3 /*break*/, 15];
|
|
10801
|
+
filename = filenames_1_1.value;
|
|
10802
|
+
_f.label = 6;
|
|
10803
|
+
case 6:
|
|
10804
|
+
_f.trys.push([6, 13, , 14]);
|
|
10503
10805
|
pipeline = void 0;
|
|
10504
|
-
if (!filename.endsWith('.ptbk.md')) return [3 /*break*/,
|
|
10806
|
+
if (!filename.endsWith('.ptbk.md')) return [3 /*break*/, 9];
|
|
10505
10807
|
return [4 /*yield*/, promises.readFile(filename, 'utf-8')];
|
|
10506
|
-
case
|
|
10808
|
+
case 7:
|
|
10507
10809
|
pipelineMarkdown = (_f.sent());
|
|
10508
10810
|
return [4 /*yield*/, pipelineStringToJson(pipelineMarkdown, tools)];
|
|
10509
|
-
case
|
|
10811
|
+
case 8:
|
|
10510
10812
|
pipeline = _f.sent();
|
|
10511
10813
|
if (isVerbose) {
|
|
10512
10814
|
console.info(colors__default["default"].green("Parsed ".concat(filename)));
|
|
10513
10815
|
}
|
|
10514
|
-
_f.label =
|
|
10515
|
-
case
|
|
10516
|
-
if (!filename.endsWith('.ptbk.json')) return [3 /*break*/,
|
|
10816
|
+
_f.label = 9;
|
|
10817
|
+
case 9:
|
|
10818
|
+
if (!filename.endsWith('.ptbk.json')) return [3 /*break*/, 11];
|
|
10517
10819
|
_c = (_b = JSON).parse;
|
|
10518
10820
|
return [4 /*yield*/, promises.readFile(filename, 'utf-8')];
|
|
10519
|
-
case 9:
|
|
10520
|
-
pipeline = _c.apply(_b, [_f.sent()]);
|
|
10521
|
-
return [3 /*break*/, 11];
|
|
10522
10821
|
case 10:
|
|
10822
|
+
pipeline = _c.apply(_b, [_f.sent()]);
|
|
10823
|
+
return [3 /*break*/, 12];
|
|
10824
|
+
case 11:
|
|
10523
10825
|
if (isVerbose) {
|
|
10524
10826
|
console.info(colors__default["default"].gray("Skipping ".concat(filename)));
|
|
10525
10827
|
}
|
|
10526
|
-
return [3 /*break*/,
|
|
10527
|
-
case
|
|
10828
|
+
return [3 /*break*/, 14];
|
|
10829
|
+
case 12:
|
|
10528
10830
|
validatePipeline(pipeline);
|
|
10529
10831
|
console.info(colors__default["default"].green("Validated ".concat(filename)));
|
|
10530
|
-
return [3 /*break*/,
|
|
10531
|
-
case
|
|
10832
|
+
return [3 /*break*/, 14];
|
|
10833
|
+
case 13:
|
|
10532
10834
|
error_1 = _f.sent();
|
|
10533
10835
|
if (!(error_1 instanceof Error)) {
|
|
10534
10836
|
throw error_1;
|
|
@@ -10537,22 +10839,22 @@
|
|
|
10537
10839
|
console.error(colors__default["default"].bgRed(error_1.name /* <- 11:11 */));
|
|
10538
10840
|
console.error(colors__default["default"].red(error_1.stack || error_1.message));
|
|
10539
10841
|
process.exit(1);
|
|
10540
|
-
return [3 /*break*/,
|
|
10541
|
-
case
|
|
10842
|
+
return [3 /*break*/, 14];
|
|
10843
|
+
case 14:
|
|
10542
10844
|
filenames_1_1 = filenames_1.next();
|
|
10543
|
-
return [3 /*break*/,
|
|
10544
|
-
case
|
|
10545
|
-
case
|
|
10845
|
+
return [3 /*break*/, 5];
|
|
10846
|
+
case 15: return [3 /*break*/, 18];
|
|
10847
|
+
case 16:
|
|
10546
10848
|
e_1_1 = _f.sent();
|
|
10547
10849
|
e_1 = { error: e_1_1 };
|
|
10548
|
-
return [3 /*break*/,
|
|
10549
|
-
case
|
|
10850
|
+
return [3 /*break*/, 18];
|
|
10851
|
+
case 17:
|
|
10550
10852
|
try {
|
|
10551
10853
|
if (filenames_1_1 && !filenames_1_1.done && (_e = filenames_1.return)) _e.call(filenames_1);
|
|
10552
10854
|
}
|
|
10553
10855
|
finally { if (e_1) throw e_1.error; }
|
|
10554
10856
|
return [7 /*endfinally*/];
|
|
10555
|
-
case
|
|
10857
|
+
case 18:
|
|
10556
10858
|
console.info(colors__default["default"].green("All pipelines are valid"));
|
|
10557
10859
|
process.exit(0);
|
|
10558
10860
|
return [2 /*return*/];
|
|
@@ -10666,6 +10968,7 @@
|
|
|
10666
10968
|
* @public exported from `@promptbook/remote-client`
|
|
10667
10969
|
*/
|
|
10668
10970
|
var RemoteLlmExecutionTools = /** @class */ (function () {
|
|
10971
|
+
/* <- TODO: [🍚] `, Destroyable` */
|
|
10669
10972
|
function RemoteLlmExecutionTools(options) {
|
|
10670
10973
|
this.options = options;
|
|
10671
10974
|
}
|
|
@@ -10715,12 +11018,16 @@
|
|
|
10715
11018
|
if (this.options.isAnonymous) {
|
|
10716
11019
|
socket.emit('listModels-request', {
|
|
10717
11020
|
isAnonymous: true,
|
|
11021
|
+
userId: this.options.userId,
|
|
10718
11022
|
llmToolsConfiguration: this.options.llmToolsConfiguration,
|
|
10719
11023
|
} /* <- TODO: [🤛] */);
|
|
10720
11024
|
}
|
|
10721
11025
|
else {
|
|
10722
11026
|
socket.emit('listModels-request', {
|
|
10723
11027
|
isAnonymous: false,
|
|
11028
|
+
appId: this.options.appId,
|
|
11029
|
+
userId: this.options.userId,
|
|
11030
|
+
customOptions: this.options.customOptions,
|
|
10724
11031
|
} /* <- TODO: [🤛] */);
|
|
10725
11032
|
}
|
|
10726
11033
|
return [4 /*yield*/, new Promise(function (resolve, reject) {
|
|
@@ -10814,7 +11121,9 @@
|
|
|
10814
11121
|
else {
|
|
10815
11122
|
socket.emit('prompt-request', {
|
|
10816
11123
|
isAnonymous: false,
|
|
11124
|
+
appId: this.options.appId,
|
|
10817
11125
|
userId: this.options.userId,
|
|
11126
|
+
customOptions: this.options.customOptions,
|
|
10818
11127
|
prompt: prompt,
|
|
10819
11128
|
} /* <- TODO: [🤛] */);
|
|
10820
11129
|
}
|
|
@@ -11296,7 +11605,7 @@
|
|
|
11296
11605
|
*/
|
|
11297
11606
|
var createAnthropicClaudeExecutionTools = Object.assign(function (options) {
|
|
11298
11607
|
if (options.isProxied) {
|
|
11299
|
-
return new RemoteLlmExecutionTools(__assign(__assign({}, options), { isAnonymous: true, llmToolsConfiguration: [
|
|
11608
|
+
return new RemoteLlmExecutionTools(__assign(__assign({}, options), { userId: null, isAnonymous: true, llmToolsConfiguration: [
|
|
11300
11609
|
{
|
|
11301
11610
|
title: 'Anthropic Claude (proxied)',
|
|
11302
11611
|
packageName: '@promptbook/anthropic-claude',
|
|
@@ -11856,12 +12165,12 @@
|
|
|
11856
12165
|
* Calls OpenAI API to use a chat model.
|
|
11857
12166
|
*/
|
|
11858
12167
|
AzureOpenAiExecutionTools.prototype.callChatModel = function (prompt) {
|
|
11859
|
-
var _a, _b;
|
|
12168
|
+
var _a, _b, _c;
|
|
11860
12169
|
return __awaiter(this, void 0, void 0, function () {
|
|
11861
12170
|
var content, parameters, modelRequirements, client, modelName, modelSettings, rawPromptContent, messages, start, complete, rawRequest, rawResponse, resultContent, usage, error_1;
|
|
11862
12171
|
var _this = this;
|
|
11863
|
-
return __generator(this, function (
|
|
11864
|
-
switch (
|
|
12172
|
+
return __generator(this, function (_d) {
|
|
12173
|
+
switch (_d.label) {
|
|
11865
12174
|
case 0:
|
|
11866
12175
|
if (this.options.isVerbose) {
|
|
11867
12176
|
console.info('💬 OpenAI callChatModel call');
|
|
@@ -11869,20 +12178,20 @@
|
|
|
11869
12178
|
content = prompt.content, parameters = prompt.parameters, modelRequirements = prompt.modelRequirements;
|
|
11870
12179
|
return [4 /*yield*/, this.getClient()];
|
|
11871
12180
|
case 1:
|
|
11872
|
-
client =
|
|
12181
|
+
client = _d.sent();
|
|
11873
12182
|
// TODO: [☂] Use here more modelRequirements
|
|
11874
12183
|
if (modelRequirements.modelVariant !== 'CHAT') {
|
|
11875
12184
|
throw new PipelineExecutionError('Use callChatModel only for CHAT variant');
|
|
11876
12185
|
}
|
|
11877
|
-
|
|
12186
|
+
_d.label = 2;
|
|
11878
12187
|
case 2:
|
|
11879
|
-
|
|
12188
|
+
_d.trys.push([2, 4, , 5]);
|
|
11880
12189
|
modelName = prompt.modelRequirements.modelName || this.options.deploymentName;
|
|
11881
12190
|
modelSettings = {
|
|
11882
12191
|
maxTokens: modelRequirements.maxTokens,
|
|
11883
12192
|
// <- TODO: [🌾] Make some global max cap for maxTokens
|
|
11884
12193
|
temperature: modelRequirements.temperature,
|
|
11885
|
-
user: this.options.
|
|
12194
|
+
user: (_a = this.options.userId) === null || _a === void 0 ? void 0 : _a.toString(),
|
|
11886
12195
|
// <- TODO: [🈁] Use `seed` here AND/OR use is `isDeterministic` for entire execution tools
|
|
11887
12196
|
// <- Note: [🧆]
|
|
11888
12197
|
};
|
|
@@ -11913,7 +12222,7 @@
|
|
|
11913
12222
|
throw error;
|
|
11914
12223
|
})];
|
|
11915
12224
|
case 3:
|
|
11916
|
-
rawResponse =
|
|
12225
|
+
rawResponse = _d.sent();
|
|
11917
12226
|
if (this.options.isVerbose) {
|
|
11918
12227
|
console.info(colors__default["default"].bgWhite('rawResponse'), JSON.stringify(rawResponse, null, 4));
|
|
11919
12228
|
}
|
|
@@ -11932,8 +12241,8 @@
|
|
|
11932
12241
|
complete = getCurrentIsoDate();
|
|
11933
12242
|
usage = {
|
|
11934
12243
|
price: uncertainNumber() /* <- TODO: [🐞] Compute usage */,
|
|
11935
|
-
input: __assign({ tokensCount: uncertainNumber((
|
|
11936
|
-
output: __assign({ tokensCount: uncertainNumber((
|
|
12244
|
+
input: __assign({ tokensCount: uncertainNumber((_b = rawResponse.usage) === null || _b === void 0 ? void 0 : _b.promptTokens) }, computeUsageCounts(prompt.content)),
|
|
12245
|
+
output: __assign({ tokensCount: uncertainNumber((_c = rawResponse.usage) === null || _c === void 0 ? void 0 : _c.completionTokens) }, computeUsageCounts(prompt.content)),
|
|
11937
12246
|
};
|
|
11938
12247
|
return [2 /*return*/, $asDeeplyFrozenSerializableJson('AzureOpenAiExecutionTools ChatPromptResult', {
|
|
11939
12248
|
content: resultContent,
|
|
@@ -11949,7 +12258,7 @@
|
|
|
11949
12258
|
// <- [🗯]
|
|
11950
12259
|
})];
|
|
11951
12260
|
case 4:
|
|
11952
|
-
error_1 =
|
|
12261
|
+
error_1 = _d.sent();
|
|
11953
12262
|
throw this.transformAzureError(error_1);
|
|
11954
12263
|
case 5: return [2 /*return*/];
|
|
11955
12264
|
}
|
|
@@ -11960,12 +12269,12 @@
|
|
|
11960
12269
|
* Calls Azure OpenAI API to use a complete model.
|
|
11961
12270
|
*/
|
|
11962
12271
|
AzureOpenAiExecutionTools.prototype.callCompletionModel = function (prompt) {
|
|
11963
|
-
var _a, _b;
|
|
12272
|
+
var _a, _b, _c;
|
|
11964
12273
|
return __awaiter(this, void 0, void 0, function () {
|
|
11965
12274
|
var content, parameters, modelRequirements, client, modelName, modelSettings, start, complete, rawPromptContent, rawRequest, rawResponse, resultContent, usage, error_2;
|
|
11966
12275
|
var _this = this;
|
|
11967
|
-
return __generator(this, function (
|
|
11968
|
-
switch (
|
|
12276
|
+
return __generator(this, function (_d) {
|
|
12277
|
+
switch (_d.label) {
|
|
11969
12278
|
case 0:
|
|
11970
12279
|
if (this.options.isVerbose) {
|
|
11971
12280
|
console.info('🖋 OpenAI callCompletionModel call');
|
|
@@ -11973,20 +12282,20 @@
|
|
|
11973
12282
|
content = prompt.content, parameters = prompt.parameters, modelRequirements = prompt.modelRequirements;
|
|
11974
12283
|
return [4 /*yield*/, this.getClient()];
|
|
11975
12284
|
case 1:
|
|
11976
|
-
client =
|
|
12285
|
+
client = _d.sent();
|
|
11977
12286
|
// TODO: [☂] Use here more modelRequirements
|
|
11978
12287
|
if (modelRequirements.modelVariant !== 'COMPLETION') {
|
|
11979
12288
|
throw new PipelineExecutionError('Use callCompletionModel only for COMPLETION variant');
|
|
11980
12289
|
}
|
|
11981
|
-
|
|
12290
|
+
_d.label = 2;
|
|
11982
12291
|
case 2:
|
|
11983
|
-
|
|
12292
|
+
_d.trys.push([2, 4, , 5]);
|
|
11984
12293
|
modelName = prompt.modelRequirements.modelName || this.options.deploymentName;
|
|
11985
12294
|
modelSettings = {
|
|
11986
12295
|
maxTokens: modelRequirements.maxTokens || 2000,
|
|
11987
12296
|
// <- TODO: [🌾] Make some global max cap for maxTokens
|
|
11988
12297
|
temperature: modelRequirements.temperature,
|
|
11989
|
-
user: this.options.
|
|
12298
|
+
user: (_a = this.options.userId) === null || _a === void 0 ? void 0 : _a.toString(),
|
|
11990
12299
|
// <- TODO: [🈁] Use `seed` here AND/OR use is `isDeterministic` for entire execution tools
|
|
11991
12300
|
// <- Note: [🧆]
|
|
11992
12301
|
};
|
|
@@ -12009,7 +12318,7 @@
|
|
|
12009
12318
|
throw error;
|
|
12010
12319
|
})];
|
|
12011
12320
|
case 3:
|
|
12012
|
-
rawResponse =
|
|
12321
|
+
rawResponse = _d.sent();
|
|
12013
12322
|
if (this.options.isVerbose) {
|
|
12014
12323
|
console.info(colors__default["default"].bgWhite('rawResponse'), JSON.stringify(rawResponse, null, 4));
|
|
12015
12324
|
}
|
|
@@ -12025,8 +12334,8 @@
|
|
|
12025
12334
|
complete = getCurrentIsoDate();
|
|
12026
12335
|
usage = {
|
|
12027
12336
|
price: uncertainNumber() /* <- TODO: [🐞] Compute usage */,
|
|
12028
|
-
input: __assign({ tokensCount: uncertainNumber((
|
|
12029
|
-
output: __assign({ tokensCount: uncertainNumber((
|
|
12337
|
+
input: __assign({ tokensCount: uncertainNumber((_b = rawResponse.usage) === null || _b === void 0 ? void 0 : _b.promptTokens) }, computeUsageCounts(prompt.content)),
|
|
12338
|
+
output: __assign({ tokensCount: uncertainNumber((_c = rawResponse.usage) === null || _c === void 0 ? void 0 : _c.completionTokens) }, computeUsageCounts(prompt.content)),
|
|
12030
12339
|
};
|
|
12031
12340
|
return [2 /*return*/, $asDeeplyFrozenSerializableJson('AzureOpenAiExecutionTools CompletionPromptResult', {
|
|
12032
12341
|
content: resultContent,
|
|
@@ -12042,7 +12351,7 @@
|
|
|
12042
12351
|
// <- [🗯]
|
|
12043
12352
|
})];
|
|
12044
12353
|
case 4:
|
|
12045
|
-
error_2 =
|
|
12354
|
+
error_2 = _d.sent();
|
|
12046
12355
|
throw this.transformAzureError(error_2);
|
|
12047
12356
|
case 5: return [2 /*return*/];
|
|
12048
12357
|
}
|
|
@@ -12267,7 +12576,6 @@
|
|
|
12267
12576
|
* @param options which are relevant are directly passed to the OpenAI client
|
|
12268
12577
|
*/
|
|
12269
12578
|
function OpenAiExecutionTools(options) {
|
|
12270
|
-
if (options === void 0) { options = {}; }
|
|
12271
12579
|
this.options = options;
|
|
12272
12580
|
/**
|
|
12273
12581
|
* OpenAI API client.
|
|
@@ -12295,8 +12603,8 @@
|
|
|
12295
12603
|
if (this.client === null) {
|
|
12296
12604
|
openAiOptions = __assign({}, this.options);
|
|
12297
12605
|
delete openAiOptions.isVerbose;
|
|
12298
|
-
delete openAiOptions.
|
|
12299
|
-
this.client = new OpenAI__default["default"](
|
|
12606
|
+
delete openAiOptions.userId;
|
|
12607
|
+
this.client = new OpenAI__default["default"](openAiOptions);
|
|
12300
12608
|
}
|
|
12301
12609
|
return [2 /*return*/, this.client];
|
|
12302
12610
|
});
|
|
@@ -12343,11 +12651,12 @@
|
|
|
12343
12651
|
* Calls OpenAI API to use a chat model.
|
|
12344
12652
|
*/
|
|
12345
12653
|
OpenAiExecutionTools.prototype.callChatModel = function (prompt) {
|
|
12654
|
+
var _a;
|
|
12346
12655
|
return __awaiter(this, void 0, void 0, function () {
|
|
12347
12656
|
var content, parameters, modelRequirements, format, client, modelName, modelSettings, rawPromptContent, rawRequest, start, complete, rawResponse, resultContent, usage;
|
|
12348
12657
|
var _this = this;
|
|
12349
|
-
return __generator(this, function (
|
|
12350
|
-
switch (
|
|
12658
|
+
return __generator(this, function (_b) {
|
|
12659
|
+
switch (_b.label) {
|
|
12351
12660
|
case 0:
|
|
12352
12661
|
if (this.options.isVerbose) {
|
|
12353
12662
|
console.info('💬 OpenAI callChatModel call', { prompt: prompt });
|
|
@@ -12355,7 +12664,7 @@
|
|
|
12355
12664
|
content = prompt.content, parameters = prompt.parameters, modelRequirements = prompt.modelRequirements, format = prompt.format;
|
|
12356
12665
|
return [4 /*yield*/, this.getClient()];
|
|
12357
12666
|
case 1:
|
|
12358
|
-
client =
|
|
12667
|
+
client = _b.sent();
|
|
12359
12668
|
// TODO: [☂] Use here more modelRequirements
|
|
12360
12669
|
if (modelRequirements.modelVariant !== 'CHAT') {
|
|
12361
12670
|
throw new PipelineExecutionError('Use callChatModel only for CHAT variant');
|
|
@@ -12387,7 +12696,7 @@
|
|
|
12387
12696
|
role: 'user',
|
|
12388
12697
|
content: rawPromptContent,
|
|
12389
12698
|
},
|
|
12390
|
-
], false), user: this.options.
|
|
12699
|
+
], false), user: (_a = this.options.userId) === null || _a === void 0 ? void 0 : _a.toString() });
|
|
12391
12700
|
start = getCurrentIsoDate();
|
|
12392
12701
|
if (this.options.isVerbose) {
|
|
12393
12702
|
console.info(colors__default["default"].bgWhite('rawRequest'), JSON.stringify(rawRequest, null, 4));
|
|
@@ -12399,7 +12708,7 @@
|
|
|
12399
12708
|
throw error;
|
|
12400
12709
|
})];
|
|
12401
12710
|
case 2:
|
|
12402
|
-
rawResponse =
|
|
12711
|
+
rawResponse = _b.sent();
|
|
12403
12712
|
if (this.options.isVerbose) {
|
|
12404
12713
|
console.info(colors__default["default"].bgWhite('rawResponse'), JSON.stringify(rawResponse, null, 4));
|
|
12405
12714
|
}
|
|
@@ -12438,11 +12747,12 @@
|
|
|
12438
12747
|
* Calls OpenAI API to use a complete model.
|
|
12439
12748
|
*/
|
|
12440
12749
|
OpenAiExecutionTools.prototype.callCompletionModel = function (prompt) {
|
|
12750
|
+
var _a;
|
|
12441
12751
|
return __awaiter(this, void 0, void 0, function () {
|
|
12442
12752
|
var content, parameters, modelRequirements, client, modelName, modelSettings, rawPromptContent, rawRequest, start, complete, rawResponse, resultContent, usage;
|
|
12443
12753
|
var _this = this;
|
|
12444
|
-
return __generator(this, function (
|
|
12445
|
-
switch (
|
|
12754
|
+
return __generator(this, function (_b) {
|
|
12755
|
+
switch (_b.label) {
|
|
12446
12756
|
case 0:
|
|
12447
12757
|
if (this.options.isVerbose) {
|
|
12448
12758
|
console.info('🖋 OpenAI callCompletionModel call', { prompt: prompt });
|
|
@@ -12450,7 +12760,7 @@
|
|
|
12450
12760
|
content = prompt.content, parameters = prompt.parameters, modelRequirements = prompt.modelRequirements;
|
|
12451
12761
|
return [4 /*yield*/, this.getClient()];
|
|
12452
12762
|
case 1:
|
|
12453
|
-
client =
|
|
12763
|
+
client = _b.sent();
|
|
12454
12764
|
// TODO: [☂] Use here more modelRequirements
|
|
12455
12765
|
if (modelRequirements.modelVariant !== 'COMPLETION') {
|
|
12456
12766
|
throw new PipelineExecutionError('Use callCompletionModel only for COMPLETION variant');
|
|
@@ -12465,7 +12775,7 @@
|
|
|
12465
12775
|
// <- Note: [🧆]
|
|
12466
12776
|
};
|
|
12467
12777
|
rawPromptContent = replaceParameters(content, __assign(__assign({}, parameters), { modelName: modelName }));
|
|
12468
|
-
rawRequest = __assign(__assign({}, modelSettings), { prompt: rawPromptContent, user: this.options.
|
|
12778
|
+
rawRequest = __assign(__assign({}, modelSettings), { prompt: rawPromptContent, user: (_a = this.options.userId) === null || _a === void 0 ? void 0 : _a.toString() });
|
|
12469
12779
|
start = getCurrentIsoDate();
|
|
12470
12780
|
if (this.options.isVerbose) {
|
|
12471
12781
|
console.info(colors__default["default"].bgWhite('rawRequest'), JSON.stringify(rawRequest, null, 4));
|
|
@@ -12477,7 +12787,7 @@
|
|
|
12477
12787
|
throw error;
|
|
12478
12788
|
})];
|
|
12479
12789
|
case 2:
|
|
12480
|
-
rawResponse =
|
|
12790
|
+
rawResponse = _b.sent();
|
|
12481
12791
|
if (this.options.isVerbose) {
|
|
12482
12792
|
console.info(colors__default["default"].bgWhite('rawResponse'), JSON.stringify(rawResponse, null, 4));
|
|
12483
12793
|
}
|
|
@@ -12703,17 +13013,16 @@
|
|
|
12703
13013
|
assistant_id: this.assistantId,
|
|
12704
13014
|
thread: {
|
|
12705
13015
|
messages: [
|
|
12706
|
-
// TODO:
|
|
13016
|
+
// TODO: [🗯] !! Allow threads to be passed
|
|
12707
13017
|
{ role: 'user', content: rawPromptContent },
|
|
12708
13018
|
],
|
|
12709
13019
|
},
|
|
12710
|
-
//
|
|
13020
|
+
// <- TODO: Add user identification here> user: this.options.user,
|
|
12711
13021
|
};
|
|
12712
13022
|
start = getCurrentIsoDate();
|
|
12713
13023
|
if (this.options.isVerbose) {
|
|
12714
13024
|
console.info(colors__default["default"].bgWhite('rawRequest'), JSON.stringify(rawRequest, null, 4));
|
|
12715
13025
|
}
|
|
12716
|
-
console.log('!!!!!! OpenAI client', client);
|
|
12717
13026
|
return [4 /*yield*/, client.beta.threads.createAndRunStream(rawRequest)];
|
|
12718
13027
|
case 2:
|
|
12719
13028
|
stream = _g.sent();
|
|
@@ -12731,7 +13040,7 @@
|
|
|
12731
13040
|
messageDelta.content[0].type === 'text') {
|
|
12732
13041
|
console.info('messageDelta', (_a = messageDelta.content[0].text) === null || _a === void 0 ? void 0 : _a.value);
|
|
12733
13042
|
}
|
|
12734
|
-
// TODO:
|
|
13043
|
+
// <- TODO: [🐚] Make streaming and running tasks working
|
|
12735
13044
|
});
|
|
12736
13045
|
stream.on('messageCreated', function (message) {
|
|
12737
13046
|
if (_this.options.isVerbose) {
|
|
@@ -12759,18 +13068,20 @@
|
|
|
12759
13068
|
throw new PipelineExecutionError("There is NOT 'text' BUT ".concat((_b = rawResponse[0].content[0]) === null || _b === void 0 ? void 0 : _b.type, " finalMessages content type from OpenAI"));
|
|
12760
13069
|
}
|
|
12761
13070
|
resultContent = (_c = rawResponse[0].content[0]) === null || _c === void 0 ? void 0 : _c.text.value;
|
|
12762
|
-
// <- TODO:
|
|
13071
|
+
// <- TODO: [🧠] There are also annotations, maybe use them
|
|
12763
13072
|
// eslint-disable-next-line prefer-const
|
|
12764
13073
|
complete = getCurrentIsoDate();
|
|
12765
13074
|
usage = UNCERTAIN_USAGE;
|
|
12766
|
-
// TODO:
|
|
13075
|
+
// <- TODO: [🥘] Compute real usage for assistant
|
|
13076
|
+
// ?> const usage = computeOpenAiUsage(content, resultContent || '', rawResponse);
|
|
12767
13077
|
if (resultContent === null) {
|
|
12768
13078
|
throw new PipelineExecutionError('No response message from OpenAI');
|
|
12769
13079
|
}
|
|
12770
13080
|
return [2 /*return*/, $asDeeplyFrozenSerializableJson('OpenAiAssistantExecutionTools ChatPromptResult', {
|
|
12771
13081
|
content: resultContent,
|
|
12772
13082
|
modelName: 'assistant',
|
|
12773
|
-
//
|
|
13083
|
+
// <- TODO: [🥘] Detect used model in assistant
|
|
13084
|
+
// ?> model: rawResponse.model || modelName,
|
|
12774
13085
|
timing: {
|
|
12775
13086
|
start: start,
|
|
12776
13087
|
complete: complete,
|
|
@@ -12788,7 +13099,6 @@
|
|
|
12788
13099
|
return OpenAiAssistantExecutionTools;
|
|
12789
13100
|
}(OpenAiExecutionTools));
|
|
12790
13101
|
/**
|
|
12791
|
-
* TODO: !!!!!! DO not use colors - can be used in browser
|
|
12792
13102
|
* TODO: [🧠][🧙♂️] Maybe there can be some wizzard for thoose who want to use just OpenAI
|
|
12793
13103
|
* TODO: Maybe make custom OpenAiError
|
|
12794
13104
|
* TODO: [🧠][🈁] Maybe use `isDeterministic` from options
|
|
@@ -12860,22 +13170,26 @@
|
|
|
12860
13170
|
/**
|
|
12861
13171
|
* Normalize options for `execCommand` and `execCommands`
|
|
12862
13172
|
*
|
|
13173
|
+
* Note: `$` is used to indicate that this function behaves differently according to `process.platform`
|
|
13174
|
+
*
|
|
12863
13175
|
* @private internal utility of `execCommand` and `execCommands`
|
|
12864
13176
|
*/
|
|
12865
|
-
function execCommandNormalizeOptions(options) {
|
|
13177
|
+
function $execCommandNormalizeOptions(options) {
|
|
12866
13178
|
var _a;
|
|
12867
|
-
var _b, _c, _d;
|
|
13179
|
+
var _b, _c, _d, _e;
|
|
12868
13180
|
var command;
|
|
12869
13181
|
var cwd;
|
|
12870
13182
|
var crashOnError;
|
|
12871
13183
|
var args = [];
|
|
12872
13184
|
var timeout;
|
|
13185
|
+
var isVerbose;
|
|
12873
13186
|
if (typeof options === 'string') {
|
|
12874
13187
|
// TODO: [1] DRY default values
|
|
12875
13188
|
command = options;
|
|
12876
13189
|
cwd = process.cwd();
|
|
12877
13190
|
crashOnError = true;
|
|
12878
|
-
timeout = Infinity;
|
|
13191
|
+
timeout = Infinity; // <- TODO: [⏳]
|
|
13192
|
+
isVerbose = DEFAULT_IS_VERBOSE;
|
|
12879
13193
|
}
|
|
12880
13194
|
else {
|
|
12881
13195
|
/*
|
|
@@ -12891,6 +13205,7 @@
|
|
|
12891
13205
|
cwd = (_b = options.cwd) !== null && _b !== void 0 ? _b : process.cwd();
|
|
12892
13206
|
crashOnError = (_c = options.crashOnError) !== null && _c !== void 0 ? _c : true;
|
|
12893
13207
|
timeout = (_d = options.timeout) !== null && _d !== void 0 ? _d : Infinity;
|
|
13208
|
+
isVerbose = (_e = options.isVerbose) !== null && _e !== void 0 ? _e : DEFAULT_IS_VERBOSE;
|
|
12894
13209
|
}
|
|
12895
13210
|
// TODO: /(-[a-zA-Z0-9-]+\s+[^\s]*)|[^\s]*/g
|
|
12896
13211
|
var _ = Array.from(command.matchAll(/(".*")|([^\s]*)/g))
|
|
@@ -12912,16 +13227,21 @@
|
|
|
12912
13227
|
if (['ts-node'].includes(humanReadableCommand)) {
|
|
12913
13228
|
humanReadableCommand += " ".concat(args[1]);
|
|
12914
13229
|
}
|
|
12915
|
-
|
|
13230
|
+
if (/^win/.test(process.platform) && ['npm', 'npx'].includes(command)) {
|
|
13231
|
+
command = "".concat(command, ".cmd");
|
|
13232
|
+
}
|
|
13233
|
+
return { command: command, humanReadableCommand: humanReadableCommand, args: args, cwd: cwd, crashOnError: crashOnError, timeout: timeout, isVerbose: isVerbose };
|
|
12916
13234
|
}
|
|
12917
13235
|
// TODO: This should show type error> execCommandNormalizeOptions({ command: '', commands: [''] });
|
|
12918
13236
|
|
|
12919
13237
|
/**
|
|
12920
13238
|
* Run one command in a shell
|
|
12921
13239
|
*
|
|
13240
|
+
*
|
|
12922
13241
|
* Note: There are 2 similar functions in the codebase:
|
|
12923
13242
|
* - `$execCommand` which runs a single command
|
|
12924
13243
|
* - `$execCommands` which runs multiple commands
|
|
13244
|
+
* Note: `$` is used to indicate that this function is not a pure function - it runs a command in a shell
|
|
12925
13245
|
*
|
|
12926
13246
|
* @public exported from `@promptbook/node`
|
|
12927
13247
|
*/
|
|
@@ -12931,7 +13251,7 @@
|
|
|
12931
13251
|
}
|
|
12932
13252
|
return new Promise(function (resolve, reject) {
|
|
12933
13253
|
// eslint-disable-next-line prefer-const
|
|
12934
|
-
var _a = execCommandNormalizeOptions(options), command = _a.command, humanReadableCommand = _a.humanReadableCommand, args = _a.args, cwd = _a.cwd, crashOnError = _a.crashOnError, timeout = _a.timeout;
|
|
13254
|
+
var _a = $execCommandNormalizeOptions(options), command = _a.command, humanReadableCommand = _a.humanReadableCommand, args = _a.args, cwd = _a.cwd, crashOnError = _a.crashOnError, timeout = _a.timeout, _b = _a.isVerbose, isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b;
|
|
12935
13255
|
if (timeout !== Infinity) {
|
|
12936
13256
|
// TODO: In waitasecond forTime(Infinity) should be equivalent to forEver()
|
|
12937
13257
|
waitasecond.forTime(timeout).then(function () {
|
|
@@ -12944,24 +13264,26 @@
|
|
|
12944
13264
|
}
|
|
12945
13265
|
});
|
|
12946
13266
|
}
|
|
12947
|
-
if (
|
|
12948
|
-
|
|
13267
|
+
if (isVerbose) {
|
|
13268
|
+
console.info(colors__default["default"].yellow(cwd) + ' ' + colors__default["default"].green(command) + ' ' + colors__default["default"].blue(args.join(' ')));
|
|
12949
13269
|
}
|
|
12950
|
-
// !!!!!! Verbose mode - to all consoles
|
|
12951
|
-
console.info(colors__default["default"].yellow(cwd) + ' ' + colors__default["default"].green(command) + ' ' + colors__default["default"].blue(args.join(' ')));
|
|
12952
13270
|
try {
|
|
12953
13271
|
var commandProcess = child_process.spawn(command, args, { cwd: cwd, shell: true });
|
|
12954
|
-
|
|
12955
|
-
|
|
12956
|
-
|
|
13272
|
+
if (isVerbose) {
|
|
13273
|
+
commandProcess.on('message', function (message) {
|
|
13274
|
+
console.info({ message: message });
|
|
13275
|
+
});
|
|
13276
|
+
}
|
|
12957
13277
|
var output_1 = [];
|
|
12958
13278
|
commandProcess.stdout.on('data', function (stdout) {
|
|
12959
13279
|
output_1.push(stdout.toString());
|
|
12960
|
-
|
|
13280
|
+
if (isVerbose) {
|
|
13281
|
+
console.info(stdout.toString());
|
|
13282
|
+
}
|
|
12961
13283
|
});
|
|
12962
13284
|
commandProcess.stderr.on('data', function (stderr) {
|
|
12963
13285
|
output_1.push(stderr.toString());
|
|
12964
|
-
if (stderr.toString().trim()) {
|
|
13286
|
+
if (isVerbose && stderr.toString().trim()) {
|
|
12965
13287
|
console.warn(stderr.toString());
|
|
12966
13288
|
}
|
|
12967
13289
|
});
|
|
@@ -12972,7 +13294,9 @@
|
|
|
12972
13294
|
"Command \"".concat(humanReadableCommand, "\" exited with code ").concat(code)));
|
|
12973
13295
|
}
|
|
12974
13296
|
else {
|
|
12975
|
-
|
|
13297
|
+
if (isVerbose) {
|
|
13298
|
+
console.warn("Command \"".concat(humanReadableCommand, "\" exited with code ").concat(code));
|
|
13299
|
+
}
|
|
12976
13300
|
resolve(spaceTrim.spaceTrim(output_1.join('\n')));
|
|
12977
13301
|
}
|
|
12978
13302
|
}
|
|
@@ -12991,7 +13315,9 @@
|
|
|
12991
13315
|
reject(new Error("Command \"".concat(humanReadableCommand, "\" failed: \n").concat(error.message)));
|
|
12992
13316
|
}
|
|
12993
13317
|
else {
|
|
12994
|
-
|
|
13318
|
+
if (isVerbose) {
|
|
13319
|
+
console.warn(error);
|
|
13320
|
+
}
|
|
12995
13321
|
resolve(spaceTrim.spaceTrim(output_1.join('\n')));
|
|
12996
13322
|
}
|
|
12997
13323
|
});
|
|
@@ -13015,12 +13341,12 @@
|
|
|
13015
13341
|
*/
|
|
13016
13342
|
function getScraperIntermediateSource(source, options) {
|
|
13017
13343
|
return __awaiter(this, void 0, void 0, function () {
|
|
13018
|
-
var sourceFilename, url, rootDirname, cacheDirname,
|
|
13344
|
+
var sourceFilename, url, rootDirname, cacheDirname, intermediateFilesStrategy, extension, isVerbose, hash, semanticName, pieces, name, cacheFilename, isDestroyed, fileHandler;
|
|
13019
13345
|
return __generator(this, function (_a) {
|
|
13020
13346
|
switch (_a.label) {
|
|
13021
13347
|
case 0:
|
|
13022
13348
|
sourceFilename = source.filename, url = source.url;
|
|
13023
|
-
rootDirname = options.rootDirname, cacheDirname = options.cacheDirname,
|
|
13349
|
+
rootDirname = options.rootDirname, cacheDirname = options.cacheDirname, intermediateFilesStrategy = options.intermediateFilesStrategy, extension = options.extension, isVerbose = options.isVerbose;
|
|
13024
13350
|
hash = cryptoJs.SHA256(
|
|
13025
13351
|
// <- TODO: [🥬] Encapsulate sha256 to some private utility function
|
|
13026
13352
|
hexEncoder__default["default"].parse(sourceFilename || url || 'untitled'))
|
|
@@ -13030,7 +13356,7 @@
|
|
|
13030
13356
|
pieces = ['intermediate', semanticName, hash].filter(function (piece) { return piece !== ''; });
|
|
13031
13357
|
name = pieces.join('-').split('--').join('-');
|
|
13032
13358
|
// <- TODO: Use MAX_FILENAME_LENGTH
|
|
13033
|
-
TODO_USE(rootDirname); // <- TODO:
|
|
13359
|
+
TODO_USE(rootDirname); // <- TODO: [😡]
|
|
13034
13360
|
cacheFilename = path.join.apply(void 0, __spreadArray(__spreadArray([process.cwd(),
|
|
13035
13361
|
cacheDirname], __read(nameToSubfolderPath(hash /* <- TODO: [🎎] Maybe add some SHA256 prefix */)), false), [name], false)).split('\\')
|
|
13036
13362
|
.join('/') +
|
|
@@ -13050,7 +13376,7 @@
|
|
|
13050
13376
|
return __generator(this, function (_a) {
|
|
13051
13377
|
switch (_a.label) {
|
|
13052
13378
|
case 0:
|
|
13053
|
-
if (!
|
|
13379
|
+
if (!(intermediateFilesStrategy === 'HIDE_AND_CLEAN')) return [3 /*break*/, 2];
|
|
13054
13380
|
if (isVerbose) {
|
|
13055
13381
|
console.info('legacyDocumentScraper: Clening cache');
|
|
13056
13382
|
}
|
|
@@ -13074,7 +13400,7 @@
|
|
|
13074
13400
|
/**
|
|
13075
13401
|
* Note: Not using `FileCacheStorage` for two reasons:
|
|
13076
13402
|
* 1) Need to store more than serialized JSONs
|
|
13077
|
-
* 2) Need to switch between a `rootDirname` and `cacheDirname` <- TODO:
|
|
13403
|
+
* 2) Need to switch between a `rootDirname` and `cacheDirname` <- TODO: [😡]
|
|
13078
13404
|
* TODO: [🐱🐉][🧠] Make some smart crop
|
|
13079
13405
|
* Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
|
|
13080
13406
|
*/
|
|
@@ -13091,7 +13417,7 @@
|
|
|
13091
13417
|
mimeTypes: ['text/markdown', 'text/plain'],
|
|
13092
13418
|
documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
|
|
13093
13419
|
isAvilableInBrowser: true,
|
|
13094
|
-
requiredExecutables: [
|
|
13420
|
+
requiredExecutables: [],
|
|
13095
13421
|
}); /* <- TODO: [🤛] */
|
|
13096
13422
|
/**
|
|
13097
13423
|
* Registration of known scraper metadata
|
|
@@ -13135,7 +13461,7 @@
|
|
|
13135
13461
|
return __generator(this, function (_k) {
|
|
13136
13462
|
switch (_k.label) {
|
|
13137
13463
|
case 0:
|
|
13138
|
-
_a = this.options, _b = _a.maxParallelCount, maxParallelCount = _b === void 0 ?
|
|
13464
|
+
_a = this.options, _b = _a.maxParallelCount, maxParallelCount = _b === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? DEFAULT_IS_VERBOSE : _c;
|
|
13139
13465
|
llm = this.tools.llm;
|
|
13140
13466
|
if (llm === undefined) {
|
|
13141
13467
|
throw new MissingToolsError('LLM tools are required for scraping external files');
|
|
@@ -13234,7 +13560,8 @@
|
|
|
13234
13560
|
embeddingResult = _c.sent();
|
|
13235
13561
|
index.push({
|
|
13236
13562
|
modelName: embeddingResult.modelName,
|
|
13237
|
-
position: embeddingResult.content,
|
|
13563
|
+
position: __spreadArray([], __read(embeddingResult.content), false),
|
|
13564
|
+
// <- TODO: [🪓] Here should be no need for spreading new array, just `position: embeddingResult.content`
|
|
13238
13565
|
});
|
|
13239
13566
|
_c.label = 6;
|
|
13240
13567
|
case 6: return [3 /*break*/, 8];
|
|
@@ -13285,7 +13612,7 @@
|
|
|
13285
13612
|
mimeTypes: ['application/vnd.openxmlformats-officedocument.wordprocessingml.document'],
|
|
13286
13613
|
documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
|
|
13287
13614
|
isAvilableInBrowser: false,
|
|
13288
|
-
requiredExecutables: ['
|
|
13615
|
+
requiredExecutables: ['Pandoc'],
|
|
13289
13616
|
}); /* <- TODO: [🤛] */
|
|
13290
13617
|
/**
|
|
13291
13618
|
* Registration of known scraper metadata
|
|
@@ -13325,12 +13652,13 @@
|
|
|
13325
13652
|
* Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object
|
|
13326
13653
|
*/
|
|
13327
13654
|
DocumentScraper.prototype.$convert = function (source) {
|
|
13655
|
+
var _a;
|
|
13328
13656
|
return __awaiter(this, void 0, void 0, function () {
|
|
13329
|
-
var
|
|
13657
|
+
var _b, _c, rootDirname, _d, cacheDirname, _e, intermediateFilesStrategy, _f, isVerbose, extension, cacheFilehandler, command_1;
|
|
13330
13658
|
return __generator(this, function (_g) {
|
|
13331
13659
|
switch (_g.label) {
|
|
13332
13660
|
case 0:
|
|
13333
|
-
|
|
13661
|
+
_b = this.options, _c = _b.rootDirname, rootDirname = _c === void 0 ? process.cwd() : _c, _d = _b.cacheDirname, cacheDirname = _d === void 0 ? DEFAULT_SCRAPE_CACHE_DIRNAME : _d, _e = _b.intermediateFilesStrategy, intermediateFilesStrategy = _e === void 0 ? DEFAULT_INTERMEDIATE_FILES_STRATEGY : _e, _f = _b.isVerbose, isVerbose = _f === void 0 ? DEFAULT_IS_VERBOSE : _f;
|
|
13334
13662
|
if (!$isRunningInNode()) {
|
|
13335
13663
|
throw new KnowledgeScrapeError('Scraping .docx files is only supported in Node environment');
|
|
13336
13664
|
}
|
|
@@ -13338,7 +13666,7 @@
|
|
|
13338
13666
|
throw new EnvironmentMismatchError('Can not scrape documents without filesystem tools');
|
|
13339
13667
|
// <- TODO: [🧠] What is the best error type here`
|
|
13340
13668
|
}
|
|
13341
|
-
if (
|
|
13669
|
+
if (((_a = this.tools.executables) === null || _a === void 0 ? void 0 : _a.pandocPath) === undefined) {
|
|
13342
13670
|
throw new MissingToolsError('Pandoc is required for scraping .docx files');
|
|
13343
13671
|
}
|
|
13344
13672
|
if (source.filename === null) {
|
|
@@ -13349,7 +13677,7 @@
|
|
|
13349
13677
|
return [4 /*yield*/, getScraperIntermediateSource(source, {
|
|
13350
13678
|
rootDirname: rootDirname,
|
|
13351
13679
|
cacheDirname: cacheDirname,
|
|
13352
|
-
|
|
13680
|
+
intermediateFilesStrategy: intermediateFilesStrategy,
|
|
13353
13681
|
extension: 'md',
|
|
13354
13682
|
isVerbose: isVerbose,
|
|
13355
13683
|
})];
|
|
@@ -13358,11 +13686,9 @@
|
|
|
13358
13686
|
return [4 /*yield*/, isFileExisting(cacheFilehandler.filename, this.tools.fs)];
|
|
13359
13687
|
case 2:
|
|
13360
13688
|
if (!!(_g.sent())) return [3 /*break*/, 5];
|
|
13361
|
-
command_1 = "\"".concat(
|
|
13362
|
-
// TODO: !!!!!! [🕊] Make execCommand standard (?node-)util of the promptbook
|
|
13689
|
+
command_1 = "\"".concat(this.tools.executables.pandocPath, "\" -f ").concat(extension, " -t markdown \"").concat(source.filename, "\" -o \"").concat(cacheFilehandler.filename, "\"");
|
|
13363
13690
|
return [4 /*yield*/, $execCommand(command_1)];
|
|
13364
13691
|
case 3:
|
|
13365
|
-
// TODO: !!!!!! [🕊] Make execCommand standard (?node-)util of the promptbook
|
|
13366
13692
|
_g.sent();
|
|
13367
13693
|
return [4 /*yield*/, isFileExisting(cacheFilehandler.filename, this.tools.fs)];
|
|
13368
13694
|
case 4:
|
|
@@ -13407,9 +13733,14 @@
|
|
|
13407
13733
|
asJson: function () {
|
|
13408
13734
|
throw new UnexpectedError('Did not expect that `markdownScraper` would need to get the content `asJson`');
|
|
13409
13735
|
},
|
|
13410
|
-
|
|
13411
|
-
|
|
13412
|
-
|
|
13736
|
+
/*
|
|
13737
|
+
TODO: [🥽]
|
|
13738
|
+
> asBlob() {
|
|
13739
|
+
> throw new UnexpectedError(
|
|
13740
|
+
> 'Did not expect that `markdownScraper` would need to get the content `asBlob`',
|
|
13741
|
+
> );
|
|
13742
|
+
> },
|
|
13743
|
+
*/
|
|
13413
13744
|
};
|
|
13414
13745
|
knowledge = this.markdownScraper.scrape(markdownSource);
|
|
13415
13746
|
return [4 /*yield*/, cacheFilehandler.destroy()];
|
|
@@ -13441,7 +13772,11 @@
|
|
|
13441
13772
|
mimeTypes: ['application/msword', 'text/rtf'],
|
|
13442
13773
|
documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
|
|
13443
13774
|
isAvilableInBrowser: false,
|
|
13444
|
-
requiredExecutables: [
|
|
13775
|
+
requiredExecutables: [
|
|
13776
|
+
'Pandoc',
|
|
13777
|
+
'LibreOffice',
|
|
13778
|
+
// <- TODO: [🧠] Should be 'LibreOffice' here, its dependency of dependency
|
|
13779
|
+
],
|
|
13445
13780
|
}); /* <- TODO: [🤛] */
|
|
13446
13781
|
/**
|
|
13447
13782
|
* Registration of known scraper metadata
|
|
@@ -13454,7 +13789,7 @@
|
|
|
13454
13789
|
var _LegacyDocumentScraperMetadataRegistration = $scrapersMetadataRegister.register(legacyDocumentScraperMetadata);
|
|
13455
13790
|
|
|
13456
13791
|
/**
|
|
13457
|
-
* Scraper for .
|
|
13792
|
+
* Scraper for old document files (like .doc and .rtf)
|
|
13458
13793
|
*
|
|
13459
13794
|
* @see `documentationUrl` for more details
|
|
13460
13795
|
* @public exported from `@promptbook/legacy-documents`
|
|
@@ -13481,12 +13816,13 @@
|
|
|
13481
13816
|
* Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object
|
|
13482
13817
|
*/
|
|
13483
13818
|
LegacyDocumentScraper.prototype.$convert = function (source) {
|
|
13819
|
+
var _a;
|
|
13484
13820
|
return __awaiter(this, void 0, void 0, function () {
|
|
13485
|
-
var
|
|
13821
|
+
var _b, _c, rootDirname, _d, cacheDirname, _e, intermediateFilesStrategy, _f, isVerbose, extension, cacheFilehandler, documentSourceOutdirPathForLibreOffice_1, command_1, files_1, file;
|
|
13486
13822
|
return __generator(this, function (_g) {
|
|
13487
13823
|
switch (_g.label) {
|
|
13488
13824
|
case 0:
|
|
13489
|
-
|
|
13825
|
+
_b = this.options, _c = _b.rootDirname, rootDirname = _c === void 0 ? process.cwd() : _c, _d = _b.cacheDirname, cacheDirname = _d === void 0 ? DEFAULT_SCRAPE_CACHE_DIRNAME : _d, _e = _b.intermediateFilesStrategy, intermediateFilesStrategy = _e === void 0 ? DEFAULT_INTERMEDIATE_FILES_STRATEGY : _e, _f = _b.isVerbose, isVerbose = _f === void 0 ? DEFAULT_IS_VERBOSE : _f;
|
|
13490
13826
|
if (!$isRunningInNode()) {
|
|
13491
13827
|
throw new KnowledgeScrapeError('Scraping .doc files is only supported in Node environment');
|
|
13492
13828
|
}
|
|
@@ -13494,7 +13830,7 @@
|
|
|
13494
13830
|
throw new EnvironmentMismatchError('Can not scrape (legacy) documents without filesystem tools');
|
|
13495
13831
|
// <- TODO: [🧠] What is the best error type here`
|
|
13496
13832
|
}
|
|
13497
|
-
if (
|
|
13833
|
+
if (((_a = this.tools.executables) === null || _a === void 0 ? void 0 : _a.libreOfficePath) === undefined) {
|
|
13498
13834
|
throw new MissingToolsError('LibreOffice is required for scraping .doc and .rtf files');
|
|
13499
13835
|
}
|
|
13500
13836
|
if (source.filename === null) {
|
|
@@ -13505,7 +13841,7 @@
|
|
|
13505
13841
|
return [4 /*yield*/, getScraperIntermediateSource(source, {
|
|
13506
13842
|
rootDirname: rootDirname,
|
|
13507
13843
|
cacheDirname: cacheDirname,
|
|
13508
|
-
|
|
13844
|
+
intermediateFilesStrategy: intermediateFilesStrategy,
|
|
13509
13845
|
extension: 'docx',
|
|
13510
13846
|
isVerbose: isVerbose,
|
|
13511
13847
|
})];
|
|
@@ -13520,11 +13856,9 @@
|
|
|
13520
13856
|
documentSourceOutdirPathForLibreOffice_1 = path.join(path.dirname(cacheFilehandler.filename), 'libreoffice')
|
|
13521
13857
|
.split('\\')
|
|
13522
13858
|
.join('/');
|
|
13523
|
-
command_1 = "\"".concat(
|
|
13524
|
-
// TODO: !!!!!! [🕊] Make execCommand standard (?node-)util of the promptbook - this should trigger build polution error
|
|
13859
|
+
command_1 = "\"".concat(this.tools.executables.libreOfficePath, "\" --headless --convert-to docx \"").concat(source.filename, "\" --outdir \"").concat(documentSourceOutdirPathForLibreOffice_1, "\"");
|
|
13525
13860
|
return [4 /*yield*/, $execCommand(command_1)];
|
|
13526
13861
|
case 3:
|
|
13527
|
-
// TODO: !!!!!! [🕊] Make execCommand standard (?node-)util of the promptbook - this should trigger build polution error
|
|
13528
13862
|
_g.sent();
|
|
13529
13863
|
return [4 /*yield*/, promises.readdir(documentSourceOutdirPathForLibreOffice_1)];
|
|
13530
13864
|
case 4:
|
|
@@ -13572,9 +13906,14 @@
|
|
|
13572
13906
|
asJson: function () {
|
|
13573
13907
|
throw new UnexpectedError('Did not expect that `documentScraper` would need to get the content `asJson`');
|
|
13574
13908
|
},
|
|
13575
|
-
|
|
13576
|
-
|
|
13577
|
-
|
|
13909
|
+
/*
|
|
13910
|
+
TODO: [🥽]
|
|
13911
|
+
> asBlob() {
|
|
13912
|
+
> throw new UnexpectedError(
|
|
13913
|
+
> 'Did not expect that `documentScraper` would need to get the content `asBlob`',
|
|
13914
|
+
> );
|
|
13915
|
+
> },
|
|
13916
|
+
*/
|
|
13578
13917
|
};
|
|
13579
13918
|
knowledge = this.documentScraper.scrape(markdownSource);
|
|
13580
13919
|
return [4 /*yield*/, cacheFilehandler.destroy()];
|
|
@@ -13681,7 +14020,7 @@
|
|
|
13681
14020
|
mimeTypes: ['application/pdf'],
|
|
13682
14021
|
documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
|
|
13683
14022
|
isAvilableInBrowser: true,
|
|
13684
|
-
requiredExecutables: [
|
|
14023
|
+
requiredExecutables: [],
|
|
13685
14024
|
}); /* <- TODO: [🤛] */
|
|
13686
14025
|
/**
|
|
13687
14026
|
* Registration of known scraper metadata
|
|
@@ -13694,7 +14033,7 @@
|
|
|
13694
14033
|
var _PdfScraperMetadataRegistration = $scrapersMetadataRegister.register(pdfScraperMetadata);
|
|
13695
14034
|
|
|
13696
14035
|
/**
|
|
13697
|
-
* Scraper for .
|
|
14036
|
+
* Scraper for .pdf files
|
|
13698
14037
|
*
|
|
13699
14038
|
* @see `documentationUrl` for more details
|
|
13700
14039
|
* @public exported from `@promptbook/pdf`
|
|
@@ -13737,9 +14076,8 @@
|
|
|
13737
14076
|
TODO_USE(this.options);
|
|
13738
14077
|
/*
|
|
13739
14078
|
const {
|
|
13740
|
-
externalProgramsPaths = {},
|
|
13741
14079
|
cacheDirname = SCRAPE_CACHE_DIRNAME,
|
|
13742
|
-
|
|
14080
|
+
intermediateFilesStrategy = DEFAULT_INTERMEDIATE_FILES_STRATEGY,
|
|
13743
14081
|
isVerbose = IS_VERBOSE,
|
|
13744
14082
|
} = options;
|
|
13745
14083
|
*/
|
|
@@ -13792,7 +14130,7 @@
|
|
|
13792
14130
|
mimeTypes: ['text/html'],
|
|
13793
14131
|
documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
|
|
13794
14132
|
isAvilableInBrowser: false,
|
|
13795
|
-
requiredExecutables: [
|
|
14133
|
+
requiredExecutables: [],
|
|
13796
14134
|
}); /* <- TODO: [🤛] */
|
|
13797
14135
|
/**
|
|
13798
14136
|
* Registration of known scraper metadata
|
|
@@ -13805,32 +14143,29 @@
|
|
|
13805
14143
|
var _WebsiteScraperMetadataRegistration = $scrapersMetadataRegister.register(websiteScraperMetadata);
|
|
13806
14144
|
|
|
13807
14145
|
/**
|
|
13808
|
-
*
|
|
14146
|
+
* Create a new showdown converter instance
|
|
13809
14147
|
*
|
|
13810
|
-
* @
|
|
13811
|
-
* @private for markdown and html knowledge scrapers
|
|
13812
|
-
*/
|
|
13813
|
-
var markdownConverter = new showdown.Converter({
|
|
13814
|
-
flavor: 'github', // <- TODO: !!!!!! Explicitly specify the flavor of promptbook markdown
|
|
13815
|
-
/*
|
|
13816
|
-
> import showdownHighlight from 'showdown-highlight';
|
|
13817
|
-
> extensions: [
|
|
13818
|
-
> showdownHighlight({
|
|
13819
|
-
> // Whether to add the classes to the <pre> tag, default is false
|
|
13820
|
-
> pre: true,
|
|
13821
|
-
> // Whether to use hljs' auto language detection, default is true
|
|
13822
|
-
> auto_detection: true,
|
|
13823
|
-
> }),
|
|
13824
|
-
> ],
|
|
13825
|
-
*/
|
|
13826
|
-
});
|
|
13827
|
-
/**
|
|
13828
|
-
* TODO: !!!!!! Figure out better name not to confuse with `Converter`
|
|
13829
|
-
* TODO: !!!!!! Lazy-make converter
|
|
14148
|
+
* @private utility of `WebsiteScraper`
|
|
13830
14149
|
*/
|
|
14150
|
+
function createShowdownConverter() {
|
|
14151
|
+
return new showdown.Converter({
|
|
14152
|
+
flavor: 'github',
|
|
14153
|
+
/*
|
|
14154
|
+
> import showdownHighlight from 'showdown-highlight';
|
|
14155
|
+
> extensions: [
|
|
14156
|
+
> showdownHighlight({
|
|
14157
|
+
> // Whether to add the classes to the <pre> tag, default is false
|
|
14158
|
+
> pre: true,
|
|
14159
|
+
> // Whether to use hljs' auto language detection, default is true
|
|
14160
|
+
> auto_detection: true,
|
|
14161
|
+
> }),
|
|
14162
|
+
> ],
|
|
14163
|
+
*/
|
|
14164
|
+
});
|
|
14165
|
+
}
|
|
13831
14166
|
|
|
13832
14167
|
/**
|
|
13833
|
-
* Scraper for
|
|
14168
|
+
* Scraper for websites
|
|
13834
14169
|
*
|
|
13835
14170
|
* @see `documentationUrl` for more details
|
|
13836
14171
|
* @public exported from `@promptbook/website-crawler`
|
|
@@ -13840,6 +14175,7 @@
|
|
|
13840
14175
|
this.tools = tools;
|
|
13841
14176
|
this.options = options;
|
|
13842
14177
|
this.markdownScraper = new MarkdownScraper(tools, options);
|
|
14178
|
+
this.showdownConverter = createShowdownConverter();
|
|
13843
14179
|
}
|
|
13844
14180
|
Object.defineProperty(WebsiteScraper.prototype, "metadata", {
|
|
13845
14181
|
/**
|
|
@@ -13860,16 +14196,17 @@
|
|
|
13860
14196
|
return __awaiter(this, void 0, void 0, function () {
|
|
13861
14197
|
var _a, _b,
|
|
13862
14198
|
// TODO: [🧠] Maybe in node use headless browser not just JSDOM
|
|
13863
|
-
|
|
13864
|
-
rootDirname, _c, cacheDirname, _d, isCacheCleaned, _e, isVerbose, jsdom$1, _f, reader, article, html, i, cacheFilehandler, markdown;
|
|
14199
|
+
rootDirname, _c, cacheDirname, _d, intermediateFilesStrategy, _e, isVerbose, jsdom$1, _f, reader, article, html, i, cacheFilehandler, markdown;
|
|
13865
14200
|
return __generator(this, function (_g) {
|
|
13866
14201
|
switch (_g.label) {
|
|
13867
14202
|
case 0:
|
|
13868
|
-
_a = this.options, _b = _a.rootDirname, rootDirname = _b === void 0 ? process.cwd() : _b, _c = _a.cacheDirname, cacheDirname = _c === void 0 ?
|
|
13869
|
-
// TODO: !!!!!! Does this work in browser? Make it work.
|
|
14203
|
+
_a = this.options, _b = _a.rootDirname, rootDirname = _b === void 0 ? process.cwd() : _b, _c = _a.cacheDirname, cacheDirname = _c === void 0 ? DEFAULT_SCRAPE_CACHE_DIRNAME : _c, _d = _a.intermediateFilesStrategy, intermediateFilesStrategy = _d === void 0 ? DEFAULT_INTERMEDIATE_FILES_STRATEGY : _d, _e = _a.isVerbose, isVerbose = _e === void 0 ? DEFAULT_IS_VERBOSE : _e;
|
|
13870
14204
|
if (source.url === null) {
|
|
13871
14205
|
throw new KnowledgeScrapeError('Website scraper requires URL');
|
|
13872
14206
|
}
|
|
14207
|
+
if (this.tools.fs === undefined) {
|
|
14208
|
+
throw new EnvironmentMismatchError('Can not scrape websites without filesystem tools');
|
|
14209
|
+
}
|
|
13873
14210
|
_f = jsdom.JSDOM.bind;
|
|
13874
14211
|
return [4 /*yield*/, source.asText()];
|
|
13875
14212
|
case 1:
|
|
@@ -13878,10 +14215,6 @@
|
|
|
13878
14215
|
}]))();
|
|
13879
14216
|
reader = new readability.Readability(jsdom$1.window.document);
|
|
13880
14217
|
article = reader.parse();
|
|
13881
|
-
console.log(article);
|
|
13882
|
-
return [4 /*yield*/, waitasecond.forTime(10000)];
|
|
13883
|
-
case 2:
|
|
13884
|
-
_g.sent();
|
|
13885
14218
|
html = (article === null || article === void 0 ? void 0 : article.content) || (article === null || article === void 0 ? void 0 : article.textContent) || jsdom$1.window.document.body.innerHTML;
|
|
13886
14219
|
// Note: Unwrap html such as it is convertable by `markdownConverter`
|
|
13887
14220
|
for (i = 0; i < 2; i++) {
|
|
@@ -13893,16 +14226,16 @@
|
|
|
13893
14226
|
return [4 /*yield*/, getScraperIntermediateSource(source, {
|
|
13894
14227
|
rootDirname: rootDirname,
|
|
13895
14228
|
cacheDirname: cacheDirname,
|
|
13896
|
-
|
|
14229
|
+
intermediateFilesStrategy: intermediateFilesStrategy,
|
|
13897
14230
|
extension: 'html',
|
|
13898
14231
|
isVerbose: isVerbose,
|
|
13899
14232
|
})];
|
|
13900
|
-
case
|
|
14233
|
+
case 2:
|
|
13901
14234
|
cacheFilehandler = _g.sent();
|
|
13902
|
-
return [4 /*yield*/,
|
|
13903
|
-
case
|
|
14235
|
+
return [4 /*yield*/, this.tools.fs.writeFile(cacheFilehandler.filename, html, 'utf-8')];
|
|
14236
|
+
case 3:
|
|
13904
14237
|
_g.sent();
|
|
13905
|
-
markdown =
|
|
14238
|
+
markdown = this.showdownConverter.makeMarkdown(html, jsdom$1.window.document);
|
|
13906
14239
|
return [2 /*return*/, __assign(__assign({}, cacheFilehandler), { markdown: markdown })];
|
|
13907
14240
|
}
|
|
13908
14241
|
});
|
|
@@ -13930,9 +14263,14 @@
|
|
|
13930
14263
|
asJson: function () {
|
|
13931
14264
|
throw new UnexpectedError('Did not expect that `markdownScraper` would need to get the content `asJson`');
|
|
13932
14265
|
},
|
|
13933
|
-
|
|
13934
|
-
|
|
13935
|
-
|
|
14266
|
+
/*
|
|
14267
|
+
TODO: [🥽]
|
|
14268
|
+
> asBlob() {
|
|
14269
|
+
> throw new UnexpectedError(
|
|
14270
|
+
> 'Did not expect that `markdownScraper` would need to get the content `asBlob`',
|
|
14271
|
+
> );
|
|
14272
|
+
> },
|
|
14273
|
+
*/
|
|
13936
14274
|
};
|
|
13937
14275
|
knowledge = this.markdownScraper.scrape(markdownSource);
|
|
13938
14276
|
return [4 /*yield*/, cacheFilehandler.destroy()];
|
|
@@ -13946,7 +14284,6 @@
|
|
|
13946
14284
|
return WebsiteScraper;
|
|
13947
14285
|
}());
|
|
13948
14286
|
/**
|
|
13949
|
-
* TODO: !!!!!! Put into separate package
|
|
13950
14287
|
* TODO: [👣] Scraped website in .md can act as cache item - there is no need to run conversion each time
|
|
13951
14288
|
* TODO: [🪂] Do it in parallel 11:11
|
|
13952
14289
|
* Note: No need to aggregate usage here, it is done by intercepting the llmTools
|