@promptbook/cli 0.72.0-9 → 0.72.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -8
- package/esm/index.es.js +987 -649
- package/esm/index.es.js.map +1 -1
- package/esm/typings/src/_packages/core.index.d.ts +24 -18
- package/esm/typings/src/_packages/node.index.d.ts +2 -0
- package/esm/typings/src/_packages/types.index.d.ts +18 -12
- package/esm/typings/src/_packages/utils.index.d.ts +2 -2
- package/esm/typings/src/collection/PipelineCollection.d.ts +1 -1
- package/esm/typings/src/collection/SimplePipelineCollection.d.ts +2 -2
- package/esm/typings/src/collection/collectionToJson.d.ts +1 -1
- package/esm/typings/src/collection/constructors/createCollectionFromJson.d.ts +1 -1
- package/esm/typings/src/collection/constructors/createCollectionFromPromise.d.ts +1 -1
- package/esm/typings/src/commands/_common/types/CommandParser.d.ts +5 -5
- package/esm/typings/src/config.d.ts +21 -14
- package/esm/typings/src/executables/$provideExecutablesForNode.d.ts +12 -0
- package/esm/typings/src/executables/apps/locateLibreoffice.d.ts +11 -0
- package/esm/typings/src/executables/apps/locatePandoc.d.ts +11 -0
- package/esm/typings/src/executables/locateApp.d.ts +33 -0
- package/esm/typings/src/executables/locateApp.test.d.ts +1 -0
- package/esm/typings/src/executables/platforms/locateAppOnLinux.d.ts +12 -0
- package/esm/typings/src/executables/platforms/locateAppOnMacOs.d.ts +12 -0
- package/esm/typings/src/executables/platforms/locateAppOnWindows.d.ts +12 -0
- package/esm/typings/src/execution/EmbeddingVector.d.ts +1 -1
- package/esm/typings/src/execution/Executables.d.ts +18 -0
- package/esm/typings/src/execution/ExecutionTools.d.ts +9 -3
- package/esm/typings/src/execution/FilesystemTools.d.ts +1 -1
- package/esm/typings/src/execution/LlmExecutionTools.d.ts +2 -2
- package/esm/typings/src/execution/PipelineExecutorResult.d.ts +2 -2
- package/esm/typings/src/execution/assertsExecutionSuccessful.d.ts +3 -2
- package/esm/typings/src/execution/createPipelineExecutor/00-CreatePipelineExecutorOptions.d.ts +29 -6
- package/esm/typings/src/execution/createPipelineExecutor/10-executePipeline.d.ts +2 -11
- package/esm/typings/src/execution/createPipelineExecutor/20-executeTemplate.d.ts +4 -13
- package/esm/typings/src/execution/createPipelineExecutor/40-executeAttempts.d.ts +9 -14
- package/esm/typings/src/execution/translation/automatic-translate/automatic-translators/LindatAutomaticTranslator.d.ts +11 -3
- package/esm/typings/src/execution/utils/addUsage.d.ts +1 -1
- package/esm/typings/src/execution/utils/forEachAsync.d.ts +1 -1
- package/esm/typings/src/formats/_common/FormatDefinition.d.ts +2 -2
- package/esm/typings/src/formats/_common/FormatSubvalueDefinition.d.ts +2 -2
- package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsForCli.d.ts +2 -2
- package/esm/typings/src/llm-providers/_common/register/LlmToolsConfiguration.d.ts +1 -1
- package/esm/typings/src/llm-providers/_common/register/createLlmToolsFromConfiguration.d.ts +7 -0
- package/esm/typings/src/llm-providers/_common/utils/cache/CacheLlmToolsOptions.d.ts +4 -1
- package/esm/typings/src/llm-providers/anthropic-claude/AnthropicClaudeExecutionTools.d.ts +1 -1
- package/esm/typings/src/llm-providers/anthropic-claude/AnthropicClaudeExecutionToolsOptions.d.ts +2 -2
- package/esm/typings/src/llm-providers/anthropic-claude/anthropic-claude-models.d.ts +1 -1
- package/esm/typings/src/llm-providers/azure-openai/AzureOpenAiExecutionTools.d.ts +1 -1
- package/esm/typings/src/llm-providers/azure-openai/AzureOpenAiExecutionToolsOptions.d.ts +2 -1
- package/esm/typings/src/llm-providers/langtail/LangtailExecutionTools.d.ts +1 -1
- package/esm/typings/src/llm-providers/mocked/$fakeTextToExpectations.d.ts +1 -1
- package/esm/typings/src/llm-providers/mocked/MockedEchoLlmExecutionTools.d.ts +1 -1
- package/esm/typings/src/llm-providers/mocked/MockedFackedLlmExecutionTools.d.ts +1 -1
- package/esm/typings/src/llm-providers/multiple/MultipleLlmExecutionTools.d.ts +4 -5
- package/esm/typings/src/llm-providers/multiple/joinLlmExecutionTools.d.ts +1 -1
- package/esm/typings/src/llm-providers/openai/OpenAiAssistantExecutionTools.d.ts +0 -1
- package/esm/typings/src/llm-providers/openai/OpenAiExecutionTools.d.ts +2 -2
- package/esm/typings/src/llm-providers/openai/OpenAiExecutionToolsOptions.d.ts +2 -2
- package/esm/typings/src/llm-providers/openai/openai-models.d.ts +1 -1
- package/esm/typings/src/llm-providers/remote/RemoteLlmExecutionTools.d.ts +4 -4
- package/esm/typings/src/llm-providers/remote/interfaces/PromptbookServer_ListModels_Request.d.ts +15 -6
- package/esm/typings/src/llm-providers/remote/interfaces/PromptbookServer_ListModels_Response.d.ts +4 -4
- package/esm/typings/src/llm-providers/remote/interfaces/PromptbookServer_Prompt_Request.d.ts +6 -12
- package/esm/typings/src/llm-providers/remote/interfaces/PromptbookServer_Prompt_Response.d.ts +3 -3
- package/esm/typings/src/llm-providers/remote/interfaces/RemoteLlmExecutionToolsOptions.d.ts +9 -14
- package/esm/typings/src/llm-providers/remote/interfaces/RemoteServerOptions.d.ts +23 -8
- package/esm/typings/src/llm-providers/remote/startRemoteServer.d.ts +1 -1
- package/esm/typings/src/prepare/PrepareAndScrapeOptions.d.ts +4 -23
- package/esm/typings/src/prepare/prepareTemplates.d.ts +1 -1
- package/esm/typings/src/scrapers/_common/Scraper.d.ts +2 -9
- package/esm/typings/src/scrapers/_common/prepareKnowledgePieces.d.ts +1 -1
- package/esm/typings/src/scrapers/_common/register/$provideScrapersForBrowser.d.ts +2 -2
- package/esm/typings/src/scrapers/_common/register/$provideScrapersForNode.d.ts +2 -2
- package/esm/typings/src/scrapers/_common/register/ScraperAndConverterMetadata.d.ts +2 -2
- package/esm/typings/src/scrapers/_common/utils/getScraperIntermediateSource.d.ts +2 -2
- package/esm/typings/src/scrapers/_common/utils/makeKnowledgeSourceHandler.d.ts +1 -1
- package/esm/typings/src/scrapers/document/DocumentScraper.d.ts +2 -2
- package/esm/typings/src/scrapers/document/createDocumentScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/document/register-metadata.d.ts +1 -1
- package/esm/typings/src/scrapers/document-legacy/LegacyDocumentScraper.d.ts +3 -3
- package/esm/typings/src/scrapers/document-legacy/createLegacyDocumentScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/document-legacy/register-metadata.d.ts +1 -1
- package/esm/typings/src/scrapers/markdown/MarkdownScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/markdown/createMarkdownScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/markdown/register-metadata.d.ts +1 -1
- package/esm/typings/src/scrapers/pdf/PdfScraper.d.ts +2 -2
- package/esm/typings/src/scrapers/pdf/createPdfScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/pdf/register-metadata.d.ts +1 -1
- package/esm/typings/src/scrapers/website/WebsiteScraper.d.ts +6 -3
- package/esm/typings/src/scrapers/website/createWebsiteScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/website/register-metadata.d.ts +1 -1
- package/esm/typings/src/scrapers/website/utils/createShowdownConverter.d.ts +7 -0
- package/esm/typings/src/scrapers/website/utils/createShowdownConverter.test.d.ts +1 -0
- package/esm/typings/src/scripting/javascript/utils/preserve.d.ts +1 -1
- package/esm/typings/src/storage/file-cache-storage/utils/nameToSubfolderPath.d.ts +1 -1
- package/esm/typings/src/types/Arrayable.d.ts +1 -1
- package/esm/typings/src/types/IntermediateFilesStrategy.d.ts +7 -0
- package/esm/typings/src/types/PipelineJson/KnowledgePieceJson.d.ts +4 -4
- package/esm/typings/src/types/PipelineJson/KnowledgeSourceJson.d.ts +1 -1
- package/esm/typings/src/types/PipelineJson/PersonaJson.d.ts +1 -1
- package/esm/typings/src/types/PipelineJson/TemplateJsonCommon.d.ts +2 -2
- package/esm/typings/src/types/Prompt.d.ts +2 -1
- package/esm/typings/src/types/execution-report/ExecutionReportJson.d.ts +1 -1
- package/esm/typings/src/types/typeAliases.d.ts +11 -8
- package/esm/typings/src/utils/$Register.d.ts +1 -1
- package/esm/typings/src/utils/FromtoItems.d.ts +1 -1
- package/esm/typings/src/utils/arrayableToArray.d.ts +1 -1
- package/esm/typings/src/utils/emojis.d.ts +1 -1
- package/esm/typings/src/utils/execCommand/$execCommand.d.ts +4 -2
- package/esm/typings/src/utils/execCommand/$execCommandNormalizeOptions.d.ts +12 -0
- package/esm/typings/src/utils/execCommand/$execCommands.d.ts +1 -0
- package/esm/typings/src/utils/execCommand/ExecCommandOptions.d.ts +45 -0
- package/esm/typings/src/utils/expectation-counters/countSentences.d.ts +1 -1
- package/esm/typings/src/utils/files/isExecutable.d.ts +11 -0
- package/esm/typings/src/utils/markdown/extractAllBlocksFromMarkdown.d.ts +1 -1
- package/esm/typings/src/utils/markdown/splitMarkdownIntoSections.d.ts +1 -1
- package/esm/typings/src/utils/normalization/IKeywords.d.ts +2 -2
- package/esm/typings/src/utils/normalization/parseKeywords.d.ts +2 -2
- package/esm/typings/src/utils/normalization/parseKeywordsFromString.d.ts +2 -2
- package/esm/typings/src/utils/normalization/searchKeywords.d.ts +2 -2
- package/esm/typings/src/utils/organization/TODO_USE.d.ts +1 -1
- package/esm/typings/src/utils/organization/keepUnused.d.ts +1 -1
- package/esm/typings/src/utils/random/$randomSeed.d.ts +1 -1
- package/esm/typings/src/utils/sets/intersection.d.ts +1 -1
- package/esm/typings/src/utils/sets/union.d.ts +1 -1
- package/esm/typings/src/utils/unwrapResult.d.ts +4 -4
- package/package.json +3 -2
- package/umd/index.umd.js +988 -651
- package/umd/index.umd.js.map +1 -1
- package/esm/typings/src/execution/createPipelineExecutor/00-CreatePipelineExecutorSettings.d.ts +0 -29
- package/esm/typings/src/scrapers/website/utils/markdownConverter.d.ts +0 -12
- package/esm/typings/src/utils/execCommand/IExecCommandOptions.d.ts +0 -23
- package/esm/typings/src/utils/execCommand/execCommandNormalizeOptions.d.ts +0 -10
- /package/esm/typings/src/{scrapers/website/WebsiteScraper.test.d.ts → executables/apps/locateLibreoffice.test.d.ts} +0 -0
- /package/esm/typings/src/{scrapers/website/utils/markdownConverter.test.d.ts → executables/apps/locatePandoc.test.d.ts} +0 -0
package/esm/index.es.js
CHANGED
|
@@ -2,7 +2,7 @@ import commander from 'commander';
|
|
|
2
2
|
import spaceTrim$1, { spaceTrim } from 'spacetrim';
|
|
3
3
|
import colors from 'colors';
|
|
4
4
|
import { forTime } from 'waitasecond';
|
|
5
|
-
import { stat, access, constants, readFile,
|
|
5
|
+
import { stat, access, constants, readFile, writeFile, readdir, mkdir, unlink, rm, rmdir, rename } from 'fs/promises';
|
|
6
6
|
import { join, basename, dirname } from 'path';
|
|
7
7
|
import { format } from 'prettier';
|
|
8
8
|
import parserHtml from 'prettier/parser-html';
|
|
@@ -10,6 +10,8 @@ import { unparse, parse } from 'papaparse';
|
|
|
10
10
|
import { SHA256 } from 'crypto-js';
|
|
11
11
|
import hexEncoder from 'crypto-js/enc-hex';
|
|
12
12
|
import { lookup } from 'mime-types';
|
|
13
|
+
import { exec as exec$2, spawn } from 'child_process';
|
|
14
|
+
import { promisify } from 'util';
|
|
13
15
|
import * as dotenv from 'dotenv';
|
|
14
16
|
import sha256 from 'crypto-js/sha256';
|
|
15
17
|
import glob from 'glob-promise';
|
|
@@ -17,7 +19,6 @@ import { io } from 'socket.io-client';
|
|
|
17
19
|
import Anthropic from '@anthropic-ai/sdk';
|
|
18
20
|
import { OpenAIClient, AzureKeyCredential } from '@azure/openai';
|
|
19
21
|
import OpenAI from 'openai';
|
|
20
|
-
import { spawn } from 'child_process';
|
|
21
22
|
import { Readability } from '@mozilla/readability';
|
|
22
23
|
import { JSDOM } from 'jsdom';
|
|
23
24
|
import { Converter } from 'showdown';
|
|
@@ -26,7 +27,7 @@ import { Converter } from 'showdown';
|
|
|
26
27
|
/**
|
|
27
28
|
* The version of the Promptbook library
|
|
28
29
|
*/
|
|
29
|
-
var PROMPTBOOK_VERSION = '0.72.0-
|
|
30
|
+
var PROMPTBOOK_VERSION = '0.72.0-34';
|
|
30
31
|
// TODO: [main] !!!! List here all the versions and annotate + put into script
|
|
31
32
|
|
|
32
33
|
/*! *****************************************************************************
|
|
@@ -399,7 +400,8 @@ var GENERATOR_WARNING = "\u26A0\uFE0F WARNING: This code has been generated so t
|
|
|
399
400
|
*
|
|
400
401
|
* @public exported from `@promptbook/core`
|
|
401
402
|
*/
|
|
402
|
-
var CLAIM = "
|
|
403
|
+
var CLAIM = "Build responsible, controlled and transparent applications on top of LLM models!";
|
|
404
|
+
// <- TODO: [🐊] Pick the best claim
|
|
403
405
|
/**
|
|
404
406
|
* Warning message for the generated sections and files files
|
|
405
407
|
*
|
|
@@ -419,6 +421,7 @@ var LOOP_LIMIT = 1000;
|
|
|
419
421
|
* @private within the repository - too low-level in comparison with other `MAX_...`
|
|
420
422
|
*/
|
|
421
423
|
var CONNECTION_TIMEOUT_MS = 7 * 1000;
|
|
424
|
+
// <- TODO: [⏳] Standartize timeouts, Make DEFAULT_TIMEOUT_MS as global constant
|
|
422
425
|
/**
|
|
423
426
|
* How many times to retry the connections
|
|
424
427
|
*
|
|
@@ -432,23 +435,30 @@ var CONNECTION_RETRIES_LIMIT = 5;
|
|
|
432
435
|
*/
|
|
433
436
|
var IMMEDIATE_TIME = 10;
|
|
434
437
|
/**
|
|
435
|
-
* The maximum
|
|
438
|
+
* The maximum length of the (generated) filename
|
|
436
439
|
*
|
|
437
440
|
* @public exported from `@promptbook/core`
|
|
438
441
|
*/
|
|
439
|
-
var
|
|
442
|
+
var MAX_FILENAME_LENGTH = 30;
|
|
440
443
|
/**
|
|
441
|
-
*
|
|
444
|
+
* Strategy for caching the intermediate results for knowledge sources
|
|
442
445
|
*
|
|
443
446
|
* @public exported from `@promptbook/core`
|
|
444
447
|
*/
|
|
445
|
-
var
|
|
448
|
+
var DEFAULT_INTERMEDIATE_FILES_STRATEGY = 'HIDE_AND_KEEP';
|
|
449
|
+
// <- TODO: [😡] Change to 'VISIBLE'
|
|
446
450
|
/**
|
|
447
|
-
* The maximum
|
|
451
|
+
* The maximum number of (LLM) tasks running in parallel
|
|
448
452
|
*
|
|
449
453
|
* @public exported from `@promptbook/core`
|
|
450
454
|
*/
|
|
451
|
-
var
|
|
455
|
+
var DEFAULT_MAX_PARALLEL_COUNT = 5; // <- TODO: [🤹♂️]
|
|
456
|
+
/**
|
|
457
|
+
* The maximum number of attempts to execute LLM task before giving up
|
|
458
|
+
*
|
|
459
|
+
* @public exported from `@promptbook/core`
|
|
460
|
+
*/
|
|
461
|
+
var DEFAULT_MAX_EXECUTION_ATTEMPTS = 3; // <- TODO: [🤹♂️]
|
|
452
462
|
/**
|
|
453
463
|
* Where to store the cache of executions for promptbook CLI
|
|
454
464
|
*
|
|
@@ -456,7 +466,7 @@ var MAX_FILENAME_LENGTH = 30;
|
|
|
456
466
|
*
|
|
457
467
|
* @public exported from `@promptbook/core`
|
|
458
468
|
*/
|
|
459
|
-
var
|
|
469
|
+
var DEFAULT_EXECUTIONS_CACHE_DIRNAME = '/.promptbook/executions-cache';
|
|
460
470
|
/**
|
|
461
471
|
* Where to store the scrape cache
|
|
462
472
|
*
|
|
@@ -464,13 +474,13 @@ var EXECUTIONS_CACHE_DIRNAME = '/.promptbook/executions-cache';
|
|
|
464
474
|
*
|
|
465
475
|
* @public exported from `@promptbook/core`
|
|
466
476
|
*/
|
|
467
|
-
var
|
|
477
|
+
var DEFAULT_SCRAPE_CACHE_DIRNAME = '/.promptbook/scrape-cache';
|
|
468
478
|
/**
|
|
469
479
|
* The name of the builded pipeline collection made by CLI `ptbk make` and for lookup in `createCollectionFromDirectory`
|
|
470
480
|
*
|
|
471
481
|
* @public exported from `@promptbook/core`
|
|
472
482
|
*/
|
|
473
|
-
var
|
|
483
|
+
var DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME = "index";
|
|
474
484
|
/**
|
|
475
485
|
* Nonce which is used for replacing things in strings
|
|
476
486
|
*
|
|
@@ -534,13 +544,13 @@ var DEFAULT_CSV_SETTINGS = Object.freeze({
|
|
|
534
544
|
*
|
|
535
545
|
* @public exported from `@promptbook/core`
|
|
536
546
|
*/
|
|
537
|
-
var
|
|
547
|
+
var DEFAULT_IS_VERBOSE = false;
|
|
538
548
|
/**
|
|
539
549
|
* @@@
|
|
540
550
|
*
|
|
541
551
|
* @public exported from `@promptbook/core`
|
|
542
552
|
*/
|
|
543
|
-
var
|
|
553
|
+
var DEFAULT_IS_AUTO_INSTALLED = false;
|
|
544
554
|
/**
|
|
545
555
|
* @@@
|
|
546
556
|
*
|
|
@@ -613,271 +623,6 @@ function initializeHelloCommand(program) {
|
|
|
613
623
|
* Note: [🟡] Code in this file should never be published outside of `@promptbook/cli`
|
|
614
624
|
*/
|
|
615
625
|
|
|
616
|
-
/**
|
|
617
|
-
* Just marks a place of place where should be something implemented
|
|
618
|
-
* No side effects.
|
|
619
|
-
*
|
|
620
|
-
* Note: It can be usefull suppressing eslint errors of unused variables
|
|
621
|
-
*
|
|
622
|
-
* @param value any values
|
|
623
|
-
* @returns void
|
|
624
|
-
* @private within the repository
|
|
625
|
-
*/
|
|
626
|
-
function TODO_USE() {
|
|
627
|
-
var value = [];
|
|
628
|
-
for (var _i = 0; _i < arguments.length; _i++) {
|
|
629
|
-
value[_i] = arguments[_i];
|
|
630
|
-
}
|
|
631
|
-
}
|
|
632
|
-
|
|
633
|
-
/**
|
|
634
|
-
* @@@
|
|
635
|
-
*
|
|
636
|
-
* @public exported from `@promptbook/node`
|
|
637
|
-
*/
|
|
638
|
-
function $provideFilesystemForNode(options) {
|
|
639
|
-
if (!$isRunningInNode()) {
|
|
640
|
-
throw new EnvironmentMismatchError('Function `$provideFilesystemForNode` works only in Node.js environment');
|
|
641
|
-
}
|
|
642
|
-
var _a = (options || {}).isVerbose, isVerbose = _a === void 0 ? IS_VERBOSE : _a;
|
|
643
|
-
TODO_USE(isVerbose);
|
|
644
|
-
return {
|
|
645
|
-
stat: stat,
|
|
646
|
-
access: access,
|
|
647
|
-
constants: constants,
|
|
648
|
-
readFile: readFile,
|
|
649
|
-
readdir: readdir,
|
|
650
|
-
};
|
|
651
|
-
}
|
|
652
|
-
/**
|
|
653
|
-
* Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
|
|
654
|
-
*/
|
|
655
|
-
|
|
656
|
-
/**
|
|
657
|
-
* This error type indicates that some part of the code is not implemented yet
|
|
658
|
-
*
|
|
659
|
-
* @public exported from `@promptbook/core`
|
|
660
|
-
*/
|
|
661
|
-
var NotYetImplementedError = /** @class */ (function (_super) {
|
|
662
|
-
__extends(NotYetImplementedError, _super);
|
|
663
|
-
function NotYetImplementedError(message) {
|
|
664
|
-
var _this = _super.call(this, spaceTrim(function (block) { return "\n ".concat(block(message), "\n\n Note: This feature is not implemented yet but it will be soon.\n\n If you want speed up the implementation or just read more, look here:\n https://github.com/webgptorg/promptbook\n\n Or contact us on me@pavolhejny.com\n\n "); })) || this;
|
|
665
|
-
_this.name = 'NotYetImplementedError';
|
|
666
|
-
Object.setPrototypeOf(_this, NotYetImplementedError.prototype);
|
|
667
|
-
return _this;
|
|
668
|
-
}
|
|
669
|
-
return NotYetImplementedError;
|
|
670
|
-
}(Error));
|
|
671
|
-
|
|
672
|
-
/**
|
|
673
|
-
* @@@
|
|
674
|
-
*
|
|
675
|
-
* Note: `$` is used to indicate that this function is not a pure function - it access global scope
|
|
676
|
-
*
|
|
677
|
-
* @private internal function of `$Register`
|
|
678
|
-
*/
|
|
679
|
-
function $getGlobalScope() {
|
|
680
|
-
return Function('return this')();
|
|
681
|
-
}
|
|
682
|
-
|
|
683
|
-
/**
|
|
684
|
-
* @@@
|
|
685
|
-
*
|
|
686
|
-
* @param text @@@
|
|
687
|
-
* @returns @@@
|
|
688
|
-
* @example 'HELLO_WORLD'
|
|
689
|
-
* @example 'I_LOVE_PROMPTBOOK'
|
|
690
|
-
* @public exported from `@promptbook/utils`
|
|
691
|
-
*/
|
|
692
|
-
function normalizeTo_SCREAMING_CASE(text) {
|
|
693
|
-
var e_1, _a;
|
|
694
|
-
var charType;
|
|
695
|
-
var lastCharType = 'OTHER';
|
|
696
|
-
var normalizedName = '';
|
|
697
|
-
try {
|
|
698
|
-
for (var text_1 = __values(text), text_1_1 = text_1.next(); !text_1_1.done; text_1_1 = text_1.next()) {
|
|
699
|
-
var char = text_1_1.value;
|
|
700
|
-
var normalizedChar = void 0;
|
|
701
|
-
if (/^[a-z]$/.test(char)) {
|
|
702
|
-
charType = 'LOWERCASE';
|
|
703
|
-
normalizedChar = char.toUpperCase();
|
|
704
|
-
}
|
|
705
|
-
else if (/^[A-Z]$/.test(char)) {
|
|
706
|
-
charType = 'UPPERCASE';
|
|
707
|
-
normalizedChar = char;
|
|
708
|
-
}
|
|
709
|
-
else if (/^[0-9]$/.test(char)) {
|
|
710
|
-
charType = 'NUMBER';
|
|
711
|
-
normalizedChar = char;
|
|
712
|
-
}
|
|
713
|
-
else {
|
|
714
|
-
charType = 'OTHER';
|
|
715
|
-
normalizedChar = '_';
|
|
716
|
-
}
|
|
717
|
-
if (charType !== lastCharType &&
|
|
718
|
-
!(lastCharType === 'UPPERCASE' && charType === 'LOWERCASE') &&
|
|
719
|
-
!(lastCharType === 'NUMBER') &&
|
|
720
|
-
!(charType === 'NUMBER')) {
|
|
721
|
-
normalizedName += '_';
|
|
722
|
-
}
|
|
723
|
-
normalizedName += normalizedChar;
|
|
724
|
-
lastCharType = charType;
|
|
725
|
-
}
|
|
726
|
-
}
|
|
727
|
-
catch (e_1_1) { e_1 = { error: e_1_1 }; }
|
|
728
|
-
finally {
|
|
729
|
-
try {
|
|
730
|
-
if (text_1_1 && !text_1_1.done && (_a = text_1.return)) _a.call(text_1);
|
|
731
|
-
}
|
|
732
|
-
finally { if (e_1) throw e_1.error; }
|
|
733
|
-
}
|
|
734
|
-
normalizedName = normalizedName.replace(/_+/g, '_');
|
|
735
|
-
normalizedName = normalizedName.replace(/_?\/_?/g, '/');
|
|
736
|
-
normalizedName = normalizedName.replace(/^_/, '');
|
|
737
|
-
normalizedName = normalizedName.replace(/_$/, '');
|
|
738
|
-
return normalizedName;
|
|
739
|
-
}
|
|
740
|
-
/**
|
|
741
|
-
* TODO: Tests
|
|
742
|
-
* > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: 'Moje tabule' })).toEqual('/VtG7sR9rRJqwNEdM2/Moje tabule');
|
|
743
|
-
* > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: 'ěščřžžýáíúů' })).toEqual('/VtG7sR9rRJqwNEdM2/escrzyaieuu');
|
|
744
|
-
* > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: ' ahoj ' })).toEqual('/VtG7sR9rRJqwNEdM2/ahoj');
|
|
745
|
-
* > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: ' ahoj_ahojAhoj ahoj ' })).toEqual('/VtG7sR9rRJqwNEdM2/ahoj-ahoj-ahoj-ahoj');
|
|
746
|
-
* TODO: [🌺] Use some intermediate util splitWords
|
|
747
|
-
*/
|
|
748
|
-
|
|
749
|
-
/**
|
|
750
|
-
* @@@
|
|
751
|
-
*
|
|
752
|
-
* @param text @@@
|
|
753
|
-
* @returns @@@
|
|
754
|
-
* @example 'hello_world'
|
|
755
|
-
* @example 'i_love_promptbook'
|
|
756
|
-
* @public exported from `@promptbook/utils`
|
|
757
|
-
*/
|
|
758
|
-
function normalizeTo_snake_case(text) {
|
|
759
|
-
return normalizeTo_SCREAMING_CASE(text).toLowerCase();
|
|
760
|
-
}
|
|
761
|
-
|
|
762
|
-
/**
|
|
763
|
-
* Register is @@@
|
|
764
|
-
*
|
|
765
|
-
* Note: `$` is used to indicate that this function is not a pure function - it accesses and adds variables in global scope.
|
|
766
|
-
*
|
|
767
|
-
* @private internal utility, exported are only signleton instances of this class
|
|
768
|
-
*/
|
|
769
|
-
var $Register = /** @class */ (function () {
|
|
770
|
-
function $Register(registerName) {
|
|
771
|
-
this.registerName = registerName;
|
|
772
|
-
var storageName = "_promptbook_".concat(normalizeTo_snake_case(registerName));
|
|
773
|
-
var globalScope = $getGlobalScope();
|
|
774
|
-
if (globalScope[storageName] === undefined) {
|
|
775
|
-
globalScope[storageName] = [];
|
|
776
|
-
}
|
|
777
|
-
else if (!Array.isArray(globalScope[storageName])) {
|
|
778
|
-
throw new UnexpectedError("Expected (global) ".concat(storageName, " to be an array, but got ").concat(typeof globalScope[storageName]));
|
|
779
|
-
}
|
|
780
|
-
this.storage = globalScope[storageName];
|
|
781
|
-
}
|
|
782
|
-
$Register.prototype.list = function () {
|
|
783
|
-
// <- TODO: ReadonlyDeep<Array<TRegistered>>
|
|
784
|
-
return this.storage;
|
|
785
|
-
};
|
|
786
|
-
$Register.prototype.register = function (registered) {
|
|
787
|
-
var packageName = registered.packageName, className = registered.className;
|
|
788
|
-
var existingRegistrationIndex = this.storage.findIndex(function (item) { return item.packageName === packageName && item.className === className; });
|
|
789
|
-
var existingRegistration = this.storage[existingRegistrationIndex];
|
|
790
|
-
if (!existingRegistration) {
|
|
791
|
-
this.storage.push(registered);
|
|
792
|
-
}
|
|
793
|
-
else {
|
|
794
|
-
this.storage[existingRegistrationIndex] = registered;
|
|
795
|
-
}
|
|
796
|
-
return {
|
|
797
|
-
registerName: this.registerName,
|
|
798
|
-
packageName: packageName,
|
|
799
|
-
className: className,
|
|
800
|
-
get isDestroyed() {
|
|
801
|
-
return false;
|
|
802
|
-
},
|
|
803
|
-
destroy: function () {
|
|
804
|
-
throw new NotYetImplementedError("Registration to ".concat(this.registerName, " is permanent in this version of Promptbook"));
|
|
805
|
-
},
|
|
806
|
-
};
|
|
807
|
-
};
|
|
808
|
-
return $Register;
|
|
809
|
-
}());
|
|
810
|
-
|
|
811
|
-
/**
|
|
812
|
-
* @@@
|
|
813
|
-
*
|
|
814
|
-
* Note: `$` is used to indicate that this interacts with the global scope
|
|
815
|
-
* @singleton Only one instance of each register is created per build, but thare can be more @@@
|
|
816
|
-
* @public exported from `@promptbook/core`
|
|
817
|
-
*/
|
|
818
|
-
var $scrapersRegister = new $Register('scraper_constructors');
|
|
819
|
-
/**
|
|
820
|
-
* TODO: [®] DRY Register logic
|
|
821
|
-
*/
|
|
822
|
-
|
|
823
|
-
/**
|
|
824
|
-
* !!!!!!
|
|
825
|
-
*
|
|
826
|
-
* 1) @@@
|
|
827
|
-
* 2) @@@
|
|
828
|
-
*
|
|
829
|
-
* @public exported from `@promptbook/node`
|
|
830
|
-
*/
|
|
831
|
-
function $provideScrapersForNode(tools, options) {
|
|
832
|
-
return __awaiter(this, void 0, void 0, function () {
|
|
833
|
-
var _a, _b, isAutoInstalled, _c, isVerbose, scrapers, _d, _e, scraperFactory, scraper, e_1_1;
|
|
834
|
-
var e_1, _f;
|
|
835
|
-
return __generator(this, function (_g) {
|
|
836
|
-
switch (_g.label) {
|
|
837
|
-
case 0:
|
|
838
|
-
if (!$isRunningInNode()) {
|
|
839
|
-
throw new EnvironmentMismatchError('Function `$getScrapersForNode` works only in Node.js environment');
|
|
840
|
-
}
|
|
841
|
-
_a = options || {}, _b = _a.isAutoInstalled, isAutoInstalled = _b === void 0 ? IS_AUTO_INSTALLED : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? IS_VERBOSE : _c;
|
|
842
|
-
TODO_USE(isAutoInstalled);
|
|
843
|
-
TODO_USE(isVerbose);
|
|
844
|
-
scrapers = [];
|
|
845
|
-
_g.label = 1;
|
|
846
|
-
case 1:
|
|
847
|
-
_g.trys.push([1, 6, 7, 8]);
|
|
848
|
-
_d = __values($scrapersRegister.list()), _e = _d.next();
|
|
849
|
-
_g.label = 2;
|
|
850
|
-
case 2:
|
|
851
|
-
if (!!_e.done) return [3 /*break*/, 5];
|
|
852
|
-
scraperFactory = _e.value;
|
|
853
|
-
return [4 /*yield*/, scraperFactory(tools, options || {})];
|
|
854
|
-
case 3:
|
|
855
|
-
scraper = _g.sent();
|
|
856
|
-
scrapers.push(scraper);
|
|
857
|
-
_g.label = 4;
|
|
858
|
-
case 4:
|
|
859
|
-
_e = _d.next();
|
|
860
|
-
return [3 /*break*/, 2];
|
|
861
|
-
case 5: return [3 /*break*/, 8];
|
|
862
|
-
case 6:
|
|
863
|
-
e_1_1 = _g.sent();
|
|
864
|
-
e_1 = { error: e_1_1 };
|
|
865
|
-
return [3 /*break*/, 8];
|
|
866
|
-
case 7:
|
|
867
|
-
try {
|
|
868
|
-
if (_e && !_e.done && (_f = _d.return)) _f.call(_d);
|
|
869
|
-
}
|
|
870
|
-
finally { if (e_1) throw e_1.error; }
|
|
871
|
-
return [7 /*endfinally*/];
|
|
872
|
-
case 8: return [2 /*return*/, scrapers];
|
|
873
|
-
}
|
|
874
|
-
});
|
|
875
|
-
});
|
|
876
|
-
}
|
|
877
|
-
/**
|
|
878
|
-
* Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
|
|
879
|
-
*/
|
|
880
|
-
|
|
881
626
|
/**
|
|
882
627
|
* Converts PipelineCollection to serialized JSON
|
|
883
628
|
*
|
|
@@ -963,7 +708,6 @@ function pipelineJsonToString(pipelineJson) {
|
|
|
963
708
|
pipelineString += '\n\n';
|
|
964
709
|
pipelineString += description;
|
|
965
710
|
}
|
|
966
|
-
// TODO:> const commands: Array<Command>
|
|
967
711
|
var commands = [];
|
|
968
712
|
if (pipelineUrl) {
|
|
969
713
|
commands.push("PIPELINE URL ".concat(pipelineUrl));
|
|
@@ -1019,7 +763,6 @@ function pipelineJsonToString(pipelineJson) {
|
|
|
1019
763
|
pipelineString += '\n\n';
|
|
1020
764
|
pipelineString += description_1;
|
|
1021
765
|
}
|
|
1022
|
-
// TODO:> const commands: Array<Command>
|
|
1023
766
|
var commands_1 = [];
|
|
1024
767
|
var contentLanguage = 'text';
|
|
1025
768
|
if (templateType === 'PROMPT_TEMPLATE') {
|
|
@@ -1318,8 +1061,7 @@ var PipelineExecutionError = /** @class */ (function (_super) {
|
|
|
1318
1061
|
* Multiple LLM Execution Tools is a proxy server that uses multiple execution tools internally and exposes the executor interface externally.
|
|
1319
1062
|
*
|
|
1320
1063
|
* Note: Internal utility of `joinLlmExecutionTools` but exposed type
|
|
1321
|
-
* @public exported from `@promptbook/
|
|
1322
|
-
* TODO: !!!!!! Export as runtime class not just type
|
|
1064
|
+
* @public exported from `@promptbook/core`
|
|
1323
1065
|
*/
|
|
1324
1066
|
var MultipleLlmExecutionTools = /** @class */ (function () {
|
|
1325
1067
|
/**
|
|
@@ -2478,6 +2220,22 @@ var LimitReachedError = /** @class */ (function (_super) {
|
|
|
2478
2220
|
return LimitReachedError;
|
|
2479
2221
|
}(Error));
|
|
2480
2222
|
|
|
2223
|
+
/**
|
|
2224
|
+
* This error type indicates that some part of the code is not implemented yet
|
|
2225
|
+
*
|
|
2226
|
+
* @public exported from `@promptbook/core`
|
|
2227
|
+
*/
|
|
2228
|
+
var NotYetImplementedError = /** @class */ (function (_super) {
|
|
2229
|
+
__extends(NotYetImplementedError, _super);
|
|
2230
|
+
function NotYetImplementedError(message) {
|
|
2231
|
+
var _this = _super.call(this, spaceTrim(function (block) { return "\n ".concat(block(message), "\n\n Note: This feature is not implemented yet but it will be soon.\n\n If you want speed up the implementation or just read more, look here:\n https://github.com/webgptorg/promptbook\n\n Or contact us on me@pavolhejny.com\n\n "); })) || this;
|
|
2232
|
+
_this.name = 'NotYetImplementedError';
|
|
2233
|
+
Object.setPrototypeOf(_this, NotYetImplementedError.prototype);
|
|
2234
|
+
return _this;
|
|
2235
|
+
}
|
|
2236
|
+
return NotYetImplementedError;
|
|
2237
|
+
}(Error));
|
|
2238
|
+
|
|
2481
2239
|
/**
|
|
2482
2240
|
* Index of all custom errors
|
|
2483
2241
|
*
|
|
@@ -2539,6 +2297,7 @@ function assertsExecutionSuccessful(executionResult) {
|
|
|
2539
2297
|
}
|
|
2540
2298
|
}
|
|
2541
2299
|
/**
|
|
2300
|
+
* TODO: [🐚] This function should be removed OR changed OR be completely rewritten
|
|
2542
2301
|
* TODO: [🧠] Can this return type be better typed than void
|
|
2543
2302
|
*/
|
|
2544
2303
|
|
|
@@ -2796,6 +2555,23 @@ function union() {
|
|
|
2796
2555
|
return union;
|
|
2797
2556
|
}
|
|
2798
2557
|
|
|
2558
|
+
/**
|
|
2559
|
+
* Just marks a place of place where should be something implemented
|
|
2560
|
+
* No side effects.
|
|
2561
|
+
*
|
|
2562
|
+
* Note: It can be usefull suppressing eslint errors of unused variables
|
|
2563
|
+
*
|
|
2564
|
+
* @param value any values
|
|
2565
|
+
* @returns void
|
|
2566
|
+
* @private within the repository
|
|
2567
|
+
*/
|
|
2568
|
+
function TODO_USE() {
|
|
2569
|
+
var value = [];
|
|
2570
|
+
for (var _i = 0; _i < arguments.length; _i++) {
|
|
2571
|
+
value[_i] = arguments[_i];
|
|
2572
|
+
}
|
|
2573
|
+
}
|
|
2574
|
+
|
|
2799
2575
|
/**
|
|
2800
2576
|
* This error indicates problems parsing the format value
|
|
2801
2577
|
*
|
|
@@ -3828,12 +3604,11 @@ function checkExpectations(expectations, value) {
|
|
|
3828
3604
|
*/
|
|
3829
3605
|
function executeAttempts(options) {
|
|
3830
3606
|
return __awaiter(this, void 0, void 0, function () {
|
|
3831
|
-
var jokerParameterNames, priority, maxAttempts, preparedContent, parameters, template, preparedPipeline, tools,
|
|
3607
|
+
var jokerParameterNames, priority, maxAttempts, preparedContent, parameters, template, preparedPipeline, tools, $executionReport, pipelineIdentification, maxExecutionAttempts, $ongoingTemplateResult, _llms, llmTools, _loop_1, attempt, state_1;
|
|
3832
3608
|
return __generator(this, function (_a) {
|
|
3833
3609
|
switch (_a.label) {
|
|
3834
3610
|
case 0:
|
|
3835
|
-
jokerParameterNames = options.jokerParameterNames, priority = options.priority, maxAttempts = options.maxAttempts, preparedContent = options.preparedContent, parameters = options.parameters, template = options.template, preparedPipeline = options.preparedPipeline, tools = options.tools,
|
|
3836
|
-
maxExecutionAttempts = settings.maxExecutionAttempts;
|
|
3611
|
+
jokerParameterNames = options.jokerParameterNames, priority = options.priority, maxAttempts = options.maxAttempts, preparedContent = options.preparedContent, parameters = options.parameters, template = options.template, preparedPipeline = options.preparedPipeline, tools = options.tools, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification, maxExecutionAttempts = options.maxExecutionAttempts;
|
|
3837
3612
|
$ongoingTemplateResult = {
|
|
3838
3613
|
$result: null,
|
|
3839
3614
|
$resultString: null,
|
|
@@ -4199,12 +3974,12 @@ function executeAttempts(options) {
|
|
|
4199
3974
|
*/
|
|
4200
3975
|
function executeFormatSubvalues(options) {
|
|
4201
3976
|
return __awaiter(this, void 0, void 0, function () {
|
|
4202
|
-
var template, jokerParameterNames, parameters, priority,
|
|
3977
|
+
var template, jokerParameterNames, parameters, priority, csvSettings, pipelineIdentification, parameterValue, formatDefinition, subvalueDefinition, formatSettings, resultString;
|
|
4203
3978
|
var _this = this;
|
|
4204
3979
|
return __generator(this, function (_a) {
|
|
4205
3980
|
switch (_a.label) {
|
|
4206
3981
|
case 0:
|
|
4207
|
-
template = options.template, jokerParameterNames = options.jokerParameterNames, parameters = options.parameters, priority = options.priority,
|
|
3982
|
+
template = options.template, jokerParameterNames = options.jokerParameterNames, parameters = options.parameters, priority = options.priority, csvSettings = options.csvSettings, pipelineIdentification = options.pipelineIdentification;
|
|
4208
3983
|
if (template.foreach === undefined) {
|
|
4209
3984
|
return [2 /*return*/, /* not await */ executeAttempts(options)];
|
|
4210
3985
|
}
|
|
@@ -4234,7 +4009,7 @@ function executeFormatSubvalues(options) {
|
|
|
4234
4009
|
.join('\n')), "\n\n [\u26F7] This should never happen because subformat name should be validated during parsing\n\n ").concat(block(pipelineIdentification), "\n "); }));
|
|
4235
4010
|
}
|
|
4236
4011
|
if (formatDefinition.formatName === 'CSV') {
|
|
4237
|
-
formatSettings =
|
|
4012
|
+
formatSettings = csvSettings;
|
|
4238
4013
|
// <- TODO: [🤹♂️] More universal, make simmilar pattern for other formats for example \n vs \r\n in text
|
|
4239
4014
|
}
|
|
4240
4015
|
return [4 /*yield*/, subvalueDefinition.mapValues(parameterValue, template.foreach.outputSubparameterName, formatSettings, function (subparameters, index) { return __awaiter(_this, void 0, void 0, function () {
|
|
@@ -4387,13 +4162,12 @@ function getReservedParametersForTemplate(options) {
|
|
|
4387
4162
|
*/
|
|
4388
4163
|
function executeTemplate(options) {
|
|
4389
4164
|
return __awaiter(this, void 0, void 0, function () {
|
|
4390
|
-
var currentTemplate, preparedPipeline, parametersToPass, tools, onProgress,
|
|
4391
|
-
var e_1,
|
|
4392
|
-
return __generator(this, function (
|
|
4393
|
-
switch (
|
|
4165
|
+
var currentTemplate, preparedPipeline, parametersToPass, tools, onProgress, $executionReport, pipelineIdentification, _a, maxExecutionAttempts, name, title, priority, usedParameterNames, dependentParameterNames, definedParameters, _b, _c, _d, definedParameterNames, parameters, _loop_1, _e, _f, parameterName, maxAttempts, jokerParameterNames, preparedContent, resultString;
|
|
4166
|
+
var e_1, _g, _h;
|
|
4167
|
+
return __generator(this, function (_j) {
|
|
4168
|
+
switch (_j.label) {
|
|
4394
4169
|
case 0:
|
|
4395
|
-
currentTemplate = options.currentTemplate, preparedPipeline = options.preparedPipeline, parametersToPass = options.parametersToPass, tools = options.tools, onProgress = options.onProgress,
|
|
4396
|
-
maxExecutionAttempts = settings.maxExecutionAttempts;
|
|
4170
|
+
currentTemplate = options.currentTemplate, preparedPipeline = options.preparedPipeline, parametersToPass = options.parametersToPass, tools = options.tools, onProgress = options.onProgress, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification, _a = options.maxExecutionAttempts, maxExecutionAttempts = _a === void 0 ? DEFAULT_MAX_EXECUTION_ATTEMPTS : _a;
|
|
4397
4171
|
name = "pipeline-executor-frame-".concat(currentTemplate.name);
|
|
4398
4172
|
title = currentTemplate.title;
|
|
4399
4173
|
priority = preparedPipeline.templates.length - preparedPipeline.templates.indexOf(currentTemplate);
|
|
@@ -4408,7 +4182,7 @@ function executeTemplate(options) {
|
|
|
4408
4182
|
// <- [🍸]
|
|
4409
4183
|
})];
|
|
4410
4184
|
case 1:
|
|
4411
|
-
|
|
4185
|
+
_j.sent();
|
|
4412
4186
|
usedParameterNames = extractParameterNamesFromTemplate(currentTemplate);
|
|
4413
4187
|
dependentParameterNames = new Set(currentTemplate.dependentParameterNames);
|
|
4414
4188
|
// TODO: [👩🏾🤝👩🏻] Use here `mapAvailableToExpectedParameters`
|
|
@@ -4419,15 +4193,15 @@ function executeTemplate(options) {
|
|
|
4419
4193
|
.map(function (name) { return "{".concat(name, "}"); })
|
|
4420
4194
|
.join(', '), "\n\n ").concat(block(pipelineIdentification), "\n\n "); }));
|
|
4421
4195
|
}
|
|
4422
|
-
|
|
4423
|
-
|
|
4196
|
+
_c = (_b = Object).freeze;
|
|
4197
|
+
_d = [{}];
|
|
4424
4198
|
return [4 /*yield*/, getReservedParametersForTemplate({
|
|
4425
4199
|
preparedPipeline: preparedPipeline,
|
|
4426
4200
|
template: currentTemplate,
|
|
4427
4201
|
pipelineIdentification: pipelineIdentification,
|
|
4428
4202
|
})];
|
|
4429
4203
|
case 2:
|
|
4430
|
-
definedParameters =
|
|
4204
|
+
definedParameters = _c.apply(_b, [__assign.apply(void 0, [__assign.apply(void 0, _d.concat([(_j.sent())])), parametersToPass])]);
|
|
4431
4205
|
definedParameterNames = new Set(Object.keys(definedParameters));
|
|
4432
4206
|
parameters = {};
|
|
4433
4207
|
_loop_1 = function (parameterName) {
|
|
@@ -4447,15 +4221,15 @@ function executeTemplate(options) {
|
|
|
4447
4221
|
try {
|
|
4448
4222
|
// Note: [2] Check that all used parameters are defined and removing unused parameters for this template
|
|
4449
4223
|
// TODO: [👩🏾🤝👩🏻] Use here `mapAvailableToExpectedParameters`
|
|
4450
|
-
for (
|
|
4451
|
-
parameterName =
|
|
4224
|
+
for (_e = __values(Array.from(union(definedParameterNames, usedParameterNames, dependentParameterNames))), _f = _e.next(); !_f.done; _f = _e.next()) {
|
|
4225
|
+
parameterName = _f.value;
|
|
4452
4226
|
_loop_1(parameterName);
|
|
4453
4227
|
}
|
|
4454
4228
|
}
|
|
4455
4229
|
catch (e_1_1) { e_1 = { error: e_1_1 }; }
|
|
4456
4230
|
finally {
|
|
4457
4231
|
try {
|
|
4458
|
-
if (
|
|
4232
|
+
if (_f && !_f.done && (_g = _e.return)) _g.call(_e);
|
|
4459
4233
|
}
|
|
4460
4234
|
finally { if (e_1) throw e_1.error; }
|
|
4461
4235
|
}
|
|
@@ -4475,12 +4249,11 @@ function executeTemplate(options) {
|
|
|
4475
4249
|
template: currentTemplate,
|
|
4476
4250
|
preparedPipeline: preparedPipeline,
|
|
4477
4251
|
tools: tools,
|
|
4478
|
-
settings: settings,
|
|
4479
4252
|
$executionReport: $executionReport,
|
|
4480
4253
|
pipelineIdentification: pipelineIdentification,
|
|
4481
4254
|
})];
|
|
4482
4255
|
case 3:
|
|
4483
|
-
resultString =
|
|
4256
|
+
resultString = _j.sent();
|
|
4484
4257
|
return [4 /*yield*/, onProgress({
|
|
4485
4258
|
name: name,
|
|
4486
4259
|
title: title,
|
|
@@ -4492,12 +4265,12 @@ function executeTemplate(options) {
|
|
|
4492
4265
|
// <- [🍸]
|
|
4493
4266
|
})];
|
|
4494
4267
|
case 4:
|
|
4495
|
-
|
|
4496
|
-
return [2 /*return*/, Object.freeze((
|
|
4497
|
-
|
|
4268
|
+
_j.sent();
|
|
4269
|
+
return [2 /*return*/, Object.freeze((_h = {},
|
|
4270
|
+
_h[currentTemplate.resultingParameterName] =
|
|
4498
4271
|
// <- Note: [👩👩👧] No need to detect parameter collision here because pipeline checks logic consistency during construction
|
|
4499
4272
|
resultString,
|
|
4500
|
-
|
|
4273
|
+
_h))];
|
|
4501
4274
|
}
|
|
4502
4275
|
});
|
|
4503
4276
|
});
|
|
@@ -4556,13 +4329,12 @@ function filterJustOutputParameters(options) {
|
|
|
4556
4329
|
*/
|
|
4557
4330
|
function executePipeline(options) {
|
|
4558
4331
|
return __awaiter(this, void 0, void 0, function () {
|
|
4559
|
-
var inputParameters, tools, onProgress, pipeline, setPreparedPipeline, pipelineIdentification,
|
|
4332
|
+
var inputParameters, tools, onProgress, pipeline, setPreparedPipeline, pipelineIdentification, maxParallelCount, rootDirname, _a, isVerbose, preparedPipeline, errors, warnings, executionReport, isReturned, _b, _c, parameter, e_1_1, _loop_1, _d, _e, parameterName, state_1, e_2_1, parametersToPass, resovedParameterNames_1, unresovedTemplates_1, resolving_1, loopLimit, _loop_2, error_1, usage_1, outputParameters_1, usage, outputParameters;
|
|
4560
4333
|
var e_1, _f, e_2, _g;
|
|
4561
4334
|
return __generator(this, function (_h) {
|
|
4562
4335
|
switch (_h.label) {
|
|
4563
4336
|
case 0:
|
|
4564
|
-
inputParameters = options.inputParameters, tools = options.tools, onProgress = options.onProgress, pipeline = options.pipeline, setPreparedPipeline = options.setPreparedPipeline, pipelineIdentification = options.pipelineIdentification,
|
|
4565
|
-
maxParallelCount = settings.maxParallelCount, rootDirname = settings.rootDirname, _a = settings.isVerbose, isVerbose = _a === void 0 ? IS_VERBOSE : _a;
|
|
4337
|
+
inputParameters = options.inputParameters, tools = options.tools, onProgress = options.onProgress, pipeline = options.pipeline, setPreparedPipeline = options.setPreparedPipeline, pipelineIdentification = options.pipelineIdentification, maxParallelCount = options.maxParallelCount, rootDirname = options.rootDirname, _a = options.isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a;
|
|
4566
4338
|
preparedPipeline = options.preparedPipeline;
|
|
4567
4339
|
if (!(preparedPipeline === undefined)) return [3 /*break*/, 2];
|
|
4568
4340
|
return [4 /*yield*/, preparePipeline(pipeline, tools, {
|
|
@@ -4747,12 +4519,7 @@ function executePipeline(options) {
|
|
|
4747
4519
|
return [3 /*break*/, 4];
|
|
4748
4520
|
case 3:
|
|
4749
4521
|
unresovedTemplates_1 = unresovedTemplates_1.filter(function (template) { return template !== currentTemplate; });
|
|
4750
|
-
work_1 = executeTemplate({
|
|
4751
|
-
currentTemplate: currentTemplate,
|
|
4752
|
-
preparedPipeline: preparedPipeline,
|
|
4753
|
-
parametersToPass: parametersToPass,
|
|
4754
|
-
tools: tools,
|
|
4755
|
-
onProgress: function (progress) {
|
|
4522
|
+
work_1 = executeTemplate(__assign(__assign({}, options), { currentTemplate: currentTemplate, preparedPipeline: preparedPipeline, parametersToPass: parametersToPass, tools: tools, onProgress: function (progress) {
|
|
4756
4523
|
if (isReturned) {
|
|
4757
4524
|
throw new UnexpectedError(spaceTrim(function (block) { return "\n Can not call `onProgress` after pipeline execution is finished\n\n ".concat(block(pipelineIdentification), "\n\n ").concat(block(JSON.stringify(progress, null, 4)
|
|
4758
4525
|
.split('\n')
|
|
@@ -4762,11 +4529,7 @@ function executePipeline(options) {
|
|
|
4762
4529
|
if (onProgress) {
|
|
4763
4530
|
onProgress(progress);
|
|
4764
4531
|
}
|
|
4765
|
-
},
|
|
4766
|
-
settings: settings,
|
|
4767
|
-
$executionReport: executionReport,
|
|
4768
|
-
pipelineIdentification: spaceTrim(function (block) { return "\n ".concat(block(pipelineIdentification), "\n Template name: ").concat(currentTemplate.name, "\n Template title: ").concat(currentTemplate.title, "\n "); }),
|
|
4769
|
-
})
|
|
4532
|
+
}, $executionReport: executionReport, pipelineIdentification: spaceTrim(function (block) { return "\n ".concat(block(pipelineIdentification), "\n Template name: ").concat(currentTemplate.name, "\n Template title: ").concat(currentTemplate.title, "\n "); }) }))
|
|
4770
4533
|
.then(function (newParametersToPass) {
|
|
4771
4534
|
parametersToPass = __assign(__assign({}, newParametersToPass), parametersToPass);
|
|
4772
4535
|
resovedParameterNames_1 = __spreadArray(__spreadArray([], __read(resovedParameterNames_1), false), [currentTemplate.resultingParameterName], false);
|
|
@@ -4870,8 +4633,7 @@ function executePipeline(options) {
|
|
|
4870
4633
|
*/
|
|
4871
4634
|
function createPipelineExecutor(options) {
|
|
4872
4635
|
var _this = this;
|
|
4873
|
-
var pipeline = options.pipeline, tools = options.tools, _a = options.
|
|
4874
|
-
var _b = settings.maxExecutionAttempts, maxExecutionAttempts = _b === void 0 ? MAX_EXECUTION_ATTEMPTS : _b, _c = settings.maxParallelCount, maxParallelCount = _c === void 0 ? MAX_PARALLEL_COUNT : _c, _d = settings.csvSettings, csvSettings = _d === void 0 ? DEFAULT_CSV_SETTINGS : _d, _e = settings.isVerbose, isVerbose = _e === void 0 ? IS_VERBOSE : _e, _f = settings.isNotPreparedWarningSupressed, isNotPreparedWarningSupressed = _f === void 0 ? false : _f, _g = settings.rootDirname, rootDirname = _g === void 0 ? null : _g;
|
|
4636
|
+
var pipeline = options.pipeline, tools = options.tools, _a = options.maxExecutionAttempts, maxExecutionAttempts = _a === void 0 ? DEFAULT_MAX_EXECUTION_ATTEMPTS : _a, _b = options.maxParallelCount, maxParallelCount = _b === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _b, _c = options.csvSettings, csvSettings = _c === void 0 ? DEFAULT_CSV_SETTINGS : _c, _d = options.isVerbose, isVerbose = _d === void 0 ? DEFAULT_IS_VERBOSE : _d, _e = options.isNotPreparedWarningSupressed, isNotPreparedWarningSupressed = _e === void 0 ? false : _e, _f = options.rootDirname, rootDirname = _f === void 0 ? null : _f;
|
|
4875
4637
|
validatePipeline(pipeline);
|
|
4876
4638
|
var pipelineIdentification = (function () {
|
|
4877
4639
|
// Note: This is a 😐 implementation of [🚞]
|
|
@@ -4905,14 +4667,12 @@ function createPipelineExecutor(options) {
|
|
|
4905
4667
|
tools: tools,
|
|
4906
4668
|
onProgress: onProgress,
|
|
4907
4669
|
pipelineIdentification: spaceTrim(function (block) { return "\n ".concat(block(pipelineIdentification), "\n ").concat(runCount === 1 ? '' : "Run #".concat(runCount), "\n "); }),
|
|
4908
|
-
|
|
4909
|
-
|
|
4910
|
-
|
|
4911
|
-
|
|
4912
|
-
|
|
4913
|
-
|
|
4914
|
-
rootDirname: rootDirname,
|
|
4915
|
-
},
|
|
4670
|
+
maxExecutionAttempts: maxExecutionAttempts,
|
|
4671
|
+
maxParallelCount: maxParallelCount,
|
|
4672
|
+
csvSettings: csvSettings,
|
|
4673
|
+
isVerbose: isVerbose,
|
|
4674
|
+
isNotPreparedWarningSupressed: isNotPreparedWarningSupressed,
|
|
4675
|
+
rootDirname: rootDirname,
|
|
4916
4676
|
})];
|
|
4917
4677
|
});
|
|
4918
4678
|
}); };
|
|
@@ -4935,7 +4695,7 @@ function preparePersona(personaDescription, tools, options) {
|
|
|
4935
4695
|
return __generator(this, function (_d) {
|
|
4936
4696
|
switch (_d.label) {
|
|
4937
4697
|
case 0:
|
|
4938
|
-
_a = options.isVerbose, isVerbose = _a === void 0 ?
|
|
4698
|
+
_a = options.isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a;
|
|
4939
4699
|
if (tools === undefined || tools.llm === undefined) {
|
|
4940
4700
|
throw new MissingToolsError('LLM tools are required for preparing persona');
|
|
4941
4701
|
}
|
|
@@ -4991,20 +4751,171 @@ function preparePersona(personaDescription, tools, options) {
|
|
|
4991
4751
|
*/
|
|
4992
4752
|
|
|
4993
4753
|
/**
|
|
4994
|
-
* This error indicates that the promptbook can not retrieve knowledge from external sources
|
|
4754
|
+
* This error indicates that the promptbook can not retrieve knowledge from external sources
|
|
4755
|
+
*
|
|
4756
|
+
* @public exported from `@promptbook/core`
|
|
4757
|
+
*/
|
|
4758
|
+
var KnowledgeScrapeError = /** @class */ (function (_super) {
|
|
4759
|
+
__extends(KnowledgeScrapeError, _super);
|
|
4760
|
+
function KnowledgeScrapeError(message) {
|
|
4761
|
+
var _this = _super.call(this, message) || this;
|
|
4762
|
+
_this.name = 'KnowledgeScrapeError';
|
|
4763
|
+
Object.setPrototypeOf(_this, KnowledgeScrapeError.prototype);
|
|
4764
|
+
return _this;
|
|
4765
|
+
}
|
|
4766
|
+
return KnowledgeScrapeError;
|
|
4767
|
+
}(Error));
|
|
4768
|
+
|
|
4769
|
+
/**
|
|
4770
|
+
* @@@
|
|
4771
|
+
*
|
|
4772
|
+
* Note: `$` is used to indicate that this function is not a pure function - it access global scope
|
|
4773
|
+
*
|
|
4774
|
+
* @private internal function of `$Register`
|
|
4775
|
+
*/
|
|
4776
|
+
function $getGlobalScope() {
|
|
4777
|
+
return Function('return this')();
|
|
4778
|
+
}
|
|
4779
|
+
|
|
4780
|
+
/**
|
|
4781
|
+
* @@@
|
|
4782
|
+
*
|
|
4783
|
+
* @param text @@@
|
|
4784
|
+
* @returns @@@
|
|
4785
|
+
* @example 'HELLO_WORLD'
|
|
4786
|
+
* @example 'I_LOVE_PROMPTBOOK'
|
|
4787
|
+
* @public exported from `@promptbook/utils`
|
|
4788
|
+
*/
|
|
4789
|
+
function normalizeTo_SCREAMING_CASE(text) {
|
|
4790
|
+
var e_1, _a;
|
|
4791
|
+
var charType;
|
|
4792
|
+
var lastCharType = 'OTHER';
|
|
4793
|
+
var normalizedName = '';
|
|
4794
|
+
try {
|
|
4795
|
+
for (var text_1 = __values(text), text_1_1 = text_1.next(); !text_1_1.done; text_1_1 = text_1.next()) {
|
|
4796
|
+
var char = text_1_1.value;
|
|
4797
|
+
var normalizedChar = void 0;
|
|
4798
|
+
if (/^[a-z]$/.test(char)) {
|
|
4799
|
+
charType = 'LOWERCASE';
|
|
4800
|
+
normalizedChar = char.toUpperCase();
|
|
4801
|
+
}
|
|
4802
|
+
else if (/^[A-Z]$/.test(char)) {
|
|
4803
|
+
charType = 'UPPERCASE';
|
|
4804
|
+
normalizedChar = char;
|
|
4805
|
+
}
|
|
4806
|
+
else if (/^[0-9]$/.test(char)) {
|
|
4807
|
+
charType = 'NUMBER';
|
|
4808
|
+
normalizedChar = char;
|
|
4809
|
+
}
|
|
4810
|
+
else {
|
|
4811
|
+
charType = 'OTHER';
|
|
4812
|
+
normalizedChar = '_';
|
|
4813
|
+
}
|
|
4814
|
+
if (charType !== lastCharType &&
|
|
4815
|
+
!(lastCharType === 'UPPERCASE' && charType === 'LOWERCASE') &&
|
|
4816
|
+
!(lastCharType === 'NUMBER') &&
|
|
4817
|
+
!(charType === 'NUMBER')) {
|
|
4818
|
+
normalizedName += '_';
|
|
4819
|
+
}
|
|
4820
|
+
normalizedName += normalizedChar;
|
|
4821
|
+
lastCharType = charType;
|
|
4822
|
+
}
|
|
4823
|
+
}
|
|
4824
|
+
catch (e_1_1) { e_1 = { error: e_1_1 }; }
|
|
4825
|
+
finally {
|
|
4826
|
+
try {
|
|
4827
|
+
if (text_1_1 && !text_1_1.done && (_a = text_1.return)) _a.call(text_1);
|
|
4828
|
+
}
|
|
4829
|
+
finally { if (e_1) throw e_1.error; }
|
|
4830
|
+
}
|
|
4831
|
+
normalizedName = normalizedName.replace(/_+/g, '_');
|
|
4832
|
+
normalizedName = normalizedName.replace(/_?\/_?/g, '/');
|
|
4833
|
+
normalizedName = normalizedName.replace(/^_/, '');
|
|
4834
|
+
normalizedName = normalizedName.replace(/_$/, '');
|
|
4835
|
+
return normalizedName;
|
|
4836
|
+
}
|
|
4837
|
+
/**
|
|
4838
|
+
* TODO: Tests
|
|
4839
|
+
* > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: 'Moje tabule' })).toEqual('/VtG7sR9rRJqwNEdM2/Moje tabule');
|
|
4840
|
+
* > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: 'ěščřžžýáíúů' })).toEqual('/VtG7sR9rRJqwNEdM2/escrzyaieuu');
|
|
4841
|
+
* > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: ' ahoj ' })).toEqual('/VtG7sR9rRJqwNEdM2/ahoj');
|
|
4842
|
+
* > expect(encodeRoutePath({ uriId: 'VtG7sR9rRJqwNEdM2', name: ' ahoj_ahojAhoj ahoj ' })).toEqual('/VtG7sR9rRJqwNEdM2/ahoj-ahoj-ahoj-ahoj');
|
|
4843
|
+
* TODO: [🌺] Use some intermediate util splitWords
|
|
4844
|
+
*/
|
|
4845
|
+
|
|
4846
|
+
/**
|
|
4847
|
+
* @@@
|
|
4848
|
+
*
|
|
4849
|
+
* @param text @@@
|
|
4850
|
+
* @returns @@@
|
|
4851
|
+
* @example 'hello_world'
|
|
4852
|
+
* @example 'i_love_promptbook'
|
|
4853
|
+
* @public exported from `@promptbook/utils`
|
|
4854
|
+
*/
|
|
4855
|
+
function normalizeTo_snake_case(text) {
|
|
4856
|
+
return normalizeTo_SCREAMING_CASE(text).toLowerCase();
|
|
4857
|
+
}
|
|
4858
|
+
|
|
4859
|
+
/**
|
|
4860
|
+
* Register is @@@
|
|
4861
|
+
*
|
|
4862
|
+
* Note: `$` is used to indicate that this function is not a pure function - it accesses and adds variables in global scope.
|
|
4863
|
+
*
|
|
4864
|
+
* @private internal utility, exported are only signleton instances of this class
|
|
4865
|
+
*/
|
|
4866
|
+
var $Register = /** @class */ (function () {
|
|
4867
|
+
function $Register(registerName) {
|
|
4868
|
+
this.registerName = registerName;
|
|
4869
|
+
var storageName = "_promptbook_".concat(normalizeTo_snake_case(registerName));
|
|
4870
|
+
var globalScope = $getGlobalScope();
|
|
4871
|
+
if (globalScope[storageName] === undefined) {
|
|
4872
|
+
globalScope[storageName] = [];
|
|
4873
|
+
}
|
|
4874
|
+
else if (!Array.isArray(globalScope[storageName])) {
|
|
4875
|
+
throw new UnexpectedError("Expected (global) ".concat(storageName, " to be an array, but got ").concat(typeof globalScope[storageName]));
|
|
4876
|
+
}
|
|
4877
|
+
this.storage = globalScope[storageName];
|
|
4878
|
+
}
|
|
4879
|
+
$Register.prototype.list = function () {
|
|
4880
|
+
// <- TODO: ReadonlyDeep<ReadonlyArray<TRegistered>>
|
|
4881
|
+
return this.storage;
|
|
4882
|
+
};
|
|
4883
|
+
$Register.prototype.register = function (registered) {
|
|
4884
|
+
var packageName = registered.packageName, className = registered.className;
|
|
4885
|
+
var existingRegistrationIndex = this.storage.findIndex(function (item) { return item.packageName === packageName && item.className === className; });
|
|
4886
|
+
var existingRegistration = this.storage[existingRegistrationIndex];
|
|
4887
|
+
if (!existingRegistration) {
|
|
4888
|
+
this.storage.push(registered);
|
|
4889
|
+
}
|
|
4890
|
+
else {
|
|
4891
|
+
this.storage[existingRegistrationIndex] = registered;
|
|
4892
|
+
}
|
|
4893
|
+
return {
|
|
4894
|
+
registerName: this.registerName,
|
|
4895
|
+
packageName: packageName,
|
|
4896
|
+
className: className,
|
|
4897
|
+
get isDestroyed() {
|
|
4898
|
+
return false;
|
|
4899
|
+
},
|
|
4900
|
+
destroy: function () {
|
|
4901
|
+
throw new NotYetImplementedError("Registration to ".concat(this.registerName, " is permanent in this version of Promptbook"));
|
|
4902
|
+
},
|
|
4903
|
+
};
|
|
4904
|
+
};
|
|
4905
|
+
return $Register;
|
|
4906
|
+
}());
|
|
4907
|
+
|
|
4908
|
+
/**
|
|
4909
|
+
* @@@
|
|
4995
4910
|
*
|
|
4911
|
+
* Note: `$` is used to indicate that this interacts with the global scope
|
|
4912
|
+
* @singleton Only one instance of each register is created per build, but thare can be more @@@
|
|
4996
4913
|
* @public exported from `@promptbook/core`
|
|
4997
4914
|
*/
|
|
4998
|
-
var
|
|
4999
|
-
|
|
5000
|
-
|
|
5001
|
-
|
|
5002
|
-
_this.name = 'KnowledgeScrapeError';
|
|
5003
|
-
Object.setPrototypeOf(_this, KnowledgeScrapeError.prototype);
|
|
5004
|
-
return _this;
|
|
5005
|
-
}
|
|
5006
|
-
return KnowledgeScrapeError;
|
|
5007
|
-
}(Error));
|
|
4915
|
+
var $scrapersMetadataRegister = new $Register('scrapers_metadata');
|
|
4916
|
+
/**
|
|
4917
|
+
* TODO: [®] DRY Register logic
|
|
4918
|
+
*/
|
|
5008
4919
|
|
|
5009
4920
|
/**
|
|
5010
4921
|
* @@@
|
|
@@ -5013,12 +4924,11 @@ var KnowledgeScrapeError = /** @class */ (function (_super) {
|
|
|
5013
4924
|
* @singleton Only one instance of each register is created per build, but thare can be more @@@
|
|
5014
4925
|
* @public exported from `@promptbook/core`
|
|
5015
4926
|
*/
|
|
5016
|
-
var $
|
|
4927
|
+
var $scrapersRegister = new $Register('scraper_constructors');
|
|
5017
4928
|
/**
|
|
5018
4929
|
* TODO: [®] DRY Register logic
|
|
5019
4930
|
*/
|
|
5020
4931
|
|
|
5021
|
-
// TODO: !!!!!! Maybe delete this function
|
|
5022
4932
|
/**
|
|
5023
4933
|
* Creates a message with all registered scrapers
|
|
5024
4934
|
*
|
|
@@ -5185,7 +5095,6 @@ function normalizeToKebabCase(text) {
|
|
|
5185
5095
|
* @private within the repository
|
|
5186
5096
|
*/
|
|
5187
5097
|
function sourceContentToName(sourceContent) {
|
|
5188
|
-
// TODO: !!!!!! Better name for source than gibberish hash
|
|
5189
5098
|
var hash = SHA256(hexEncoder.parse(JSON.stringify(sourceContent)))
|
|
5190
5099
|
// <- TODO: [🥬] Encapsulate sha256 to some private utility function
|
|
5191
5100
|
.toString( /* hex */)
|
|
@@ -5285,20 +5194,20 @@ function isValidFilePath(filename) {
|
|
|
5285
5194
|
/**
|
|
5286
5195
|
* @@@
|
|
5287
5196
|
*
|
|
5288
|
-
* @
|
|
5197
|
+
* @public exported from `@promptbook/core`
|
|
5289
5198
|
*/
|
|
5290
5199
|
function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
|
|
5291
5200
|
var _a;
|
|
5292
5201
|
return __awaiter(this, void 0, void 0, function () {
|
|
5293
5202
|
var sourceContent, name, _b, _c, rootDirname, _d,
|
|
5294
5203
|
// <- TODO: process.cwd() if running in Node.js
|
|
5295
|
-
isVerbose, url, response_1, mimeType, filename_1, fileExtension,
|
|
5204
|
+
isVerbose, url, response_1, mimeType, filename_1, fileExtension, mimeType;
|
|
5296
5205
|
return __generator(this, function (_e) {
|
|
5297
5206
|
switch (_e.label) {
|
|
5298
5207
|
case 0:
|
|
5299
5208
|
sourceContent = knowledgeSource.sourceContent;
|
|
5300
5209
|
name = knowledgeSource.name;
|
|
5301
|
-
_b = options || {}, _c = _b.rootDirname, rootDirname = _c === void 0 ? null : _c, _d = _b.isVerbose, isVerbose = _d === void 0 ?
|
|
5210
|
+
_b = options || {}, _c = _b.rootDirname, rootDirname = _c === void 0 ? null : _c, _d = _b.isVerbose, isVerbose = _d === void 0 ? DEFAULT_IS_VERBOSE : _d;
|
|
5302
5211
|
TODO_USE(isVerbose);
|
|
5303
5212
|
if (!name) {
|
|
5304
5213
|
name = sourceContentToName(sourceContent);
|
|
@@ -5314,19 +5223,14 @@ function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
|
|
|
5314
5223
|
filename: null,
|
|
5315
5224
|
url: url,
|
|
5316
5225
|
mimeType: mimeType,
|
|
5317
|
-
|
|
5318
|
-
|
|
5319
|
-
|
|
5320
|
-
|
|
5321
|
-
|
|
5322
|
-
|
|
5323
|
-
|
|
5324
|
-
|
|
5325
|
-
return [2 /*return*/, content];
|
|
5326
|
-
}
|
|
5327
|
-
});
|
|
5328
|
-
});
|
|
5329
|
-
},
|
|
5226
|
+
/*
|
|
5227
|
+
TODO: [🥽]
|
|
5228
|
+
> async asBlob() {
|
|
5229
|
+
> // TODO: [👨🏻🤝👨🏻] This can be called multiple times BUT when called second time, response in already consumed
|
|
5230
|
+
> const content = await response.blob();
|
|
5231
|
+
> return content;
|
|
5232
|
+
> },
|
|
5233
|
+
*/
|
|
5330
5234
|
asJson: function () {
|
|
5331
5235
|
return __awaiter(this, void 0, void 0, function () {
|
|
5332
5236
|
var content;
|
|
@@ -5366,34 +5270,31 @@ function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
|
|
|
5366
5270
|
}
|
|
5367
5271
|
filename_1 = join(rootDirname, sourceContent).split('\\').join('/');
|
|
5368
5272
|
fileExtension = getFileExtension(filename_1);
|
|
5369
|
-
|
|
5273
|
+
mimeType = extensionToMimeType(fileExtension || '');
|
|
5370
5274
|
return [4 /*yield*/, isFileExisting(filename_1, tools.fs)];
|
|
5371
5275
|
case 3:
|
|
5372
5276
|
if (!(_e.sent())) {
|
|
5373
5277
|
throw new NotFoundError(spaceTrim$1(function (block) { return "\n Can not make source handler for file which does not exist:\n\n File:\n ".concat(block(filename_1), "\n "); }));
|
|
5374
5278
|
}
|
|
5375
|
-
// TODO:
|
|
5279
|
+
// TODO: [🧠][😿] Test security file - file is scoped to the project (BUT maybe do this in `filesystemTools`)
|
|
5376
5280
|
return [2 /*return*/, {
|
|
5377
5281
|
source: name,
|
|
5378
5282
|
filename: filename_1,
|
|
5379
5283
|
url: null,
|
|
5380
|
-
mimeType:
|
|
5381
|
-
|
|
5382
|
-
|
|
5383
|
-
|
|
5384
|
-
|
|
5385
|
-
|
|
5386
|
-
|
|
5387
|
-
|
|
5388
|
-
|
|
5389
|
-
|
|
5390
|
-
|
|
5391
|
-
|
|
5392
|
-
|
|
5393
|
-
|
|
5394
|
-
});
|
|
5395
|
-
});
|
|
5396
|
-
},
|
|
5284
|
+
mimeType: mimeType,
|
|
5285
|
+
/*
|
|
5286
|
+
TODO: [🥽]
|
|
5287
|
+
> async asBlob() {
|
|
5288
|
+
> const content = await tools.fs!.readFile(filename);
|
|
5289
|
+
> return new Blob(
|
|
5290
|
+
> [
|
|
5291
|
+
> content,
|
|
5292
|
+
> // <- TODO: [🥽] This is NOT tested, test it
|
|
5293
|
+
> ],
|
|
5294
|
+
> { type: mimeType },
|
|
5295
|
+
> );
|
|
5296
|
+
> },
|
|
5297
|
+
*/
|
|
5397
5298
|
asJson: function () {
|
|
5398
5299
|
return __awaiter(this, void 0, void 0, function () {
|
|
5399
5300
|
var _a, _b;
|
|
@@ -5429,9 +5330,14 @@ function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
|
|
|
5429
5330
|
asJson: function () {
|
|
5430
5331
|
throw new UnexpectedError('Did not expect that `markdownScraper` would need to get the content `asJson`');
|
|
5431
5332
|
},
|
|
5432
|
-
|
|
5433
|
-
|
|
5434
|
-
|
|
5333
|
+
/*
|
|
5334
|
+
TODO: [🥽]
|
|
5335
|
+
> asBlob() {
|
|
5336
|
+
> throw new UnexpectedError(
|
|
5337
|
+
> 'Did not expect that `markdownScraper` would need to get the content `asBlob`',
|
|
5338
|
+
> );
|
|
5339
|
+
> },
|
|
5340
|
+
*/
|
|
5435
5341
|
}];
|
|
5436
5342
|
}
|
|
5437
5343
|
});
|
|
@@ -5451,7 +5357,7 @@ function prepareKnowledgePieces(knowledgeSources, tools, options) {
|
|
|
5451
5357
|
return __generator(this, function (_c) {
|
|
5452
5358
|
switch (_c.label) {
|
|
5453
5359
|
case 0:
|
|
5454
|
-
_a = options.maxParallelCount, maxParallelCount = _a === void 0 ?
|
|
5360
|
+
_a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a, rootDirname = options.rootDirname, _b = options.isVerbose, isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b;
|
|
5455
5361
|
knowledgePreparedUnflatten = new Array(knowledgeSources.length);
|
|
5456
5362
|
return [4 /*yield*/, forEachAsync(knowledgeSources, { maxParallelCount: maxParallelCount }, function (knowledgeSource, index) { return __awaiter(_this, void 0, void 0, function () {
|
|
5457
5363
|
var partialPieces, sourceHandler, _a, _b, scraper, partialPiecesUnchecked, e_1_1, pieces;
|
|
@@ -5480,7 +5386,8 @@ function prepareKnowledgePieces(knowledgeSources, tools, options) {
|
|
|
5480
5386
|
case 4:
|
|
5481
5387
|
partialPiecesUnchecked = _d.sent();
|
|
5482
5388
|
if (partialPiecesUnchecked !== null) {
|
|
5483
|
-
partialPieces = partialPiecesUnchecked;
|
|
5389
|
+
partialPieces = __spreadArray([], __read(partialPiecesUnchecked), false);
|
|
5390
|
+
// <- TODO: [🪓] Here should be no need for spreading new array, just `partialPieces = partialPiecesUnchecked`
|
|
5484
5391
|
return [3 /*break*/, 6];
|
|
5485
5392
|
}
|
|
5486
5393
|
_d.label = 5;
|
|
@@ -5528,7 +5435,7 @@ TODO: [🧊] This is how it can look in future
|
|
|
5528
5435
|
> /**
|
|
5529
5436
|
> * Unprepared knowledge
|
|
5530
5437
|
> * /
|
|
5531
|
-
> readonly knowledgeSources:
|
|
5438
|
+
> readonly knowledgeSources: ReadonlyArray<KnowledgeSourceJson>;
|
|
5532
5439
|
> };
|
|
5533
5440
|
>
|
|
5534
5441
|
> export async function prepareKnowledgePieces(
|
|
@@ -5586,7 +5493,7 @@ function prepareTemplates(pipeline, tools, options) {
|
|
|
5586
5493
|
return __generator(this, function (_b) {
|
|
5587
5494
|
switch (_b.label) {
|
|
5588
5495
|
case 0:
|
|
5589
|
-
_a = options.maxParallelCount, maxParallelCount = _a === void 0 ?
|
|
5496
|
+
_a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a;
|
|
5590
5497
|
templates = pipeline.templates, parameters = pipeline.parameters, knowledgePiecesCount = pipeline.knowledgePiecesCount;
|
|
5591
5498
|
// TODO: [main] !!!!! Apply samples to each template (if missing and is for the template defined)
|
|
5592
5499
|
TODO_USE(parameters);
|
|
@@ -5648,7 +5555,7 @@ function preparePipeline(pipeline, tools, options) {
|
|
|
5648
5555
|
if (isPipelinePrepared(pipeline)) {
|
|
5649
5556
|
return [2 /*return*/, pipeline];
|
|
5650
5557
|
}
|
|
5651
|
-
rootDirname = options.rootDirname, _a = options.maxParallelCount, maxParallelCount = _a === void 0 ?
|
|
5558
|
+
rootDirname = options.rootDirname, _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a, _b = options.isVerbose, isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b;
|
|
5652
5559
|
parameters = pipeline.parameters, templates = pipeline.templates, knowledgeSources = pipeline.knowledgeSources, personas = pipeline.personas;
|
|
5653
5560
|
if (tools === undefined || tools.llm === undefined) {
|
|
5654
5561
|
throw new MissingToolsError('LLM tools are required for preparing the pipeline');
|
|
@@ -5706,7 +5613,9 @@ function preparePipeline(pipeline, tools, options) {
|
|
|
5706
5613
|
// ----- /Templates preparation -----
|
|
5707
5614
|
// Note: Count total usage
|
|
5708
5615
|
currentPreparation.usage = llmToolsWithUsage.getTotalUsage();
|
|
5709
|
-
return [2 /*return*/, $asDeeplyFrozenSerializableJson('Prepared PipelineJson', __assign(__assign({}, clonePipeline(pipeline)), { templates:
|
|
5616
|
+
return [2 /*return*/, $asDeeplyFrozenSerializableJson('Prepared PipelineJson', __assign(__assign({}, clonePipeline(pipeline)), { templates: __spreadArray([], __read(templatesPrepared), false),
|
|
5617
|
+
// <- TODO: [🪓] Here should be no need for spreading new array, just ` templates: templatesPrepared`
|
|
5618
|
+
knowledgeSources: knowledgeSourcesPrepared, knowledgePieces: knowledgePiecesPrepared, personas: preparedPersonas, preparations: __spreadArray([], __read(preparations), false) }))];
|
|
5710
5619
|
}
|
|
5711
5620
|
});
|
|
5712
5621
|
});
|
|
@@ -5753,6 +5662,7 @@ var knowledgeCommandParser = {
|
|
|
5753
5662
|
'KNOWLEDGE ./hejny-cv.md',
|
|
5754
5663
|
'KNOWLEDGE ./hejny-cv.pdf',
|
|
5755
5664
|
'KNOWLEDGE ./hejny-cv.docx',
|
|
5665
|
+
// <- TODO: [😿] Allow ONLY files scoped in the (sub)directory NOT ../ and test it
|
|
5756
5666
|
],
|
|
5757
5667
|
/**
|
|
5758
5668
|
* Parses the KNOWLEDGE command
|
|
@@ -8427,38 +8337,359 @@ function pipelineStringToJsonSync(pipelineString) {
|
|
|
8427
8337
|
* Note: This function does not validate logic of the pipeline only the parsing
|
|
8428
8338
|
* Note: This function acts as compilation process
|
|
8429
8339
|
*
|
|
8430
|
-
* @param pipelineString {Promptbook} in string markdown format (.ptbk.md)
|
|
8431
|
-
* @param tools - Tools for the preparation and scraping - if not provided together with `llm`, the preparation will be skipped
|
|
8432
|
-
* @param options - Options and tools for the compilation
|
|
8433
|
-
* @returns {Promptbook} compiled in JSON format (.ptbk.json)
|
|
8434
|
-
* @throws {ParseError} if the promptbook string is not valid
|
|
8435
|
-
* @public exported from `@promptbook/core`
|
|
8340
|
+
* @param pipelineString {Promptbook} in string markdown format (.ptbk.md)
|
|
8341
|
+
* @param tools - Tools for the preparation and scraping - if not provided together with `llm`, the preparation will be skipped
|
|
8342
|
+
* @param options - Options and tools for the compilation
|
|
8343
|
+
* @returns {Promptbook} compiled in JSON format (.ptbk.json)
|
|
8344
|
+
* @throws {ParseError} if the promptbook string is not valid
|
|
8345
|
+
* @public exported from `@promptbook/core`
|
|
8346
|
+
*/
|
|
8347
|
+
function pipelineStringToJson(pipelineString, tools, options) {
|
|
8348
|
+
return __awaiter(this, void 0, void 0, function () {
|
|
8349
|
+
var pipelineJson;
|
|
8350
|
+
return __generator(this, function (_a) {
|
|
8351
|
+
switch (_a.label) {
|
|
8352
|
+
case 0:
|
|
8353
|
+
pipelineJson = pipelineStringToJsonSync(pipelineString);
|
|
8354
|
+
if (!(tools !== undefined && tools.llm !== undefined)) return [3 /*break*/, 2];
|
|
8355
|
+
return [4 /*yield*/, preparePipeline(pipelineJson, tools, options || {
|
|
8356
|
+
rootDirname: null,
|
|
8357
|
+
})];
|
|
8358
|
+
case 1:
|
|
8359
|
+
pipelineJson = _a.sent();
|
|
8360
|
+
_a.label = 2;
|
|
8361
|
+
case 2:
|
|
8362
|
+
// Note: No need to use `$asDeeplyFrozenSerializableJson` because `pipelineStringToJsonSync` and `preparePipeline` already do that
|
|
8363
|
+
return [2 /*return*/, pipelineJson];
|
|
8364
|
+
}
|
|
8365
|
+
});
|
|
8366
|
+
});
|
|
8367
|
+
}
|
|
8368
|
+
/**
|
|
8369
|
+
* TODO: [🏏] Leverage the batch API and build queues @see https://platform.openai.com/docs/guides/batch
|
|
8370
|
+
* TODO: [🛠] Actions, instruments (and maybe knowledge) => Functions and tools
|
|
8371
|
+
* TODO: [🧠] Should be in generated JSON file GENERATOR_WARNING
|
|
8372
|
+
*/
|
|
8373
|
+
|
|
8374
|
+
// Note: We want to use the `exec` as async function
|
|
8375
|
+
var exec$1 = promisify(exec$2);
|
|
8376
|
+
/**
|
|
8377
|
+
* @@@
|
|
8378
|
+
*
|
|
8379
|
+
* @private within the repository
|
|
8380
|
+
*/
|
|
8381
|
+
function locateAppOnLinux(_a) {
|
|
8382
|
+
var appName = _a.appName, linuxWhich = _a.linuxWhich;
|
|
8383
|
+
return __awaiter(this, void 0, void 0, function () {
|
|
8384
|
+
var _b, stderr, stdout, error_1;
|
|
8385
|
+
return __generator(this, function (_c) {
|
|
8386
|
+
switch (_c.label) {
|
|
8387
|
+
case 0:
|
|
8388
|
+
_c.trys.push([0, 2, , 3]);
|
|
8389
|
+
return [4 /*yield*/, exec$1("which ".concat(linuxWhich))];
|
|
8390
|
+
case 1:
|
|
8391
|
+
_b = _c.sent(), stderr = _b.stderr, stdout = _b.stdout;
|
|
8392
|
+
if (!stderr && stdout) {
|
|
8393
|
+
return [2 /*return*/, stdout.trim()];
|
|
8394
|
+
}
|
|
8395
|
+
throw new Error("Can not locate app ".concat(appName, " on Linux.\n ").concat(stderr));
|
|
8396
|
+
case 2:
|
|
8397
|
+
error_1 = _c.sent();
|
|
8398
|
+
if (!(error_1 instanceof Error)) {
|
|
8399
|
+
throw error_1;
|
|
8400
|
+
}
|
|
8401
|
+
return [2 /*return*/, null];
|
|
8402
|
+
case 3: return [2 /*return*/];
|
|
8403
|
+
}
|
|
8404
|
+
});
|
|
8405
|
+
});
|
|
8406
|
+
}
|
|
8407
|
+
/**
|
|
8408
|
+
* TODO: [🧠][♿] Maybe export through `@promptbook/node`
|
|
8409
|
+
* Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
|
|
8410
|
+
*/
|
|
8411
|
+
|
|
8412
|
+
/**
|
|
8413
|
+
* @@@
|
|
8414
|
+
*
|
|
8415
|
+
* @public exported from `@promptbook/node`
|
|
8416
|
+
*/
|
|
8417
|
+
function $provideFilesystemForNode(options) {
|
|
8418
|
+
if (!$isRunningInNode()) {
|
|
8419
|
+
throw new EnvironmentMismatchError('Function `$provideFilesystemForNode` works only in Node.js environment');
|
|
8420
|
+
}
|
|
8421
|
+
var _a = (options || {}).isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a;
|
|
8422
|
+
TODO_USE(isVerbose);
|
|
8423
|
+
return {
|
|
8424
|
+
stat: stat,
|
|
8425
|
+
access: access,
|
|
8426
|
+
constants: constants,
|
|
8427
|
+
readFile: readFile,
|
|
8428
|
+
writeFile: writeFile,
|
|
8429
|
+
readdir: readdir,
|
|
8430
|
+
};
|
|
8431
|
+
}
|
|
8432
|
+
/**
|
|
8433
|
+
* Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
|
|
8434
|
+
*/
|
|
8435
|
+
|
|
8436
|
+
/**
|
|
8437
|
+
* Checks if the file is executable
|
|
8438
|
+
*
|
|
8439
|
+
* @private within the repository
|
|
8440
|
+
*/
|
|
8441
|
+
function isExecutable(path, fs) {
|
|
8442
|
+
return __awaiter(this, void 0, void 0, function () {
|
|
8443
|
+
return __generator(this, function (_a) {
|
|
8444
|
+
switch (_a.label) {
|
|
8445
|
+
case 0:
|
|
8446
|
+
_a.trys.push([0, 2, , 3]);
|
|
8447
|
+
return [4 /*yield*/, fs.access(path, fs.constants.X_OK)];
|
|
8448
|
+
case 1:
|
|
8449
|
+
_a.sent();
|
|
8450
|
+
return [2 /*return*/, true];
|
|
8451
|
+
case 2:
|
|
8452
|
+
_a.sent();
|
|
8453
|
+
return [2 /*return*/, false];
|
|
8454
|
+
case 3: return [2 /*return*/];
|
|
8455
|
+
}
|
|
8456
|
+
});
|
|
8457
|
+
});
|
|
8458
|
+
}
|
|
8459
|
+
/**
|
|
8460
|
+
* Note: Not [~🟢~] because it is not directly dependent on `fs
|
|
8461
|
+
* TODO: [🖇] What about symlinks?
|
|
8462
|
+
*/
|
|
8463
|
+
|
|
8464
|
+
// Note: Module `userhome` has no types available, so it is imported using `require`
|
|
8465
|
+
// @see https://stackoverflow.com/questions/37000981/how-to-import-node-module-in-typescript-without-type-definitions
|
|
8466
|
+
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
|
8467
|
+
var userhome = require('userhome');
|
|
8468
|
+
// Note: We want to use the `exec` as async function
|
|
8469
|
+
var exec = promisify(exec$2);
|
|
8470
|
+
/**
|
|
8471
|
+
* @@@
|
|
8472
|
+
*
|
|
8473
|
+
* @private within the repository
|
|
8474
|
+
*/
|
|
8475
|
+
function locateAppOnMacOs(_a) {
|
|
8476
|
+
var appName = _a.appName, macOsName = _a.macOsName;
|
|
8477
|
+
return __awaiter(this, void 0, void 0, function () {
|
|
8478
|
+
var toExec, regPath, altPath, _b, stderr, stdout, error_1;
|
|
8479
|
+
return __generator(this, function (_c) {
|
|
8480
|
+
switch (_c.label) {
|
|
8481
|
+
case 0:
|
|
8482
|
+
_c.trys.push([0, 6, , 7]);
|
|
8483
|
+
toExec = "/Contents/MacOS/".concat(macOsName);
|
|
8484
|
+
regPath = "/Applications/".concat(macOsName, ".app") + toExec;
|
|
8485
|
+
altPath = userhome(regPath.slice(1));
|
|
8486
|
+
return [4 /*yield*/, isExecutable(regPath, $provideFilesystemForNode())];
|
|
8487
|
+
case 1:
|
|
8488
|
+
if (!_c.sent()) return [3 /*break*/, 2];
|
|
8489
|
+
return [2 /*return*/, regPath];
|
|
8490
|
+
case 2: return [4 /*yield*/, isExecutable(altPath, $provideFilesystemForNode())];
|
|
8491
|
+
case 3:
|
|
8492
|
+
if (_c.sent()) {
|
|
8493
|
+
return [2 /*return*/, altPath];
|
|
8494
|
+
}
|
|
8495
|
+
_c.label = 4;
|
|
8496
|
+
case 4: return [4 /*yield*/, exec("mdfind 'kMDItemDisplayName == \"".concat(macOsName, "\" && kMDItemKind == Application'"))];
|
|
8497
|
+
case 5:
|
|
8498
|
+
_b = _c.sent(), stderr = _b.stderr, stdout = _b.stdout;
|
|
8499
|
+
if (!stderr && stdout) {
|
|
8500
|
+
return [2 /*return*/, stdout.trim() + toExec];
|
|
8501
|
+
}
|
|
8502
|
+
throw new Error("Can not locate app ".concat(appName, " on macOS.\n ").concat(stderr));
|
|
8503
|
+
case 6:
|
|
8504
|
+
error_1 = _c.sent();
|
|
8505
|
+
if (!(error_1 instanceof Error)) {
|
|
8506
|
+
throw error_1;
|
|
8507
|
+
}
|
|
8508
|
+
return [2 /*return*/, null];
|
|
8509
|
+
case 7: return [2 /*return*/];
|
|
8510
|
+
}
|
|
8511
|
+
});
|
|
8512
|
+
});
|
|
8513
|
+
}
|
|
8514
|
+
/**
|
|
8515
|
+
* TODO: [🧠][♿] Maybe export through `@promptbook/node`
|
|
8516
|
+
* Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
|
|
8517
|
+
*/
|
|
8518
|
+
|
|
8519
|
+
/**
|
|
8520
|
+
* @@@
|
|
8521
|
+
*
|
|
8522
|
+
* @private within the repository
|
|
8523
|
+
*/
|
|
8524
|
+
function locateAppOnWindows(_a) {
|
|
8525
|
+
var appName = _a.appName, windowsSuffix = _a.windowsSuffix;
|
|
8526
|
+
return __awaiter(this, void 0, void 0, function () {
|
|
8527
|
+
var prefixes, prefixes_1, prefixes_1_1, prefix, path, e_1_1, error_1;
|
|
8528
|
+
var e_1, _b;
|
|
8529
|
+
return __generator(this, function (_c) {
|
|
8530
|
+
switch (_c.label) {
|
|
8531
|
+
case 0:
|
|
8532
|
+
_c.trys.push([0, 9, , 10]);
|
|
8533
|
+
prefixes = [
|
|
8534
|
+
process.env.LOCALAPPDATA,
|
|
8535
|
+
join(process.env.LOCALAPPDATA || '', 'Programs'),
|
|
8536
|
+
process.env.PROGRAMFILES,
|
|
8537
|
+
process.env['PROGRAMFILES(X86)'],
|
|
8538
|
+
];
|
|
8539
|
+
_c.label = 1;
|
|
8540
|
+
case 1:
|
|
8541
|
+
_c.trys.push([1, 6, 7, 8]);
|
|
8542
|
+
prefixes_1 = __values(prefixes), prefixes_1_1 = prefixes_1.next();
|
|
8543
|
+
_c.label = 2;
|
|
8544
|
+
case 2:
|
|
8545
|
+
if (!!prefixes_1_1.done) return [3 /*break*/, 5];
|
|
8546
|
+
prefix = prefixes_1_1.value;
|
|
8547
|
+
path = prefix + windowsSuffix;
|
|
8548
|
+
return [4 /*yield*/, isExecutable(path, $provideFilesystemForNode())];
|
|
8549
|
+
case 3:
|
|
8550
|
+
if (_c.sent()) {
|
|
8551
|
+
return [2 /*return*/, path];
|
|
8552
|
+
}
|
|
8553
|
+
_c.label = 4;
|
|
8554
|
+
case 4:
|
|
8555
|
+
prefixes_1_1 = prefixes_1.next();
|
|
8556
|
+
return [3 /*break*/, 2];
|
|
8557
|
+
case 5: return [3 /*break*/, 8];
|
|
8558
|
+
case 6:
|
|
8559
|
+
e_1_1 = _c.sent();
|
|
8560
|
+
e_1 = { error: e_1_1 };
|
|
8561
|
+
return [3 /*break*/, 8];
|
|
8562
|
+
case 7:
|
|
8563
|
+
try {
|
|
8564
|
+
if (prefixes_1_1 && !prefixes_1_1.done && (_b = prefixes_1.return)) _b.call(prefixes_1);
|
|
8565
|
+
}
|
|
8566
|
+
finally { if (e_1) throw e_1.error; }
|
|
8567
|
+
return [7 /*endfinally*/];
|
|
8568
|
+
case 8: throw new Error("Can not locate app ".concat(appName, " on Windows."));
|
|
8569
|
+
case 9:
|
|
8570
|
+
error_1 = _c.sent();
|
|
8571
|
+
if (!(error_1 instanceof Error)) {
|
|
8572
|
+
throw error_1;
|
|
8573
|
+
}
|
|
8574
|
+
return [2 /*return*/, null];
|
|
8575
|
+
case 10: return [2 /*return*/];
|
|
8576
|
+
}
|
|
8577
|
+
});
|
|
8578
|
+
});
|
|
8579
|
+
}
|
|
8580
|
+
/**
|
|
8581
|
+
* TODO: [🧠][♿] Maybe export through `@promptbook/node`
|
|
8582
|
+
* Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
|
|
8583
|
+
*/
|
|
8584
|
+
|
|
8585
|
+
/**
|
|
8586
|
+
* Locates an application on the system
|
|
8587
|
+
*
|
|
8588
|
+
* @private within the repository
|
|
8589
|
+
*/
|
|
8590
|
+
function locateApp(options) {
|
|
8591
|
+
if (!$isRunningInNode()) {
|
|
8592
|
+
throw new EnvironmentMismatchError('Locating apps works only in Node.js environment');
|
|
8593
|
+
}
|
|
8594
|
+
var appName = options.appName, linuxWhich = options.linuxWhich, windowsSuffix = options.windowsSuffix, macOsName = options.macOsName;
|
|
8595
|
+
if (process.platform === 'win32') {
|
|
8596
|
+
if (windowsSuffix) {
|
|
8597
|
+
return locateAppOnWindows({ appName: appName, windowsSuffix: windowsSuffix });
|
|
8598
|
+
}
|
|
8599
|
+
else {
|
|
8600
|
+
throw new Error("".concat(appName, " is not available on Windows."));
|
|
8601
|
+
}
|
|
8602
|
+
}
|
|
8603
|
+
else if (process.platform === 'darwin') {
|
|
8604
|
+
if (macOsName) {
|
|
8605
|
+
return locateAppOnMacOs({ appName: appName, macOsName: macOsName });
|
|
8606
|
+
}
|
|
8607
|
+
else {
|
|
8608
|
+
throw new Error("".concat(appName, " is not available on macOS."));
|
|
8609
|
+
}
|
|
8610
|
+
}
|
|
8611
|
+
else {
|
|
8612
|
+
if (linuxWhich) {
|
|
8613
|
+
return locateAppOnLinux({ appName: appName, linuxWhich: linuxWhich });
|
|
8614
|
+
}
|
|
8615
|
+
else {
|
|
8616
|
+
throw new Error("".concat(appName, " is not available on Linux."));
|
|
8617
|
+
}
|
|
8618
|
+
}
|
|
8619
|
+
}
|
|
8620
|
+
/**
|
|
8621
|
+
* TODO: [🧠][♿] Maybe export through `@promptbook/node`
|
|
8622
|
+
* Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
|
|
8623
|
+
*/
|
|
8624
|
+
|
|
8625
|
+
/**
|
|
8626
|
+
* @@@
|
|
8627
|
+
*
|
|
8628
|
+
* @private within the repository
|
|
8629
|
+
*/
|
|
8630
|
+
function locateLibreoffice() {
|
|
8631
|
+
return locateApp({
|
|
8632
|
+
appName: 'Libreoffice',
|
|
8633
|
+
linuxWhich: 'libreoffice',
|
|
8634
|
+
windowsSuffix: '\\LibreOffice\\program\\soffice.exe',
|
|
8635
|
+
macOsName: 'LibreOffice',
|
|
8636
|
+
});
|
|
8637
|
+
}
|
|
8638
|
+
/**
|
|
8639
|
+
* TODO: [🧠][♿] Maybe export through `@promptbook/node` OR `@promptbook/legacy-documents`
|
|
8640
|
+
* Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
|
|
8641
|
+
*/
|
|
8642
|
+
|
|
8643
|
+
/**
|
|
8644
|
+
* @@@
|
|
8645
|
+
*
|
|
8646
|
+
* @private within the repository
|
|
8647
|
+
*/
|
|
8648
|
+
function locatePandoc() {
|
|
8649
|
+
return locateApp({
|
|
8650
|
+
appName: 'Pandoc',
|
|
8651
|
+
linuxWhich: 'pandoc',
|
|
8652
|
+
windowsSuffix: '\\Pandoc\\pandoc.exe',
|
|
8653
|
+
macOsName: 'Pandoc',
|
|
8654
|
+
});
|
|
8655
|
+
}
|
|
8656
|
+
/**
|
|
8657
|
+
* TODO: [🧠][♿] Maybe export through `@promptbook/node` OR `@promptbook/documents`
|
|
8658
|
+
* Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
|
|
8659
|
+
*/
|
|
8660
|
+
|
|
8661
|
+
/**
|
|
8662
|
+
* @@@
|
|
8663
|
+
*
|
|
8664
|
+
* @public exported from `@promptbook/node`
|
|
8436
8665
|
*/
|
|
8437
|
-
function
|
|
8666
|
+
function $provideExecutablesForNode(options) {
|
|
8438
8667
|
return __awaiter(this, void 0, void 0, function () {
|
|
8439
|
-
var
|
|
8440
|
-
|
|
8441
|
-
|
|
8668
|
+
var _a, _b, isAutoInstalled, _c, isVerbose;
|
|
8669
|
+
var _d;
|
|
8670
|
+
return __generator(this, function (_e) {
|
|
8671
|
+
switch (_e.label) {
|
|
8442
8672
|
case 0:
|
|
8443
|
-
|
|
8444
|
-
|
|
8445
|
-
|
|
8446
|
-
|
|
8447
|
-
|
|
8673
|
+
if (!$isRunningInNode()) {
|
|
8674
|
+
throw new EnvironmentMismatchError('Function `$getScrapersForNode` works only in Node.js environment');
|
|
8675
|
+
}
|
|
8676
|
+
_a = options || {}, _b = _a.isAutoInstalled, isAutoInstalled = _b === void 0 ? DEFAULT_IS_AUTO_INSTALLED : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? DEFAULT_IS_VERBOSE : _c;
|
|
8677
|
+
TODO_USE(isAutoInstalled); // <- TODO: [🔱][🧠] Auto-install the executables
|
|
8678
|
+
TODO_USE(isVerbose);
|
|
8679
|
+
_d = {};
|
|
8680
|
+
return [4 /*yield*/, locatePandoc()];
|
|
8448
8681
|
case 1:
|
|
8449
|
-
|
|
8450
|
-
|
|
8451
|
-
case 2:
|
|
8452
|
-
|
|
8453
|
-
return [2 /*return*/, pipelineJson];
|
|
8682
|
+
_d.pandocPath = (_e.sent()) || undefined;
|
|
8683
|
+
return [4 /*yield*/, locateLibreoffice()];
|
|
8684
|
+
case 2: return [2 /*return*/, (_d.libreOfficePath = (_e.sent()) || undefined,
|
|
8685
|
+
_d)];
|
|
8454
8686
|
}
|
|
8455
8687
|
});
|
|
8456
8688
|
});
|
|
8457
8689
|
}
|
|
8458
8690
|
/**
|
|
8459
|
-
* TODO: [
|
|
8460
|
-
*
|
|
8461
|
-
* TODO: [🧠] Should be in generated JSON file GENERATOR_WARNING
|
|
8691
|
+
* TODO: [🧠] Allow to override the executables without need to call `locatePandoc` / `locateLibreoffice` in case of provided
|
|
8692
|
+
* Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
|
|
8462
8693
|
*/
|
|
8463
8694
|
|
|
8464
8695
|
/**
|
|
@@ -8630,7 +8861,7 @@ function $registeredLlmToolsMessage() {
|
|
|
8630
8861
|
*/
|
|
8631
8862
|
function createLlmToolsFromConfiguration(configuration, options) {
|
|
8632
8863
|
if (options === void 0) { options = {}; }
|
|
8633
|
-
var _a = options.isVerbose, isVerbose = _a === void 0 ?
|
|
8864
|
+
var _a = options.isVerbose, isVerbose = _a === void 0 ? DEFAULT_IS_VERBOSE : _a, userId = options.userId;
|
|
8634
8865
|
var llmTools = configuration.map(function (llmConfiguration) {
|
|
8635
8866
|
var registeredItem = $llmToolsRegister
|
|
8636
8867
|
.list()
|
|
@@ -8641,7 +8872,7 @@ function createLlmToolsFromConfiguration(configuration, options) {
|
|
|
8641
8872
|
if (registeredItem === undefined) {
|
|
8642
8873
|
throw new Error(spaceTrim$1(function (block) { return "\n There is no constructor for LLM provider `".concat(llmConfiguration.className, "` from `").concat(llmConfiguration.packageName, "`\n\n You have probably forgotten install and import the provider package.\n To fix this issue, you can:\n\n Install:\n\n > npm install ").concat(llmConfiguration.packageName, "\n\n And import:\n\n > import '").concat(llmConfiguration.packageName, "';\n\n\n ").concat(block($registeredLlmToolsMessage()), "\n "); }));
|
|
8643
8874
|
}
|
|
8644
|
-
return registeredItem(__assign({ isVerbose: isVerbose }, llmConfiguration.options));
|
|
8875
|
+
return registeredItem(__assign({ isVerbose: isVerbose, userId: userId }, llmConfiguration.options));
|
|
8645
8876
|
});
|
|
8646
8877
|
return joinLlmExecutionTools.apply(void 0, __spreadArray([], __read(llmTools), false));
|
|
8647
8878
|
}
|
|
@@ -8691,6 +8922,64 @@ function $provideLlmToolsFromEnv(options) {
|
|
|
8691
8922
|
* TODO: [®] DRY Register logic
|
|
8692
8923
|
*/
|
|
8693
8924
|
|
|
8925
|
+
/**
|
|
8926
|
+
* @@@
|
|
8927
|
+
*
|
|
8928
|
+
* 1) @@@
|
|
8929
|
+
* 2) @@@
|
|
8930
|
+
*
|
|
8931
|
+
* @public exported from `@promptbook/node`
|
|
8932
|
+
*/
|
|
8933
|
+
function $provideScrapersForNode(tools, options) {
|
|
8934
|
+
return __awaiter(this, void 0, void 0, function () {
|
|
8935
|
+
var _a, _b, isAutoInstalled, _c, isVerbose, scrapers, _d, _e, scraperFactory, scraper, e_1_1;
|
|
8936
|
+
var e_1, _f;
|
|
8937
|
+
return __generator(this, function (_g) {
|
|
8938
|
+
switch (_g.label) {
|
|
8939
|
+
case 0:
|
|
8940
|
+
if (!$isRunningInNode()) {
|
|
8941
|
+
throw new EnvironmentMismatchError('Function `$getScrapersForNode` works only in Node.js environment');
|
|
8942
|
+
}
|
|
8943
|
+
_a = options || {}, _b = _a.isAutoInstalled, isAutoInstalled = _b === void 0 ? DEFAULT_IS_AUTO_INSTALLED : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? DEFAULT_IS_VERBOSE : _c;
|
|
8944
|
+
TODO_USE(isAutoInstalled);
|
|
8945
|
+
TODO_USE(isVerbose);
|
|
8946
|
+
scrapers = [];
|
|
8947
|
+
_g.label = 1;
|
|
8948
|
+
case 1:
|
|
8949
|
+
_g.trys.push([1, 6, 7, 8]);
|
|
8950
|
+
_d = __values($scrapersRegister.list()), _e = _d.next();
|
|
8951
|
+
_g.label = 2;
|
|
8952
|
+
case 2:
|
|
8953
|
+
if (!!_e.done) return [3 /*break*/, 5];
|
|
8954
|
+
scraperFactory = _e.value;
|
|
8955
|
+
return [4 /*yield*/, scraperFactory(tools, options || {})];
|
|
8956
|
+
case 3:
|
|
8957
|
+
scraper = _g.sent();
|
|
8958
|
+
scrapers.push(scraper);
|
|
8959
|
+
_g.label = 4;
|
|
8960
|
+
case 4:
|
|
8961
|
+
_e = _d.next();
|
|
8962
|
+
return [3 /*break*/, 2];
|
|
8963
|
+
case 5: return [3 /*break*/, 8];
|
|
8964
|
+
case 6:
|
|
8965
|
+
e_1_1 = _g.sent();
|
|
8966
|
+
e_1 = { error: e_1_1 };
|
|
8967
|
+
return [3 /*break*/, 8];
|
|
8968
|
+
case 7:
|
|
8969
|
+
try {
|
|
8970
|
+
if (_e && !_e.done && (_f = _d.return)) _f.call(_d);
|
|
8971
|
+
}
|
|
8972
|
+
finally { if (e_1) throw e_1.error; }
|
|
8973
|
+
return [7 /*endfinally*/];
|
|
8974
|
+
case 8: return [2 /*return*/, scrapers];
|
|
8975
|
+
}
|
|
8976
|
+
});
|
|
8977
|
+
});
|
|
8978
|
+
}
|
|
8979
|
+
/**
|
|
8980
|
+
* Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
|
|
8981
|
+
*/
|
|
8982
|
+
|
|
8694
8983
|
/**
|
|
8695
8984
|
* Extracts code block from markdown.
|
|
8696
8985
|
*
|
|
@@ -9145,7 +9434,7 @@ var JavascriptExecutionTools = JavascriptEvalExecutionTools;
|
|
|
9145
9434
|
*/
|
|
9146
9435
|
function $provideExecutionToolsForNode(options) {
|
|
9147
9436
|
return __awaiter(this, void 0, void 0, function () {
|
|
9148
|
-
var fs, llm, tools;
|
|
9437
|
+
var fs, llm, executables, tools;
|
|
9149
9438
|
var _a;
|
|
9150
9439
|
return __generator(this, function (_b) {
|
|
9151
9440
|
switch (_b.label) {
|
|
@@ -9155,12 +9444,16 @@ function $provideExecutionToolsForNode(options) {
|
|
|
9155
9444
|
}
|
|
9156
9445
|
fs = $provideFilesystemForNode();
|
|
9157
9446
|
llm = $provideLlmToolsFromEnv(options);
|
|
9447
|
+
return [4 /*yield*/, $provideExecutablesForNode(options)];
|
|
9448
|
+
case 1:
|
|
9449
|
+
executables = _b.sent();
|
|
9158
9450
|
_a = {
|
|
9159
9451
|
llm: llm,
|
|
9160
|
-
fs: fs
|
|
9452
|
+
fs: fs,
|
|
9453
|
+
executables: executables
|
|
9161
9454
|
};
|
|
9162
|
-
return [4 /*yield*/, $provideScrapersForNode({ fs: fs, llm: llm }, options)];
|
|
9163
|
-
case
|
|
9455
|
+
return [4 /*yield*/, $provideScrapersForNode({ fs: fs, llm: llm, executables: executables }, options)];
|
|
9456
|
+
case 2:
|
|
9164
9457
|
tools = (_a.scrapers = _b.sent(),
|
|
9165
9458
|
_a.script = [new JavascriptExecutionTools(options)],
|
|
9166
9459
|
_a);
|
|
@@ -9398,7 +9691,9 @@ function createCollectionFromDirectory(path, tools, options) {
|
|
|
9398
9691
|
throw new EnvironmentMismatchError('Can not create collection without filesystem tools');
|
|
9399
9692
|
// <- TODO: [🧠] What is the best error type here`
|
|
9400
9693
|
}
|
|
9401
|
-
makedLibraryFilePath = join(path, "".concat(
|
|
9694
|
+
makedLibraryFilePath = join(path, "".concat(DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME
|
|
9695
|
+
// <- TODO: [🦒] Allow to override (pass different value into the function)
|
|
9696
|
+
, ".json"));
|
|
9402
9697
|
return [4 /*yield*/, isFileExisting(makedLibraryFilePath, tools.fs)];
|
|
9403
9698
|
case 3:
|
|
9404
9699
|
if (!(_f.sent())) {
|
|
@@ -9409,7 +9704,7 @@ function createCollectionFromDirectory(path, tools, options) {
|
|
|
9409
9704
|
// TODO: !! Implement;
|
|
9410
9705
|
// TODO: [🌗]
|
|
9411
9706
|
}
|
|
9412
|
-
_a = options || {}, _b = _a.isRecursive, isRecursive = _b === void 0 ? true : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ?
|
|
9707
|
+
_a = options || {}, _b = _a.isRecursive, isRecursive = _b === void 0 ? true : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? DEFAULT_IS_VERBOSE : _c, _d = _a.isLazyLoaded, isLazyLoaded = _d === void 0 ? false : _d, _e = _a.isCrashedOnError, isCrashedOnError = _e === void 0 ? true : _e;
|
|
9413
9708
|
collection = createCollectionFromPromise(function () { return __awaiter(_this, void 0, void 0, function () {
|
|
9414
9709
|
var fileNames, collection, _loop_1, fileNames_1, fileNames_1_1, fileName, e_1_1;
|
|
9415
9710
|
var e_1, _a;
|
|
@@ -9881,7 +10176,7 @@ function $currentDate() {
|
|
|
9881
10176
|
function cacheLlmTools(llmTools, options) {
|
|
9882
10177
|
var _this = this;
|
|
9883
10178
|
if (options === void 0) { options = {}; }
|
|
9884
|
-
var _a = options.storage, storage = _a === void 0 ? new MemoryStorage() : _a, _b = options.
|
|
10179
|
+
var _a = options.storage, storage = _a === void 0 ? new MemoryStorage() : _a, _b = options.isCacheReloaded, isCacheReloaded = _b === void 0 ? false : _b;
|
|
9885
10180
|
var proxyTools = __assign(__assign({}, llmTools), {
|
|
9886
10181
|
// <- Note: [🥫]
|
|
9887
10182
|
get title() {
|
|
@@ -9903,7 +10198,7 @@ function cacheLlmTools(llmTools, options) {
|
|
|
9903
10198
|
key = titleToName(prompt.title.substring(0, MAX_FILENAME_LENGTH - 10) +
|
|
9904
10199
|
'-' +
|
|
9905
10200
|
sha256(hexEncoder.parse(JSON.stringify(prompt.parameters))).toString( /* hex */));
|
|
9906
|
-
if (!!
|
|
10201
|
+
if (!!isCacheReloaded) return [3 /*break*/, 2];
|
|
9907
10202
|
return [4 /*yield*/, storage.getItem(key)];
|
|
9908
10203
|
case 1:
|
|
9909
10204
|
_a = _c.sent();
|
|
@@ -9989,12 +10284,14 @@ function $provideLlmToolsForCli(options) {
|
|
|
9989
10284
|
if (!$isRunningInNode()) {
|
|
9990
10285
|
throw new EnvironmentMismatchError('Function `$provideLlmToolsForTestingAndScriptsAndPlayground` works only in Node.js environment');
|
|
9991
10286
|
}
|
|
9992
|
-
var
|
|
10287
|
+
var isCacheReloaded = (options !== null && options !== void 0 ? options : {}).isCacheReloaded;
|
|
9993
10288
|
return cacheLlmTools(countTotalUsage(
|
|
9994
10289
|
// <- Note: for example here we don`t want the [🌯]
|
|
9995
10290
|
$provideLlmToolsFromEnv()), {
|
|
9996
|
-
storage: new FileCacheStorage({ fs: $provideFilesystemForNode() }, {
|
|
9997
|
-
|
|
10291
|
+
storage: new FileCacheStorage({ fs: $provideFilesystemForNode() }, {
|
|
10292
|
+
rootFolderPath: join(process.cwd(), DEFAULT_EXECUTIONS_CACHE_DIRNAME),
|
|
10293
|
+
}),
|
|
10294
|
+
isCacheReloaded: isCacheReloaded,
|
|
9998
10295
|
});
|
|
9999
10296
|
}
|
|
10000
10297
|
/**
|
|
@@ -10021,13 +10318,13 @@ function initializeMakeCommand(program) {
|
|
|
10021
10318
|
makeCommand.option('-f, --format <format>', spaceTrim$1("\n Output format of builded collection \"javascript\", \"typescript\" or \"json\"\n\n Note: You can use multiple formats separated by comma\n "), 'javascript' /* <- Note: [🏳🌈] */);
|
|
10022
10319
|
makeCommand.option('--no-validation', "Do not validate logic of pipelines in collection", true);
|
|
10023
10320
|
makeCommand.option('--validation', "Types of validations separated by comma (options \"logic\",\"imports\")", 'logic,imports');
|
|
10024
|
-
makeCommand.option('--reload
|
|
10321
|
+
makeCommand.option('--reload', "Call LLM models even if same prompt with result is in the cache", false);
|
|
10025
10322
|
makeCommand.option('--verbose', "Is output verbose", false);
|
|
10026
|
-
makeCommand.option('-o, --out-file <path>', spaceTrim$1("\n Where to save the builded collection\n\n Note: If you keep it \"".concat(
|
|
10323
|
+
makeCommand.option('-o, --out-file <path>', spaceTrim$1("\n Where to save the builded collection\n\n Note: If you keep it \"".concat(DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME, "\" it will be saved in the root of the promptbook directory\n If you set it to a path, it will be saved in that path\n BUT you can use only one format and set correct extension\n ")), DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME);
|
|
10027
10324
|
makeCommand.action(function (path, _a) {
|
|
10028
|
-
var projectName = _a.projectName, format = _a.format, validation = _a.validation,
|
|
10325
|
+
var projectName = _a.projectName, format = _a.format, validation = _a.validation, isCacheReloaded = _a.reloadCache, isVerbose = _a.verbose, outFile = _a.outFile;
|
|
10029
10326
|
return __awaiter(_this, void 0, void 0, function () {
|
|
10030
|
-
var formats, validations, options, fs, llm, tools, collection, validations_1, validations_1_1, validation_1, _b, _c, pipelineUrl, pipeline, e_1_1, e_2_1, collectionJson, collectionJsonString, collectionJsonItems, saveFile;
|
|
10327
|
+
var formats, validations, options, fs, llm, executables, tools, collection, validations_1, validations_1_1, validation_1, _b, _c, pipelineUrl, pipeline, e_1_1, e_2_1, collectionJson, collectionJsonString, collectionJsonItems, saveFile;
|
|
10031
10328
|
var _d, e_2, _e, e_1, _f;
|
|
10032
10329
|
var _this = this;
|
|
10033
10330
|
return __generator(this, function (_g) {
|
|
@@ -10041,22 +10338,25 @@ function initializeMakeCommand(program) {
|
|
|
10041
10338
|
.split(',')
|
|
10042
10339
|
.map(function (_) { return _.trim(); })
|
|
10043
10340
|
.filter(function (_) { return _ !== ''; });
|
|
10044
|
-
if (outFile !==
|
|
10341
|
+
if (outFile !== DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME && formats.length !== 1) {
|
|
10045
10342
|
console.error(colors.red("You can only use one format if you specify --out-file"));
|
|
10046
10343
|
process.exit(1);
|
|
10047
10344
|
}
|
|
10048
10345
|
options = {
|
|
10049
10346
|
isVerbose: isVerbose,
|
|
10050
|
-
|
|
10347
|
+
isCacheReloaded: isCacheReloaded,
|
|
10051
10348
|
};
|
|
10052
10349
|
fs = $provideFilesystemForNode(options);
|
|
10053
10350
|
llm = $provideLlmToolsForCli(options);
|
|
10351
|
+
return [4 /*yield*/, $provideExecutablesForNode(options)];
|
|
10352
|
+
case 1:
|
|
10353
|
+
executables = _g.sent();
|
|
10054
10354
|
_d = {
|
|
10055
10355
|
llm: llm,
|
|
10056
10356
|
fs: fs
|
|
10057
10357
|
};
|
|
10058
|
-
return [4 /*yield*/, $provideScrapersForNode({ fs: fs, llm: llm }, options)];
|
|
10059
|
-
case
|
|
10358
|
+
return [4 /*yield*/, $provideScrapersForNode({ fs: fs, llm: llm, executables: executables }, options)];
|
|
10359
|
+
case 2:
|
|
10060
10360
|
tools = (_d.scrapers = _g.sent(),
|
|
10061
10361
|
_d.script = [
|
|
10062
10362
|
/*new JavascriptExecutionTools(options)*/
|
|
@@ -10065,31 +10365,31 @@ function initializeMakeCommand(program) {
|
|
|
10065
10365
|
return [4 /*yield*/, createCollectionFromDirectory(path, tools, {
|
|
10066
10366
|
isVerbose: isVerbose,
|
|
10067
10367
|
isRecursive: true,
|
|
10068
|
-
// <- TODO: [🍖]
|
|
10368
|
+
// <- TODO: [🍖] Add `intermediateFilesStrategy`
|
|
10069
10369
|
})];
|
|
10070
|
-
case 2:
|
|
10071
|
-
collection = _g.sent();
|
|
10072
|
-
_g.label = 3;
|
|
10073
10370
|
case 3:
|
|
10074
|
-
_g.
|
|
10075
|
-
validations_1 = __values(validations), validations_1_1 = validations_1.next();
|
|
10371
|
+
collection = _g.sent();
|
|
10076
10372
|
_g.label = 4;
|
|
10077
10373
|
case 4:
|
|
10078
|
-
|
|
10079
|
-
|
|
10374
|
+
_g.trys.push([4, 16, 17, 18]);
|
|
10375
|
+
validations_1 = __values(validations), validations_1_1 = validations_1.next();
|
|
10080
10376
|
_g.label = 5;
|
|
10081
10377
|
case 5:
|
|
10082
|
-
|
|
10378
|
+
if (!!validations_1_1.done) return [3 /*break*/, 15];
|
|
10379
|
+
validation_1 = validations_1_1.value;
|
|
10380
|
+
_g.label = 6;
|
|
10381
|
+
case 6:
|
|
10382
|
+
_g.trys.push([6, 12, 13, 14]);
|
|
10083
10383
|
e_1 = void 0;
|
|
10084
10384
|
return [4 /*yield*/, collection.listPipelines()];
|
|
10085
|
-
case 6:
|
|
10086
|
-
_b = (__values.apply(void 0, [_g.sent()])), _c = _b.next();
|
|
10087
|
-
_g.label = 7;
|
|
10088
10385
|
case 7:
|
|
10089
|
-
|
|
10386
|
+
_b = (__values.apply(void 0, [_g.sent()])), _c = _b.next();
|
|
10387
|
+
_g.label = 8;
|
|
10388
|
+
case 8:
|
|
10389
|
+
if (!!_c.done) return [3 /*break*/, 11];
|
|
10090
10390
|
pipelineUrl = _c.value;
|
|
10091
10391
|
return [4 /*yield*/, collection.getPipelineByUrl(pipelineUrl)];
|
|
10092
|
-
case
|
|
10392
|
+
case 9:
|
|
10093
10393
|
pipeline = _g.sent();
|
|
10094
10394
|
if (validation_1 === 'logic') {
|
|
10095
10395
|
validatePipeline(pipeline);
|
|
@@ -10097,37 +10397,37 @@ function initializeMakeCommand(program) {
|
|
|
10097
10397
|
console.info(colors.cyan("Validated logic of ".concat(pipeline.pipelineUrl)));
|
|
10098
10398
|
}
|
|
10099
10399
|
}
|
|
10100
|
-
_g.label =
|
|
10101
|
-
case
|
|
10400
|
+
_g.label = 10;
|
|
10401
|
+
case 10:
|
|
10102
10402
|
_c = _b.next();
|
|
10103
|
-
return [3 /*break*/,
|
|
10104
|
-
case
|
|
10105
|
-
case
|
|
10403
|
+
return [3 /*break*/, 8];
|
|
10404
|
+
case 11: return [3 /*break*/, 14];
|
|
10405
|
+
case 12:
|
|
10106
10406
|
e_1_1 = _g.sent();
|
|
10107
10407
|
e_1 = { error: e_1_1 };
|
|
10108
|
-
return [3 /*break*/,
|
|
10109
|
-
case
|
|
10408
|
+
return [3 /*break*/, 14];
|
|
10409
|
+
case 13:
|
|
10110
10410
|
try {
|
|
10111
10411
|
if (_c && !_c.done && (_f = _b.return)) _f.call(_b);
|
|
10112
10412
|
}
|
|
10113
10413
|
finally { if (e_1) throw e_1.error; }
|
|
10114
10414
|
return [7 /*endfinally*/];
|
|
10115
|
-
case
|
|
10415
|
+
case 14:
|
|
10116
10416
|
validations_1_1 = validations_1.next();
|
|
10117
|
-
return [3 /*break*/,
|
|
10118
|
-
case
|
|
10119
|
-
case
|
|
10417
|
+
return [3 /*break*/, 5];
|
|
10418
|
+
case 15: return [3 /*break*/, 18];
|
|
10419
|
+
case 16:
|
|
10120
10420
|
e_2_1 = _g.sent();
|
|
10121
10421
|
e_2 = { error: e_2_1 };
|
|
10122
|
-
return [3 /*break*/,
|
|
10123
|
-
case
|
|
10422
|
+
return [3 /*break*/, 18];
|
|
10423
|
+
case 17:
|
|
10124
10424
|
try {
|
|
10125
10425
|
if (validations_1_1 && !validations_1_1.done && (_e = validations_1.return)) _e.call(validations_1);
|
|
10126
10426
|
}
|
|
10127
10427
|
finally { if (e_2) throw e_2.error; }
|
|
10128
10428
|
return [7 /*endfinally*/];
|
|
10129
|
-
case
|
|
10130
|
-
case
|
|
10429
|
+
case 18: return [4 /*yield*/, collectionToJson(collection)];
|
|
10430
|
+
case 19:
|
|
10131
10431
|
collectionJson = _g.sent();
|
|
10132
10432
|
collectionJsonString = stringifyPipelineJson(collectionJson).trim();
|
|
10133
10433
|
collectionJsonItems = (function () {
|
|
@@ -10146,9 +10446,9 @@ function initializeMakeCommand(program) {
|
|
|
10146
10446
|
return __generator(this, function (_a) {
|
|
10147
10447
|
switch (_a.label) {
|
|
10148
10448
|
case 0:
|
|
10149
|
-
filename = outFile !==
|
|
10449
|
+
filename = outFile !== DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME
|
|
10150
10450
|
? outFile
|
|
10151
|
-
: join(path, "".concat(
|
|
10451
|
+
: join(path, "".concat(DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME, ".").concat(extension));
|
|
10152
10452
|
if (!outFile.endsWith(".".concat(extension))) {
|
|
10153
10453
|
console.warn(colors.yellow("Warning: Extension of output file should be \"".concat(extension, "\"")));
|
|
10154
10454
|
}
|
|
@@ -10164,27 +10464,27 @@ function initializeMakeCommand(program) {
|
|
|
10164
10464
|
}
|
|
10165
10465
|
});
|
|
10166
10466
|
}); };
|
|
10167
|
-
if (!formats.includes('json')) return [3 /*break*/,
|
|
10467
|
+
if (!formats.includes('json')) return [3 /*break*/, 21];
|
|
10168
10468
|
formats = formats.filter(function (format) { return format !== 'json'; });
|
|
10169
10469
|
return [4 /*yield*/, saveFile('json', collectionJsonString)];
|
|
10170
|
-
case 19:
|
|
10171
|
-
_g.sent();
|
|
10172
|
-
_g.label = 20;
|
|
10173
10470
|
case 20:
|
|
10174
|
-
|
|
10471
|
+
_g.sent();
|
|
10472
|
+
_g.label = 21;
|
|
10473
|
+
case 21:
|
|
10474
|
+
if (!(formats.includes('javascript') || formats.includes('js'))) return [3 /*break*/, 23];
|
|
10175
10475
|
formats = formats.filter(function (format) { return format !== 'javascript' && format !== 'js'; });
|
|
10176
10476
|
return [4 /*yield*/, saveFile('js', spaceTrim$1(function (block) { return "\n // ".concat(block(GENERATOR_WARNING_BY_PROMPTBOOK_CLI), "\n\n import { createCollectionFromJson } from '@promptbook/core';\n\n /**\n * Pipeline collection for ").concat(projectName, "\n *\n * ").concat(block(GENERATOR_WARNING_BY_PROMPTBOOK_CLI), "\n *\n * @private internal cache for `getPipelineCollection`\n */\n let pipelineCollection = null;\n\n\n /**\n * Get pipeline collection for ").concat(projectName, "\n *\n * ").concat(block(GENERATOR_WARNING_BY_PROMPTBOOK_CLI), "\n *\n * @returns {PipelineCollection} Library of promptbooks for ").concat(projectName, "\n */\n export function getPipelineCollection(){\n if(pipelineCollection===null){\n pipelineCollection = createCollectionFromJson(\n ").concat(block(collectionJsonItems), "\n );\n }\n\n return pipelineCollection;\n }\n "); }))];
|
|
10177
|
-
case 21:
|
|
10178
|
-
(_g.sent()) + '\n';
|
|
10179
|
-
_g.label = 22;
|
|
10180
10477
|
case 22:
|
|
10181
|
-
|
|
10478
|
+
(_g.sent()) + '\n';
|
|
10479
|
+
_g.label = 23;
|
|
10480
|
+
case 23:
|
|
10481
|
+
if (!(formats.includes('typescript') || formats.includes('ts'))) return [3 /*break*/, 25];
|
|
10182
10482
|
formats = formats.filter(function (format) { return format !== 'typescript' && format !== 'ts'; });
|
|
10183
10483
|
return [4 /*yield*/, saveFile('ts', spaceTrim$1(function (block) { return "\n // ".concat(block(GENERATOR_WARNING_BY_PROMPTBOOK_CLI), "\n\n import { createCollectionFromJson } from '@promptbook/core';\n import type { PipelineCollection } from '@promptbook/types';\n\n /**\n * Pipeline collection for ").concat(projectName, "\n *\n * ").concat(block(GENERATOR_WARNING_BY_PROMPTBOOK_CLI), "\n *\n * @private internal cache for `getPipelineCollection`\n */\n let pipelineCollection: null | PipelineCollection = null;\n\n\n /**\n * Get pipeline collection for ").concat(projectName, "\n *\n * ").concat(block(GENERATOR_WARNING_BY_PROMPTBOOK_CLI), "\n *\n * @returns {PipelineCollection} Library of promptbooks for ").concat(projectName, "\n */\n export function getPipelineCollection(): PipelineCollection{\n if(pipelineCollection===null){\n pipelineCollection = createCollectionFromJson(\n ").concat(block(collectionJsonItems), "\n );\n }\n\n return pipelineCollection;\n }\n "); }) + '\n')];
|
|
10184
|
-
case 23:
|
|
10185
|
-
_g.sent();
|
|
10186
|
-
_g.label = 24;
|
|
10187
10484
|
case 24:
|
|
10485
|
+
_g.sent();
|
|
10486
|
+
_g.label = 25;
|
|
10487
|
+
case 25:
|
|
10188
10488
|
if (formats.length > 0) {
|
|
10189
10489
|
console.warn(colors.yellow("Format ".concat(formats.join(' and '), " is not supported")));
|
|
10190
10490
|
}
|
|
@@ -10446,76 +10746,79 @@ function initializeTestCommand(program) {
|
|
|
10446
10746
|
// <- TODO: [🧟♂️] Unite path to promptbook collection argument
|
|
10447
10747
|
'Pipelines to test as glob pattern');
|
|
10448
10748
|
testCommand.option('-i, --ignore <glob>', "Ignore as glob pattern");
|
|
10449
|
-
testCommand.option('--reload
|
|
10749
|
+
testCommand.option('--reload', "Call LLM models even if same prompt with result is in the cache ", false);
|
|
10450
10750
|
testCommand.option('-v, --verbose', "Is output verbose", false);
|
|
10451
10751
|
testCommand.action(function (filesGlob, _a) {
|
|
10452
|
-
var ignore = _a.ignore,
|
|
10752
|
+
var ignore = _a.ignore, isCacheReloaded = _a.reloadCache, isVerbose = _a.verbose;
|
|
10453
10753
|
return __awaiter(_this, void 0, void 0, function () {
|
|
10454
|
-
var options, fs, llm, tools, filenames, filenames_1, filenames_1_1, filename, pipeline, pipelineMarkdown, _b, _c, error_1, e_1_1;
|
|
10754
|
+
var options, fs, llm, executables, tools, filenames, filenames_1, filenames_1_1, filename, pipeline, pipelineMarkdown, _b, _c, error_1, e_1_1;
|
|
10455
10755
|
var _d, e_1, _e;
|
|
10456
10756
|
return __generator(this, function (_f) {
|
|
10457
10757
|
switch (_f.label) {
|
|
10458
10758
|
case 0:
|
|
10459
10759
|
options = {
|
|
10460
10760
|
isVerbose: isVerbose,
|
|
10461
|
-
|
|
10761
|
+
isCacheReloaded: isCacheReloaded,
|
|
10462
10762
|
};
|
|
10463
10763
|
fs = $provideFilesystemForNode(options);
|
|
10464
10764
|
llm = $provideLlmToolsForCli(options);
|
|
10765
|
+
return [4 /*yield*/, $provideExecutablesForNode(options)];
|
|
10766
|
+
case 1:
|
|
10767
|
+
executables = _f.sent();
|
|
10465
10768
|
_d = {
|
|
10466
10769
|
llm: llm,
|
|
10467
10770
|
fs: fs
|
|
10468
10771
|
};
|
|
10469
|
-
return [4 /*yield*/, $provideScrapersForNode({ fs: fs, llm: llm }, options)];
|
|
10470
|
-
case
|
|
10772
|
+
return [4 /*yield*/, $provideScrapersForNode({ fs: fs, llm: llm, executables: executables }, options)];
|
|
10773
|
+
case 2:
|
|
10471
10774
|
tools = (_d.scrapers = _f.sent(),
|
|
10472
10775
|
_d.script = [
|
|
10473
10776
|
/*new JavascriptExecutionTools(options)*/
|
|
10474
10777
|
],
|
|
10475
10778
|
_d);
|
|
10476
10779
|
return [4 /*yield*/, glob(filesGlob, { ignore: ignore })];
|
|
10477
|
-
case 2:
|
|
10478
|
-
filenames = _f.sent();
|
|
10479
|
-
_f.label = 3;
|
|
10480
10780
|
case 3:
|
|
10481
|
-
_f.
|
|
10482
|
-
filenames_1 = __values(filenames), filenames_1_1 = filenames_1.next();
|
|
10781
|
+
filenames = _f.sent();
|
|
10483
10782
|
_f.label = 4;
|
|
10484
10783
|
case 4:
|
|
10485
|
-
|
|
10486
|
-
|
|
10784
|
+
_f.trys.push([4, 16, 17, 18]);
|
|
10785
|
+
filenames_1 = __values(filenames), filenames_1_1 = filenames_1.next();
|
|
10487
10786
|
_f.label = 5;
|
|
10488
10787
|
case 5:
|
|
10489
|
-
|
|
10788
|
+
if (!!filenames_1_1.done) return [3 /*break*/, 15];
|
|
10789
|
+
filename = filenames_1_1.value;
|
|
10790
|
+
_f.label = 6;
|
|
10791
|
+
case 6:
|
|
10792
|
+
_f.trys.push([6, 13, , 14]);
|
|
10490
10793
|
pipeline = void 0;
|
|
10491
|
-
if (!filename.endsWith('.ptbk.md')) return [3 /*break*/,
|
|
10794
|
+
if (!filename.endsWith('.ptbk.md')) return [3 /*break*/, 9];
|
|
10492
10795
|
return [4 /*yield*/, readFile(filename, 'utf-8')];
|
|
10493
|
-
case
|
|
10796
|
+
case 7:
|
|
10494
10797
|
pipelineMarkdown = (_f.sent());
|
|
10495
10798
|
return [4 /*yield*/, pipelineStringToJson(pipelineMarkdown, tools)];
|
|
10496
|
-
case
|
|
10799
|
+
case 8:
|
|
10497
10800
|
pipeline = _f.sent();
|
|
10498
10801
|
if (isVerbose) {
|
|
10499
10802
|
console.info(colors.green("Parsed ".concat(filename)));
|
|
10500
10803
|
}
|
|
10501
|
-
_f.label =
|
|
10502
|
-
case
|
|
10503
|
-
if (!filename.endsWith('.ptbk.json')) return [3 /*break*/,
|
|
10804
|
+
_f.label = 9;
|
|
10805
|
+
case 9:
|
|
10806
|
+
if (!filename.endsWith('.ptbk.json')) return [3 /*break*/, 11];
|
|
10504
10807
|
_c = (_b = JSON).parse;
|
|
10505
10808
|
return [4 /*yield*/, readFile(filename, 'utf-8')];
|
|
10506
|
-
case 9:
|
|
10507
|
-
pipeline = _c.apply(_b, [_f.sent()]);
|
|
10508
|
-
return [3 /*break*/, 11];
|
|
10509
10809
|
case 10:
|
|
10810
|
+
pipeline = _c.apply(_b, [_f.sent()]);
|
|
10811
|
+
return [3 /*break*/, 12];
|
|
10812
|
+
case 11:
|
|
10510
10813
|
if (isVerbose) {
|
|
10511
10814
|
console.info(colors.gray("Skipping ".concat(filename)));
|
|
10512
10815
|
}
|
|
10513
|
-
return [3 /*break*/,
|
|
10514
|
-
case
|
|
10816
|
+
return [3 /*break*/, 14];
|
|
10817
|
+
case 12:
|
|
10515
10818
|
validatePipeline(pipeline);
|
|
10516
10819
|
console.info(colors.green("Validated ".concat(filename)));
|
|
10517
|
-
return [3 /*break*/,
|
|
10518
|
-
case
|
|
10820
|
+
return [3 /*break*/, 14];
|
|
10821
|
+
case 13:
|
|
10519
10822
|
error_1 = _f.sent();
|
|
10520
10823
|
if (!(error_1 instanceof Error)) {
|
|
10521
10824
|
throw error_1;
|
|
@@ -10524,22 +10827,22 @@ function initializeTestCommand(program) {
|
|
|
10524
10827
|
console.error(colors.bgRed(error_1.name /* <- 11:11 */));
|
|
10525
10828
|
console.error(colors.red(error_1.stack || error_1.message));
|
|
10526
10829
|
process.exit(1);
|
|
10527
|
-
return [3 /*break*/,
|
|
10528
|
-
case
|
|
10830
|
+
return [3 /*break*/, 14];
|
|
10831
|
+
case 14:
|
|
10529
10832
|
filenames_1_1 = filenames_1.next();
|
|
10530
|
-
return [3 /*break*/,
|
|
10531
|
-
case
|
|
10532
|
-
case
|
|
10833
|
+
return [3 /*break*/, 5];
|
|
10834
|
+
case 15: return [3 /*break*/, 18];
|
|
10835
|
+
case 16:
|
|
10533
10836
|
e_1_1 = _f.sent();
|
|
10534
10837
|
e_1 = { error: e_1_1 };
|
|
10535
|
-
return [3 /*break*/,
|
|
10536
|
-
case
|
|
10838
|
+
return [3 /*break*/, 18];
|
|
10839
|
+
case 17:
|
|
10537
10840
|
try {
|
|
10538
10841
|
if (filenames_1_1 && !filenames_1_1.done && (_e = filenames_1.return)) _e.call(filenames_1);
|
|
10539
10842
|
}
|
|
10540
10843
|
finally { if (e_1) throw e_1.error; }
|
|
10541
10844
|
return [7 /*endfinally*/];
|
|
10542
|
-
case
|
|
10845
|
+
case 18:
|
|
10543
10846
|
console.info(colors.green("All pipelines are valid"));
|
|
10544
10847
|
process.exit(0);
|
|
10545
10848
|
return [2 /*return*/];
|
|
@@ -10653,6 +10956,7 @@ var _AnthropicClaudeMetadataRegistration = $llmToolsMetadataRegister.register({
|
|
|
10653
10956
|
* @public exported from `@promptbook/remote-client`
|
|
10654
10957
|
*/
|
|
10655
10958
|
var RemoteLlmExecutionTools = /** @class */ (function () {
|
|
10959
|
+
/* <- TODO: [🍚] `, Destroyable` */
|
|
10656
10960
|
function RemoteLlmExecutionTools(options) {
|
|
10657
10961
|
this.options = options;
|
|
10658
10962
|
}
|
|
@@ -10702,12 +11006,16 @@ var RemoteLlmExecutionTools = /** @class */ (function () {
|
|
|
10702
11006
|
if (this.options.isAnonymous) {
|
|
10703
11007
|
socket.emit('listModels-request', {
|
|
10704
11008
|
isAnonymous: true,
|
|
11009
|
+
userId: this.options.userId,
|
|
10705
11010
|
llmToolsConfiguration: this.options.llmToolsConfiguration,
|
|
10706
11011
|
} /* <- TODO: [🤛] */);
|
|
10707
11012
|
}
|
|
10708
11013
|
else {
|
|
10709
11014
|
socket.emit('listModels-request', {
|
|
10710
11015
|
isAnonymous: false,
|
|
11016
|
+
appId: this.options.appId,
|
|
11017
|
+
userId: this.options.userId,
|
|
11018
|
+
customOptions: this.options.customOptions,
|
|
10711
11019
|
} /* <- TODO: [🤛] */);
|
|
10712
11020
|
}
|
|
10713
11021
|
return [4 /*yield*/, new Promise(function (resolve, reject) {
|
|
@@ -10801,7 +11109,9 @@ var RemoteLlmExecutionTools = /** @class */ (function () {
|
|
|
10801
11109
|
else {
|
|
10802
11110
|
socket.emit('prompt-request', {
|
|
10803
11111
|
isAnonymous: false,
|
|
11112
|
+
appId: this.options.appId,
|
|
10804
11113
|
userId: this.options.userId,
|
|
11114
|
+
customOptions: this.options.customOptions,
|
|
10805
11115
|
prompt: prompt,
|
|
10806
11116
|
} /* <- TODO: [🤛] */);
|
|
10807
11117
|
}
|
|
@@ -11283,7 +11593,7 @@ var AnthropicClaudeExecutionTools = /** @class */ (function () {
|
|
|
11283
11593
|
*/
|
|
11284
11594
|
var createAnthropicClaudeExecutionTools = Object.assign(function (options) {
|
|
11285
11595
|
if (options.isProxied) {
|
|
11286
|
-
return new RemoteLlmExecutionTools(__assign(__assign({}, options), { isAnonymous: true, llmToolsConfiguration: [
|
|
11596
|
+
return new RemoteLlmExecutionTools(__assign(__assign({}, options), { userId: null, isAnonymous: true, llmToolsConfiguration: [
|
|
11287
11597
|
{
|
|
11288
11598
|
title: 'Anthropic Claude (proxied)',
|
|
11289
11599
|
packageName: '@promptbook/anthropic-claude',
|
|
@@ -11843,12 +12153,12 @@ var AzureOpenAiExecutionTools = /** @class */ (function () {
|
|
|
11843
12153
|
* Calls OpenAI API to use a chat model.
|
|
11844
12154
|
*/
|
|
11845
12155
|
AzureOpenAiExecutionTools.prototype.callChatModel = function (prompt) {
|
|
11846
|
-
var _a, _b;
|
|
12156
|
+
var _a, _b, _c;
|
|
11847
12157
|
return __awaiter(this, void 0, void 0, function () {
|
|
11848
12158
|
var content, parameters, modelRequirements, client, modelName, modelSettings, rawPromptContent, messages, start, complete, rawRequest, rawResponse, resultContent, usage, error_1;
|
|
11849
12159
|
var _this = this;
|
|
11850
|
-
return __generator(this, function (
|
|
11851
|
-
switch (
|
|
12160
|
+
return __generator(this, function (_d) {
|
|
12161
|
+
switch (_d.label) {
|
|
11852
12162
|
case 0:
|
|
11853
12163
|
if (this.options.isVerbose) {
|
|
11854
12164
|
console.info('💬 OpenAI callChatModel call');
|
|
@@ -11856,20 +12166,20 @@ var AzureOpenAiExecutionTools = /** @class */ (function () {
|
|
|
11856
12166
|
content = prompt.content, parameters = prompt.parameters, modelRequirements = prompt.modelRequirements;
|
|
11857
12167
|
return [4 /*yield*/, this.getClient()];
|
|
11858
12168
|
case 1:
|
|
11859
|
-
client =
|
|
12169
|
+
client = _d.sent();
|
|
11860
12170
|
// TODO: [☂] Use here more modelRequirements
|
|
11861
12171
|
if (modelRequirements.modelVariant !== 'CHAT') {
|
|
11862
12172
|
throw new PipelineExecutionError('Use callChatModel only for CHAT variant');
|
|
11863
12173
|
}
|
|
11864
|
-
|
|
12174
|
+
_d.label = 2;
|
|
11865
12175
|
case 2:
|
|
11866
|
-
|
|
12176
|
+
_d.trys.push([2, 4, , 5]);
|
|
11867
12177
|
modelName = prompt.modelRequirements.modelName || this.options.deploymentName;
|
|
11868
12178
|
modelSettings = {
|
|
11869
12179
|
maxTokens: modelRequirements.maxTokens,
|
|
11870
12180
|
// <- TODO: [🌾] Make some global max cap for maxTokens
|
|
11871
12181
|
temperature: modelRequirements.temperature,
|
|
11872
|
-
user: this.options.
|
|
12182
|
+
user: (_a = this.options.userId) === null || _a === void 0 ? void 0 : _a.toString(),
|
|
11873
12183
|
// <- TODO: [🈁] Use `seed` here AND/OR use is `isDeterministic` for entire execution tools
|
|
11874
12184
|
// <- Note: [🧆]
|
|
11875
12185
|
};
|
|
@@ -11900,7 +12210,7 @@ var AzureOpenAiExecutionTools = /** @class */ (function () {
|
|
|
11900
12210
|
throw error;
|
|
11901
12211
|
})];
|
|
11902
12212
|
case 3:
|
|
11903
|
-
rawResponse =
|
|
12213
|
+
rawResponse = _d.sent();
|
|
11904
12214
|
if (this.options.isVerbose) {
|
|
11905
12215
|
console.info(colors.bgWhite('rawResponse'), JSON.stringify(rawResponse, null, 4));
|
|
11906
12216
|
}
|
|
@@ -11919,8 +12229,8 @@ var AzureOpenAiExecutionTools = /** @class */ (function () {
|
|
|
11919
12229
|
complete = getCurrentIsoDate();
|
|
11920
12230
|
usage = {
|
|
11921
12231
|
price: uncertainNumber() /* <- TODO: [🐞] Compute usage */,
|
|
11922
|
-
input: __assign({ tokensCount: uncertainNumber((
|
|
11923
|
-
output: __assign({ tokensCount: uncertainNumber((
|
|
12232
|
+
input: __assign({ tokensCount: uncertainNumber((_b = rawResponse.usage) === null || _b === void 0 ? void 0 : _b.promptTokens) }, computeUsageCounts(prompt.content)),
|
|
12233
|
+
output: __assign({ tokensCount: uncertainNumber((_c = rawResponse.usage) === null || _c === void 0 ? void 0 : _c.completionTokens) }, computeUsageCounts(prompt.content)),
|
|
11924
12234
|
};
|
|
11925
12235
|
return [2 /*return*/, $asDeeplyFrozenSerializableJson('AzureOpenAiExecutionTools ChatPromptResult', {
|
|
11926
12236
|
content: resultContent,
|
|
@@ -11936,7 +12246,7 @@ var AzureOpenAiExecutionTools = /** @class */ (function () {
|
|
|
11936
12246
|
// <- [🗯]
|
|
11937
12247
|
})];
|
|
11938
12248
|
case 4:
|
|
11939
|
-
error_1 =
|
|
12249
|
+
error_1 = _d.sent();
|
|
11940
12250
|
throw this.transformAzureError(error_1);
|
|
11941
12251
|
case 5: return [2 /*return*/];
|
|
11942
12252
|
}
|
|
@@ -11947,12 +12257,12 @@ var AzureOpenAiExecutionTools = /** @class */ (function () {
|
|
|
11947
12257
|
* Calls Azure OpenAI API to use a complete model.
|
|
11948
12258
|
*/
|
|
11949
12259
|
AzureOpenAiExecutionTools.prototype.callCompletionModel = function (prompt) {
|
|
11950
|
-
var _a, _b;
|
|
12260
|
+
var _a, _b, _c;
|
|
11951
12261
|
return __awaiter(this, void 0, void 0, function () {
|
|
11952
12262
|
var content, parameters, modelRequirements, client, modelName, modelSettings, start, complete, rawPromptContent, rawRequest, rawResponse, resultContent, usage, error_2;
|
|
11953
12263
|
var _this = this;
|
|
11954
|
-
return __generator(this, function (
|
|
11955
|
-
switch (
|
|
12264
|
+
return __generator(this, function (_d) {
|
|
12265
|
+
switch (_d.label) {
|
|
11956
12266
|
case 0:
|
|
11957
12267
|
if (this.options.isVerbose) {
|
|
11958
12268
|
console.info('🖋 OpenAI callCompletionModel call');
|
|
@@ -11960,20 +12270,20 @@ var AzureOpenAiExecutionTools = /** @class */ (function () {
|
|
|
11960
12270
|
content = prompt.content, parameters = prompt.parameters, modelRequirements = prompt.modelRequirements;
|
|
11961
12271
|
return [4 /*yield*/, this.getClient()];
|
|
11962
12272
|
case 1:
|
|
11963
|
-
client =
|
|
12273
|
+
client = _d.sent();
|
|
11964
12274
|
// TODO: [☂] Use here more modelRequirements
|
|
11965
12275
|
if (modelRequirements.modelVariant !== 'COMPLETION') {
|
|
11966
12276
|
throw new PipelineExecutionError('Use callCompletionModel only for COMPLETION variant');
|
|
11967
12277
|
}
|
|
11968
|
-
|
|
12278
|
+
_d.label = 2;
|
|
11969
12279
|
case 2:
|
|
11970
|
-
|
|
12280
|
+
_d.trys.push([2, 4, , 5]);
|
|
11971
12281
|
modelName = prompt.modelRequirements.modelName || this.options.deploymentName;
|
|
11972
12282
|
modelSettings = {
|
|
11973
12283
|
maxTokens: modelRequirements.maxTokens || 2000,
|
|
11974
12284
|
// <- TODO: [🌾] Make some global max cap for maxTokens
|
|
11975
12285
|
temperature: modelRequirements.temperature,
|
|
11976
|
-
user: this.options.
|
|
12286
|
+
user: (_a = this.options.userId) === null || _a === void 0 ? void 0 : _a.toString(),
|
|
11977
12287
|
// <- TODO: [🈁] Use `seed` here AND/OR use is `isDeterministic` for entire execution tools
|
|
11978
12288
|
// <- Note: [🧆]
|
|
11979
12289
|
};
|
|
@@ -11996,7 +12306,7 @@ var AzureOpenAiExecutionTools = /** @class */ (function () {
|
|
|
11996
12306
|
throw error;
|
|
11997
12307
|
})];
|
|
11998
12308
|
case 3:
|
|
11999
|
-
rawResponse =
|
|
12309
|
+
rawResponse = _d.sent();
|
|
12000
12310
|
if (this.options.isVerbose) {
|
|
12001
12311
|
console.info(colors.bgWhite('rawResponse'), JSON.stringify(rawResponse, null, 4));
|
|
12002
12312
|
}
|
|
@@ -12012,8 +12322,8 @@ var AzureOpenAiExecutionTools = /** @class */ (function () {
|
|
|
12012
12322
|
complete = getCurrentIsoDate();
|
|
12013
12323
|
usage = {
|
|
12014
12324
|
price: uncertainNumber() /* <- TODO: [🐞] Compute usage */,
|
|
12015
|
-
input: __assign({ tokensCount: uncertainNumber((
|
|
12016
|
-
output: __assign({ tokensCount: uncertainNumber((
|
|
12325
|
+
input: __assign({ tokensCount: uncertainNumber((_b = rawResponse.usage) === null || _b === void 0 ? void 0 : _b.promptTokens) }, computeUsageCounts(prompt.content)),
|
|
12326
|
+
output: __assign({ tokensCount: uncertainNumber((_c = rawResponse.usage) === null || _c === void 0 ? void 0 : _c.completionTokens) }, computeUsageCounts(prompt.content)),
|
|
12017
12327
|
};
|
|
12018
12328
|
return [2 /*return*/, $asDeeplyFrozenSerializableJson('AzureOpenAiExecutionTools CompletionPromptResult', {
|
|
12019
12329
|
content: resultContent,
|
|
@@ -12029,7 +12339,7 @@ var AzureOpenAiExecutionTools = /** @class */ (function () {
|
|
|
12029
12339
|
// <- [🗯]
|
|
12030
12340
|
})];
|
|
12031
12341
|
case 4:
|
|
12032
|
-
error_2 =
|
|
12342
|
+
error_2 = _d.sent();
|
|
12033
12343
|
throw this.transformAzureError(error_2);
|
|
12034
12344
|
case 5: return [2 /*return*/];
|
|
12035
12345
|
}
|
|
@@ -12254,7 +12564,6 @@ var OpenAiExecutionTools = /** @class */ (function () {
|
|
|
12254
12564
|
* @param options which are relevant are directly passed to the OpenAI client
|
|
12255
12565
|
*/
|
|
12256
12566
|
function OpenAiExecutionTools(options) {
|
|
12257
|
-
if (options === void 0) { options = {}; }
|
|
12258
12567
|
this.options = options;
|
|
12259
12568
|
/**
|
|
12260
12569
|
* OpenAI API client.
|
|
@@ -12282,8 +12591,8 @@ var OpenAiExecutionTools = /** @class */ (function () {
|
|
|
12282
12591
|
if (this.client === null) {
|
|
12283
12592
|
openAiOptions = __assign({}, this.options);
|
|
12284
12593
|
delete openAiOptions.isVerbose;
|
|
12285
|
-
delete openAiOptions.
|
|
12286
|
-
this.client = new OpenAI(
|
|
12594
|
+
delete openAiOptions.userId;
|
|
12595
|
+
this.client = new OpenAI(openAiOptions);
|
|
12287
12596
|
}
|
|
12288
12597
|
return [2 /*return*/, this.client];
|
|
12289
12598
|
});
|
|
@@ -12330,11 +12639,12 @@ var OpenAiExecutionTools = /** @class */ (function () {
|
|
|
12330
12639
|
* Calls OpenAI API to use a chat model.
|
|
12331
12640
|
*/
|
|
12332
12641
|
OpenAiExecutionTools.prototype.callChatModel = function (prompt) {
|
|
12642
|
+
var _a;
|
|
12333
12643
|
return __awaiter(this, void 0, void 0, function () {
|
|
12334
12644
|
var content, parameters, modelRequirements, format, client, modelName, modelSettings, rawPromptContent, rawRequest, start, complete, rawResponse, resultContent, usage;
|
|
12335
12645
|
var _this = this;
|
|
12336
|
-
return __generator(this, function (
|
|
12337
|
-
switch (
|
|
12646
|
+
return __generator(this, function (_b) {
|
|
12647
|
+
switch (_b.label) {
|
|
12338
12648
|
case 0:
|
|
12339
12649
|
if (this.options.isVerbose) {
|
|
12340
12650
|
console.info('💬 OpenAI callChatModel call', { prompt: prompt });
|
|
@@ -12342,7 +12652,7 @@ var OpenAiExecutionTools = /** @class */ (function () {
|
|
|
12342
12652
|
content = prompt.content, parameters = prompt.parameters, modelRequirements = prompt.modelRequirements, format = prompt.format;
|
|
12343
12653
|
return [4 /*yield*/, this.getClient()];
|
|
12344
12654
|
case 1:
|
|
12345
|
-
client =
|
|
12655
|
+
client = _b.sent();
|
|
12346
12656
|
// TODO: [☂] Use here more modelRequirements
|
|
12347
12657
|
if (modelRequirements.modelVariant !== 'CHAT') {
|
|
12348
12658
|
throw new PipelineExecutionError('Use callChatModel only for CHAT variant');
|
|
@@ -12374,7 +12684,7 @@ var OpenAiExecutionTools = /** @class */ (function () {
|
|
|
12374
12684
|
role: 'user',
|
|
12375
12685
|
content: rawPromptContent,
|
|
12376
12686
|
},
|
|
12377
|
-
], false), user: this.options.
|
|
12687
|
+
], false), user: (_a = this.options.userId) === null || _a === void 0 ? void 0 : _a.toString() });
|
|
12378
12688
|
start = getCurrentIsoDate();
|
|
12379
12689
|
if (this.options.isVerbose) {
|
|
12380
12690
|
console.info(colors.bgWhite('rawRequest'), JSON.stringify(rawRequest, null, 4));
|
|
@@ -12386,7 +12696,7 @@ var OpenAiExecutionTools = /** @class */ (function () {
|
|
|
12386
12696
|
throw error;
|
|
12387
12697
|
})];
|
|
12388
12698
|
case 2:
|
|
12389
|
-
rawResponse =
|
|
12699
|
+
rawResponse = _b.sent();
|
|
12390
12700
|
if (this.options.isVerbose) {
|
|
12391
12701
|
console.info(colors.bgWhite('rawResponse'), JSON.stringify(rawResponse, null, 4));
|
|
12392
12702
|
}
|
|
@@ -12425,11 +12735,12 @@ var OpenAiExecutionTools = /** @class */ (function () {
|
|
|
12425
12735
|
* Calls OpenAI API to use a complete model.
|
|
12426
12736
|
*/
|
|
12427
12737
|
OpenAiExecutionTools.prototype.callCompletionModel = function (prompt) {
|
|
12738
|
+
var _a;
|
|
12428
12739
|
return __awaiter(this, void 0, void 0, function () {
|
|
12429
12740
|
var content, parameters, modelRequirements, client, modelName, modelSettings, rawPromptContent, rawRequest, start, complete, rawResponse, resultContent, usage;
|
|
12430
12741
|
var _this = this;
|
|
12431
|
-
return __generator(this, function (
|
|
12432
|
-
switch (
|
|
12742
|
+
return __generator(this, function (_b) {
|
|
12743
|
+
switch (_b.label) {
|
|
12433
12744
|
case 0:
|
|
12434
12745
|
if (this.options.isVerbose) {
|
|
12435
12746
|
console.info('🖋 OpenAI callCompletionModel call', { prompt: prompt });
|
|
@@ -12437,7 +12748,7 @@ var OpenAiExecutionTools = /** @class */ (function () {
|
|
|
12437
12748
|
content = prompt.content, parameters = prompt.parameters, modelRequirements = prompt.modelRequirements;
|
|
12438
12749
|
return [4 /*yield*/, this.getClient()];
|
|
12439
12750
|
case 1:
|
|
12440
|
-
client =
|
|
12751
|
+
client = _b.sent();
|
|
12441
12752
|
// TODO: [☂] Use here more modelRequirements
|
|
12442
12753
|
if (modelRequirements.modelVariant !== 'COMPLETION') {
|
|
12443
12754
|
throw new PipelineExecutionError('Use callCompletionModel only for COMPLETION variant');
|
|
@@ -12452,7 +12763,7 @@ var OpenAiExecutionTools = /** @class */ (function () {
|
|
|
12452
12763
|
// <- Note: [🧆]
|
|
12453
12764
|
};
|
|
12454
12765
|
rawPromptContent = replaceParameters(content, __assign(__assign({}, parameters), { modelName: modelName }));
|
|
12455
|
-
rawRequest = __assign(__assign({}, modelSettings), { prompt: rawPromptContent, user: this.options.
|
|
12766
|
+
rawRequest = __assign(__assign({}, modelSettings), { prompt: rawPromptContent, user: (_a = this.options.userId) === null || _a === void 0 ? void 0 : _a.toString() });
|
|
12456
12767
|
start = getCurrentIsoDate();
|
|
12457
12768
|
if (this.options.isVerbose) {
|
|
12458
12769
|
console.info(colors.bgWhite('rawRequest'), JSON.stringify(rawRequest, null, 4));
|
|
@@ -12464,7 +12775,7 @@ var OpenAiExecutionTools = /** @class */ (function () {
|
|
|
12464
12775
|
throw error;
|
|
12465
12776
|
})];
|
|
12466
12777
|
case 2:
|
|
12467
|
-
rawResponse =
|
|
12778
|
+
rawResponse = _b.sent();
|
|
12468
12779
|
if (this.options.isVerbose) {
|
|
12469
12780
|
console.info(colors.bgWhite('rawResponse'), JSON.stringify(rawResponse, null, 4));
|
|
12470
12781
|
}
|
|
@@ -12690,17 +13001,16 @@ var OpenAiAssistantExecutionTools = /** @class */ (function (_super) {
|
|
|
12690
13001
|
assistant_id: this.assistantId,
|
|
12691
13002
|
thread: {
|
|
12692
13003
|
messages: [
|
|
12693
|
-
// TODO:
|
|
13004
|
+
// TODO: [🗯] !! Allow threads to be passed
|
|
12694
13005
|
{ role: 'user', content: rawPromptContent },
|
|
12695
13006
|
],
|
|
12696
13007
|
},
|
|
12697
|
-
//
|
|
13008
|
+
// <- TODO: Add user identification here> user: this.options.user,
|
|
12698
13009
|
};
|
|
12699
13010
|
start = getCurrentIsoDate();
|
|
12700
13011
|
if (this.options.isVerbose) {
|
|
12701
13012
|
console.info(colors.bgWhite('rawRequest'), JSON.stringify(rawRequest, null, 4));
|
|
12702
13013
|
}
|
|
12703
|
-
console.log('!!!!!! OpenAI client', client);
|
|
12704
13014
|
return [4 /*yield*/, client.beta.threads.createAndRunStream(rawRequest)];
|
|
12705
13015
|
case 2:
|
|
12706
13016
|
stream = _g.sent();
|
|
@@ -12718,7 +13028,7 @@ var OpenAiAssistantExecutionTools = /** @class */ (function (_super) {
|
|
|
12718
13028
|
messageDelta.content[0].type === 'text') {
|
|
12719
13029
|
console.info('messageDelta', (_a = messageDelta.content[0].text) === null || _a === void 0 ? void 0 : _a.value);
|
|
12720
13030
|
}
|
|
12721
|
-
// TODO:
|
|
13031
|
+
// <- TODO: [🐚] Make streaming and running tasks working
|
|
12722
13032
|
});
|
|
12723
13033
|
stream.on('messageCreated', function (message) {
|
|
12724
13034
|
if (_this.options.isVerbose) {
|
|
@@ -12746,18 +13056,20 @@ var OpenAiAssistantExecutionTools = /** @class */ (function (_super) {
|
|
|
12746
13056
|
throw new PipelineExecutionError("There is NOT 'text' BUT ".concat((_b = rawResponse[0].content[0]) === null || _b === void 0 ? void 0 : _b.type, " finalMessages content type from OpenAI"));
|
|
12747
13057
|
}
|
|
12748
13058
|
resultContent = (_c = rawResponse[0].content[0]) === null || _c === void 0 ? void 0 : _c.text.value;
|
|
12749
|
-
// <- TODO:
|
|
13059
|
+
// <- TODO: [🧠] There are also annotations, maybe use them
|
|
12750
13060
|
// eslint-disable-next-line prefer-const
|
|
12751
13061
|
complete = getCurrentIsoDate();
|
|
12752
13062
|
usage = UNCERTAIN_USAGE;
|
|
12753
|
-
// TODO:
|
|
13063
|
+
// <- TODO: [🥘] Compute real usage for assistant
|
|
13064
|
+
// ?> const usage = computeOpenAiUsage(content, resultContent || '', rawResponse);
|
|
12754
13065
|
if (resultContent === null) {
|
|
12755
13066
|
throw new PipelineExecutionError('No response message from OpenAI');
|
|
12756
13067
|
}
|
|
12757
13068
|
return [2 /*return*/, $asDeeplyFrozenSerializableJson('OpenAiAssistantExecutionTools ChatPromptResult', {
|
|
12758
13069
|
content: resultContent,
|
|
12759
13070
|
modelName: 'assistant',
|
|
12760
|
-
//
|
|
13071
|
+
// <- TODO: [🥘] Detect used model in assistant
|
|
13072
|
+
// ?> model: rawResponse.model || modelName,
|
|
12761
13073
|
timing: {
|
|
12762
13074
|
start: start,
|
|
12763
13075
|
complete: complete,
|
|
@@ -12775,7 +13087,6 @@ var OpenAiAssistantExecutionTools = /** @class */ (function (_super) {
|
|
|
12775
13087
|
return OpenAiAssistantExecutionTools;
|
|
12776
13088
|
}(OpenAiExecutionTools));
|
|
12777
13089
|
/**
|
|
12778
|
-
* TODO: !!!!!! DO not use colors - can be used in browser
|
|
12779
13090
|
* TODO: [🧠][🧙♂️] Maybe there can be some wizzard for thoose who want to use just OpenAI
|
|
12780
13091
|
* TODO: Maybe make custom OpenAiError
|
|
12781
13092
|
* TODO: [🧠][🈁] Maybe use `isDeterministic` from options
|
|
@@ -12847,22 +13158,26 @@ var _OpenAiAssistantRegistration = $llmToolsRegister.register(createOpenAiAssist
|
|
|
12847
13158
|
/**
|
|
12848
13159
|
* Normalize options for `execCommand` and `execCommands`
|
|
12849
13160
|
*
|
|
13161
|
+
* Note: `$` is used to indicate that this function behaves differently according to `process.platform`
|
|
13162
|
+
*
|
|
12850
13163
|
* @private internal utility of `execCommand` and `execCommands`
|
|
12851
13164
|
*/
|
|
12852
|
-
function execCommandNormalizeOptions(options) {
|
|
13165
|
+
function $execCommandNormalizeOptions(options) {
|
|
12853
13166
|
var _a;
|
|
12854
|
-
var _b, _c, _d;
|
|
13167
|
+
var _b, _c, _d, _e;
|
|
12855
13168
|
var command;
|
|
12856
13169
|
var cwd;
|
|
12857
13170
|
var crashOnError;
|
|
12858
13171
|
var args = [];
|
|
12859
13172
|
var timeout;
|
|
13173
|
+
var isVerbose;
|
|
12860
13174
|
if (typeof options === 'string') {
|
|
12861
13175
|
// TODO: [1] DRY default values
|
|
12862
13176
|
command = options;
|
|
12863
13177
|
cwd = process.cwd();
|
|
12864
13178
|
crashOnError = true;
|
|
12865
|
-
timeout = Infinity;
|
|
13179
|
+
timeout = Infinity; // <- TODO: [⏳]
|
|
13180
|
+
isVerbose = DEFAULT_IS_VERBOSE;
|
|
12866
13181
|
}
|
|
12867
13182
|
else {
|
|
12868
13183
|
/*
|
|
@@ -12878,6 +13193,7 @@ function execCommandNormalizeOptions(options) {
|
|
|
12878
13193
|
cwd = (_b = options.cwd) !== null && _b !== void 0 ? _b : process.cwd();
|
|
12879
13194
|
crashOnError = (_c = options.crashOnError) !== null && _c !== void 0 ? _c : true;
|
|
12880
13195
|
timeout = (_d = options.timeout) !== null && _d !== void 0 ? _d : Infinity;
|
|
13196
|
+
isVerbose = (_e = options.isVerbose) !== null && _e !== void 0 ? _e : DEFAULT_IS_VERBOSE;
|
|
12881
13197
|
}
|
|
12882
13198
|
// TODO: /(-[a-zA-Z0-9-]+\s+[^\s]*)|[^\s]*/g
|
|
12883
13199
|
var _ = Array.from(command.matchAll(/(".*")|([^\s]*)/g))
|
|
@@ -12899,16 +13215,21 @@ function execCommandNormalizeOptions(options) {
|
|
|
12899
13215
|
if (['ts-node'].includes(humanReadableCommand)) {
|
|
12900
13216
|
humanReadableCommand += " ".concat(args[1]);
|
|
12901
13217
|
}
|
|
12902
|
-
|
|
13218
|
+
if (/^win/.test(process.platform) && ['npm', 'npx'].includes(command)) {
|
|
13219
|
+
command = "".concat(command, ".cmd");
|
|
13220
|
+
}
|
|
13221
|
+
return { command: command, humanReadableCommand: humanReadableCommand, args: args, cwd: cwd, crashOnError: crashOnError, timeout: timeout, isVerbose: isVerbose };
|
|
12903
13222
|
}
|
|
12904
13223
|
// TODO: This should show type error> execCommandNormalizeOptions({ command: '', commands: [''] });
|
|
12905
13224
|
|
|
12906
13225
|
/**
|
|
12907
13226
|
* Run one command in a shell
|
|
12908
13227
|
*
|
|
13228
|
+
*
|
|
12909
13229
|
* Note: There are 2 similar functions in the codebase:
|
|
12910
13230
|
* - `$execCommand` which runs a single command
|
|
12911
13231
|
* - `$execCommands` which runs multiple commands
|
|
13232
|
+
* Note: `$` is used to indicate that this function is not a pure function - it runs a command in a shell
|
|
12912
13233
|
*
|
|
12913
13234
|
* @public exported from `@promptbook/node`
|
|
12914
13235
|
*/
|
|
@@ -12918,7 +13239,7 @@ function $execCommand(options) {
|
|
|
12918
13239
|
}
|
|
12919
13240
|
return new Promise(function (resolve, reject) {
|
|
12920
13241
|
// eslint-disable-next-line prefer-const
|
|
12921
|
-
var _a = execCommandNormalizeOptions(options), command = _a.command, humanReadableCommand = _a.humanReadableCommand, args = _a.args, cwd = _a.cwd, crashOnError = _a.crashOnError, timeout = _a.timeout;
|
|
13242
|
+
var _a = $execCommandNormalizeOptions(options), command = _a.command, humanReadableCommand = _a.humanReadableCommand, args = _a.args, cwd = _a.cwd, crashOnError = _a.crashOnError, timeout = _a.timeout, _b = _a.isVerbose, isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b;
|
|
12922
13243
|
if (timeout !== Infinity) {
|
|
12923
13244
|
// TODO: In waitasecond forTime(Infinity) should be equivalent to forEver()
|
|
12924
13245
|
forTime(timeout).then(function () {
|
|
@@ -12931,24 +13252,26 @@ function $execCommand(options) {
|
|
|
12931
13252
|
}
|
|
12932
13253
|
});
|
|
12933
13254
|
}
|
|
12934
|
-
if (
|
|
12935
|
-
|
|
13255
|
+
if (isVerbose) {
|
|
13256
|
+
console.info(colors.yellow(cwd) + ' ' + colors.green(command) + ' ' + colors.blue(args.join(' ')));
|
|
12936
13257
|
}
|
|
12937
|
-
// !!!!!! Verbose mode - to all consoles
|
|
12938
|
-
console.info(colors.yellow(cwd) + ' ' + colors.green(command) + ' ' + colors.blue(args.join(' ')));
|
|
12939
13258
|
try {
|
|
12940
13259
|
var commandProcess = spawn(command, args, { cwd: cwd, shell: true });
|
|
12941
|
-
|
|
12942
|
-
|
|
12943
|
-
|
|
13260
|
+
if (isVerbose) {
|
|
13261
|
+
commandProcess.on('message', function (message) {
|
|
13262
|
+
console.info({ message: message });
|
|
13263
|
+
});
|
|
13264
|
+
}
|
|
12944
13265
|
var output_1 = [];
|
|
12945
13266
|
commandProcess.stdout.on('data', function (stdout) {
|
|
12946
13267
|
output_1.push(stdout.toString());
|
|
12947
|
-
|
|
13268
|
+
if (isVerbose) {
|
|
13269
|
+
console.info(stdout.toString());
|
|
13270
|
+
}
|
|
12948
13271
|
});
|
|
12949
13272
|
commandProcess.stderr.on('data', function (stderr) {
|
|
12950
13273
|
output_1.push(stderr.toString());
|
|
12951
|
-
if (stderr.toString().trim()) {
|
|
13274
|
+
if (isVerbose && stderr.toString().trim()) {
|
|
12952
13275
|
console.warn(stderr.toString());
|
|
12953
13276
|
}
|
|
12954
13277
|
});
|
|
@@ -12959,7 +13282,9 @@ function $execCommand(options) {
|
|
|
12959
13282
|
"Command \"".concat(humanReadableCommand, "\" exited with code ").concat(code)));
|
|
12960
13283
|
}
|
|
12961
13284
|
else {
|
|
12962
|
-
|
|
13285
|
+
if (isVerbose) {
|
|
13286
|
+
console.warn("Command \"".concat(humanReadableCommand, "\" exited with code ").concat(code));
|
|
13287
|
+
}
|
|
12963
13288
|
resolve(spaceTrim(output_1.join('\n')));
|
|
12964
13289
|
}
|
|
12965
13290
|
}
|
|
@@ -12978,7 +13303,9 @@ function $execCommand(options) {
|
|
|
12978
13303
|
reject(new Error("Command \"".concat(humanReadableCommand, "\" failed: \n").concat(error.message)));
|
|
12979
13304
|
}
|
|
12980
13305
|
else {
|
|
12981
|
-
|
|
13306
|
+
if (isVerbose) {
|
|
13307
|
+
console.warn(error);
|
|
13308
|
+
}
|
|
12982
13309
|
resolve(spaceTrim(output_1.join('\n')));
|
|
12983
13310
|
}
|
|
12984
13311
|
});
|
|
@@ -13002,12 +13329,12 @@ function $execCommand(options) {
|
|
|
13002
13329
|
*/
|
|
13003
13330
|
function getScraperIntermediateSource(source, options) {
|
|
13004
13331
|
return __awaiter(this, void 0, void 0, function () {
|
|
13005
|
-
var sourceFilename, url, rootDirname, cacheDirname,
|
|
13332
|
+
var sourceFilename, url, rootDirname, cacheDirname, intermediateFilesStrategy, extension, isVerbose, hash, semanticName, pieces, name, cacheFilename, isDestroyed, fileHandler;
|
|
13006
13333
|
return __generator(this, function (_a) {
|
|
13007
13334
|
switch (_a.label) {
|
|
13008
13335
|
case 0:
|
|
13009
13336
|
sourceFilename = source.filename, url = source.url;
|
|
13010
|
-
rootDirname = options.rootDirname, cacheDirname = options.cacheDirname,
|
|
13337
|
+
rootDirname = options.rootDirname, cacheDirname = options.cacheDirname, intermediateFilesStrategy = options.intermediateFilesStrategy, extension = options.extension, isVerbose = options.isVerbose;
|
|
13011
13338
|
hash = SHA256(
|
|
13012
13339
|
// <- TODO: [🥬] Encapsulate sha256 to some private utility function
|
|
13013
13340
|
hexEncoder.parse(sourceFilename || url || 'untitled'))
|
|
@@ -13017,7 +13344,7 @@ function getScraperIntermediateSource(source, options) {
|
|
|
13017
13344
|
pieces = ['intermediate', semanticName, hash].filter(function (piece) { return piece !== ''; });
|
|
13018
13345
|
name = pieces.join('-').split('--').join('-');
|
|
13019
13346
|
// <- TODO: Use MAX_FILENAME_LENGTH
|
|
13020
|
-
TODO_USE(rootDirname); // <- TODO:
|
|
13347
|
+
TODO_USE(rootDirname); // <- TODO: [😡]
|
|
13021
13348
|
cacheFilename = join.apply(void 0, __spreadArray(__spreadArray([process.cwd(),
|
|
13022
13349
|
cacheDirname], __read(nameToSubfolderPath(hash /* <- TODO: [🎎] Maybe add some SHA256 prefix */)), false), [name], false)).split('\\')
|
|
13023
13350
|
.join('/') +
|
|
@@ -13037,7 +13364,7 @@ function getScraperIntermediateSource(source, options) {
|
|
|
13037
13364
|
return __generator(this, function (_a) {
|
|
13038
13365
|
switch (_a.label) {
|
|
13039
13366
|
case 0:
|
|
13040
|
-
if (!
|
|
13367
|
+
if (!(intermediateFilesStrategy === 'HIDE_AND_CLEAN')) return [3 /*break*/, 2];
|
|
13041
13368
|
if (isVerbose) {
|
|
13042
13369
|
console.info('legacyDocumentScraper: Clening cache');
|
|
13043
13370
|
}
|
|
@@ -13061,7 +13388,7 @@ function getScraperIntermediateSource(source, options) {
|
|
|
13061
13388
|
/**
|
|
13062
13389
|
* Note: Not using `FileCacheStorage` for two reasons:
|
|
13063
13390
|
* 1) Need to store more than serialized JSONs
|
|
13064
|
-
* 2) Need to switch between a `rootDirname` and `cacheDirname` <- TODO:
|
|
13391
|
+
* 2) Need to switch between a `rootDirname` and `cacheDirname` <- TODO: [😡]
|
|
13065
13392
|
* TODO: [🐱🐉][🧠] Make some smart crop
|
|
13066
13393
|
* Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
|
|
13067
13394
|
*/
|
|
@@ -13078,7 +13405,7 @@ var markdownScraperMetadata = $deepFreeze({
|
|
|
13078
13405
|
mimeTypes: ['text/markdown', 'text/plain'],
|
|
13079
13406
|
documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
|
|
13080
13407
|
isAvilableInBrowser: true,
|
|
13081
|
-
requiredExecutables: [
|
|
13408
|
+
requiredExecutables: [],
|
|
13082
13409
|
}); /* <- TODO: [🤛] */
|
|
13083
13410
|
/**
|
|
13084
13411
|
* Registration of known scraper metadata
|
|
@@ -13122,7 +13449,7 @@ var MarkdownScraper = /** @class */ (function () {
|
|
|
13122
13449
|
return __generator(this, function (_k) {
|
|
13123
13450
|
switch (_k.label) {
|
|
13124
13451
|
case 0:
|
|
13125
|
-
_a = this.options, _b = _a.maxParallelCount, maxParallelCount = _b === void 0 ?
|
|
13452
|
+
_a = this.options, _b = _a.maxParallelCount, maxParallelCount = _b === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _b, _c = _a.isVerbose, isVerbose = _c === void 0 ? DEFAULT_IS_VERBOSE : _c;
|
|
13126
13453
|
llm = this.tools.llm;
|
|
13127
13454
|
if (llm === undefined) {
|
|
13128
13455
|
throw new MissingToolsError('LLM tools are required for scraping external files');
|
|
@@ -13221,7 +13548,8 @@ var MarkdownScraper = /** @class */ (function () {
|
|
|
13221
13548
|
embeddingResult = _c.sent();
|
|
13222
13549
|
index.push({
|
|
13223
13550
|
modelName: embeddingResult.modelName,
|
|
13224
|
-
position: embeddingResult.content,
|
|
13551
|
+
position: __spreadArray([], __read(embeddingResult.content), false),
|
|
13552
|
+
// <- TODO: [🪓] Here should be no need for spreading new array, just `position: embeddingResult.content`
|
|
13225
13553
|
});
|
|
13226
13554
|
_c.label = 6;
|
|
13227
13555
|
case 6: return [3 /*break*/, 8];
|
|
@@ -13272,7 +13600,7 @@ var documentScraperMetadata = $deepFreeze({
|
|
|
13272
13600
|
mimeTypes: ['application/vnd.openxmlformats-officedocument.wordprocessingml.document'],
|
|
13273
13601
|
documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
|
|
13274
13602
|
isAvilableInBrowser: false,
|
|
13275
|
-
requiredExecutables: ['
|
|
13603
|
+
requiredExecutables: ['Pandoc'],
|
|
13276
13604
|
}); /* <- TODO: [🤛] */
|
|
13277
13605
|
/**
|
|
13278
13606
|
* Registration of known scraper metadata
|
|
@@ -13312,12 +13640,13 @@ var DocumentScraper = /** @class */ (function () {
|
|
|
13312
13640
|
* Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object
|
|
13313
13641
|
*/
|
|
13314
13642
|
DocumentScraper.prototype.$convert = function (source) {
|
|
13643
|
+
var _a;
|
|
13315
13644
|
return __awaiter(this, void 0, void 0, function () {
|
|
13316
|
-
var
|
|
13645
|
+
var _b, _c, rootDirname, _d, cacheDirname, _e, intermediateFilesStrategy, _f, isVerbose, extension, cacheFilehandler, command_1;
|
|
13317
13646
|
return __generator(this, function (_g) {
|
|
13318
13647
|
switch (_g.label) {
|
|
13319
13648
|
case 0:
|
|
13320
|
-
|
|
13649
|
+
_b = this.options, _c = _b.rootDirname, rootDirname = _c === void 0 ? process.cwd() : _c, _d = _b.cacheDirname, cacheDirname = _d === void 0 ? DEFAULT_SCRAPE_CACHE_DIRNAME : _d, _e = _b.intermediateFilesStrategy, intermediateFilesStrategy = _e === void 0 ? DEFAULT_INTERMEDIATE_FILES_STRATEGY : _e, _f = _b.isVerbose, isVerbose = _f === void 0 ? DEFAULT_IS_VERBOSE : _f;
|
|
13321
13650
|
if (!$isRunningInNode()) {
|
|
13322
13651
|
throw new KnowledgeScrapeError('Scraping .docx files is only supported in Node environment');
|
|
13323
13652
|
}
|
|
@@ -13325,7 +13654,7 @@ var DocumentScraper = /** @class */ (function () {
|
|
|
13325
13654
|
throw new EnvironmentMismatchError('Can not scrape documents without filesystem tools');
|
|
13326
13655
|
// <- TODO: [🧠] What is the best error type here`
|
|
13327
13656
|
}
|
|
13328
|
-
if (
|
|
13657
|
+
if (((_a = this.tools.executables) === null || _a === void 0 ? void 0 : _a.pandocPath) === undefined) {
|
|
13329
13658
|
throw new MissingToolsError('Pandoc is required for scraping .docx files');
|
|
13330
13659
|
}
|
|
13331
13660
|
if (source.filename === null) {
|
|
@@ -13336,7 +13665,7 @@ var DocumentScraper = /** @class */ (function () {
|
|
|
13336
13665
|
return [4 /*yield*/, getScraperIntermediateSource(source, {
|
|
13337
13666
|
rootDirname: rootDirname,
|
|
13338
13667
|
cacheDirname: cacheDirname,
|
|
13339
|
-
|
|
13668
|
+
intermediateFilesStrategy: intermediateFilesStrategy,
|
|
13340
13669
|
extension: 'md',
|
|
13341
13670
|
isVerbose: isVerbose,
|
|
13342
13671
|
})];
|
|
@@ -13345,11 +13674,9 @@ var DocumentScraper = /** @class */ (function () {
|
|
|
13345
13674
|
return [4 /*yield*/, isFileExisting(cacheFilehandler.filename, this.tools.fs)];
|
|
13346
13675
|
case 2:
|
|
13347
13676
|
if (!!(_g.sent())) return [3 /*break*/, 5];
|
|
13348
|
-
command_1 = "\"".concat(
|
|
13349
|
-
// TODO: !!!!!! [🕊] Make execCommand standard (?node-)util of the promptbook
|
|
13677
|
+
command_1 = "\"".concat(this.tools.executables.pandocPath, "\" -f ").concat(extension, " -t markdown \"").concat(source.filename, "\" -o \"").concat(cacheFilehandler.filename, "\"");
|
|
13350
13678
|
return [4 /*yield*/, $execCommand(command_1)];
|
|
13351
13679
|
case 3:
|
|
13352
|
-
// TODO: !!!!!! [🕊] Make execCommand standard (?node-)util of the promptbook
|
|
13353
13680
|
_g.sent();
|
|
13354
13681
|
return [4 /*yield*/, isFileExisting(cacheFilehandler.filename, this.tools.fs)];
|
|
13355
13682
|
case 4:
|
|
@@ -13394,9 +13721,14 @@ var DocumentScraper = /** @class */ (function () {
|
|
|
13394
13721
|
asJson: function () {
|
|
13395
13722
|
throw new UnexpectedError('Did not expect that `markdownScraper` would need to get the content `asJson`');
|
|
13396
13723
|
},
|
|
13397
|
-
|
|
13398
|
-
|
|
13399
|
-
|
|
13724
|
+
/*
|
|
13725
|
+
TODO: [🥽]
|
|
13726
|
+
> asBlob() {
|
|
13727
|
+
> throw new UnexpectedError(
|
|
13728
|
+
> 'Did not expect that `markdownScraper` would need to get the content `asBlob`',
|
|
13729
|
+
> );
|
|
13730
|
+
> },
|
|
13731
|
+
*/
|
|
13400
13732
|
};
|
|
13401
13733
|
knowledge = this.markdownScraper.scrape(markdownSource);
|
|
13402
13734
|
return [4 /*yield*/, cacheFilehandler.destroy()];
|
|
@@ -13428,7 +13760,11 @@ var legacyDocumentScraperMetadata = $deepFreeze({
|
|
|
13428
13760
|
mimeTypes: ['application/msword', 'text/rtf'],
|
|
13429
13761
|
documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
|
|
13430
13762
|
isAvilableInBrowser: false,
|
|
13431
|
-
requiredExecutables: [
|
|
13763
|
+
requiredExecutables: [
|
|
13764
|
+
'Pandoc',
|
|
13765
|
+
'LibreOffice',
|
|
13766
|
+
// <- TODO: [🧠] Should be 'LibreOffice' here, its dependency of dependency
|
|
13767
|
+
],
|
|
13432
13768
|
}); /* <- TODO: [🤛] */
|
|
13433
13769
|
/**
|
|
13434
13770
|
* Registration of known scraper metadata
|
|
@@ -13441,7 +13777,7 @@ var legacyDocumentScraperMetadata = $deepFreeze({
|
|
|
13441
13777
|
var _LegacyDocumentScraperMetadataRegistration = $scrapersMetadataRegister.register(legacyDocumentScraperMetadata);
|
|
13442
13778
|
|
|
13443
13779
|
/**
|
|
13444
|
-
* Scraper for .
|
|
13780
|
+
* Scraper for old document files (like .doc and .rtf)
|
|
13445
13781
|
*
|
|
13446
13782
|
* @see `documentationUrl` for more details
|
|
13447
13783
|
* @public exported from `@promptbook/legacy-documents`
|
|
@@ -13468,12 +13804,13 @@ var LegacyDocumentScraper = /** @class */ (function () {
|
|
|
13468
13804
|
* Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object
|
|
13469
13805
|
*/
|
|
13470
13806
|
LegacyDocumentScraper.prototype.$convert = function (source) {
|
|
13807
|
+
var _a;
|
|
13471
13808
|
return __awaiter(this, void 0, void 0, function () {
|
|
13472
|
-
var
|
|
13809
|
+
var _b, _c, rootDirname, _d, cacheDirname, _e, intermediateFilesStrategy, _f, isVerbose, extension, cacheFilehandler, documentSourceOutdirPathForLibreOffice_1, command_1, files_1, file;
|
|
13473
13810
|
return __generator(this, function (_g) {
|
|
13474
13811
|
switch (_g.label) {
|
|
13475
13812
|
case 0:
|
|
13476
|
-
|
|
13813
|
+
_b = this.options, _c = _b.rootDirname, rootDirname = _c === void 0 ? process.cwd() : _c, _d = _b.cacheDirname, cacheDirname = _d === void 0 ? DEFAULT_SCRAPE_CACHE_DIRNAME : _d, _e = _b.intermediateFilesStrategy, intermediateFilesStrategy = _e === void 0 ? DEFAULT_INTERMEDIATE_FILES_STRATEGY : _e, _f = _b.isVerbose, isVerbose = _f === void 0 ? DEFAULT_IS_VERBOSE : _f;
|
|
13477
13814
|
if (!$isRunningInNode()) {
|
|
13478
13815
|
throw new KnowledgeScrapeError('Scraping .doc files is only supported in Node environment');
|
|
13479
13816
|
}
|
|
@@ -13481,7 +13818,7 @@ var LegacyDocumentScraper = /** @class */ (function () {
|
|
|
13481
13818
|
throw new EnvironmentMismatchError('Can not scrape (legacy) documents without filesystem tools');
|
|
13482
13819
|
// <- TODO: [🧠] What is the best error type here`
|
|
13483
13820
|
}
|
|
13484
|
-
if (
|
|
13821
|
+
if (((_a = this.tools.executables) === null || _a === void 0 ? void 0 : _a.libreOfficePath) === undefined) {
|
|
13485
13822
|
throw new MissingToolsError('LibreOffice is required for scraping .doc and .rtf files');
|
|
13486
13823
|
}
|
|
13487
13824
|
if (source.filename === null) {
|
|
@@ -13492,7 +13829,7 @@ var LegacyDocumentScraper = /** @class */ (function () {
|
|
|
13492
13829
|
return [4 /*yield*/, getScraperIntermediateSource(source, {
|
|
13493
13830
|
rootDirname: rootDirname,
|
|
13494
13831
|
cacheDirname: cacheDirname,
|
|
13495
|
-
|
|
13832
|
+
intermediateFilesStrategy: intermediateFilesStrategy,
|
|
13496
13833
|
extension: 'docx',
|
|
13497
13834
|
isVerbose: isVerbose,
|
|
13498
13835
|
})];
|
|
@@ -13507,11 +13844,9 @@ var LegacyDocumentScraper = /** @class */ (function () {
|
|
|
13507
13844
|
documentSourceOutdirPathForLibreOffice_1 = join(dirname(cacheFilehandler.filename), 'libreoffice')
|
|
13508
13845
|
.split('\\')
|
|
13509
13846
|
.join('/');
|
|
13510
|
-
command_1 = "\"".concat(
|
|
13511
|
-
// TODO: !!!!!! [🕊] Make execCommand standard (?node-)util of the promptbook - this should trigger build polution error
|
|
13847
|
+
command_1 = "\"".concat(this.tools.executables.libreOfficePath, "\" --headless --convert-to docx \"").concat(source.filename, "\" --outdir \"").concat(documentSourceOutdirPathForLibreOffice_1, "\"");
|
|
13512
13848
|
return [4 /*yield*/, $execCommand(command_1)];
|
|
13513
13849
|
case 3:
|
|
13514
|
-
// TODO: !!!!!! [🕊] Make execCommand standard (?node-)util of the promptbook - this should trigger build polution error
|
|
13515
13850
|
_g.sent();
|
|
13516
13851
|
return [4 /*yield*/, readdir(documentSourceOutdirPathForLibreOffice_1)];
|
|
13517
13852
|
case 4:
|
|
@@ -13559,9 +13894,14 @@ var LegacyDocumentScraper = /** @class */ (function () {
|
|
|
13559
13894
|
asJson: function () {
|
|
13560
13895
|
throw new UnexpectedError('Did not expect that `documentScraper` would need to get the content `asJson`');
|
|
13561
13896
|
},
|
|
13562
|
-
|
|
13563
|
-
|
|
13564
|
-
|
|
13897
|
+
/*
|
|
13898
|
+
TODO: [🥽]
|
|
13899
|
+
> asBlob() {
|
|
13900
|
+
> throw new UnexpectedError(
|
|
13901
|
+
> 'Did not expect that `documentScraper` would need to get the content `asBlob`',
|
|
13902
|
+
> );
|
|
13903
|
+
> },
|
|
13904
|
+
*/
|
|
13565
13905
|
};
|
|
13566
13906
|
knowledge = this.documentScraper.scrape(markdownSource);
|
|
13567
13907
|
return [4 /*yield*/, cacheFilehandler.destroy()];
|
|
@@ -13668,7 +14008,7 @@ var pdfScraperMetadata = $deepFreeze({
|
|
|
13668
14008
|
mimeTypes: ['application/pdf'],
|
|
13669
14009
|
documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
|
|
13670
14010
|
isAvilableInBrowser: true,
|
|
13671
|
-
requiredExecutables: [
|
|
14011
|
+
requiredExecutables: [],
|
|
13672
14012
|
}); /* <- TODO: [🤛] */
|
|
13673
14013
|
/**
|
|
13674
14014
|
* Registration of known scraper metadata
|
|
@@ -13681,7 +14021,7 @@ var pdfScraperMetadata = $deepFreeze({
|
|
|
13681
14021
|
var _PdfScraperMetadataRegistration = $scrapersMetadataRegister.register(pdfScraperMetadata);
|
|
13682
14022
|
|
|
13683
14023
|
/**
|
|
13684
|
-
* Scraper for .
|
|
14024
|
+
* Scraper for .pdf files
|
|
13685
14025
|
*
|
|
13686
14026
|
* @see `documentationUrl` for more details
|
|
13687
14027
|
* @public exported from `@promptbook/pdf`
|
|
@@ -13724,9 +14064,8 @@ var PdfScraper = /** @class */ (function () {
|
|
|
13724
14064
|
TODO_USE(this.options);
|
|
13725
14065
|
/*
|
|
13726
14066
|
const {
|
|
13727
|
-
externalProgramsPaths = {},
|
|
13728
14067
|
cacheDirname = SCRAPE_CACHE_DIRNAME,
|
|
13729
|
-
|
|
14068
|
+
intermediateFilesStrategy = DEFAULT_INTERMEDIATE_FILES_STRATEGY,
|
|
13730
14069
|
isVerbose = IS_VERBOSE,
|
|
13731
14070
|
} = options;
|
|
13732
14071
|
*/
|
|
@@ -13779,7 +14118,7 @@ var websiteScraperMetadata = $deepFreeze({
|
|
|
13779
14118
|
mimeTypes: ['text/html'],
|
|
13780
14119
|
documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
|
|
13781
14120
|
isAvilableInBrowser: false,
|
|
13782
|
-
requiredExecutables: [
|
|
14121
|
+
requiredExecutables: [],
|
|
13783
14122
|
}); /* <- TODO: [🤛] */
|
|
13784
14123
|
/**
|
|
13785
14124
|
* Registration of known scraper metadata
|
|
@@ -13792,32 +14131,29 @@ var websiteScraperMetadata = $deepFreeze({
|
|
|
13792
14131
|
var _WebsiteScraperMetadataRegistration = $scrapersMetadataRegister.register(websiteScraperMetadata);
|
|
13793
14132
|
|
|
13794
14133
|
/**
|
|
13795
|
-
*
|
|
14134
|
+
* Create a new showdown converter instance
|
|
13796
14135
|
*
|
|
13797
|
-
* @
|
|
13798
|
-
* @private for markdown and html knowledge scrapers
|
|
13799
|
-
*/
|
|
13800
|
-
var markdownConverter = new Converter({
|
|
13801
|
-
flavor: 'github', // <- TODO: !!!!!! Explicitly specify the flavor of promptbook markdown
|
|
13802
|
-
/*
|
|
13803
|
-
> import showdownHighlight from 'showdown-highlight';
|
|
13804
|
-
> extensions: [
|
|
13805
|
-
> showdownHighlight({
|
|
13806
|
-
> // Whether to add the classes to the <pre> tag, default is false
|
|
13807
|
-
> pre: true,
|
|
13808
|
-
> // Whether to use hljs' auto language detection, default is true
|
|
13809
|
-
> auto_detection: true,
|
|
13810
|
-
> }),
|
|
13811
|
-
> ],
|
|
13812
|
-
*/
|
|
13813
|
-
});
|
|
13814
|
-
/**
|
|
13815
|
-
* TODO: !!!!!! Figure out better name not to confuse with `Converter`
|
|
13816
|
-
* TODO: !!!!!! Lazy-make converter
|
|
14136
|
+
* @private utility of `WebsiteScraper`
|
|
13817
14137
|
*/
|
|
14138
|
+
function createShowdownConverter() {
|
|
14139
|
+
return new Converter({
|
|
14140
|
+
flavor: 'github',
|
|
14141
|
+
/*
|
|
14142
|
+
> import showdownHighlight from 'showdown-highlight';
|
|
14143
|
+
> extensions: [
|
|
14144
|
+
> showdownHighlight({
|
|
14145
|
+
> // Whether to add the classes to the <pre> tag, default is false
|
|
14146
|
+
> pre: true,
|
|
14147
|
+
> // Whether to use hljs' auto language detection, default is true
|
|
14148
|
+
> auto_detection: true,
|
|
14149
|
+
> }),
|
|
14150
|
+
> ],
|
|
14151
|
+
*/
|
|
14152
|
+
});
|
|
14153
|
+
}
|
|
13818
14154
|
|
|
13819
14155
|
/**
|
|
13820
|
-
* Scraper for
|
|
14156
|
+
* Scraper for websites
|
|
13821
14157
|
*
|
|
13822
14158
|
* @see `documentationUrl` for more details
|
|
13823
14159
|
* @public exported from `@promptbook/website-crawler`
|
|
@@ -13827,6 +14163,7 @@ var WebsiteScraper = /** @class */ (function () {
|
|
|
13827
14163
|
this.tools = tools;
|
|
13828
14164
|
this.options = options;
|
|
13829
14165
|
this.markdownScraper = new MarkdownScraper(tools, options);
|
|
14166
|
+
this.showdownConverter = createShowdownConverter();
|
|
13830
14167
|
}
|
|
13831
14168
|
Object.defineProperty(WebsiteScraper.prototype, "metadata", {
|
|
13832
14169
|
/**
|
|
@@ -13847,16 +14184,17 @@ var WebsiteScraper = /** @class */ (function () {
|
|
|
13847
14184
|
return __awaiter(this, void 0, void 0, function () {
|
|
13848
14185
|
var _a, _b,
|
|
13849
14186
|
// TODO: [🧠] Maybe in node use headless browser not just JSDOM
|
|
13850
|
-
|
|
13851
|
-
rootDirname, _c, cacheDirname, _d, isCacheCleaned, _e, isVerbose, jsdom, _f, reader, article, html, i, cacheFilehandler, markdown;
|
|
14187
|
+
rootDirname, _c, cacheDirname, _d, intermediateFilesStrategy, _e, isVerbose, jsdom, _f, reader, article, html, i, cacheFilehandler, markdown;
|
|
13852
14188
|
return __generator(this, function (_g) {
|
|
13853
14189
|
switch (_g.label) {
|
|
13854
14190
|
case 0:
|
|
13855
|
-
_a = this.options, _b = _a.rootDirname, rootDirname = _b === void 0 ? process.cwd() : _b, _c = _a.cacheDirname, cacheDirname = _c === void 0 ?
|
|
13856
|
-
// TODO: !!!!!! Does this work in browser? Make it work.
|
|
14191
|
+
_a = this.options, _b = _a.rootDirname, rootDirname = _b === void 0 ? process.cwd() : _b, _c = _a.cacheDirname, cacheDirname = _c === void 0 ? DEFAULT_SCRAPE_CACHE_DIRNAME : _c, _d = _a.intermediateFilesStrategy, intermediateFilesStrategy = _d === void 0 ? DEFAULT_INTERMEDIATE_FILES_STRATEGY : _d, _e = _a.isVerbose, isVerbose = _e === void 0 ? DEFAULT_IS_VERBOSE : _e;
|
|
13857
14192
|
if (source.url === null) {
|
|
13858
14193
|
throw new KnowledgeScrapeError('Website scraper requires URL');
|
|
13859
14194
|
}
|
|
14195
|
+
if (this.tools.fs === undefined) {
|
|
14196
|
+
throw new EnvironmentMismatchError('Can not scrape websites without filesystem tools');
|
|
14197
|
+
}
|
|
13860
14198
|
_f = JSDOM.bind;
|
|
13861
14199
|
return [4 /*yield*/, source.asText()];
|
|
13862
14200
|
case 1:
|
|
@@ -13865,10 +14203,6 @@ var WebsiteScraper = /** @class */ (function () {
|
|
|
13865
14203
|
}]))();
|
|
13866
14204
|
reader = new Readability(jsdom.window.document);
|
|
13867
14205
|
article = reader.parse();
|
|
13868
|
-
console.log(article);
|
|
13869
|
-
return [4 /*yield*/, forTime(10000)];
|
|
13870
|
-
case 2:
|
|
13871
|
-
_g.sent();
|
|
13872
14206
|
html = (article === null || article === void 0 ? void 0 : article.content) || (article === null || article === void 0 ? void 0 : article.textContent) || jsdom.window.document.body.innerHTML;
|
|
13873
14207
|
// Note: Unwrap html such as it is convertable by `markdownConverter`
|
|
13874
14208
|
for (i = 0; i < 2; i++) {
|
|
@@ -13880,16 +14214,16 @@ var WebsiteScraper = /** @class */ (function () {
|
|
|
13880
14214
|
return [4 /*yield*/, getScraperIntermediateSource(source, {
|
|
13881
14215
|
rootDirname: rootDirname,
|
|
13882
14216
|
cacheDirname: cacheDirname,
|
|
13883
|
-
|
|
14217
|
+
intermediateFilesStrategy: intermediateFilesStrategy,
|
|
13884
14218
|
extension: 'html',
|
|
13885
14219
|
isVerbose: isVerbose,
|
|
13886
14220
|
})];
|
|
13887
|
-
case
|
|
14221
|
+
case 2:
|
|
13888
14222
|
cacheFilehandler = _g.sent();
|
|
13889
|
-
return [4 /*yield*/, writeFile(cacheFilehandler.filename, html, 'utf-8')];
|
|
13890
|
-
case
|
|
14223
|
+
return [4 /*yield*/, this.tools.fs.writeFile(cacheFilehandler.filename, html, 'utf-8')];
|
|
14224
|
+
case 3:
|
|
13891
14225
|
_g.sent();
|
|
13892
|
-
markdown =
|
|
14226
|
+
markdown = this.showdownConverter.makeMarkdown(html, jsdom.window.document);
|
|
13893
14227
|
return [2 /*return*/, __assign(__assign({}, cacheFilehandler), { markdown: markdown })];
|
|
13894
14228
|
}
|
|
13895
14229
|
});
|
|
@@ -13917,9 +14251,14 @@ var WebsiteScraper = /** @class */ (function () {
|
|
|
13917
14251
|
asJson: function () {
|
|
13918
14252
|
throw new UnexpectedError('Did not expect that `markdownScraper` would need to get the content `asJson`');
|
|
13919
14253
|
},
|
|
13920
|
-
|
|
13921
|
-
|
|
13922
|
-
|
|
14254
|
+
/*
|
|
14255
|
+
TODO: [🥽]
|
|
14256
|
+
> asBlob() {
|
|
14257
|
+
> throw new UnexpectedError(
|
|
14258
|
+
> 'Did not expect that `markdownScraper` would need to get the content `asBlob`',
|
|
14259
|
+
> );
|
|
14260
|
+
> },
|
|
14261
|
+
*/
|
|
13923
14262
|
};
|
|
13924
14263
|
knowledge = this.markdownScraper.scrape(markdownSource);
|
|
13925
14264
|
return [4 /*yield*/, cacheFilehandler.destroy()];
|
|
@@ -13933,7 +14272,6 @@ var WebsiteScraper = /** @class */ (function () {
|
|
|
13933
14272
|
return WebsiteScraper;
|
|
13934
14273
|
}());
|
|
13935
14274
|
/**
|
|
13936
|
-
* TODO: !!!!!! Put into separate package
|
|
13937
14275
|
* TODO: [👣] Scraped website in .md can act as cache item - there is no need to run conversion each time
|
|
13938
14276
|
* TODO: [🪂] Do it in parallel 11:11
|
|
13939
14277
|
* Note: No need to aggregate usage here, it is done by intercepting the llmTools
|