@promptbook/remote-server 0.88.0 → 0.89.0-11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -1
- package/esm/index.es.js +375 -98
- package/esm/index.es.js.map +1 -1
- package/esm/typings/src/_packages/core.index.d.ts +18 -6
- package/esm/typings/src/_packages/remote-client.index.d.ts +6 -8
- package/esm/typings/src/_packages/remote-server.index.d.ts +6 -6
- package/esm/typings/src/_packages/types.index.d.ts +18 -10
- package/esm/typings/src/cli/cli-commands/login.d.ts +15 -0
- package/esm/typings/src/cli/common/$addGlobalOptionsToCommand.d.ts +7 -0
- package/esm/typings/src/cli/common/$provideLlmToolsForCli.d.ts +15 -0
- package/esm/typings/src/config.d.ts +15 -8
- package/esm/typings/src/errors/0-index.d.ts +6 -0
- package/esm/typings/src/errors/AuthenticationError.d.ts +9 -0
- package/esm/typings/src/errors/PromptbookFetchError.d.ts +9 -0
- package/esm/typings/src/execution/PipelineExecutorResult.d.ts +2 -2
- package/esm/typings/src/execution/PromptResult.d.ts +2 -2
- package/esm/typings/src/execution/{PromptResultUsage.d.ts → Usage.d.ts} +5 -5
- package/esm/typings/src/execution/utils/addUsage.d.ts +2 -2
- package/esm/typings/src/execution/utils/computeUsageCounts.d.ts +3 -3
- package/esm/typings/src/execution/utils/usage-constants.d.ts +77 -60
- package/esm/typings/src/execution/utils/usageToHuman.d.ts +5 -5
- package/esm/typings/src/execution/utils/usageToWorktime.d.ts +5 -5
- package/esm/typings/src/llm-providers/_common/register/$provideEnvFilename.d.ts +12 -0
- package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsConfigurationFromEnv.d.ts +2 -8
- package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground.d.ts +2 -0
- package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsForWizzardOrCli.d.ts +36 -1
- package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsFromEnv.d.ts +1 -0
- package/esm/typings/src/llm-providers/_common/utils/count-total-usage/LlmExecutionToolsWithTotalUsage.d.ts +9 -2
- package/esm/typings/src/llm-providers/_common/utils/count-total-usage/{countTotalUsage.d.ts → countUsage.d.ts} +1 -1
- package/esm/typings/src/llm-providers/_common/utils/count-total-usage/limitTotalUsage.d.ts +2 -2
- package/esm/typings/src/llm-providers/anthropic-claude/AnthropicClaudeExecutionToolsOptions.d.ts +1 -1
- package/esm/typings/src/llm-providers/anthropic-claude/computeAnthropicClaudeUsage.d.ts +2 -2
- package/esm/typings/src/llm-providers/anthropic-claude/register-configuration.d.ts +1 -1
- package/esm/typings/src/llm-providers/openai/OpenAiExecutionTools.d.ts +0 -9
- package/esm/typings/src/llm-providers/openai/computeOpenAiUsage.d.ts +2 -2
- package/esm/typings/src/pipeline/PipelineJson/PreparationJson.d.ts +2 -2
- package/esm/typings/src/playground/playground.d.ts +5 -0
- package/esm/typings/src/remote-server/RemoteServer.d.ts +23 -0
- package/esm/typings/src/remote-server/socket-types/_subtypes/{PromptbookServer_Identification.d.ts → Identification.d.ts} +5 -4
- package/esm/typings/src/remote-server/socket-types/listModels/PromptbookServer_ListModels_Request.d.ts +2 -2
- package/esm/typings/src/remote-server/socket-types/prepare/PromptbookServer_PreparePipeline_Request.d.ts +2 -2
- package/esm/typings/src/remote-server/socket-types/prompt/PromptbookServer_Prompt_Request.d.ts +2 -2
- package/esm/typings/src/remote-server/startRemoteServer.d.ts +2 -2
- package/esm/typings/src/remote-server/types/RemoteClientOptions.d.ts +4 -12
- package/esm/typings/src/remote-server/types/RemoteServerOptions.d.ts +88 -6
- package/esm/typings/src/scrapers/_common/utils/{scraperFetch.d.ts → promptbookFetch.d.ts} +2 -2
- package/esm/typings/src/storage/env-storage/$EnvStorage.d.ts +37 -0
- package/esm/typings/src/types/typeAliases.d.ts +8 -2
- package/esm/typings/src/utils/organization/TODO_narrow.d.ts +6 -0
- package/package.json +4 -2
- package/umd/index.umd.js +379 -102
- package/umd/index.umd.js.map +1 -1
package/esm/index.es.js
CHANGED
|
@@ -3,6 +3,8 @@ import express from 'express';
|
|
|
3
3
|
import http from 'http';
|
|
4
4
|
import { Server } from 'socket.io';
|
|
5
5
|
import spaceTrim$1, { spaceTrim } from 'spacetrim';
|
|
6
|
+
import swaggerJsdoc from 'swagger-jsdoc';
|
|
7
|
+
import swaggerUi from 'swagger-ui-express';
|
|
6
8
|
import { forTime } from 'waitasecond';
|
|
7
9
|
import { randomBytes } from 'crypto';
|
|
8
10
|
import { spawn } from 'child_process';
|
|
@@ -31,7 +33,7 @@ const BOOK_LANGUAGE_VERSION = '1.0.0';
|
|
|
31
33
|
* @generated
|
|
32
34
|
* @see https://github.com/webgptorg/promptbook
|
|
33
35
|
*/
|
|
34
|
-
const PROMPTBOOK_ENGINE_VERSION = '0.
|
|
36
|
+
const PROMPTBOOK_ENGINE_VERSION = '0.89.0-11';
|
|
35
37
|
/**
|
|
36
38
|
* TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
|
|
37
39
|
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
@@ -156,6 +158,7 @@ const DEFAULT_MAX_PARALLEL_COUNT = 5; // <- TODO: [🤹♂️]
|
|
|
156
158
|
*/
|
|
157
159
|
const DEFAULT_MAX_EXECUTION_ATTEMPTS = 10; // <- TODO: [🤹♂️]
|
|
158
160
|
// <- TODO: [🕝] Make also `BOOKS_DIRNAME_ALTERNATIVES`
|
|
161
|
+
// TODO: !!!!!! Just .promptbook dir, hardocode others
|
|
159
162
|
/**
|
|
160
163
|
* Where to store the temporary downloads
|
|
161
164
|
*
|
|
@@ -207,9 +210,22 @@ const IS_PIPELINE_LOGIC_VALIDATED = just(
|
|
|
207
210
|
true);
|
|
208
211
|
/**
|
|
209
212
|
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
210
|
-
* TODO: [🧠][🧜♂️] Maybe join
|
|
213
|
+
* TODO: [🧠][🧜♂️] Maybe join remoteServerUrl and path into single value
|
|
211
214
|
*/
|
|
212
215
|
|
|
216
|
+
/**
|
|
217
|
+
* AuthenticationError is thrown from login function which is dependency of remote server
|
|
218
|
+
*
|
|
219
|
+
* @public exported from `@promptbook/core`
|
|
220
|
+
*/
|
|
221
|
+
class AuthenticationError extends Error {
|
|
222
|
+
constructor(message) {
|
|
223
|
+
super(message);
|
|
224
|
+
this.name = 'AuthenticationError';
|
|
225
|
+
Object.setPrototypeOf(this, AuthenticationError.prototype);
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
|
|
213
229
|
/**
|
|
214
230
|
* Generates random token
|
|
215
231
|
*
|
|
@@ -438,6 +454,19 @@ class PipelineUrlError extends Error {
|
|
|
438
454
|
}
|
|
439
455
|
}
|
|
440
456
|
|
|
457
|
+
/**
|
|
458
|
+
* Error thrown when a fetch request fails
|
|
459
|
+
*
|
|
460
|
+
* @public exported from `@promptbook/core`
|
|
461
|
+
*/
|
|
462
|
+
class PromptbookFetchError extends Error {
|
|
463
|
+
constructor(message) {
|
|
464
|
+
super(message);
|
|
465
|
+
this.name = 'PromptbookFetchError';
|
|
466
|
+
Object.setPrototypeOf(this, PromptbookFetchError.prototype);
|
|
467
|
+
}
|
|
468
|
+
}
|
|
469
|
+
|
|
441
470
|
/**
|
|
442
471
|
* Make error report URL for the given error
|
|
443
472
|
*
|
|
@@ -543,6 +572,8 @@ const COMMON_JAVASCRIPT_ERRORS = {
|
|
|
543
572
|
TypeError,
|
|
544
573
|
URIError,
|
|
545
574
|
AggregateError,
|
|
575
|
+
AuthenticationError,
|
|
576
|
+
PromptbookFetchError,
|
|
546
577
|
/*
|
|
547
578
|
Note: Not widely supported
|
|
548
579
|
> InternalError,
|
|
@@ -578,6 +609,10 @@ function serializeError(error) {
|
|
|
578
609
|
|
|
579
610
|
Cannot serialize error with name "${name}"
|
|
580
611
|
|
|
612
|
+
Authors of Promptbook probably forgot to add this error into the list of errors:
|
|
613
|
+
https://github.com/webgptorg/promptbook/blob/main/src/errors/0-index.ts
|
|
614
|
+
|
|
615
|
+
|
|
581
616
|
${block(stack || message)}
|
|
582
617
|
|
|
583
618
|
`));
|
|
@@ -694,6 +729,7 @@ function $execCommand(options) {
|
|
|
694
729
|
}
|
|
695
730
|
else {
|
|
696
731
|
console.warn(`Command "${humanReadableCommand}" exceeded time limit of ${timeout}ms but continues running`);
|
|
732
|
+
// <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
|
|
697
733
|
resolve('Command exceeded time limit');
|
|
698
734
|
}
|
|
699
735
|
});
|
|
@@ -719,6 +755,7 @@ function $execCommand(options) {
|
|
|
719
755
|
output.push(stderr.toString());
|
|
720
756
|
if (isVerbose && stderr.toString().trim()) {
|
|
721
757
|
console.warn(stderr.toString());
|
|
758
|
+
// <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
|
|
722
759
|
}
|
|
723
760
|
});
|
|
724
761
|
const finishWithCode = (code) => {
|
|
@@ -730,6 +767,7 @@ function $execCommand(options) {
|
|
|
730
767
|
else {
|
|
731
768
|
if (isVerbose) {
|
|
732
769
|
console.warn(`Command "${humanReadableCommand}" exited with code ${code}`);
|
|
770
|
+
// <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
|
|
733
771
|
}
|
|
734
772
|
resolve(spaceTrim(output.join('\n')));
|
|
735
773
|
}
|
|
@@ -751,6 +789,7 @@ function $execCommand(options) {
|
|
|
751
789
|
else {
|
|
752
790
|
if (isVerbose) {
|
|
753
791
|
console.warn(error);
|
|
792
|
+
// <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
|
|
754
793
|
}
|
|
755
794
|
resolve(spaceTrim(output.join('\n')));
|
|
756
795
|
}
|
|
@@ -1819,6 +1858,7 @@ function assertsTaskSuccessful(executionResult) {
|
|
|
1819
1858
|
const { isSuccessful, errors, warnings } = executionResult;
|
|
1820
1859
|
for (const warning of warnings) {
|
|
1821
1860
|
console.warn(warning.message);
|
|
1861
|
+
// <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
|
|
1822
1862
|
}
|
|
1823
1863
|
if (isSuccessful === true) {
|
|
1824
1864
|
return;
|
|
@@ -2420,30 +2460,42 @@ async function forEachAsync(array, options, callbackfunction) {
|
|
|
2420
2460
|
await Promise.all(tasks);
|
|
2421
2461
|
}
|
|
2422
2462
|
|
|
2463
|
+
/**
|
|
2464
|
+
* Represents the uncertain value
|
|
2465
|
+
*
|
|
2466
|
+
* @public exported from `@promptbook/core`
|
|
2467
|
+
*/
|
|
2468
|
+
const ZERO_VALUE = $deepFreeze({ value: 0 });
|
|
2469
|
+
/**
|
|
2470
|
+
* Represents the uncertain value
|
|
2471
|
+
*
|
|
2472
|
+
* @public exported from `@promptbook/core`
|
|
2473
|
+
*/
|
|
2474
|
+
const UNCERTAIN_ZERO_VALUE = $deepFreeze({ value: 0, isUncertain: true });
|
|
2423
2475
|
/**
|
|
2424
2476
|
* Represents the usage with no resources consumed
|
|
2425
2477
|
*
|
|
2426
2478
|
* @public exported from `@promptbook/core`
|
|
2427
2479
|
*/
|
|
2428
2480
|
const ZERO_USAGE = $deepFreeze({
|
|
2429
|
-
price:
|
|
2481
|
+
price: ZERO_VALUE,
|
|
2430
2482
|
input: {
|
|
2431
|
-
tokensCount:
|
|
2432
|
-
charactersCount:
|
|
2433
|
-
wordsCount:
|
|
2434
|
-
sentencesCount:
|
|
2435
|
-
linesCount:
|
|
2436
|
-
paragraphsCount:
|
|
2437
|
-
pagesCount:
|
|
2483
|
+
tokensCount: ZERO_VALUE,
|
|
2484
|
+
charactersCount: ZERO_VALUE,
|
|
2485
|
+
wordsCount: ZERO_VALUE,
|
|
2486
|
+
sentencesCount: ZERO_VALUE,
|
|
2487
|
+
linesCount: ZERO_VALUE,
|
|
2488
|
+
paragraphsCount: ZERO_VALUE,
|
|
2489
|
+
pagesCount: ZERO_VALUE,
|
|
2438
2490
|
},
|
|
2439
2491
|
output: {
|
|
2440
|
-
tokensCount:
|
|
2441
|
-
charactersCount:
|
|
2442
|
-
wordsCount:
|
|
2443
|
-
sentencesCount:
|
|
2444
|
-
linesCount:
|
|
2445
|
-
paragraphsCount:
|
|
2446
|
-
pagesCount:
|
|
2492
|
+
tokensCount: ZERO_VALUE,
|
|
2493
|
+
charactersCount: ZERO_VALUE,
|
|
2494
|
+
wordsCount: ZERO_VALUE,
|
|
2495
|
+
sentencesCount: ZERO_VALUE,
|
|
2496
|
+
linesCount: ZERO_VALUE,
|
|
2497
|
+
paragraphsCount: ZERO_VALUE,
|
|
2498
|
+
pagesCount: ZERO_VALUE,
|
|
2447
2499
|
},
|
|
2448
2500
|
});
|
|
2449
2501
|
/**
|
|
@@ -2452,24 +2504,24 @@ const ZERO_USAGE = $deepFreeze({
|
|
|
2452
2504
|
* @public exported from `@promptbook/core`
|
|
2453
2505
|
*/
|
|
2454
2506
|
$deepFreeze({
|
|
2455
|
-
price:
|
|
2507
|
+
price: UNCERTAIN_ZERO_VALUE,
|
|
2456
2508
|
input: {
|
|
2457
|
-
tokensCount:
|
|
2458
|
-
charactersCount:
|
|
2459
|
-
wordsCount:
|
|
2460
|
-
sentencesCount:
|
|
2461
|
-
linesCount:
|
|
2462
|
-
paragraphsCount:
|
|
2463
|
-
pagesCount:
|
|
2509
|
+
tokensCount: UNCERTAIN_ZERO_VALUE,
|
|
2510
|
+
charactersCount: UNCERTAIN_ZERO_VALUE,
|
|
2511
|
+
wordsCount: UNCERTAIN_ZERO_VALUE,
|
|
2512
|
+
sentencesCount: UNCERTAIN_ZERO_VALUE,
|
|
2513
|
+
linesCount: UNCERTAIN_ZERO_VALUE,
|
|
2514
|
+
paragraphsCount: UNCERTAIN_ZERO_VALUE,
|
|
2515
|
+
pagesCount: UNCERTAIN_ZERO_VALUE,
|
|
2464
2516
|
},
|
|
2465
2517
|
output: {
|
|
2466
|
-
tokensCount:
|
|
2467
|
-
charactersCount:
|
|
2468
|
-
wordsCount:
|
|
2469
|
-
sentencesCount:
|
|
2470
|
-
linesCount:
|
|
2471
|
-
paragraphsCount:
|
|
2472
|
-
pagesCount:
|
|
2518
|
+
tokensCount: UNCERTAIN_ZERO_VALUE,
|
|
2519
|
+
charactersCount: UNCERTAIN_ZERO_VALUE,
|
|
2520
|
+
wordsCount: UNCERTAIN_ZERO_VALUE,
|
|
2521
|
+
sentencesCount: UNCERTAIN_ZERO_VALUE,
|
|
2522
|
+
linesCount: UNCERTAIN_ZERO_VALUE,
|
|
2523
|
+
paragraphsCount: UNCERTAIN_ZERO_VALUE,
|
|
2524
|
+
pagesCount: UNCERTAIN_ZERO_VALUE,
|
|
2473
2525
|
},
|
|
2474
2526
|
});
|
|
2475
2527
|
/**
|
|
@@ -2530,8 +2582,9 @@ function addUsage(...usageItems) {
|
|
|
2530
2582
|
* @returns LLM tools with same functionality with added total cost counting
|
|
2531
2583
|
* @public exported from `@promptbook/core`
|
|
2532
2584
|
*/
|
|
2533
|
-
function
|
|
2585
|
+
function countUsage(llmTools) {
|
|
2534
2586
|
let totalUsage = ZERO_USAGE;
|
|
2587
|
+
const spending = new Subject();
|
|
2535
2588
|
const proxyTools = {
|
|
2536
2589
|
get title() {
|
|
2537
2590
|
// TODO: [🧠] Maybe put here some suffix
|
|
@@ -2541,12 +2594,15 @@ function countTotalUsage(llmTools) {
|
|
|
2541
2594
|
// TODO: [🧠] Maybe put here some suffix
|
|
2542
2595
|
return llmTools.description;
|
|
2543
2596
|
},
|
|
2544
|
-
|
|
2597
|
+
checkConfiguration() {
|
|
2545
2598
|
return /* not await */ llmTools.checkConfiguration();
|
|
2546
2599
|
},
|
|
2547
2600
|
listModels() {
|
|
2548
2601
|
return /* not await */ llmTools.listModels();
|
|
2549
2602
|
},
|
|
2603
|
+
spending() {
|
|
2604
|
+
return spending.asObservable();
|
|
2605
|
+
},
|
|
2550
2606
|
getTotalUsage() {
|
|
2551
2607
|
// <- Note: [🥫] Not using getter `get totalUsage` but `getTotalUsage` to allow this object to be proxied
|
|
2552
2608
|
return totalUsage;
|
|
@@ -2557,6 +2613,7 @@ function countTotalUsage(llmTools) {
|
|
|
2557
2613
|
// console.info('[🚕] callChatModel through countTotalUsage');
|
|
2558
2614
|
const promptResult = await llmTools.callChatModel(prompt);
|
|
2559
2615
|
totalUsage = addUsage(totalUsage, promptResult.usage);
|
|
2616
|
+
spending.next(promptResult.usage);
|
|
2560
2617
|
return promptResult;
|
|
2561
2618
|
};
|
|
2562
2619
|
}
|
|
@@ -2565,6 +2622,7 @@ function countTotalUsage(llmTools) {
|
|
|
2565
2622
|
// console.info('[🚕] callCompletionModel through countTotalUsage');
|
|
2566
2623
|
const promptResult = await llmTools.callCompletionModel(prompt);
|
|
2567
2624
|
totalUsage = addUsage(totalUsage, promptResult.usage);
|
|
2625
|
+
spending.next(promptResult.usage);
|
|
2568
2626
|
return promptResult;
|
|
2569
2627
|
};
|
|
2570
2628
|
}
|
|
@@ -2573,6 +2631,7 @@ function countTotalUsage(llmTools) {
|
|
|
2573
2631
|
// console.info('[🚕] callEmbeddingModel through countTotalUsage');
|
|
2574
2632
|
const promptResult = await llmTools.callEmbeddingModel(prompt);
|
|
2575
2633
|
totalUsage = addUsage(totalUsage, promptResult.usage);
|
|
2634
|
+
spending.next(promptResult.usage);
|
|
2576
2635
|
return promptResult;
|
|
2577
2636
|
};
|
|
2578
2637
|
}
|
|
@@ -2750,6 +2809,7 @@ function joinLlmExecutionTools(...llmExecutionTools) {
|
|
|
2750
2809
|
`);
|
|
2751
2810
|
// TODO: [🟥] Detect browser / node and make it colorfull
|
|
2752
2811
|
console.warn(warningMessage);
|
|
2812
|
+
// <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
|
|
2753
2813
|
/*
|
|
2754
2814
|
return {
|
|
2755
2815
|
async listModels() {
|
|
@@ -3521,17 +3581,24 @@ function titleToName(value) {
|
|
|
3521
3581
|
/**
|
|
3522
3582
|
* The built-in `fetch' function with a lightweight error handling wrapper as default fetch function used in Promptbook scrapers
|
|
3523
3583
|
*
|
|
3524
|
-
* @
|
|
3584
|
+
* @public exported from `@promptbook/core`
|
|
3525
3585
|
*/
|
|
3526
|
-
const
|
|
3586
|
+
const promptbookFetch = async (urlOrRequest, init) => {
|
|
3527
3587
|
try {
|
|
3528
|
-
return await fetch(
|
|
3588
|
+
return await fetch(urlOrRequest, init);
|
|
3529
3589
|
}
|
|
3530
3590
|
catch (error) {
|
|
3531
3591
|
if (!(error instanceof Error)) {
|
|
3532
3592
|
throw error;
|
|
3533
3593
|
}
|
|
3534
|
-
|
|
3594
|
+
let url;
|
|
3595
|
+
if (typeof urlOrRequest === 'string') {
|
|
3596
|
+
url = urlOrRequest;
|
|
3597
|
+
}
|
|
3598
|
+
else if (urlOrRequest instanceof Request) {
|
|
3599
|
+
url = urlOrRequest.url;
|
|
3600
|
+
}
|
|
3601
|
+
throw new PromptbookFetchError(spaceTrim$1((block) => `
|
|
3535
3602
|
Can not fetch "${url}"
|
|
3536
3603
|
|
|
3537
3604
|
Fetch error:
|
|
@@ -3552,7 +3619,7 @@ const scraperFetch = async (url, init) => {
|
|
|
3552
3619
|
async function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
|
|
3553
3620
|
// console.log('!! makeKnowledgeSourceHandler', knowledgeSource);
|
|
3554
3621
|
var _a;
|
|
3555
|
-
const { fetch =
|
|
3622
|
+
const { fetch = promptbookFetch } = tools;
|
|
3556
3623
|
const { knowledgeSourceContent } = knowledgeSource;
|
|
3557
3624
|
let { name } = knowledgeSource;
|
|
3558
3625
|
const { rootDirname = null,
|
|
@@ -3693,63 +3760,73 @@ async function prepareKnowledgePieces(knowledgeSources, tools, options) {
|
|
|
3693
3760
|
const { maxParallelCount = DEFAULT_MAX_PARALLEL_COUNT, rootDirname, isVerbose = DEFAULT_IS_VERBOSE } = options;
|
|
3694
3761
|
const knowledgePreparedUnflatten = new Array(knowledgeSources.length);
|
|
3695
3762
|
await forEachAsync(knowledgeSources, { maxParallelCount }, async (knowledgeSource, index) => {
|
|
3696
|
-
|
|
3697
|
-
|
|
3698
|
-
|
|
3699
|
-
|
|
3700
|
-
|
|
3701
|
-
|
|
3702
|
-
|
|
3703
|
-
|
|
3704
|
-
|
|
3705
|
-
|
|
3706
|
-
|
|
3707
|
-
|
|
3708
|
-
|
|
3709
|
-
|
|
3710
|
-
|
|
3711
|
-
|
|
3712
|
-
|
|
3763
|
+
try {
|
|
3764
|
+
let partialPieces = null;
|
|
3765
|
+
const sourceHandler = await makeKnowledgeSourceHandler(knowledgeSource, tools, { rootDirname, isVerbose });
|
|
3766
|
+
const scrapers = arrayableToArray(tools.scrapers);
|
|
3767
|
+
for (const scraper of scrapers) {
|
|
3768
|
+
if (!scraper.metadata.mimeTypes.includes(sourceHandler.mimeType)
|
|
3769
|
+
// <- TODO: [🦔] Implement mime-type wildcards
|
|
3770
|
+
) {
|
|
3771
|
+
continue;
|
|
3772
|
+
}
|
|
3773
|
+
const partialPiecesUnchecked = await scraper.scrape(sourceHandler);
|
|
3774
|
+
if (partialPiecesUnchecked !== null) {
|
|
3775
|
+
partialPieces = [...partialPiecesUnchecked];
|
|
3776
|
+
// <- TODO: [🪓] Here should be no need for spreading new array, just `partialPieces = partialPiecesUnchecked`
|
|
3777
|
+
break;
|
|
3778
|
+
}
|
|
3779
|
+
console.warn(spaceTrim$1((block) => `
|
|
3780
|
+
Cannot scrape knowledge from source despite the scraper \`${scraper.metadata.className}\` supports the mime type "${sourceHandler.mimeType}".
|
|
3713
3781
|
|
|
3714
|
-
|
|
3715
|
-
|
|
3716
|
-
|
|
3717
|
-
|
|
3718
|
-
|
|
3782
|
+
The source:
|
|
3783
|
+
${block(knowledgeSource.knowledgeSourceContent
|
|
3784
|
+
.split('\n')
|
|
3785
|
+
.map((line) => `> ${line}`)
|
|
3786
|
+
.join('\n'))}
|
|
3719
3787
|
|
|
3720
|
-
|
|
3788
|
+
${block($registeredScrapersMessage(scrapers))}
|
|
3721
3789
|
|
|
3722
3790
|
|
|
3723
|
-
|
|
3724
|
-
|
|
3725
|
-
|
|
3726
|
-
|
|
3727
|
-
|
|
3791
|
+
`));
|
|
3792
|
+
// <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
|
|
3793
|
+
}
|
|
3794
|
+
if (partialPieces === null) {
|
|
3795
|
+
throw new KnowledgeScrapeError(spaceTrim$1((block) => `
|
|
3796
|
+
Cannot scrape knowledge
|
|
3728
3797
|
|
|
3729
|
-
|
|
3730
|
-
|
|
3731
|
-
|
|
3732
|
-
|
|
3733
|
-
|
|
3798
|
+
The source:
|
|
3799
|
+
> ${block(knowledgeSource.knowledgeSourceContent
|
|
3800
|
+
.split('\n')
|
|
3801
|
+
.map((line) => `> ${line}`)
|
|
3802
|
+
.join('\n'))}
|
|
3734
3803
|
|
|
3735
|
-
|
|
3804
|
+
No scraper found for the mime type "${sourceHandler.mimeType}"
|
|
3736
3805
|
|
|
3737
|
-
|
|
3806
|
+
${block($registeredScrapersMessage(scrapers))}
|
|
3738
3807
|
|
|
3739
3808
|
|
|
3740
|
-
|
|
3809
|
+
`));
|
|
3810
|
+
}
|
|
3811
|
+
const pieces = partialPieces.map((partialPiece) => ({
|
|
3812
|
+
...partialPiece,
|
|
3813
|
+
sources: [
|
|
3814
|
+
{
|
|
3815
|
+
name: knowledgeSource.name,
|
|
3816
|
+
// line, column <- TODO: [☀]
|
|
3817
|
+
// <- TODO: [❎]
|
|
3818
|
+
},
|
|
3819
|
+
],
|
|
3820
|
+
}));
|
|
3821
|
+
knowledgePreparedUnflatten[index] = pieces;
|
|
3822
|
+
}
|
|
3823
|
+
catch (error) {
|
|
3824
|
+
if (!(error instanceof Error)) {
|
|
3825
|
+
throw error;
|
|
3826
|
+
}
|
|
3827
|
+
console.warn(error);
|
|
3828
|
+
// <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
|
|
3741
3829
|
}
|
|
3742
|
-
const pieces = partialPieces.map((partialPiece) => ({
|
|
3743
|
-
...partialPiece,
|
|
3744
|
-
sources: [
|
|
3745
|
-
{
|
|
3746
|
-
name: knowledgeSource.name,
|
|
3747
|
-
// line, column <- TODO: [☀]
|
|
3748
|
-
// <- TODO: [❎]
|
|
3749
|
-
},
|
|
3750
|
-
],
|
|
3751
|
-
}));
|
|
3752
|
-
knowledgePreparedUnflatten[index] = pieces;
|
|
3753
3830
|
});
|
|
3754
3831
|
const knowledgePrepared = knowledgePreparedUnflatten.flat();
|
|
3755
3832
|
return knowledgePrepared;
|
|
@@ -3855,7 +3932,7 @@ async function preparePipeline(pipeline, tools, options) {
|
|
|
3855
3932
|
// TODO: [🚐] Make arrayable LLMs -> single LLM DRY
|
|
3856
3933
|
const _llms = arrayableToArray(tools.llm);
|
|
3857
3934
|
const llmTools = _llms.length === 1 ? _llms[0] : joinLlmExecutionTools(..._llms);
|
|
3858
|
-
const llmToolsWithUsage =
|
|
3935
|
+
const llmToolsWithUsage = countUsage(llmTools);
|
|
3859
3936
|
// <- TODO: [🌯]
|
|
3860
3937
|
/*
|
|
3861
3938
|
TODO: [🧠][🪑][🔃] Should this be done or not
|
|
@@ -4167,7 +4244,7 @@ function extractParameterNamesFromTask(task) {
|
|
|
4167
4244
|
if (parameterNames.has(subparameterName)) {
|
|
4168
4245
|
parameterNames.delete(subparameterName);
|
|
4169
4246
|
parameterNames.add(foreach.parameterName);
|
|
4170
|
-
// <- TODO: [
|
|
4247
|
+
// <- TODO: [🏮] Warn/logic error when `subparameterName` not used
|
|
4171
4248
|
}
|
|
4172
4249
|
}
|
|
4173
4250
|
}
|
|
@@ -5780,6 +5857,7 @@ function createPipelineExecutor(options) {
|
|
|
5780
5857
|
|
|
5781
5858
|
@see more at https://ptbk.io/prepare-pipeline
|
|
5782
5859
|
`));
|
|
5860
|
+
// <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
|
|
5783
5861
|
}
|
|
5784
5862
|
let runCount = 0;
|
|
5785
5863
|
const pipelineExecutorWithCallback = async (inputParameters, onProgress) => {
|
|
@@ -6675,11 +6753,12 @@ async function $provideScriptingForNode(options) {
|
|
|
6675
6753
|
* @public exported from `@promptbook/remote-server`
|
|
6676
6754
|
*/
|
|
6677
6755
|
function startRemoteServer(options) {
|
|
6678
|
-
const { port, collection, createLlmExecutionTools, isAnonymousModeAllowed, isApplicationModeAllowed, isVerbose = DEFAULT_IS_VERBOSE, } = {
|
|
6756
|
+
const { port, collection, createLlmExecutionTools, isAnonymousModeAllowed, isApplicationModeAllowed, isVerbose = DEFAULT_IS_VERBOSE, login, } = {
|
|
6679
6757
|
isAnonymousModeAllowed: false,
|
|
6680
6758
|
isApplicationModeAllowed: false,
|
|
6681
6759
|
collection: null,
|
|
6682
6760
|
createLlmExecutionTools: null,
|
|
6761
|
+
login: null,
|
|
6683
6762
|
...options,
|
|
6684
6763
|
};
|
|
6685
6764
|
// <- TODO: [🦪] Some helper type to be able to use discriminant union types with destructuring
|
|
@@ -6746,9 +6825,38 @@ function startRemoteServer(options) {
|
|
|
6746
6825
|
response.setHeader('X-Powered-By', 'Promptbook engine');
|
|
6747
6826
|
next();
|
|
6748
6827
|
});
|
|
6828
|
+
const swaggerOptions = {
|
|
6829
|
+
definition: {
|
|
6830
|
+
openapi: '3.0.0',
|
|
6831
|
+
info: {
|
|
6832
|
+
title: 'Promptbook Remote Server API',
|
|
6833
|
+
version: '1.0.0',
|
|
6834
|
+
description: 'API documentation for the Promptbook Remote Server',
|
|
6835
|
+
},
|
|
6836
|
+
servers: [
|
|
6837
|
+
{
|
|
6838
|
+
url: `http://localhost:${port}${rootPath}`,
|
|
6839
|
+
// <- TODO: !!!!! Probbably: Pass `remoteServerUrl` instead of `port` and `rootPath`
|
|
6840
|
+
},
|
|
6841
|
+
],
|
|
6842
|
+
},
|
|
6843
|
+
apis: ['./src/remote-server/**/*.ts'], // Adjust path as needed
|
|
6844
|
+
};
|
|
6845
|
+
const swaggerSpec = swaggerJsdoc(swaggerOptions);
|
|
6846
|
+
app.use([`/api-docs`, `${rootPath}/api-docs`], swaggerUi.serve, swaggerUi.setup(swaggerSpec));
|
|
6749
6847
|
const runningExecutionTasks = [];
|
|
6750
6848
|
// <- TODO: [🤬] Identify the users
|
|
6751
6849
|
// TODO: [🧠] Do here some garbage collection of finished tasks
|
|
6850
|
+
/**
|
|
6851
|
+
* @swagger
|
|
6852
|
+
* /:
|
|
6853
|
+
* get:
|
|
6854
|
+
* summary: Get server details
|
|
6855
|
+
* description: Returns details about the Promptbook server.
|
|
6856
|
+
* responses:
|
|
6857
|
+
* 200:
|
|
6858
|
+
* description: Server details in markdown format.
|
|
6859
|
+
*/
|
|
6752
6860
|
app.get(['/', rootPath], async (request, response) => {
|
|
6753
6861
|
var _a;
|
|
6754
6862
|
if ((_a = request.url) === null || _a === void 0 ? void 0 : _a.includes('socket.io')) {
|
|
@@ -6785,9 +6893,12 @@ function startRemoteServer(options) {
|
|
|
6785
6893
|
|
|
6786
6894
|
## Paths
|
|
6787
6895
|
|
|
6788
|
-
${block(
|
|
6789
|
-
.
|
|
6790
|
-
|
|
6896
|
+
${block([
|
|
6897
|
+
...app._router.stack
|
|
6898
|
+
.map(({ route }) => (route === null || route === void 0 ? void 0 : route.path) || null)
|
|
6899
|
+
.filter((path) => path !== null),
|
|
6900
|
+
'/api-docs',
|
|
6901
|
+
]
|
|
6791
6902
|
.map((path) => `- ${path}`)
|
|
6792
6903
|
.join('\n'))}
|
|
6793
6904
|
|
|
@@ -6805,7 +6916,96 @@ function startRemoteServer(options) {
|
|
|
6805
6916
|
https://github.com/webgptorg/promptbook
|
|
6806
6917
|
`));
|
|
6807
6918
|
});
|
|
6808
|
-
|
|
6919
|
+
/**
|
|
6920
|
+
* @swagger
|
|
6921
|
+
*
|
|
6922
|
+
* /login:
|
|
6923
|
+
* post:
|
|
6924
|
+
* summary: Login to the server
|
|
6925
|
+
* description: Login to the server and get identification.
|
|
6926
|
+
* requestBody:
|
|
6927
|
+
* required: true
|
|
6928
|
+
* content:
|
|
6929
|
+
* application/json:
|
|
6930
|
+
* schema:
|
|
6931
|
+
* type: object
|
|
6932
|
+
* properties:
|
|
6933
|
+
* username:
|
|
6934
|
+
* type: string
|
|
6935
|
+
* password:
|
|
6936
|
+
* type: string
|
|
6937
|
+
* appId:
|
|
6938
|
+
* type: string
|
|
6939
|
+
* responses:
|
|
6940
|
+
* 200:
|
|
6941
|
+
* description: Successful login
|
|
6942
|
+
* content:
|
|
6943
|
+
* application/json:
|
|
6944
|
+
* schema:
|
|
6945
|
+
* type: object
|
|
6946
|
+
* properties:
|
|
6947
|
+
* identification:
|
|
6948
|
+
* type: object
|
|
6949
|
+
*/
|
|
6950
|
+
app.post([`/login`, `${rootPath}/login`], async (request, response) => {
|
|
6951
|
+
if (!isApplicationModeAllowed || login === null) {
|
|
6952
|
+
response.status(400).send('Application mode is not allowed');
|
|
6953
|
+
return;
|
|
6954
|
+
}
|
|
6955
|
+
try {
|
|
6956
|
+
const username = request.body.username;
|
|
6957
|
+
const password = request.body.password;
|
|
6958
|
+
const appId = request.body.appId;
|
|
6959
|
+
const { isSuccess, error, message, identification } = await login({
|
|
6960
|
+
username,
|
|
6961
|
+
password,
|
|
6962
|
+
appId,
|
|
6963
|
+
rawRequest: request,
|
|
6964
|
+
rawResponse: response,
|
|
6965
|
+
});
|
|
6966
|
+
response.status(201).send({
|
|
6967
|
+
isSuccess,
|
|
6968
|
+
message,
|
|
6969
|
+
error: error ? serializeError(error) : undefined,
|
|
6970
|
+
identification,
|
|
6971
|
+
});
|
|
6972
|
+
return;
|
|
6973
|
+
}
|
|
6974
|
+
catch (error) {
|
|
6975
|
+
if (!(error instanceof Error)) {
|
|
6976
|
+
throw error;
|
|
6977
|
+
}
|
|
6978
|
+
if (error instanceof AuthenticationError) {
|
|
6979
|
+
response.status(401).send({
|
|
6980
|
+
isSuccess: false,
|
|
6981
|
+
message: error.message,
|
|
6982
|
+
error: serializeError(error),
|
|
6983
|
+
});
|
|
6984
|
+
}
|
|
6985
|
+
console.warn(`Login function thrown different error than AuthenticationError`, {
|
|
6986
|
+
error,
|
|
6987
|
+
serializedError: serializeError(error),
|
|
6988
|
+
});
|
|
6989
|
+
response.status(400).send({ error: serializeError(error) });
|
|
6990
|
+
}
|
|
6991
|
+
});
|
|
6992
|
+
/**
|
|
6993
|
+
* @swagger
|
|
6994
|
+
* /books:
|
|
6995
|
+
* get:
|
|
6996
|
+
* summary: List all books
|
|
6997
|
+
* description: Returns a list of all available books in the collection.
|
|
6998
|
+
* responses:
|
|
6999
|
+
* 200:
|
|
7000
|
+
* description: A list of books.
|
|
7001
|
+
* content:
|
|
7002
|
+
* application/json:
|
|
7003
|
+
* schema:
|
|
7004
|
+
* type: array
|
|
7005
|
+
* items:
|
|
7006
|
+
* type: string
|
|
7007
|
+
*/
|
|
7008
|
+
app.get([`/books`, `${rootPath}/books`], async (request, response) => {
|
|
6809
7009
|
if (collection === null) {
|
|
6810
7010
|
response.status(500).send('No collection available');
|
|
6811
7011
|
return;
|
|
@@ -6815,7 +7015,30 @@ function startRemoteServer(options) {
|
|
|
6815
7015
|
response.send(pipelines);
|
|
6816
7016
|
});
|
|
6817
7017
|
// TODO: [🧠] Is it secure / good idea to expose source codes of hosted books
|
|
6818
|
-
|
|
7018
|
+
/**
|
|
7019
|
+
* @swagger
|
|
7020
|
+
* /books/{bookId}:
|
|
7021
|
+
* get:
|
|
7022
|
+
* summary: Get book content
|
|
7023
|
+
* description: Returns the content of a specific book.
|
|
7024
|
+
* parameters:
|
|
7025
|
+
* - in: path
|
|
7026
|
+
* name: bookId
|
|
7027
|
+
* required: true
|
|
7028
|
+
* schema:
|
|
7029
|
+
* type: string
|
|
7030
|
+
* description: The ID of the book to retrieve.
|
|
7031
|
+
* responses:
|
|
7032
|
+
* 200:
|
|
7033
|
+
* description: The content of the book.
|
|
7034
|
+
* content:
|
|
7035
|
+
* text/markdown:
|
|
7036
|
+
* schema:
|
|
7037
|
+
* type: string
|
|
7038
|
+
* 404:
|
|
7039
|
+
* description: Book not found.
|
|
7040
|
+
*/
|
|
7041
|
+
app.get([`/books/*`, `${rootPath}/books/*`], async (request, response) => {
|
|
6819
7042
|
try {
|
|
6820
7043
|
if (collection === null) {
|
|
6821
7044
|
response.status(500).send('No collection nor books available');
|
|
@@ -6869,10 +7092,26 @@ function startRemoteServer(options) {
|
|
|
6869
7092
|
};
|
|
6870
7093
|
}
|
|
6871
7094
|
}
|
|
6872
|
-
|
|
7095
|
+
/**
|
|
7096
|
+
* @swagger
|
|
7097
|
+
* /executions:
|
|
7098
|
+
* get:
|
|
7099
|
+
* summary: List all executions
|
|
7100
|
+
* description: Returns a list of all running execution tasks.
|
|
7101
|
+
* responses:
|
|
7102
|
+
* 200:
|
|
7103
|
+
* description: A list of execution tasks.
|
|
7104
|
+
* content:
|
|
7105
|
+
* application/json:
|
|
7106
|
+
* schema:
|
|
7107
|
+
* type: array
|
|
7108
|
+
* items:
|
|
7109
|
+
* type: object
|
|
7110
|
+
*/
|
|
7111
|
+
app.get([`/executions`, `${rootPath}/executions`], async (request, response) => {
|
|
6873
7112
|
response.send(runningExecutionTasks.map((runningExecutionTask) => exportExecutionTask(runningExecutionTask, false)));
|
|
6874
7113
|
});
|
|
6875
|
-
app.get(`${rootPath}/executions/last
|
|
7114
|
+
app.get([`/executions/last`, `${rootPath}/executions/last`], async (request, response) => {
|
|
6876
7115
|
// TODO: [🤬] Filter only for user
|
|
6877
7116
|
if (runningExecutionTasks.length === 0) {
|
|
6878
7117
|
response.status(404).send('No execution tasks found');
|
|
@@ -6881,7 +7120,7 @@ function startRemoteServer(options) {
|
|
|
6881
7120
|
const lastExecutionTask = runningExecutionTasks[runningExecutionTasks.length - 1];
|
|
6882
7121
|
response.send(exportExecutionTask(lastExecutionTask, true));
|
|
6883
7122
|
});
|
|
6884
|
-
app.get(`${rootPath}/executions/:taskId
|
|
7123
|
+
app.get([`/executions/:taskId`, `${rootPath}/executions/:taskId`], async (request, response) => {
|
|
6885
7124
|
const { taskId } = request.params;
|
|
6886
7125
|
// TODO: [🤬] Filter only for user
|
|
6887
7126
|
const executionTask = runningExecutionTasks.find((executionTask) => executionTask.taskId === taskId);
|
|
@@ -6893,7 +7132,36 @@ function startRemoteServer(options) {
|
|
|
6893
7132
|
}
|
|
6894
7133
|
response.send(exportExecutionTask(executionTask, true));
|
|
6895
7134
|
});
|
|
6896
|
-
|
|
7135
|
+
/**
|
|
7136
|
+
* @swagger
|
|
7137
|
+
* /executions/new:
|
|
7138
|
+
* post:
|
|
7139
|
+
* summary: Start a new execution
|
|
7140
|
+
* description: Starts a new execution task for a given pipeline.
|
|
7141
|
+
* requestBody:
|
|
7142
|
+
* required: true
|
|
7143
|
+
* content:
|
|
7144
|
+
* application/json:
|
|
7145
|
+
* schema:
|
|
7146
|
+
* type: object
|
|
7147
|
+
* properties:
|
|
7148
|
+
* pipelineUrl:
|
|
7149
|
+
* type: string
|
|
7150
|
+
* inputParameters:
|
|
7151
|
+
* type: object
|
|
7152
|
+
* identification:
|
|
7153
|
+
* type: object
|
|
7154
|
+
* responses:
|
|
7155
|
+
* 200:
|
|
7156
|
+
* description: The newly created execution task.
|
|
7157
|
+
* content:
|
|
7158
|
+
* application/json:
|
|
7159
|
+
* schema:
|
|
7160
|
+
* type: object
|
|
7161
|
+
* 400:
|
|
7162
|
+
* description: Invalid input.
|
|
7163
|
+
*/
|
|
7164
|
+
app.post([`/executions/new`, `${rootPath}/executions/new`], async (request, response) => {
|
|
6897
7165
|
try {
|
|
6898
7166
|
const { inputParameters, identification /* <- [🤬] */ } = request.body;
|
|
6899
7167
|
const pipelineUrl = request.body.pipelineUrl || request.body.book;
|
|
@@ -7070,6 +7338,15 @@ function startRemoteServer(options) {
|
|
|
7070
7338
|
}
|
|
7071
7339
|
let isDestroyed = false;
|
|
7072
7340
|
return {
|
|
7341
|
+
get httpServer() {
|
|
7342
|
+
return httpServer;
|
|
7343
|
+
},
|
|
7344
|
+
get expressApp() {
|
|
7345
|
+
return app;
|
|
7346
|
+
},
|
|
7347
|
+
get socketIoServer() {
|
|
7348
|
+
return server;
|
|
7349
|
+
},
|
|
7073
7350
|
get isDestroyed() {
|
|
7074
7351
|
return isDestroyed;
|
|
7075
7352
|
},
|