@promptbook/remote-server 0.88.0 → 0.89.0-11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/README.md +7 -1
  2. package/esm/index.es.js +375 -98
  3. package/esm/index.es.js.map +1 -1
  4. package/esm/typings/src/_packages/core.index.d.ts +18 -6
  5. package/esm/typings/src/_packages/remote-client.index.d.ts +6 -8
  6. package/esm/typings/src/_packages/remote-server.index.d.ts +6 -6
  7. package/esm/typings/src/_packages/types.index.d.ts +18 -10
  8. package/esm/typings/src/cli/cli-commands/login.d.ts +15 -0
  9. package/esm/typings/src/cli/common/$addGlobalOptionsToCommand.d.ts +7 -0
  10. package/esm/typings/src/cli/common/$provideLlmToolsForCli.d.ts +15 -0
  11. package/esm/typings/src/config.d.ts +15 -8
  12. package/esm/typings/src/errors/0-index.d.ts +6 -0
  13. package/esm/typings/src/errors/AuthenticationError.d.ts +9 -0
  14. package/esm/typings/src/errors/PromptbookFetchError.d.ts +9 -0
  15. package/esm/typings/src/execution/PipelineExecutorResult.d.ts +2 -2
  16. package/esm/typings/src/execution/PromptResult.d.ts +2 -2
  17. package/esm/typings/src/execution/{PromptResultUsage.d.ts → Usage.d.ts} +5 -5
  18. package/esm/typings/src/execution/utils/addUsage.d.ts +2 -2
  19. package/esm/typings/src/execution/utils/computeUsageCounts.d.ts +3 -3
  20. package/esm/typings/src/execution/utils/usage-constants.d.ts +77 -60
  21. package/esm/typings/src/execution/utils/usageToHuman.d.ts +5 -5
  22. package/esm/typings/src/execution/utils/usageToWorktime.d.ts +5 -5
  23. package/esm/typings/src/llm-providers/_common/register/$provideEnvFilename.d.ts +12 -0
  24. package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsConfigurationFromEnv.d.ts +2 -8
  25. package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground.d.ts +2 -0
  26. package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsForWizzardOrCli.d.ts +36 -1
  27. package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsFromEnv.d.ts +1 -0
  28. package/esm/typings/src/llm-providers/_common/utils/count-total-usage/LlmExecutionToolsWithTotalUsage.d.ts +9 -2
  29. package/esm/typings/src/llm-providers/_common/utils/count-total-usage/{countTotalUsage.d.ts → countUsage.d.ts} +1 -1
  30. package/esm/typings/src/llm-providers/_common/utils/count-total-usage/limitTotalUsage.d.ts +2 -2
  31. package/esm/typings/src/llm-providers/anthropic-claude/AnthropicClaudeExecutionToolsOptions.d.ts +1 -1
  32. package/esm/typings/src/llm-providers/anthropic-claude/computeAnthropicClaudeUsage.d.ts +2 -2
  33. package/esm/typings/src/llm-providers/anthropic-claude/register-configuration.d.ts +1 -1
  34. package/esm/typings/src/llm-providers/openai/OpenAiExecutionTools.d.ts +0 -9
  35. package/esm/typings/src/llm-providers/openai/computeOpenAiUsage.d.ts +2 -2
  36. package/esm/typings/src/pipeline/PipelineJson/PreparationJson.d.ts +2 -2
  37. package/esm/typings/src/playground/playground.d.ts +5 -0
  38. package/esm/typings/src/remote-server/RemoteServer.d.ts +23 -0
  39. package/esm/typings/src/remote-server/socket-types/_subtypes/{PromptbookServer_Identification.d.ts → Identification.d.ts} +5 -4
  40. package/esm/typings/src/remote-server/socket-types/listModels/PromptbookServer_ListModels_Request.d.ts +2 -2
  41. package/esm/typings/src/remote-server/socket-types/prepare/PromptbookServer_PreparePipeline_Request.d.ts +2 -2
  42. package/esm/typings/src/remote-server/socket-types/prompt/PromptbookServer_Prompt_Request.d.ts +2 -2
  43. package/esm/typings/src/remote-server/startRemoteServer.d.ts +2 -2
  44. package/esm/typings/src/remote-server/types/RemoteClientOptions.d.ts +4 -12
  45. package/esm/typings/src/remote-server/types/RemoteServerOptions.d.ts +88 -6
  46. package/esm/typings/src/scrapers/_common/utils/{scraperFetch.d.ts → promptbookFetch.d.ts} +2 -2
  47. package/esm/typings/src/storage/env-storage/$EnvStorage.d.ts +37 -0
  48. package/esm/typings/src/types/typeAliases.d.ts +8 -2
  49. package/esm/typings/src/utils/organization/TODO_narrow.d.ts +6 -0
  50. package/package.json +4 -2
  51. package/umd/index.umd.js +379 -102
  52. package/umd/index.umd.js.map +1 -1
package/umd/index.umd.js CHANGED
@@ -1,8 +1,8 @@
1
1
  (function (global, factory) {
2
- typeof exports === 'object' && typeof module !== 'undefined' ? factory(exports, require('colors'), require('express'), require('http'), require('socket.io'), require('spacetrim'), require('waitasecond'), require('crypto'), require('child_process'), require('fs/promises'), require('path'), require('rxjs'), require('prettier'), require('prettier/parser-html'), require('crypto-js/enc-hex'), require('crypto-js/sha256'), require('crypto-js'), require('mime-types'), require('papaparse')) :
3
- typeof define === 'function' && define.amd ? define(['exports', 'colors', 'express', 'http', 'socket.io', 'spacetrim', 'waitasecond', 'crypto', 'child_process', 'fs/promises', 'path', 'rxjs', 'prettier', 'prettier/parser-html', 'crypto-js/enc-hex', 'crypto-js/sha256', 'crypto-js', 'mime-types', 'papaparse'], factory) :
4
- (global = typeof globalThis !== 'undefined' ? globalThis : global || self, factory(global["promptbook-remote-server"] = {}, global.colors, global.express, global.http, global.socket_io, global.spaceTrim, global.waitasecond, global.crypto, global.child_process, global.promises, global.path, global.rxjs, global.prettier, global.parserHtml, global.hexEncoder, global.sha256, global.cryptoJs, global.mimeTypes, global.papaparse));
5
- })(this, (function (exports, colors, express, http, socket_io, spaceTrim, waitasecond, crypto, child_process, promises, path, rxjs, prettier, parserHtml, hexEncoder, sha256, cryptoJs, mimeTypes, papaparse) { 'use strict';
2
+ typeof exports === 'object' && typeof module !== 'undefined' ? factory(exports, require('colors'), require('express'), require('http'), require('socket.io'), require('spacetrim'), require('swagger-jsdoc'), require('swagger-ui-express'), require('waitasecond'), require('crypto'), require('child_process'), require('fs/promises'), require('path'), require('rxjs'), require('prettier'), require('prettier/parser-html'), require('crypto-js/enc-hex'), require('crypto-js/sha256'), require('crypto-js'), require('mime-types'), require('papaparse')) :
3
+ typeof define === 'function' && define.amd ? define(['exports', 'colors', 'express', 'http', 'socket.io', 'spacetrim', 'swagger-jsdoc', 'swagger-ui-express', 'waitasecond', 'crypto', 'child_process', 'fs/promises', 'path', 'rxjs', 'prettier', 'prettier/parser-html', 'crypto-js/enc-hex', 'crypto-js/sha256', 'crypto-js', 'mime-types', 'papaparse'], factory) :
4
+ (global = typeof globalThis !== 'undefined' ? globalThis : global || self, factory(global["promptbook-remote-server"] = {}, global.colors, global.express, global.http, global.socket_io, global.spaceTrim, global.swaggerJsdoc, global.swaggerUi, global.waitasecond, global.crypto, global.child_process, global.promises, global.path, global.rxjs, global.prettier, global.parserHtml, global.hexEncoder, global.sha256, global.cryptoJs, global.mimeTypes, global.papaparse));
5
+ })(this, (function (exports, colors, express, http, socket_io, spaceTrim, swaggerJsdoc, swaggerUi, waitasecond, crypto, child_process, promises, path, rxjs, prettier, parserHtml, hexEncoder, sha256, cryptoJs, mimeTypes, papaparse) { 'use strict';
6
6
 
7
7
  function _interopDefaultLegacy (e) { return e && typeof e === 'object' && 'default' in e ? e : { 'default': e }; }
8
8
 
@@ -10,6 +10,8 @@
10
10
  var express__default = /*#__PURE__*/_interopDefaultLegacy(express);
11
11
  var http__default = /*#__PURE__*/_interopDefaultLegacy(http);
12
12
  var spaceTrim__default = /*#__PURE__*/_interopDefaultLegacy(spaceTrim);
13
+ var swaggerJsdoc__default = /*#__PURE__*/_interopDefaultLegacy(swaggerJsdoc);
14
+ var swaggerUi__default = /*#__PURE__*/_interopDefaultLegacy(swaggerUi);
13
15
  var parserHtml__default = /*#__PURE__*/_interopDefaultLegacy(parserHtml);
14
16
  var hexEncoder__default = /*#__PURE__*/_interopDefaultLegacy(hexEncoder);
15
17
  var sha256__default = /*#__PURE__*/_interopDefaultLegacy(sha256);
@@ -28,7 +30,7 @@
28
30
  * @generated
29
31
  * @see https://github.com/webgptorg/promptbook
30
32
  */
31
- const PROMPTBOOK_ENGINE_VERSION = '0.88.0';
33
+ const PROMPTBOOK_ENGINE_VERSION = '0.89.0-11';
32
34
  /**
33
35
  * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
34
36
  * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -153,6 +155,7 @@
153
155
  */
154
156
  const DEFAULT_MAX_EXECUTION_ATTEMPTS = 10; // <- TODO: [🤹‍♂️]
155
157
  // <- TODO: [🕝] Make also `BOOKS_DIRNAME_ALTERNATIVES`
158
+ // TODO: !!!!!! Just .promptbook dir, hardocode others
156
159
  /**
157
160
  * Where to store the temporary downloads
158
161
  *
@@ -204,9 +207,22 @@
204
207
  true);
205
208
  /**
206
209
  * Note: [💞] Ignore a discrepancy between file name and entity name
207
- * TODO: [🧠][🧜‍♂️] Maybe join remoteUrl and path into single value
210
+ * TODO: [🧠][🧜‍♂️] Maybe join remoteServerUrl and path into single value
208
211
  */
209
212
 
213
+ /**
214
+ * AuthenticationError is thrown from login function which is dependency of remote server
215
+ *
216
+ * @public exported from `@promptbook/core`
217
+ */
218
+ class AuthenticationError extends Error {
219
+ constructor(message) {
220
+ super(message);
221
+ this.name = 'AuthenticationError';
222
+ Object.setPrototypeOf(this, AuthenticationError.prototype);
223
+ }
224
+ }
225
+
210
226
  /**
211
227
  * Generates random token
212
228
  *
@@ -435,6 +451,19 @@
435
451
  }
436
452
  }
437
453
 
454
+ /**
455
+ * Error thrown when a fetch request fails
456
+ *
457
+ * @public exported from `@promptbook/core`
458
+ */
459
+ class PromptbookFetchError extends Error {
460
+ constructor(message) {
461
+ super(message);
462
+ this.name = 'PromptbookFetchError';
463
+ Object.setPrototypeOf(this, PromptbookFetchError.prototype);
464
+ }
465
+ }
466
+
438
467
  /**
439
468
  * Make error report URL for the given error
440
469
  *
@@ -540,6 +569,8 @@
540
569
  TypeError,
541
570
  URIError,
542
571
  AggregateError,
572
+ AuthenticationError,
573
+ PromptbookFetchError,
543
574
  /*
544
575
  Note: Not widely supported
545
576
  > InternalError,
@@ -575,6 +606,10 @@
575
606
 
576
607
  Cannot serialize error with name "${name}"
577
608
 
609
+ Authors of Promptbook probably forgot to add this error into the list of errors:
610
+ https://github.com/webgptorg/promptbook/blob/main/src/errors/0-index.ts
611
+
612
+
578
613
  ${block(stack || message)}
579
614
 
580
615
  `));
@@ -691,6 +726,7 @@
691
726
  }
692
727
  else {
693
728
  console.warn(`Command "${humanReadableCommand}" exceeded time limit of ${timeout}ms but continues running`);
729
+ // <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
694
730
  resolve('Command exceeded time limit');
695
731
  }
696
732
  });
@@ -716,6 +752,7 @@
716
752
  output.push(stderr.toString());
717
753
  if (isVerbose && stderr.toString().trim()) {
718
754
  console.warn(stderr.toString());
755
+ // <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
719
756
  }
720
757
  });
721
758
  const finishWithCode = (code) => {
@@ -727,6 +764,7 @@
727
764
  else {
728
765
  if (isVerbose) {
729
766
  console.warn(`Command "${humanReadableCommand}" exited with code ${code}`);
767
+ // <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
730
768
  }
731
769
  resolve(spaceTrim.spaceTrim(output.join('\n')));
732
770
  }
@@ -748,6 +786,7 @@
748
786
  else {
749
787
  if (isVerbose) {
750
788
  console.warn(error);
789
+ // <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
751
790
  }
752
791
  resolve(spaceTrim.spaceTrim(output.join('\n')));
753
792
  }
@@ -1816,6 +1855,7 @@
1816
1855
  const { isSuccessful, errors, warnings } = executionResult;
1817
1856
  for (const warning of warnings) {
1818
1857
  console.warn(warning.message);
1858
+ // <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
1819
1859
  }
1820
1860
  if (isSuccessful === true) {
1821
1861
  return;
@@ -2417,30 +2457,42 @@
2417
2457
  await Promise.all(tasks);
2418
2458
  }
2419
2459
 
2460
+ /**
2461
+ * Represents the uncertain value
2462
+ *
2463
+ * @public exported from `@promptbook/core`
2464
+ */
2465
+ const ZERO_VALUE = $deepFreeze({ value: 0 });
2466
+ /**
2467
+ * Represents the uncertain value
2468
+ *
2469
+ * @public exported from `@promptbook/core`
2470
+ */
2471
+ const UNCERTAIN_ZERO_VALUE = $deepFreeze({ value: 0, isUncertain: true });
2420
2472
  /**
2421
2473
  * Represents the usage with no resources consumed
2422
2474
  *
2423
2475
  * @public exported from `@promptbook/core`
2424
2476
  */
2425
2477
  const ZERO_USAGE = $deepFreeze({
2426
- price: { value: 0 },
2478
+ price: ZERO_VALUE,
2427
2479
  input: {
2428
- tokensCount: { value: 0 },
2429
- charactersCount: { value: 0 },
2430
- wordsCount: { value: 0 },
2431
- sentencesCount: { value: 0 },
2432
- linesCount: { value: 0 },
2433
- paragraphsCount: { value: 0 },
2434
- pagesCount: { value: 0 },
2480
+ tokensCount: ZERO_VALUE,
2481
+ charactersCount: ZERO_VALUE,
2482
+ wordsCount: ZERO_VALUE,
2483
+ sentencesCount: ZERO_VALUE,
2484
+ linesCount: ZERO_VALUE,
2485
+ paragraphsCount: ZERO_VALUE,
2486
+ pagesCount: ZERO_VALUE,
2435
2487
  },
2436
2488
  output: {
2437
- tokensCount: { value: 0 },
2438
- charactersCount: { value: 0 },
2439
- wordsCount: { value: 0 },
2440
- sentencesCount: { value: 0 },
2441
- linesCount: { value: 0 },
2442
- paragraphsCount: { value: 0 },
2443
- pagesCount: { value: 0 },
2489
+ tokensCount: ZERO_VALUE,
2490
+ charactersCount: ZERO_VALUE,
2491
+ wordsCount: ZERO_VALUE,
2492
+ sentencesCount: ZERO_VALUE,
2493
+ linesCount: ZERO_VALUE,
2494
+ paragraphsCount: ZERO_VALUE,
2495
+ pagesCount: ZERO_VALUE,
2444
2496
  },
2445
2497
  });
2446
2498
  /**
@@ -2449,24 +2501,24 @@
2449
2501
  * @public exported from `@promptbook/core`
2450
2502
  */
2451
2503
  $deepFreeze({
2452
- price: { value: 0, isUncertain: true },
2504
+ price: UNCERTAIN_ZERO_VALUE,
2453
2505
  input: {
2454
- tokensCount: { value: 0, isUncertain: true },
2455
- charactersCount: { value: 0, isUncertain: true },
2456
- wordsCount: { value: 0, isUncertain: true },
2457
- sentencesCount: { value: 0, isUncertain: true },
2458
- linesCount: { value: 0, isUncertain: true },
2459
- paragraphsCount: { value: 0, isUncertain: true },
2460
- pagesCount: { value: 0, isUncertain: true },
2506
+ tokensCount: UNCERTAIN_ZERO_VALUE,
2507
+ charactersCount: UNCERTAIN_ZERO_VALUE,
2508
+ wordsCount: UNCERTAIN_ZERO_VALUE,
2509
+ sentencesCount: UNCERTAIN_ZERO_VALUE,
2510
+ linesCount: UNCERTAIN_ZERO_VALUE,
2511
+ paragraphsCount: UNCERTAIN_ZERO_VALUE,
2512
+ pagesCount: UNCERTAIN_ZERO_VALUE,
2461
2513
  },
2462
2514
  output: {
2463
- tokensCount: { value: 0, isUncertain: true },
2464
- charactersCount: { value: 0, isUncertain: true },
2465
- wordsCount: { value: 0, isUncertain: true },
2466
- sentencesCount: { value: 0, isUncertain: true },
2467
- linesCount: { value: 0, isUncertain: true },
2468
- paragraphsCount: { value: 0, isUncertain: true },
2469
- pagesCount: { value: 0, isUncertain: true },
2515
+ tokensCount: UNCERTAIN_ZERO_VALUE,
2516
+ charactersCount: UNCERTAIN_ZERO_VALUE,
2517
+ wordsCount: UNCERTAIN_ZERO_VALUE,
2518
+ sentencesCount: UNCERTAIN_ZERO_VALUE,
2519
+ linesCount: UNCERTAIN_ZERO_VALUE,
2520
+ paragraphsCount: UNCERTAIN_ZERO_VALUE,
2521
+ pagesCount: UNCERTAIN_ZERO_VALUE,
2470
2522
  },
2471
2523
  });
2472
2524
  /**
@@ -2527,8 +2579,9 @@
2527
2579
  * @returns LLM tools with same functionality with added total cost counting
2528
2580
  * @public exported from `@promptbook/core`
2529
2581
  */
2530
- function countTotalUsage(llmTools) {
2582
+ function countUsage(llmTools) {
2531
2583
  let totalUsage = ZERO_USAGE;
2584
+ const spending = new rxjs.Subject();
2532
2585
  const proxyTools = {
2533
2586
  get title() {
2534
2587
  // TODO: [🧠] Maybe put here some suffix
@@ -2538,12 +2591,15 @@
2538
2591
  // TODO: [🧠] Maybe put here some suffix
2539
2592
  return llmTools.description;
2540
2593
  },
2541
- async checkConfiguration() {
2594
+ checkConfiguration() {
2542
2595
  return /* not await */ llmTools.checkConfiguration();
2543
2596
  },
2544
2597
  listModels() {
2545
2598
  return /* not await */ llmTools.listModels();
2546
2599
  },
2600
+ spending() {
2601
+ return spending.asObservable();
2602
+ },
2547
2603
  getTotalUsage() {
2548
2604
  // <- Note: [🥫] Not using getter `get totalUsage` but `getTotalUsage` to allow this object to be proxied
2549
2605
  return totalUsage;
@@ -2554,6 +2610,7 @@
2554
2610
  // console.info('[🚕] callChatModel through countTotalUsage');
2555
2611
  const promptResult = await llmTools.callChatModel(prompt);
2556
2612
  totalUsage = addUsage(totalUsage, promptResult.usage);
2613
+ spending.next(promptResult.usage);
2557
2614
  return promptResult;
2558
2615
  };
2559
2616
  }
@@ -2562,6 +2619,7 @@
2562
2619
  // console.info('[🚕] callCompletionModel through countTotalUsage');
2563
2620
  const promptResult = await llmTools.callCompletionModel(prompt);
2564
2621
  totalUsage = addUsage(totalUsage, promptResult.usage);
2622
+ spending.next(promptResult.usage);
2565
2623
  return promptResult;
2566
2624
  };
2567
2625
  }
@@ -2570,6 +2628,7 @@
2570
2628
  // console.info('[🚕] callEmbeddingModel through countTotalUsage');
2571
2629
  const promptResult = await llmTools.callEmbeddingModel(prompt);
2572
2630
  totalUsage = addUsage(totalUsage, promptResult.usage);
2631
+ spending.next(promptResult.usage);
2573
2632
  return promptResult;
2574
2633
  };
2575
2634
  }
@@ -2747,6 +2806,7 @@
2747
2806
  `);
2748
2807
  // TODO: [🟥] Detect browser / node and make it colorfull
2749
2808
  console.warn(warningMessage);
2809
+ // <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
2750
2810
  /*
2751
2811
  return {
2752
2812
  async listModels() {
@@ -3518,17 +3578,24 @@
3518
3578
  /**
3519
3579
  * The built-in `fetch' function with a lightweight error handling wrapper as default fetch function used in Promptbook scrapers
3520
3580
  *
3521
- * @private as default `fetch` function used in Promptbook scrapers
3581
+ * @public exported from `@promptbook/core`
3522
3582
  */
3523
- const scraperFetch = async (url, init) => {
3583
+ const promptbookFetch = async (urlOrRequest, init) => {
3524
3584
  try {
3525
- return await fetch(url, init);
3585
+ return await fetch(urlOrRequest, init);
3526
3586
  }
3527
3587
  catch (error) {
3528
3588
  if (!(error instanceof Error)) {
3529
3589
  throw error;
3530
3590
  }
3531
- throw new KnowledgeScrapeError(spaceTrim__default["default"]((block) => `
3591
+ let url;
3592
+ if (typeof urlOrRequest === 'string') {
3593
+ url = urlOrRequest;
3594
+ }
3595
+ else if (urlOrRequest instanceof Request) {
3596
+ url = urlOrRequest.url;
3597
+ }
3598
+ throw new PromptbookFetchError(spaceTrim__default["default"]((block) => `
3532
3599
  Can not fetch "${url}"
3533
3600
 
3534
3601
  Fetch error:
@@ -3549,7 +3616,7 @@
3549
3616
  async function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
3550
3617
  // console.log('!! makeKnowledgeSourceHandler', knowledgeSource);
3551
3618
  var _a;
3552
- const { fetch = scraperFetch } = tools;
3619
+ const { fetch = promptbookFetch } = tools;
3553
3620
  const { knowledgeSourceContent } = knowledgeSource;
3554
3621
  let { name } = knowledgeSource;
3555
3622
  const { rootDirname = null,
@@ -3690,63 +3757,73 @@
3690
3757
  const { maxParallelCount = DEFAULT_MAX_PARALLEL_COUNT, rootDirname, isVerbose = DEFAULT_IS_VERBOSE } = options;
3691
3758
  const knowledgePreparedUnflatten = new Array(knowledgeSources.length);
3692
3759
  await forEachAsync(knowledgeSources, { maxParallelCount }, async (knowledgeSource, index) => {
3693
- let partialPieces = null;
3694
- const sourceHandler = await makeKnowledgeSourceHandler(knowledgeSource, tools, { rootDirname, isVerbose });
3695
- const scrapers = arrayableToArray(tools.scrapers);
3696
- for (const scraper of scrapers) {
3697
- if (!scraper.metadata.mimeTypes.includes(sourceHandler.mimeType)
3698
- // <- TODO: [🦔] Implement mime-type wildcards
3699
- ) {
3700
- continue;
3701
- }
3702
- const partialPiecesUnchecked = await scraper.scrape(sourceHandler);
3703
- if (partialPiecesUnchecked !== null) {
3704
- partialPieces = [...partialPiecesUnchecked];
3705
- // <- TODO: [🪓] Here should be no need for spreading new array, just `partialPieces = partialPiecesUnchecked`
3706
- break;
3707
- }
3708
- console.warn(spaceTrim__default["default"]((block) => `
3709
- Cannot scrape knowledge from source despite the scraper \`${scraper.metadata.className}\` supports the mime type "${sourceHandler.mimeType}".
3760
+ try {
3761
+ let partialPieces = null;
3762
+ const sourceHandler = await makeKnowledgeSourceHandler(knowledgeSource, tools, { rootDirname, isVerbose });
3763
+ const scrapers = arrayableToArray(tools.scrapers);
3764
+ for (const scraper of scrapers) {
3765
+ if (!scraper.metadata.mimeTypes.includes(sourceHandler.mimeType)
3766
+ // <- TODO: [🦔] Implement mime-type wildcards
3767
+ ) {
3768
+ continue;
3769
+ }
3770
+ const partialPiecesUnchecked = await scraper.scrape(sourceHandler);
3771
+ if (partialPiecesUnchecked !== null) {
3772
+ partialPieces = [...partialPiecesUnchecked];
3773
+ // <- TODO: [🪓] Here should be no need for spreading new array, just `partialPieces = partialPiecesUnchecked`
3774
+ break;
3775
+ }
3776
+ console.warn(spaceTrim__default["default"]((block) => `
3777
+ Cannot scrape knowledge from source despite the scraper \`${scraper.metadata.className}\` supports the mime type "${sourceHandler.mimeType}".
3710
3778
 
3711
- The source:
3712
- ${block(knowledgeSource.knowledgeSourceContent
3713
- .split('\n')
3714
- .map((line) => `> ${line}`)
3715
- .join('\n'))}
3779
+ The source:
3780
+ ${block(knowledgeSource.knowledgeSourceContent
3781
+ .split('\n')
3782
+ .map((line) => `> ${line}`)
3783
+ .join('\n'))}
3716
3784
 
3717
- ${block($registeredScrapersMessage(scrapers))}
3785
+ ${block($registeredScrapersMessage(scrapers))}
3718
3786
 
3719
3787
 
3720
- `));
3721
- }
3722
- if (partialPieces === null) {
3723
- throw new KnowledgeScrapeError(spaceTrim__default["default"]((block) => `
3724
- Cannot scrape knowledge
3788
+ `));
3789
+ // <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
3790
+ }
3791
+ if (partialPieces === null) {
3792
+ throw new KnowledgeScrapeError(spaceTrim__default["default"]((block) => `
3793
+ Cannot scrape knowledge
3725
3794
 
3726
- The source:
3727
- > ${block(knowledgeSource.knowledgeSourceContent
3728
- .split('\n')
3729
- .map((line) => `> ${line}`)
3730
- .join('\n'))}
3795
+ The source:
3796
+ > ${block(knowledgeSource.knowledgeSourceContent
3797
+ .split('\n')
3798
+ .map((line) => `> ${line}`)
3799
+ .join('\n'))}
3731
3800
 
3732
- No scraper found for the mime type "${sourceHandler.mimeType}"
3801
+ No scraper found for the mime type "${sourceHandler.mimeType}"
3733
3802
 
3734
- ${block($registeredScrapersMessage(scrapers))}
3803
+ ${block($registeredScrapersMessage(scrapers))}
3735
3804
 
3736
3805
 
3737
- `));
3806
+ `));
3807
+ }
3808
+ const pieces = partialPieces.map((partialPiece) => ({
3809
+ ...partialPiece,
3810
+ sources: [
3811
+ {
3812
+ name: knowledgeSource.name,
3813
+ // line, column <- TODO: [☀]
3814
+ // <- TODO: [❎]
3815
+ },
3816
+ ],
3817
+ }));
3818
+ knowledgePreparedUnflatten[index] = pieces;
3819
+ }
3820
+ catch (error) {
3821
+ if (!(error instanceof Error)) {
3822
+ throw error;
3823
+ }
3824
+ console.warn(error);
3825
+ // <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
3738
3826
  }
3739
- const pieces = partialPieces.map((partialPiece) => ({
3740
- ...partialPiece,
3741
- sources: [
3742
- {
3743
- name: knowledgeSource.name,
3744
- // line, column <- TODO: [☀]
3745
- // <- TODO: [❎]
3746
- },
3747
- ],
3748
- }));
3749
- knowledgePreparedUnflatten[index] = pieces;
3750
3827
  });
3751
3828
  const knowledgePrepared = knowledgePreparedUnflatten.flat();
3752
3829
  return knowledgePrepared;
@@ -3852,7 +3929,7 @@
3852
3929
  // TODO: [🚐] Make arrayable LLMs -> single LLM DRY
3853
3930
  const _llms = arrayableToArray(tools.llm);
3854
3931
  const llmTools = _llms.length === 1 ? _llms[0] : joinLlmExecutionTools(..._llms);
3855
- const llmToolsWithUsage = countTotalUsage(llmTools);
3932
+ const llmToolsWithUsage = countUsage(llmTools);
3856
3933
  // <- TODO: [🌯]
3857
3934
  /*
3858
3935
  TODO: [🧠][🪑][🔃] Should this be done or not
@@ -4164,7 +4241,7 @@
4164
4241
  if (parameterNames.has(subparameterName)) {
4165
4242
  parameterNames.delete(subparameterName);
4166
4243
  parameterNames.add(foreach.parameterName);
4167
- // <- TODO: [🚎] Warn/logic error when `subparameterName` not used
4244
+ // <- TODO: [🏮] Warn/logic error when `subparameterName` not used
4168
4245
  }
4169
4246
  }
4170
4247
  }
@@ -5777,6 +5854,7 @@
5777
5854
 
5778
5855
  @see more at https://ptbk.io/prepare-pipeline
5779
5856
  `));
5857
+ // <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
5780
5858
  }
5781
5859
  let runCount = 0;
5782
5860
  const pipelineExecutorWithCallback = async (inputParameters, onProgress) => {
@@ -6672,11 +6750,12 @@
6672
6750
  * @public exported from `@promptbook/remote-server`
6673
6751
  */
6674
6752
  function startRemoteServer(options) {
6675
- const { port, collection, createLlmExecutionTools, isAnonymousModeAllowed, isApplicationModeAllowed, isVerbose = DEFAULT_IS_VERBOSE, } = {
6753
+ const { port, collection, createLlmExecutionTools, isAnonymousModeAllowed, isApplicationModeAllowed, isVerbose = DEFAULT_IS_VERBOSE, login, } = {
6676
6754
  isAnonymousModeAllowed: false,
6677
6755
  isApplicationModeAllowed: false,
6678
6756
  collection: null,
6679
6757
  createLlmExecutionTools: null,
6758
+ login: null,
6680
6759
  ...options,
6681
6760
  };
6682
6761
  // <- TODO: [🦪] Some helper type to be able to use discriminant union types with destructuring
@@ -6743,9 +6822,38 @@
6743
6822
  response.setHeader('X-Powered-By', 'Promptbook engine');
6744
6823
  next();
6745
6824
  });
6825
+ const swaggerOptions = {
6826
+ definition: {
6827
+ openapi: '3.0.0',
6828
+ info: {
6829
+ title: 'Promptbook Remote Server API',
6830
+ version: '1.0.0',
6831
+ description: 'API documentation for the Promptbook Remote Server',
6832
+ },
6833
+ servers: [
6834
+ {
6835
+ url: `http://localhost:${port}${rootPath}`,
6836
+ // <- TODO: !!!!! Probbably: Pass `remoteServerUrl` instead of `port` and `rootPath`
6837
+ },
6838
+ ],
6839
+ },
6840
+ apis: ['./src/remote-server/**/*.ts'], // Adjust path as needed
6841
+ };
6842
+ const swaggerSpec = swaggerJsdoc__default["default"](swaggerOptions);
6843
+ app.use([`/api-docs`, `${rootPath}/api-docs`], swaggerUi__default["default"].serve, swaggerUi__default["default"].setup(swaggerSpec));
6746
6844
  const runningExecutionTasks = [];
6747
6845
  // <- TODO: [🤬] Identify the users
6748
6846
  // TODO: [🧠] Do here some garbage collection of finished tasks
6847
+ /**
6848
+ * @swagger
6849
+ * /:
6850
+ * get:
6851
+ * summary: Get server details
6852
+ * description: Returns details about the Promptbook server.
6853
+ * responses:
6854
+ * 200:
6855
+ * description: Server details in markdown format.
6856
+ */
6749
6857
  app.get(['/', rootPath], async (request, response) => {
6750
6858
  var _a;
6751
6859
  if ((_a = request.url) === null || _a === void 0 ? void 0 : _a.includes('socket.io')) {
@@ -6782,9 +6890,12 @@
6782
6890
 
6783
6891
  ## Paths
6784
6892
 
6785
- ${block(app._router.stack
6786
- .map(({ route }) => (route === null || route === void 0 ? void 0 : route.path) || null)
6787
- .filter((path) => path !== null)
6893
+ ${block([
6894
+ ...app._router.stack
6895
+ .map(({ route }) => (route === null || route === void 0 ? void 0 : route.path) || null)
6896
+ .filter((path) => path !== null),
6897
+ '/api-docs',
6898
+ ]
6788
6899
  .map((path) => `- ${path}`)
6789
6900
  .join('\n'))}
6790
6901
 
@@ -6802,7 +6913,96 @@
6802
6913
  https://github.com/webgptorg/promptbook
6803
6914
  `));
6804
6915
  });
6805
- app.get(`${rootPath}/books`, async (request, response) => {
6916
+ /**
6917
+ * @swagger
6918
+ *
6919
+ * /login:
6920
+ * post:
6921
+ * summary: Login to the server
6922
+ * description: Login to the server and get identification.
6923
+ * requestBody:
6924
+ * required: true
6925
+ * content:
6926
+ * application/json:
6927
+ * schema:
6928
+ * type: object
6929
+ * properties:
6930
+ * username:
6931
+ * type: string
6932
+ * password:
6933
+ * type: string
6934
+ * appId:
6935
+ * type: string
6936
+ * responses:
6937
+ * 200:
6938
+ * description: Successful login
6939
+ * content:
6940
+ * application/json:
6941
+ * schema:
6942
+ * type: object
6943
+ * properties:
6944
+ * identification:
6945
+ * type: object
6946
+ */
6947
+ app.post([`/login`, `${rootPath}/login`], async (request, response) => {
6948
+ if (!isApplicationModeAllowed || login === null) {
6949
+ response.status(400).send('Application mode is not allowed');
6950
+ return;
6951
+ }
6952
+ try {
6953
+ const username = request.body.username;
6954
+ const password = request.body.password;
6955
+ const appId = request.body.appId;
6956
+ const { isSuccess, error, message, identification } = await login({
6957
+ username,
6958
+ password,
6959
+ appId,
6960
+ rawRequest: request,
6961
+ rawResponse: response,
6962
+ });
6963
+ response.status(201).send({
6964
+ isSuccess,
6965
+ message,
6966
+ error: error ? serializeError(error) : undefined,
6967
+ identification,
6968
+ });
6969
+ return;
6970
+ }
6971
+ catch (error) {
6972
+ if (!(error instanceof Error)) {
6973
+ throw error;
6974
+ }
6975
+ if (error instanceof AuthenticationError) {
6976
+ response.status(401).send({
6977
+ isSuccess: false,
6978
+ message: error.message,
6979
+ error: serializeError(error),
6980
+ });
6981
+ }
6982
+ console.warn(`Login function thrown different error than AuthenticationError`, {
6983
+ error,
6984
+ serializedError: serializeError(error),
6985
+ });
6986
+ response.status(400).send({ error: serializeError(error) });
6987
+ }
6988
+ });
6989
+ /**
6990
+ * @swagger
6991
+ * /books:
6992
+ * get:
6993
+ * summary: List all books
6994
+ * description: Returns a list of all available books in the collection.
6995
+ * responses:
6996
+ * 200:
6997
+ * description: A list of books.
6998
+ * content:
6999
+ * application/json:
7000
+ * schema:
7001
+ * type: array
7002
+ * items:
7003
+ * type: string
7004
+ */
7005
+ app.get([`/books`, `${rootPath}/books`], async (request, response) => {
6806
7006
  if (collection === null) {
6807
7007
  response.status(500).send('No collection available');
6808
7008
  return;
@@ -6812,7 +7012,30 @@
6812
7012
  response.send(pipelines);
6813
7013
  });
6814
7014
  // TODO: [🧠] Is it secure / good idea to expose source codes of hosted books
6815
- app.get(`${rootPath}/books/*`, async (request, response) => {
7015
+ /**
7016
+ * @swagger
7017
+ * /books/{bookId}:
7018
+ * get:
7019
+ * summary: Get book content
7020
+ * description: Returns the content of a specific book.
7021
+ * parameters:
7022
+ * - in: path
7023
+ * name: bookId
7024
+ * required: true
7025
+ * schema:
7026
+ * type: string
7027
+ * description: The ID of the book to retrieve.
7028
+ * responses:
7029
+ * 200:
7030
+ * description: The content of the book.
7031
+ * content:
7032
+ * text/markdown:
7033
+ * schema:
7034
+ * type: string
7035
+ * 404:
7036
+ * description: Book not found.
7037
+ */
7038
+ app.get([`/books/*`, `${rootPath}/books/*`], async (request, response) => {
6816
7039
  try {
6817
7040
  if (collection === null) {
6818
7041
  response.status(500).send('No collection nor books available');
@@ -6866,10 +7089,26 @@
6866
7089
  };
6867
7090
  }
6868
7091
  }
6869
- app.get(`${rootPath}/executions`, async (request, response) => {
7092
+ /**
7093
+ * @swagger
7094
+ * /executions:
7095
+ * get:
7096
+ * summary: List all executions
7097
+ * description: Returns a list of all running execution tasks.
7098
+ * responses:
7099
+ * 200:
7100
+ * description: A list of execution tasks.
7101
+ * content:
7102
+ * application/json:
7103
+ * schema:
7104
+ * type: array
7105
+ * items:
7106
+ * type: object
7107
+ */
7108
+ app.get([`/executions`, `${rootPath}/executions`], async (request, response) => {
6870
7109
  response.send(runningExecutionTasks.map((runningExecutionTask) => exportExecutionTask(runningExecutionTask, false)));
6871
7110
  });
6872
- app.get(`${rootPath}/executions/last`, async (request, response) => {
7111
+ app.get([`/executions/last`, `${rootPath}/executions/last`], async (request, response) => {
6873
7112
  // TODO: [🤬] Filter only for user
6874
7113
  if (runningExecutionTasks.length === 0) {
6875
7114
  response.status(404).send('No execution tasks found');
@@ -6878,7 +7117,7 @@
6878
7117
  const lastExecutionTask = runningExecutionTasks[runningExecutionTasks.length - 1];
6879
7118
  response.send(exportExecutionTask(lastExecutionTask, true));
6880
7119
  });
6881
- app.get(`${rootPath}/executions/:taskId`, async (request, response) => {
7120
+ app.get([`/executions/:taskId`, `${rootPath}/executions/:taskId`], async (request, response) => {
6882
7121
  const { taskId } = request.params;
6883
7122
  // TODO: [🤬] Filter only for user
6884
7123
  const executionTask = runningExecutionTasks.find((executionTask) => executionTask.taskId === taskId);
@@ -6890,7 +7129,36 @@
6890
7129
  }
6891
7130
  response.send(exportExecutionTask(executionTask, true));
6892
7131
  });
6893
- app.post(`${rootPath}/executions/new`, async (request, response) => {
7132
+ /**
7133
+ * @swagger
7134
+ * /executions/new:
7135
+ * post:
7136
+ * summary: Start a new execution
7137
+ * description: Starts a new execution task for a given pipeline.
7138
+ * requestBody:
7139
+ * required: true
7140
+ * content:
7141
+ * application/json:
7142
+ * schema:
7143
+ * type: object
7144
+ * properties:
7145
+ * pipelineUrl:
7146
+ * type: string
7147
+ * inputParameters:
7148
+ * type: object
7149
+ * identification:
7150
+ * type: object
7151
+ * responses:
7152
+ * 200:
7153
+ * description: The newly created execution task.
7154
+ * content:
7155
+ * application/json:
7156
+ * schema:
7157
+ * type: object
7158
+ * 400:
7159
+ * description: Invalid input.
7160
+ */
7161
+ app.post([`/executions/new`, `${rootPath}/executions/new`], async (request, response) => {
6894
7162
  try {
6895
7163
  const { inputParameters, identification /* <- [🤬] */ } = request.body;
6896
7164
  const pipelineUrl = request.body.pipelineUrl || request.body.book;
@@ -7067,6 +7335,15 @@
7067
7335
  }
7068
7336
  let isDestroyed = false;
7069
7337
  return {
7338
+ get httpServer() {
7339
+ return httpServer;
7340
+ },
7341
+ get expressApp() {
7342
+ return app;
7343
+ },
7344
+ get socketIoServer() {
7345
+ return server;
7346
+ },
7070
7347
  get isDestroyed() {
7071
7348
  return isDestroyed;
7072
7349
  },