@promptbook/remote-server 0.88.0 → 0.89.0-11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/README.md +7 -1
  2. package/esm/index.es.js +375 -98
  3. package/esm/index.es.js.map +1 -1
  4. package/esm/typings/src/_packages/core.index.d.ts +18 -6
  5. package/esm/typings/src/_packages/remote-client.index.d.ts +6 -8
  6. package/esm/typings/src/_packages/remote-server.index.d.ts +6 -6
  7. package/esm/typings/src/_packages/types.index.d.ts +18 -10
  8. package/esm/typings/src/cli/cli-commands/login.d.ts +15 -0
  9. package/esm/typings/src/cli/common/$addGlobalOptionsToCommand.d.ts +7 -0
  10. package/esm/typings/src/cli/common/$provideLlmToolsForCli.d.ts +15 -0
  11. package/esm/typings/src/config.d.ts +15 -8
  12. package/esm/typings/src/errors/0-index.d.ts +6 -0
  13. package/esm/typings/src/errors/AuthenticationError.d.ts +9 -0
  14. package/esm/typings/src/errors/PromptbookFetchError.d.ts +9 -0
  15. package/esm/typings/src/execution/PipelineExecutorResult.d.ts +2 -2
  16. package/esm/typings/src/execution/PromptResult.d.ts +2 -2
  17. package/esm/typings/src/execution/{PromptResultUsage.d.ts → Usage.d.ts} +5 -5
  18. package/esm/typings/src/execution/utils/addUsage.d.ts +2 -2
  19. package/esm/typings/src/execution/utils/computeUsageCounts.d.ts +3 -3
  20. package/esm/typings/src/execution/utils/usage-constants.d.ts +77 -60
  21. package/esm/typings/src/execution/utils/usageToHuman.d.ts +5 -5
  22. package/esm/typings/src/execution/utils/usageToWorktime.d.ts +5 -5
  23. package/esm/typings/src/llm-providers/_common/register/$provideEnvFilename.d.ts +12 -0
  24. package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsConfigurationFromEnv.d.ts +2 -8
  25. package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground.d.ts +2 -0
  26. package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsForWizzardOrCli.d.ts +36 -1
  27. package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsFromEnv.d.ts +1 -0
  28. package/esm/typings/src/llm-providers/_common/utils/count-total-usage/LlmExecutionToolsWithTotalUsage.d.ts +9 -2
  29. package/esm/typings/src/llm-providers/_common/utils/count-total-usage/{countTotalUsage.d.ts → countUsage.d.ts} +1 -1
  30. package/esm/typings/src/llm-providers/_common/utils/count-total-usage/limitTotalUsage.d.ts +2 -2
  31. package/esm/typings/src/llm-providers/anthropic-claude/AnthropicClaudeExecutionToolsOptions.d.ts +1 -1
  32. package/esm/typings/src/llm-providers/anthropic-claude/computeAnthropicClaudeUsage.d.ts +2 -2
  33. package/esm/typings/src/llm-providers/anthropic-claude/register-configuration.d.ts +1 -1
  34. package/esm/typings/src/llm-providers/openai/OpenAiExecutionTools.d.ts +0 -9
  35. package/esm/typings/src/llm-providers/openai/computeOpenAiUsage.d.ts +2 -2
  36. package/esm/typings/src/pipeline/PipelineJson/PreparationJson.d.ts +2 -2
  37. package/esm/typings/src/playground/playground.d.ts +5 -0
  38. package/esm/typings/src/remote-server/RemoteServer.d.ts +23 -0
  39. package/esm/typings/src/remote-server/socket-types/_subtypes/{PromptbookServer_Identification.d.ts → Identification.d.ts} +5 -4
  40. package/esm/typings/src/remote-server/socket-types/listModels/PromptbookServer_ListModels_Request.d.ts +2 -2
  41. package/esm/typings/src/remote-server/socket-types/prepare/PromptbookServer_PreparePipeline_Request.d.ts +2 -2
  42. package/esm/typings/src/remote-server/socket-types/prompt/PromptbookServer_Prompt_Request.d.ts +2 -2
  43. package/esm/typings/src/remote-server/startRemoteServer.d.ts +2 -2
  44. package/esm/typings/src/remote-server/types/RemoteClientOptions.d.ts +4 -12
  45. package/esm/typings/src/remote-server/types/RemoteServerOptions.d.ts +88 -6
  46. package/esm/typings/src/scrapers/_common/utils/{scraperFetch.d.ts → promptbookFetch.d.ts} +2 -2
  47. package/esm/typings/src/storage/env-storage/$EnvStorage.d.ts +37 -0
  48. package/esm/typings/src/types/typeAliases.d.ts +8 -2
  49. package/esm/typings/src/utils/organization/TODO_narrow.d.ts +6 -0
  50. package/package.json +4 -2
  51. package/umd/index.umd.js +379 -102
  52. package/umd/index.umd.js.map +1 -1
package/esm/index.es.js CHANGED
@@ -3,6 +3,8 @@ import express from 'express';
3
3
  import http from 'http';
4
4
  import { Server } from 'socket.io';
5
5
  import spaceTrim$1, { spaceTrim } from 'spacetrim';
6
+ import swaggerJsdoc from 'swagger-jsdoc';
7
+ import swaggerUi from 'swagger-ui-express';
6
8
  import { forTime } from 'waitasecond';
7
9
  import { randomBytes } from 'crypto';
8
10
  import { spawn } from 'child_process';
@@ -31,7 +33,7 @@ const BOOK_LANGUAGE_VERSION = '1.0.0';
31
33
  * @generated
32
34
  * @see https://github.com/webgptorg/promptbook
33
35
  */
34
- const PROMPTBOOK_ENGINE_VERSION = '0.88.0';
36
+ const PROMPTBOOK_ENGINE_VERSION = '0.89.0-11';
35
37
  /**
36
38
  * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
37
39
  * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -156,6 +158,7 @@ const DEFAULT_MAX_PARALLEL_COUNT = 5; // <- TODO: [🤹‍♂️]
156
158
  */
157
159
  const DEFAULT_MAX_EXECUTION_ATTEMPTS = 10; // <- TODO: [🤹‍♂️]
158
160
  // <- TODO: [🕝] Make also `BOOKS_DIRNAME_ALTERNATIVES`
161
+ // TODO: !!!!!! Just .promptbook dir, hardocode others
159
162
  /**
160
163
  * Where to store the temporary downloads
161
164
  *
@@ -207,9 +210,22 @@ const IS_PIPELINE_LOGIC_VALIDATED = just(
207
210
  true);
208
211
  /**
209
212
  * Note: [💞] Ignore a discrepancy between file name and entity name
210
- * TODO: [🧠][🧜‍♂️] Maybe join remoteUrl and path into single value
213
+ * TODO: [🧠][🧜‍♂️] Maybe join remoteServerUrl and path into single value
211
214
  */
212
215
 
216
+ /**
217
+ * AuthenticationError is thrown from login function which is dependency of remote server
218
+ *
219
+ * @public exported from `@promptbook/core`
220
+ */
221
+ class AuthenticationError extends Error {
222
+ constructor(message) {
223
+ super(message);
224
+ this.name = 'AuthenticationError';
225
+ Object.setPrototypeOf(this, AuthenticationError.prototype);
226
+ }
227
+ }
228
+
213
229
  /**
214
230
  * Generates random token
215
231
  *
@@ -438,6 +454,19 @@ class PipelineUrlError extends Error {
438
454
  }
439
455
  }
440
456
 
457
+ /**
458
+ * Error thrown when a fetch request fails
459
+ *
460
+ * @public exported from `@promptbook/core`
461
+ */
462
+ class PromptbookFetchError extends Error {
463
+ constructor(message) {
464
+ super(message);
465
+ this.name = 'PromptbookFetchError';
466
+ Object.setPrototypeOf(this, PromptbookFetchError.prototype);
467
+ }
468
+ }
469
+
441
470
  /**
442
471
  * Make error report URL for the given error
443
472
  *
@@ -543,6 +572,8 @@ const COMMON_JAVASCRIPT_ERRORS = {
543
572
  TypeError,
544
573
  URIError,
545
574
  AggregateError,
575
+ AuthenticationError,
576
+ PromptbookFetchError,
546
577
  /*
547
578
  Note: Not widely supported
548
579
  > InternalError,
@@ -578,6 +609,10 @@ function serializeError(error) {
578
609
 
579
610
  Cannot serialize error with name "${name}"
580
611
 
612
+ Authors of Promptbook probably forgot to add this error into the list of errors:
613
+ https://github.com/webgptorg/promptbook/blob/main/src/errors/0-index.ts
614
+
615
+
581
616
  ${block(stack || message)}
582
617
 
583
618
  `));
@@ -694,6 +729,7 @@ function $execCommand(options) {
694
729
  }
695
730
  else {
696
731
  console.warn(`Command "${humanReadableCommand}" exceeded time limit of ${timeout}ms but continues running`);
732
+ // <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
697
733
  resolve('Command exceeded time limit');
698
734
  }
699
735
  });
@@ -719,6 +755,7 @@ function $execCommand(options) {
719
755
  output.push(stderr.toString());
720
756
  if (isVerbose && stderr.toString().trim()) {
721
757
  console.warn(stderr.toString());
758
+ // <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
722
759
  }
723
760
  });
724
761
  const finishWithCode = (code) => {
@@ -730,6 +767,7 @@ function $execCommand(options) {
730
767
  else {
731
768
  if (isVerbose) {
732
769
  console.warn(`Command "${humanReadableCommand}" exited with code ${code}`);
770
+ // <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
733
771
  }
734
772
  resolve(spaceTrim(output.join('\n')));
735
773
  }
@@ -751,6 +789,7 @@ function $execCommand(options) {
751
789
  else {
752
790
  if (isVerbose) {
753
791
  console.warn(error);
792
+ // <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
754
793
  }
755
794
  resolve(spaceTrim(output.join('\n')));
756
795
  }
@@ -1819,6 +1858,7 @@ function assertsTaskSuccessful(executionResult) {
1819
1858
  const { isSuccessful, errors, warnings } = executionResult;
1820
1859
  for (const warning of warnings) {
1821
1860
  console.warn(warning.message);
1861
+ // <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
1822
1862
  }
1823
1863
  if (isSuccessful === true) {
1824
1864
  return;
@@ -2420,30 +2460,42 @@ async function forEachAsync(array, options, callbackfunction) {
2420
2460
  await Promise.all(tasks);
2421
2461
  }
2422
2462
 
2463
+ /**
2464
+ * Represents the uncertain value
2465
+ *
2466
+ * @public exported from `@promptbook/core`
2467
+ */
2468
+ const ZERO_VALUE = $deepFreeze({ value: 0 });
2469
+ /**
2470
+ * Represents the uncertain value
2471
+ *
2472
+ * @public exported from `@promptbook/core`
2473
+ */
2474
+ const UNCERTAIN_ZERO_VALUE = $deepFreeze({ value: 0, isUncertain: true });
2423
2475
  /**
2424
2476
  * Represents the usage with no resources consumed
2425
2477
  *
2426
2478
  * @public exported from `@promptbook/core`
2427
2479
  */
2428
2480
  const ZERO_USAGE = $deepFreeze({
2429
- price: { value: 0 },
2481
+ price: ZERO_VALUE,
2430
2482
  input: {
2431
- tokensCount: { value: 0 },
2432
- charactersCount: { value: 0 },
2433
- wordsCount: { value: 0 },
2434
- sentencesCount: { value: 0 },
2435
- linesCount: { value: 0 },
2436
- paragraphsCount: { value: 0 },
2437
- pagesCount: { value: 0 },
2483
+ tokensCount: ZERO_VALUE,
2484
+ charactersCount: ZERO_VALUE,
2485
+ wordsCount: ZERO_VALUE,
2486
+ sentencesCount: ZERO_VALUE,
2487
+ linesCount: ZERO_VALUE,
2488
+ paragraphsCount: ZERO_VALUE,
2489
+ pagesCount: ZERO_VALUE,
2438
2490
  },
2439
2491
  output: {
2440
- tokensCount: { value: 0 },
2441
- charactersCount: { value: 0 },
2442
- wordsCount: { value: 0 },
2443
- sentencesCount: { value: 0 },
2444
- linesCount: { value: 0 },
2445
- paragraphsCount: { value: 0 },
2446
- pagesCount: { value: 0 },
2492
+ tokensCount: ZERO_VALUE,
2493
+ charactersCount: ZERO_VALUE,
2494
+ wordsCount: ZERO_VALUE,
2495
+ sentencesCount: ZERO_VALUE,
2496
+ linesCount: ZERO_VALUE,
2497
+ paragraphsCount: ZERO_VALUE,
2498
+ pagesCount: ZERO_VALUE,
2447
2499
  },
2448
2500
  });
2449
2501
  /**
@@ -2452,24 +2504,24 @@ const ZERO_USAGE = $deepFreeze({
2452
2504
  * @public exported from `@promptbook/core`
2453
2505
  */
2454
2506
  $deepFreeze({
2455
- price: { value: 0, isUncertain: true },
2507
+ price: UNCERTAIN_ZERO_VALUE,
2456
2508
  input: {
2457
- tokensCount: { value: 0, isUncertain: true },
2458
- charactersCount: { value: 0, isUncertain: true },
2459
- wordsCount: { value: 0, isUncertain: true },
2460
- sentencesCount: { value: 0, isUncertain: true },
2461
- linesCount: { value: 0, isUncertain: true },
2462
- paragraphsCount: { value: 0, isUncertain: true },
2463
- pagesCount: { value: 0, isUncertain: true },
2509
+ tokensCount: UNCERTAIN_ZERO_VALUE,
2510
+ charactersCount: UNCERTAIN_ZERO_VALUE,
2511
+ wordsCount: UNCERTAIN_ZERO_VALUE,
2512
+ sentencesCount: UNCERTAIN_ZERO_VALUE,
2513
+ linesCount: UNCERTAIN_ZERO_VALUE,
2514
+ paragraphsCount: UNCERTAIN_ZERO_VALUE,
2515
+ pagesCount: UNCERTAIN_ZERO_VALUE,
2464
2516
  },
2465
2517
  output: {
2466
- tokensCount: { value: 0, isUncertain: true },
2467
- charactersCount: { value: 0, isUncertain: true },
2468
- wordsCount: { value: 0, isUncertain: true },
2469
- sentencesCount: { value: 0, isUncertain: true },
2470
- linesCount: { value: 0, isUncertain: true },
2471
- paragraphsCount: { value: 0, isUncertain: true },
2472
- pagesCount: { value: 0, isUncertain: true },
2518
+ tokensCount: UNCERTAIN_ZERO_VALUE,
2519
+ charactersCount: UNCERTAIN_ZERO_VALUE,
2520
+ wordsCount: UNCERTAIN_ZERO_VALUE,
2521
+ sentencesCount: UNCERTAIN_ZERO_VALUE,
2522
+ linesCount: UNCERTAIN_ZERO_VALUE,
2523
+ paragraphsCount: UNCERTAIN_ZERO_VALUE,
2524
+ pagesCount: UNCERTAIN_ZERO_VALUE,
2473
2525
  },
2474
2526
  });
2475
2527
  /**
@@ -2530,8 +2582,9 @@ function addUsage(...usageItems) {
2530
2582
  * @returns LLM tools with same functionality with added total cost counting
2531
2583
  * @public exported from `@promptbook/core`
2532
2584
  */
2533
- function countTotalUsage(llmTools) {
2585
+ function countUsage(llmTools) {
2534
2586
  let totalUsage = ZERO_USAGE;
2587
+ const spending = new Subject();
2535
2588
  const proxyTools = {
2536
2589
  get title() {
2537
2590
  // TODO: [🧠] Maybe put here some suffix
@@ -2541,12 +2594,15 @@ function countTotalUsage(llmTools) {
2541
2594
  // TODO: [🧠] Maybe put here some suffix
2542
2595
  return llmTools.description;
2543
2596
  },
2544
- async checkConfiguration() {
2597
+ checkConfiguration() {
2545
2598
  return /* not await */ llmTools.checkConfiguration();
2546
2599
  },
2547
2600
  listModels() {
2548
2601
  return /* not await */ llmTools.listModels();
2549
2602
  },
2603
+ spending() {
2604
+ return spending.asObservable();
2605
+ },
2550
2606
  getTotalUsage() {
2551
2607
  // <- Note: [🥫] Not using getter `get totalUsage` but `getTotalUsage` to allow this object to be proxied
2552
2608
  return totalUsage;
@@ -2557,6 +2613,7 @@ function countTotalUsage(llmTools) {
2557
2613
  // console.info('[🚕] callChatModel through countTotalUsage');
2558
2614
  const promptResult = await llmTools.callChatModel(prompt);
2559
2615
  totalUsage = addUsage(totalUsage, promptResult.usage);
2616
+ spending.next(promptResult.usage);
2560
2617
  return promptResult;
2561
2618
  };
2562
2619
  }
@@ -2565,6 +2622,7 @@ function countTotalUsage(llmTools) {
2565
2622
  // console.info('[🚕] callCompletionModel through countTotalUsage');
2566
2623
  const promptResult = await llmTools.callCompletionModel(prompt);
2567
2624
  totalUsage = addUsage(totalUsage, promptResult.usage);
2625
+ spending.next(promptResult.usage);
2568
2626
  return promptResult;
2569
2627
  };
2570
2628
  }
@@ -2573,6 +2631,7 @@ function countTotalUsage(llmTools) {
2573
2631
  // console.info('[🚕] callEmbeddingModel through countTotalUsage');
2574
2632
  const promptResult = await llmTools.callEmbeddingModel(prompt);
2575
2633
  totalUsage = addUsage(totalUsage, promptResult.usage);
2634
+ spending.next(promptResult.usage);
2576
2635
  return promptResult;
2577
2636
  };
2578
2637
  }
@@ -2750,6 +2809,7 @@ function joinLlmExecutionTools(...llmExecutionTools) {
2750
2809
  `);
2751
2810
  // TODO: [🟥] Detect browser / node and make it colorfull
2752
2811
  console.warn(warningMessage);
2812
+ // <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
2753
2813
  /*
2754
2814
  return {
2755
2815
  async listModels() {
@@ -3521,17 +3581,24 @@ function titleToName(value) {
3521
3581
  /**
3522
3582
  * The built-in `fetch' function with a lightweight error handling wrapper as default fetch function used in Promptbook scrapers
3523
3583
  *
3524
- * @private as default `fetch` function used in Promptbook scrapers
3584
+ * @public exported from `@promptbook/core`
3525
3585
  */
3526
- const scraperFetch = async (url, init) => {
3586
+ const promptbookFetch = async (urlOrRequest, init) => {
3527
3587
  try {
3528
- return await fetch(url, init);
3588
+ return await fetch(urlOrRequest, init);
3529
3589
  }
3530
3590
  catch (error) {
3531
3591
  if (!(error instanceof Error)) {
3532
3592
  throw error;
3533
3593
  }
3534
- throw new KnowledgeScrapeError(spaceTrim$1((block) => `
3594
+ let url;
3595
+ if (typeof urlOrRequest === 'string') {
3596
+ url = urlOrRequest;
3597
+ }
3598
+ else if (urlOrRequest instanceof Request) {
3599
+ url = urlOrRequest.url;
3600
+ }
3601
+ throw new PromptbookFetchError(spaceTrim$1((block) => `
3535
3602
  Can not fetch "${url}"
3536
3603
 
3537
3604
  Fetch error:
@@ -3552,7 +3619,7 @@ const scraperFetch = async (url, init) => {
3552
3619
  async function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
3553
3620
  // console.log('!! makeKnowledgeSourceHandler', knowledgeSource);
3554
3621
  var _a;
3555
- const { fetch = scraperFetch } = tools;
3622
+ const { fetch = promptbookFetch } = tools;
3556
3623
  const { knowledgeSourceContent } = knowledgeSource;
3557
3624
  let { name } = knowledgeSource;
3558
3625
  const { rootDirname = null,
@@ -3693,63 +3760,73 @@ async function prepareKnowledgePieces(knowledgeSources, tools, options) {
3693
3760
  const { maxParallelCount = DEFAULT_MAX_PARALLEL_COUNT, rootDirname, isVerbose = DEFAULT_IS_VERBOSE } = options;
3694
3761
  const knowledgePreparedUnflatten = new Array(knowledgeSources.length);
3695
3762
  await forEachAsync(knowledgeSources, { maxParallelCount }, async (knowledgeSource, index) => {
3696
- let partialPieces = null;
3697
- const sourceHandler = await makeKnowledgeSourceHandler(knowledgeSource, tools, { rootDirname, isVerbose });
3698
- const scrapers = arrayableToArray(tools.scrapers);
3699
- for (const scraper of scrapers) {
3700
- if (!scraper.metadata.mimeTypes.includes(sourceHandler.mimeType)
3701
- // <- TODO: [🦔] Implement mime-type wildcards
3702
- ) {
3703
- continue;
3704
- }
3705
- const partialPiecesUnchecked = await scraper.scrape(sourceHandler);
3706
- if (partialPiecesUnchecked !== null) {
3707
- partialPieces = [...partialPiecesUnchecked];
3708
- // <- TODO: [🪓] Here should be no need for spreading new array, just `partialPieces = partialPiecesUnchecked`
3709
- break;
3710
- }
3711
- console.warn(spaceTrim$1((block) => `
3712
- Cannot scrape knowledge from source despite the scraper \`${scraper.metadata.className}\` supports the mime type "${sourceHandler.mimeType}".
3763
+ try {
3764
+ let partialPieces = null;
3765
+ const sourceHandler = await makeKnowledgeSourceHandler(knowledgeSource, tools, { rootDirname, isVerbose });
3766
+ const scrapers = arrayableToArray(tools.scrapers);
3767
+ for (const scraper of scrapers) {
3768
+ if (!scraper.metadata.mimeTypes.includes(sourceHandler.mimeType)
3769
+ // <- TODO: [🦔] Implement mime-type wildcards
3770
+ ) {
3771
+ continue;
3772
+ }
3773
+ const partialPiecesUnchecked = await scraper.scrape(sourceHandler);
3774
+ if (partialPiecesUnchecked !== null) {
3775
+ partialPieces = [...partialPiecesUnchecked];
3776
+ // <- TODO: [🪓] Here should be no need for spreading new array, just `partialPieces = partialPiecesUnchecked`
3777
+ break;
3778
+ }
3779
+ console.warn(spaceTrim$1((block) => `
3780
+ Cannot scrape knowledge from source despite the scraper \`${scraper.metadata.className}\` supports the mime type "${sourceHandler.mimeType}".
3713
3781
 
3714
- The source:
3715
- ${block(knowledgeSource.knowledgeSourceContent
3716
- .split('\n')
3717
- .map((line) => `> ${line}`)
3718
- .join('\n'))}
3782
+ The source:
3783
+ ${block(knowledgeSource.knowledgeSourceContent
3784
+ .split('\n')
3785
+ .map((line) => `> ${line}`)
3786
+ .join('\n'))}
3719
3787
 
3720
- ${block($registeredScrapersMessage(scrapers))}
3788
+ ${block($registeredScrapersMessage(scrapers))}
3721
3789
 
3722
3790
 
3723
- `));
3724
- }
3725
- if (partialPieces === null) {
3726
- throw new KnowledgeScrapeError(spaceTrim$1((block) => `
3727
- Cannot scrape knowledge
3791
+ `));
3792
+ // <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
3793
+ }
3794
+ if (partialPieces === null) {
3795
+ throw new KnowledgeScrapeError(spaceTrim$1((block) => `
3796
+ Cannot scrape knowledge
3728
3797
 
3729
- The source:
3730
- > ${block(knowledgeSource.knowledgeSourceContent
3731
- .split('\n')
3732
- .map((line) => `> ${line}`)
3733
- .join('\n'))}
3798
+ The source:
3799
+ > ${block(knowledgeSource.knowledgeSourceContent
3800
+ .split('\n')
3801
+ .map((line) => `> ${line}`)
3802
+ .join('\n'))}
3734
3803
 
3735
- No scraper found for the mime type "${sourceHandler.mimeType}"
3804
+ No scraper found for the mime type "${sourceHandler.mimeType}"
3736
3805
 
3737
- ${block($registeredScrapersMessage(scrapers))}
3806
+ ${block($registeredScrapersMessage(scrapers))}
3738
3807
 
3739
3808
 
3740
- `));
3809
+ `));
3810
+ }
3811
+ const pieces = partialPieces.map((partialPiece) => ({
3812
+ ...partialPiece,
3813
+ sources: [
3814
+ {
3815
+ name: knowledgeSource.name,
3816
+ // line, column <- TODO: [☀]
3817
+ // <- TODO: [❎]
3818
+ },
3819
+ ],
3820
+ }));
3821
+ knowledgePreparedUnflatten[index] = pieces;
3822
+ }
3823
+ catch (error) {
3824
+ if (!(error instanceof Error)) {
3825
+ throw error;
3826
+ }
3827
+ console.warn(error);
3828
+ // <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
3741
3829
  }
3742
- const pieces = partialPieces.map((partialPiece) => ({
3743
- ...partialPiece,
3744
- sources: [
3745
- {
3746
- name: knowledgeSource.name,
3747
- // line, column <- TODO: [☀]
3748
- // <- TODO: [❎]
3749
- },
3750
- ],
3751
- }));
3752
- knowledgePreparedUnflatten[index] = pieces;
3753
3830
  });
3754
3831
  const knowledgePrepared = knowledgePreparedUnflatten.flat();
3755
3832
  return knowledgePrepared;
@@ -3855,7 +3932,7 @@ async function preparePipeline(pipeline, tools, options) {
3855
3932
  // TODO: [🚐] Make arrayable LLMs -> single LLM DRY
3856
3933
  const _llms = arrayableToArray(tools.llm);
3857
3934
  const llmTools = _llms.length === 1 ? _llms[0] : joinLlmExecutionTools(..._llms);
3858
- const llmToolsWithUsage = countTotalUsage(llmTools);
3935
+ const llmToolsWithUsage = countUsage(llmTools);
3859
3936
  // <- TODO: [🌯]
3860
3937
  /*
3861
3938
  TODO: [🧠][🪑][🔃] Should this be done or not
@@ -4167,7 +4244,7 @@ function extractParameterNamesFromTask(task) {
4167
4244
  if (parameterNames.has(subparameterName)) {
4168
4245
  parameterNames.delete(subparameterName);
4169
4246
  parameterNames.add(foreach.parameterName);
4170
- // <- TODO: [🚎] Warn/logic error when `subparameterName` not used
4247
+ // <- TODO: [🏮] Warn/logic error when `subparameterName` not used
4171
4248
  }
4172
4249
  }
4173
4250
  }
@@ -5780,6 +5857,7 @@ function createPipelineExecutor(options) {
5780
5857
 
5781
5858
  @see more at https://ptbk.io/prepare-pipeline
5782
5859
  `));
5860
+ // <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
5783
5861
  }
5784
5862
  let runCount = 0;
5785
5863
  const pipelineExecutorWithCallback = async (inputParameters, onProgress) => {
@@ -6675,11 +6753,12 @@ async function $provideScriptingForNode(options) {
6675
6753
  * @public exported from `@promptbook/remote-server`
6676
6754
  */
6677
6755
  function startRemoteServer(options) {
6678
- const { port, collection, createLlmExecutionTools, isAnonymousModeAllowed, isApplicationModeAllowed, isVerbose = DEFAULT_IS_VERBOSE, } = {
6756
+ const { port, collection, createLlmExecutionTools, isAnonymousModeAllowed, isApplicationModeAllowed, isVerbose = DEFAULT_IS_VERBOSE, login, } = {
6679
6757
  isAnonymousModeAllowed: false,
6680
6758
  isApplicationModeAllowed: false,
6681
6759
  collection: null,
6682
6760
  createLlmExecutionTools: null,
6761
+ login: null,
6683
6762
  ...options,
6684
6763
  };
6685
6764
  // <- TODO: [🦪] Some helper type to be able to use discriminant union types with destructuring
@@ -6746,9 +6825,38 @@ function startRemoteServer(options) {
6746
6825
  response.setHeader('X-Powered-By', 'Promptbook engine');
6747
6826
  next();
6748
6827
  });
6828
+ const swaggerOptions = {
6829
+ definition: {
6830
+ openapi: '3.0.0',
6831
+ info: {
6832
+ title: 'Promptbook Remote Server API',
6833
+ version: '1.0.0',
6834
+ description: 'API documentation for the Promptbook Remote Server',
6835
+ },
6836
+ servers: [
6837
+ {
6838
+ url: `http://localhost:${port}${rootPath}`,
6839
+ // <- TODO: !!!!! Probbably: Pass `remoteServerUrl` instead of `port` and `rootPath`
6840
+ },
6841
+ ],
6842
+ },
6843
+ apis: ['./src/remote-server/**/*.ts'], // Adjust path as needed
6844
+ };
6845
+ const swaggerSpec = swaggerJsdoc(swaggerOptions);
6846
+ app.use([`/api-docs`, `${rootPath}/api-docs`], swaggerUi.serve, swaggerUi.setup(swaggerSpec));
6749
6847
  const runningExecutionTasks = [];
6750
6848
  // <- TODO: [🤬] Identify the users
6751
6849
  // TODO: [🧠] Do here some garbage collection of finished tasks
6850
+ /**
6851
+ * @swagger
6852
+ * /:
6853
+ * get:
6854
+ * summary: Get server details
6855
+ * description: Returns details about the Promptbook server.
6856
+ * responses:
6857
+ * 200:
6858
+ * description: Server details in markdown format.
6859
+ */
6752
6860
  app.get(['/', rootPath], async (request, response) => {
6753
6861
  var _a;
6754
6862
  if ((_a = request.url) === null || _a === void 0 ? void 0 : _a.includes('socket.io')) {
@@ -6785,9 +6893,12 @@ function startRemoteServer(options) {
6785
6893
 
6786
6894
  ## Paths
6787
6895
 
6788
- ${block(app._router.stack
6789
- .map(({ route }) => (route === null || route === void 0 ? void 0 : route.path) || null)
6790
- .filter((path) => path !== null)
6896
+ ${block([
6897
+ ...app._router.stack
6898
+ .map(({ route }) => (route === null || route === void 0 ? void 0 : route.path) || null)
6899
+ .filter((path) => path !== null),
6900
+ '/api-docs',
6901
+ ]
6791
6902
  .map((path) => `- ${path}`)
6792
6903
  .join('\n'))}
6793
6904
 
@@ -6805,7 +6916,96 @@ function startRemoteServer(options) {
6805
6916
  https://github.com/webgptorg/promptbook
6806
6917
  `));
6807
6918
  });
6808
- app.get(`${rootPath}/books`, async (request, response) => {
6919
+ /**
6920
+ * @swagger
6921
+ *
6922
+ * /login:
6923
+ * post:
6924
+ * summary: Login to the server
6925
+ * description: Login to the server and get identification.
6926
+ * requestBody:
6927
+ * required: true
6928
+ * content:
6929
+ * application/json:
6930
+ * schema:
6931
+ * type: object
6932
+ * properties:
6933
+ * username:
6934
+ * type: string
6935
+ * password:
6936
+ * type: string
6937
+ * appId:
6938
+ * type: string
6939
+ * responses:
6940
+ * 200:
6941
+ * description: Successful login
6942
+ * content:
6943
+ * application/json:
6944
+ * schema:
6945
+ * type: object
6946
+ * properties:
6947
+ * identification:
6948
+ * type: object
6949
+ */
6950
+ app.post([`/login`, `${rootPath}/login`], async (request, response) => {
6951
+ if (!isApplicationModeAllowed || login === null) {
6952
+ response.status(400).send('Application mode is not allowed');
6953
+ return;
6954
+ }
6955
+ try {
6956
+ const username = request.body.username;
6957
+ const password = request.body.password;
6958
+ const appId = request.body.appId;
6959
+ const { isSuccess, error, message, identification } = await login({
6960
+ username,
6961
+ password,
6962
+ appId,
6963
+ rawRequest: request,
6964
+ rawResponse: response,
6965
+ });
6966
+ response.status(201).send({
6967
+ isSuccess,
6968
+ message,
6969
+ error: error ? serializeError(error) : undefined,
6970
+ identification,
6971
+ });
6972
+ return;
6973
+ }
6974
+ catch (error) {
6975
+ if (!(error instanceof Error)) {
6976
+ throw error;
6977
+ }
6978
+ if (error instanceof AuthenticationError) {
6979
+ response.status(401).send({
6980
+ isSuccess: false,
6981
+ message: error.message,
6982
+ error: serializeError(error),
6983
+ });
6984
+ }
6985
+ console.warn(`Login function thrown different error than AuthenticationError`, {
6986
+ error,
6987
+ serializedError: serializeError(error),
6988
+ });
6989
+ response.status(400).send({ error: serializeError(error) });
6990
+ }
6991
+ });
6992
+ /**
6993
+ * @swagger
6994
+ * /books:
6995
+ * get:
6996
+ * summary: List all books
6997
+ * description: Returns a list of all available books in the collection.
6998
+ * responses:
6999
+ * 200:
7000
+ * description: A list of books.
7001
+ * content:
7002
+ * application/json:
7003
+ * schema:
7004
+ * type: array
7005
+ * items:
7006
+ * type: string
7007
+ */
7008
+ app.get([`/books`, `${rootPath}/books`], async (request, response) => {
6809
7009
  if (collection === null) {
6810
7010
  response.status(500).send('No collection available');
6811
7011
  return;
@@ -6815,7 +7015,30 @@ function startRemoteServer(options) {
6815
7015
  response.send(pipelines);
6816
7016
  });
6817
7017
  // TODO: [🧠] Is it secure / good idea to expose source codes of hosted books
6818
- app.get(`${rootPath}/books/*`, async (request, response) => {
7018
+ /**
7019
+ * @swagger
7020
+ * /books/{bookId}:
7021
+ * get:
7022
+ * summary: Get book content
7023
+ * description: Returns the content of a specific book.
7024
+ * parameters:
7025
+ * - in: path
7026
+ * name: bookId
7027
+ * required: true
7028
+ * schema:
7029
+ * type: string
7030
+ * description: The ID of the book to retrieve.
7031
+ * responses:
7032
+ * 200:
7033
+ * description: The content of the book.
7034
+ * content:
7035
+ * text/markdown:
7036
+ * schema:
7037
+ * type: string
7038
+ * 404:
7039
+ * description: Book not found.
7040
+ */
7041
+ app.get([`/books/*`, `${rootPath}/books/*`], async (request, response) => {
6819
7042
  try {
6820
7043
  if (collection === null) {
6821
7044
  response.status(500).send('No collection nor books available');
@@ -6869,10 +7092,26 @@ function startRemoteServer(options) {
6869
7092
  };
6870
7093
  }
6871
7094
  }
6872
- app.get(`${rootPath}/executions`, async (request, response) => {
7095
+ /**
7096
+ * @swagger
7097
+ * /executions:
7098
+ * get:
7099
+ * summary: List all executions
7100
+ * description: Returns a list of all running execution tasks.
7101
+ * responses:
7102
+ * 200:
7103
+ * description: A list of execution tasks.
7104
+ * content:
7105
+ * application/json:
7106
+ * schema:
7107
+ * type: array
7108
+ * items:
7109
+ * type: object
7110
+ */
7111
+ app.get([`/executions`, `${rootPath}/executions`], async (request, response) => {
6873
7112
  response.send(runningExecutionTasks.map((runningExecutionTask) => exportExecutionTask(runningExecutionTask, false)));
6874
7113
  });
6875
- app.get(`${rootPath}/executions/last`, async (request, response) => {
7114
+ app.get([`/executions/last`, `${rootPath}/executions/last`], async (request, response) => {
6876
7115
  // TODO: [🤬] Filter only for user
6877
7116
  if (runningExecutionTasks.length === 0) {
6878
7117
  response.status(404).send('No execution tasks found');
@@ -6881,7 +7120,7 @@ function startRemoteServer(options) {
6881
7120
  const lastExecutionTask = runningExecutionTasks[runningExecutionTasks.length - 1];
6882
7121
  response.send(exportExecutionTask(lastExecutionTask, true));
6883
7122
  });
6884
- app.get(`${rootPath}/executions/:taskId`, async (request, response) => {
7123
+ app.get([`/executions/:taskId`, `${rootPath}/executions/:taskId`], async (request, response) => {
6885
7124
  const { taskId } = request.params;
6886
7125
  // TODO: [🤬] Filter only for user
6887
7126
  const executionTask = runningExecutionTasks.find((executionTask) => executionTask.taskId === taskId);
@@ -6893,7 +7132,36 @@ function startRemoteServer(options) {
6893
7132
  }
6894
7133
  response.send(exportExecutionTask(executionTask, true));
6895
7134
  });
6896
- app.post(`${rootPath}/executions/new`, async (request, response) => {
7135
+ /**
7136
+ * @swagger
7137
+ * /executions/new:
7138
+ * post:
7139
+ * summary: Start a new execution
7140
+ * description: Starts a new execution task for a given pipeline.
7141
+ * requestBody:
7142
+ * required: true
7143
+ * content:
7144
+ * application/json:
7145
+ * schema:
7146
+ * type: object
7147
+ * properties:
7148
+ * pipelineUrl:
7149
+ * type: string
7150
+ * inputParameters:
7151
+ * type: object
7152
+ * identification:
7153
+ * type: object
7154
+ * responses:
7155
+ * 200:
7156
+ * description: The newly created execution task.
7157
+ * content:
7158
+ * application/json:
7159
+ * schema:
7160
+ * type: object
7161
+ * 400:
7162
+ * description: Invalid input.
7163
+ */
7164
+ app.post([`/executions/new`, `${rootPath}/executions/new`], async (request, response) => {
6897
7165
  try {
6898
7166
  const { inputParameters, identification /* <- [🤬] */ } = request.body;
6899
7167
  const pipelineUrl = request.body.pipelineUrl || request.body.book;
@@ -7070,6 +7338,15 @@ function startRemoteServer(options) {
7070
7338
  }
7071
7339
  let isDestroyed = false;
7072
7340
  return {
7341
+ get httpServer() {
7342
+ return httpServer;
7343
+ },
7344
+ get expressApp() {
7345
+ return app;
7346
+ },
7347
+ get socketIoServer() {
7348
+ return server;
7349
+ },
7073
7350
  get isDestroyed() {
7074
7351
  return isDestroyed;
7075
7352
  },