@promptbook/core 0.84.0-9 → 0.85.0-0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. package/README.md +21 -5
  2. package/esm/index.es.js +544 -288
  3. package/esm/index.es.js.map +1 -1
  4. package/esm/typings/src/_packages/cli.index.d.ts +4 -0
  5. package/esm/typings/src/_packages/core.index.d.ts +12 -4
  6. package/esm/typings/src/_packages/deepseek.index.d.ts +8 -0
  7. package/esm/typings/src/_packages/types.index.d.ts +16 -2
  8. package/esm/typings/src/_packages/utils.index.d.ts +2 -2
  9. package/esm/typings/src/_packages/wizzard.index.d.ts +4 -0
  10. package/esm/typings/src/cli/cli-commands/_boilerplate.d.ts +13 -0
  11. package/esm/typings/src/cli/cli-commands/about.d.ts +4 -1
  12. package/esm/typings/src/cli/cli-commands/hello.d.ts +3 -1
  13. package/esm/typings/src/cli/cli-commands/list-models.d.ts +3 -1
  14. package/esm/typings/src/cli/cli-commands/list-scrapers.d.ts +13 -0
  15. package/esm/typings/src/cli/cli-commands/make.d.ts +3 -1
  16. package/esm/typings/src/cli/cli-commands/prettify.d.ts +3 -1
  17. package/esm/typings/src/cli/cli-commands/run.d.ts +3 -1
  18. package/esm/typings/src/cli/cli-commands/runInteractiveChatbot.d.ts +1 -1
  19. package/esm/typings/src/cli/cli-commands/start-server.d.ts +13 -0
  20. package/esm/typings/src/cli/cli-commands/test-command.d.ts +3 -1
  21. package/esm/typings/src/config.d.ts +27 -1
  22. package/esm/typings/src/conversion/compilePipelineOnRemoteServer.d.ts +1 -1
  23. package/esm/typings/src/execution/AbstractTaskResult.d.ts +25 -0
  24. package/esm/typings/src/execution/ExecutionTask.d.ts +71 -0
  25. package/esm/typings/src/execution/FilesystemTools.d.ts +1 -1
  26. package/esm/typings/src/execution/PipelineExecutor.d.ts +2 -5
  27. package/esm/typings/src/execution/PipelineExecutorResult.d.ts +2 -15
  28. package/esm/typings/src/execution/PromptbookFetch.d.ts +8 -1
  29. package/esm/typings/src/execution/{assertsExecutionSuccessful.d.ts → assertsTaskSuccessful.d.ts} +4 -3
  30. package/esm/typings/src/execution/createPipelineExecutor/00-createPipelineExecutor.d.ts +0 -3
  31. package/esm/typings/src/execution/createPipelineExecutor/10-executePipeline.d.ts +2 -6
  32. package/esm/typings/src/execution/createPipelineExecutor/20-executeTask.d.ts +3 -6
  33. package/esm/typings/src/llm-providers/_common/register/$registeredLlmToolsMessage.d.ts +9 -0
  34. package/esm/typings/src/llm-providers/deepseek/DeepseekExecutionToolsOptions.d.ts +9 -0
  35. package/esm/typings/src/llm-providers/deepseek/createDeepseekExecutionTools.d.ts +14 -0
  36. package/esm/typings/src/llm-providers/deepseek/register-configuration.d.ts +14 -0
  37. package/esm/typings/src/llm-providers/deepseek/register-constructor.d.ts +15 -0
  38. package/esm/typings/src/pipeline/book-notation.d.ts +3 -2
  39. package/esm/typings/src/pipeline/prompt-notation.d.ts +18 -5
  40. package/esm/typings/src/prepare/preparePipelineOnRemoteServer.d.ts +1 -1
  41. package/esm/typings/src/remote-server/socket-types/_subtypes/PromptbookServer_Identification.d.ts +5 -2
  42. package/esm/typings/src/remote-server/startRemoteServer.d.ts +1 -0
  43. package/esm/typings/src/types/typeAliases.d.ts +2 -0
  44. package/esm/typings/src/utils/editable/edit-pipeline-string/deflatePipeline.test.d.ts +1 -0
  45. package/esm/typings/src/utils/editable/utils/isFlatPipeline.test.d.ts +1 -0
  46. package/esm/typings/src/utils/environment/$isRunningInBrowser.d.ts +3 -0
  47. package/esm/typings/src/utils/environment/$isRunningInJest.d.ts +3 -0
  48. package/esm/typings/src/utils/environment/$isRunningInNode.d.ts +3 -0
  49. package/esm/typings/src/utils/environment/$isRunningInWebWorker.d.ts +3 -0
  50. package/esm/typings/src/utils/files/mimeTypeToExtension.d.ts +10 -0
  51. package/esm/typings/src/utils/files/mimeTypeToExtension.test.d.ts +1 -0
  52. package/esm/typings/src/utils/random/$randomSeed.d.ts +2 -1
  53. package/esm/typings/src/utils/random/$randomToken.d.ts +13 -0
  54. package/esm/typings/src/wizzard/wizzard.d.ts +8 -3
  55. package/package.json +10 -15
  56. package/umd/index.umd.js +548 -290
  57. package/umd/index.umd.js.map +1 -1
  58. package/esm/typings/src/remote-server/socket-types/_common/PromptbookServer_Progress.d.ts +0 -10
  59. package/esm/typings/src/types/TaskProgress.d.ts +0 -43
package/umd/index.umd.js CHANGED
@@ -1,17 +1,17 @@
1
1
  (function (global, factory) {
2
- typeof exports === 'object' && typeof module !== 'undefined' ? factory(exports, require('spacetrim'), require('prettier'), require('prettier/parser-html'), require('waitasecond'), require('papaparse'), require('path'), require('crypto-js'), require('crypto-js/enc-hex'), require('mime-types'), require('moment'), require('colors'), require('crypto-js/sha256')) :
3
- typeof define === 'function' && define.amd ? define(['exports', 'spacetrim', 'prettier', 'prettier/parser-html', 'waitasecond', 'papaparse', 'path', 'crypto-js', 'crypto-js/enc-hex', 'mime-types', 'moment', 'colors', 'crypto-js/sha256'], factory) :
4
- (global = typeof globalThis !== 'undefined' ? globalThis : global || self, factory(global["promptbook-core"] = {}, global.spaceTrim, global.prettier, global.parserHtml, global.waitasecond, global.papaparse, global.path, global.cryptoJs, global.hexEncoder, global.mimeTypes, global.moment, global.colors, global.sha256));
5
- })(this, (function (exports, spaceTrim, prettier, parserHtml, waitasecond, papaparse, path, cryptoJs, hexEncoder, mimeTypes, moment, colors, sha256) { 'use strict';
2
+ typeof exports === 'object' && typeof module !== 'undefined' ? factory(exports, require('spacetrim'), require('prettier'), require('prettier/parser-html'), require('rxjs'), require('crypto'), require('waitasecond'), require('papaparse'), require('crypto-js/enc-hex'), require('crypto-js/sha256'), require('path'), require('crypto-js'), require('mime-types'), require('moment'), require('colors')) :
3
+ typeof define === 'function' && define.amd ? define(['exports', 'spacetrim', 'prettier', 'prettier/parser-html', 'rxjs', 'crypto', 'waitasecond', 'papaparse', 'crypto-js/enc-hex', 'crypto-js/sha256', 'path', 'crypto-js', 'mime-types', 'moment', 'colors'], factory) :
4
+ (global = typeof globalThis !== 'undefined' ? globalThis : global || self, factory(global["promptbook-core"] = {}, global.spaceTrim, global.prettier, global.parserHtml, global.rxjs, global.crypto, global.waitasecond, global.papaparse, global.hexEncoder, global.sha256, global.path, global.cryptoJs, global.mimeTypes, global.moment, global.colors));
5
+ })(this, (function (exports, spaceTrim, prettier, parserHtml, rxjs, crypto, waitasecond, papaparse, hexEncoder, sha256, path, cryptoJs, mimeTypes, moment, colors) { 'use strict';
6
6
 
7
7
  function _interopDefaultLegacy (e) { return e && typeof e === 'object' && 'default' in e ? e : { 'default': e }; }
8
8
 
9
9
  var spaceTrim__default = /*#__PURE__*/_interopDefaultLegacy(spaceTrim);
10
10
  var parserHtml__default = /*#__PURE__*/_interopDefaultLegacy(parserHtml);
11
11
  var hexEncoder__default = /*#__PURE__*/_interopDefaultLegacy(hexEncoder);
12
+ var sha256__default = /*#__PURE__*/_interopDefaultLegacy(sha256);
12
13
  var moment__default = /*#__PURE__*/_interopDefaultLegacy(moment);
13
14
  var colors__default = /*#__PURE__*/_interopDefaultLegacy(colors);
14
- var sha256__default = /*#__PURE__*/_interopDefaultLegacy(sha256);
15
15
 
16
16
  // ⚠️ WARNING: This code has been generated so that any manual changes will be overwritten
17
17
  /**
@@ -27,7 +27,7 @@
27
27
  * @generated
28
28
  * @see https://github.com/webgptorg/promptbook
29
29
  */
30
- var PROMPTBOOK_ENGINE_VERSION = '0.84.0-8';
30
+ var PROMPTBOOK_ENGINE_VERSION = '0.84.0';
31
31
  /**
32
32
  * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
33
33
  * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -180,22 +180,94 @@
180
180
  */
181
181
 
182
182
  /**
183
- * Function isValidJsonString will tell you if the string is valid JSON or not
183
+ * Checks if value is valid email
184
184
  *
185
185
  * @public exported from `@promptbook/utils`
186
186
  */
187
- function isValidJsonString(value /* <- [👨‍⚖️] */) {
188
- try {
189
- JSON.parse(value);
187
+ function isValidEmail(email) {
188
+ if (typeof email !== 'string') {
189
+ return false;
190
+ }
191
+ if (email.split('\n').length > 1) {
192
+ return false;
193
+ }
194
+ return /^.+@.+\..+$/.test(email);
195
+ }
196
+
197
+ /**
198
+ * Tests if given string is valid URL.
199
+ *
200
+ * Note: This does not check if the file exists only if the path is valid
201
+ * @public exported from `@promptbook/utils`
202
+ */
203
+ function isValidFilePath(filename) {
204
+ if (typeof filename !== 'string') {
205
+ return false;
206
+ }
207
+ if (filename.split('\n').length > 1) {
208
+ return false;
209
+ }
210
+ if (filename.split(' ').length >
211
+ 5 /* <- TODO: [🧠][🈷] Make some better non-arbitrary way how to distinct filenames from informational texts */) {
212
+ return false;
213
+ }
214
+ var filenameSlashes = filename.split('\\').join('/');
215
+ // Absolute Unix path: /hello.txt
216
+ if (/^(\/)/i.test(filenameSlashes)) {
217
+ // console.log(filename, 'Absolute Unix path: /hello.txt');
190
218
  return true;
191
219
  }
192
- catch (error) {
193
- if (!(error instanceof Error)) {
194
- throw error;
220
+ // Absolute Windows path: /hello.txt
221
+ if (/^([A-Z]{1,2}:\/?)\//i.test(filenameSlashes)) {
222
+ // console.log(filename, 'Absolute Windows path: /hello.txt');
223
+ return true;
224
+ }
225
+ // Relative path: ./hello.txt
226
+ if (/^(\.\.?\/)+/i.test(filenameSlashes)) {
227
+ // console.log(filename, 'Relative path: ./hello.txt');
228
+ return true;
229
+ }
230
+ // Allow paths like foo/hello
231
+ if (/^[^/]+\/[^/]+/i.test(filenameSlashes)) {
232
+ // console.log(filename, 'Allow paths like foo/hello');
233
+ return true;
234
+ }
235
+ // Allow paths like hello.book
236
+ if (/^[^/]+\.[^/]+$/i.test(filenameSlashes)) {
237
+ // console.log(filename, 'Allow paths like hello.book');
238
+ return true;
239
+ }
240
+ return false;
241
+ }
242
+ /**
243
+ * TODO: [🍏] Implement for MacOs
244
+ */
245
+
246
+ /**
247
+ * Tests if given string is valid URL.
248
+ *
249
+ * Note: Dataurl are considered perfectly valid.
250
+ * Note: There are two simmilar functions:
251
+ * - `isValidUrl` which tests any URL
252
+ * - `isValidPipelineUrl` *(this one)* which tests just promptbook URL
253
+ *
254
+ * @public exported from `@promptbook/utils`
255
+ */
256
+ function isValidUrl(url) {
257
+ if (typeof url !== 'string') {
258
+ return false;
259
+ }
260
+ try {
261
+ if (url.startsWith('blob:')) {
262
+ url = url.replace(/^blob:/, '');
195
263
  }
196
- if (error.message.includes('Unexpected token')) {
264
+ var urlObject = new URL(url /* because fail is handled */);
265
+ if (!['http:', 'https:', 'data:'].includes(urlObject.protocol)) {
197
266
  return false;
198
267
  }
268
+ return true;
269
+ }
270
+ catch (error) {
199
271
  return false;
200
272
  }
201
273
  }
@@ -219,6 +291,27 @@
219
291
  * TODO: Maybe split `ParseError` and `ApplyError`
220
292
  */
221
293
 
294
+ /**
295
+ * Function isValidJsonString will tell you if the string is valid JSON or not
296
+ *
297
+ * @public exported from `@promptbook/utils`
298
+ */
299
+ function isValidJsonString(value /* <- [👨‍⚖️] */) {
300
+ try {
301
+ JSON.parse(value);
302
+ return true;
303
+ }
304
+ catch (error) {
305
+ if (!(error instanceof Error)) {
306
+ throw error;
307
+ }
308
+ if (error.message.includes('Unexpected token')) {
309
+ return false;
310
+ }
311
+ return false;
312
+ }
313
+ }
314
+
222
315
  /**
223
316
  * Function `validatePipelineString` will validate the if the string is a valid pipeline string
224
317
  * It does not check if the string is fully logically correct, but if it is a string that can be a pipeline string or the string looks completely different.
@@ -232,6 +325,15 @@
232
325
  if (isValidJsonString(pipelineString)) {
233
326
  throw new ParseError('Expected a book, but got a JSON string');
234
327
  }
328
+ else if (isValidUrl(pipelineString)) {
329
+ throw new ParseError("Expected a book, but got just the URL \"".concat(pipelineString, "\""));
330
+ }
331
+ else if (isValidFilePath(pipelineString)) {
332
+ throw new ParseError("Expected a book, but got just the file path \"".concat(pipelineString, "\""));
333
+ }
334
+ else if (isValidEmail(pipelineString)) {
335
+ throw new ParseError("Expected a book, but got just the email \"".concat(pipelineString, "\""));
336
+ }
235
337
  // <- TODO: Implement the validation + add tests when the pipeline logic considered as invalid
236
338
  return pipelineString;
237
339
  }
@@ -572,6 +674,24 @@
572
674
  * @public exported from `@promptbook/core`
573
675
  */
574
676
  var DEFAULT_TASK_TITLE = "Task";
677
+ /**
678
+ * When the title of the prompt task is not provided, the default title is used
679
+ *
680
+ * @public exported from `@promptbook/core`
681
+ */
682
+ var DEFAULT_PROMPT_TASK_TITLE = "Prompt";
683
+ /**
684
+ * When the pipeline is flat and no name of return parameter is provided, this name is used
685
+ *
686
+ * @public exported from `@promptbook/core`
687
+ */
688
+ var DEFAULT_BOOK_OUTPUT_PARAMETER_NAME = 'result';
689
+ /**
690
+ * Maximum file size limit
691
+ *
692
+ * @public exported from `@promptbook/core`
693
+ */
694
+ var DEFAULT_MAX_FILE_SIZE = 100 * 1024 * 1024; // 100MB
575
695
  // <- TODO: [🧠] Better system for generator warnings - not always "code" and "by `@promptbook/cli`"
576
696
  /**
577
697
  * The maximum number of iterations for a loops
@@ -652,6 +772,14 @@
652
772
  */
653
773
  var DEFAULT_BOOKS_DIRNAME = './books';
654
774
  // <- TODO: [🕝] Make also `BOOKS_DIRNAME_ALTERNATIVES`
775
+ /**
776
+ * Where to store the temporary downloads
777
+ *
778
+ * Note: When the folder does not exist, it is created recursively
779
+ *
780
+ * @public exported from `@promptbook/core`
781
+ */
782
+ var DEFAULT_DOWNLOAD_CACHE_DIRNAME = './.promptbook/download-cache';
655
783
  /**
656
784
  * Where to store the cache of executions for promptbook CLI
657
785
  *
@@ -659,7 +787,7 @@
659
787
  *
660
788
  * @public exported from `@promptbook/core`
661
789
  */
662
- var DEFAULT_EXECUTIONS_CACHE_DIRNAME = './.promptbook/executions-cache';
790
+ var DEFAULT_EXECUTION_CACHE_DIRNAME = './.promptbook/execution-cache';
663
791
  /**
664
792
  * Where to store the scrape cache
665
793
  *
@@ -1185,35 +1313,6 @@
1185
1313
  return isHostnameOnPrivateNetwork(url.hostname);
1186
1314
  }
1187
1315
 
1188
- /**
1189
- * Tests if given string is valid URL.
1190
- *
1191
- * Note: Dataurl are considered perfectly valid.
1192
- * Note: There are two simmilar functions:
1193
- * - `isValidUrl` which tests any URL
1194
- * - `isValidPipelineUrl` *(this one)* which tests just promptbook URL
1195
- *
1196
- * @public exported from `@promptbook/utils`
1197
- */
1198
- function isValidUrl(url) {
1199
- if (typeof url !== 'string') {
1200
- return false;
1201
- }
1202
- try {
1203
- if (url.startsWith('blob:')) {
1204
- url = url.replace(/^blob:/, '');
1205
- }
1206
- var urlObject = new URL(url /* because fail is handled */);
1207
- if (!['http:', 'https:', 'data:'].includes(urlObject.protocol)) {
1208
- return false;
1209
- }
1210
- return true;
1211
- }
1212
- catch (error) {
1213
- return false;
1214
- }
1215
- }
1216
-
1217
1316
  /**
1218
1317
  * Tests if given string is valid pipeline URL URL.
1219
1318
  *
@@ -1943,6 +2042,58 @@
1943
2042
  return MissingToolsError;
1944
2043
  }(Error));
1945
2044
 
2045
+ /**
2046
+ * Determine if the pipeline is fully prepared
2047
+ *
2048
+ * @see https://github.com/webgptorg/promptbook/discussions/196
2049
+ *
2050
+ * @public exported from `@promptbook/core`
2051
+ */
2052
+ function isPipelinePrepared(pipeline) {
2053
+ // Note: Ignoring `pipeline.preparations` @@@
2054
+ // Note: Ignoring `pipeline.knowledgePieces` @@@
2055
+ if (pipeline.title === undefined || pipeline.title === '' || pipeline.title === DEFAULT_BOOK_TITLE) {
2056
+ return false;
2057
+ }
2058
+ if (!pipeline.personas.every(function (persona) { return persona.modelRequirements !== undefined; })) {
2059
+ return false;
2060
+ }
2061
+ if (!pipeline.knowledgeSources.every(function (knowledgeSource) { return knowledgeSource.preparationIds !== undefined; })) {
2062
+ return false;
2063
+ }
2064
+ /*
2065
+ TODO: [🧠][🍫] `tasks` can not be determined if they are fully prepared SO ignoring them
2066
+ > if (!pipeline.tasks.every(({ preparedContent }) => preparedContent === undefined)) {
2067
+ > return false;
2068
+ > }
2069
+ */
2070
+ return true;
2071
+ }
2072
+ /**
2073
+ * TODO: [🔃][main] If the pipeline was prepared with different version or different set of models, prepare it once again
2074
+ * TODO: [🐠] Maybe base this on `makeValidator`
2075
+ * TODO: [🧊] Pipeline can be partially prepared, this should return true ONLY if fully prepared
2076
+ * TODO: [🧿] Maybe do same process with same granularity and subfinctions as `preparePipeline`
2077
+ * - [🏍] ? Is context in each task
2078
+ * - [♨] Are examples prepared
2079
+ * - [♨] Are tasks prepared
2080
+ */
2081
+
2082
+ /**
2083
+ * Generates random token
2084
+ *
2085
+ * Note: This function is cryptographically secure (it uses crypto.randomBytes internally)
2086
+ *
2087
+ * @private internal helper function
2088
+ * @returns secure random token
2089
+ */
2090
+ function $randomToken(randomness) {
2091
+ return crypto.randomBytes(randomness).toString('hex');
2092
+ }
2093
+ /**
2094
+ * TODO: Maybe use nanoid instead https://github.com/ai/nanoid
2095
+ */
2096
+
1946
2097
  /**
1947
2098
  * This error indicates errors during the execution of the pipeline
1948
2099
  *
@@ -2172,12 +2323,28 @@
2172
2323
  /**
2173
2324
  * Asserts that the execution of a Promptbook is successful
2174
2325
  *
2326
+ * Note: If there are only warnings, the execution is still successful but the warnings are logged in the console
2327
+ *
2175
2328
  * @param executionResult - The partial result of the Promptbook execution
2176
2329
  * @throws {PipelineExecutionError} If the execution is not successful or if multiple errors occurred
2177
- * @public exported from `@promptbook/core`
2330
+ * @private internal helper function of `asPromise` method of `ExecutionTask`
2178
2331
  */
2179
- function assertsExecutionSuccessful(executionResult) {
2180
- var isSuccessful = executionResult.isSuccessful, errors = executionResult.errors;
2332
+ function assertsTaskSuccessful(executionResult) {
2333
+ var e_1, _a;
2334
+ var isSuccessful = executionResult.isSuccessful, errors = executionResult.errors, warnings = executionResult.warnings;
2335
+ try {
2336
+ for (var warnings_1 = __values(warnings), warnings_1_1 = warnings_1.next(); !warnings_1_1.done; warnings_1_1 = warnings_1.next()) {
2337
+ var warning = warnings_1_1.value;
2338
+ console.warn(warning.message);
2339
+ }
2340
+ }
2341
+ catch (e_1_1) { e_1 = { error: e_1_1 }; }
2342
+ finally {
2343
+ try {
2344
+ if (warnings_1_1 && !warnings_1_1.done && (_a = warnings_1.return)) _a.call(warnings_1);
2345
+ }
2346
+ finally { if (e_1) throw e_1.error; }
2347
+ }
2181
2348
  if (isSuccessful === true) {
2182
2349
  return;
2183
2350
  }
@@ -2197,46 +2364,71 @@
2197
2364
  }
2198
2365
  }
2199
2366
  /**
2200
- * TODO: [🐚] This function should be removed OR changed OR be completely rewritten
2201
2367
  * TODO: [🧠] Can this return type be better typed than void
2202
2368
  */
2203
2369
 
2204
2370
  /**
2205
- * Determine if the pipeline is fully prepared
2371
+ * Helper to create a new task
2206
2372
  *
2207
- * @see https://github.com/webgptorg/promptbook/discussions/196
2208
- *
2209
- * @public exported from `@promptbook/core`
2373
+ * @private internal helper function
2210
2374
  */
2211
- function isPipelinePrepared(pipeline) {
2212
- // Note: Ignoring `pipeline.preparations` @@@
2213
- // Note: Ignoring `pipeline.knowledgePieces` @@@
2214
- if (pipeline.title === undefined || pipeline.title === '' || pipeline.title === DEFAULT_BOOK_TITLE) {
2215
- return false;
2216
- }
2217
- if (!pipeline.personas.every(function (persona) { return persona.modelRequirements !== undefined; })) {
2218
- return false;
2219
- }
2220
- if (!pipeline.knowledgeSources.every(function (knowledgeSource) { return knowledgeSource.preparationIds !== undefined; })) {
2221
- return false;
2375
+ function createTask(options) {
2376
+ var taskType = options.taskType, taskProcessCallback = options.taskProcessCallback;
2377
+ var taskId = "".concat(taskType.toLowerCase(), "-").concat($randomToken(256 /* <- TODO: !!! To global config */));
2378
+ var resultSubject = new rxjs.BehaviorSubject({});
2379
+ var finalResultPromise = /* not await */ taskProcessCallback(function (newOngoingResult) {
2380
+ resultSubject.next(newOngoingResult);
2381
+ });
2382
+ function asPromise(options) {
2383
+ return __awaiter(this, void 0, void 0, function () {
2384
+ var _a, isCrashedOnError, finalResult;
2385
+ return __generator(this, function (_b) {
2386
+ switch (_b.label) {
2387
+ case 0:
2388
+ _a = (options || {}).isCrashedOnError, isCrashedOnError = _a === void 0 ? true : _a;
2389
+ return [4 /*yield*/, finalResultPromise];
2390
+ case 1:
2391
+ finalResult = _b.sent();
2392
+ if (isCrashedOnError) {
2393
+ assertsTaskSuccessful(finalResult);
2394
+ }
2395
+ return [2 /*return*/, finalResult];
2396
+ }
2397
+ });
2398
+ });
2222
2399
  }
2223
- /*
2224
- TODO: [🧠][🍫] `tasks` can not be determined if they are fully prepared SO ignoring them
2225
- > if (!pipeline.tasks.every(({ preparedContent }) => preparedContent === undefined)) {
2226
- > return false;
2227
- > }
2228
- */
2229
- return true;
2400
+ return {
2401
+ taskType: taskType,
2402
+ taskId: taskId,
2403
+ asPromise: asPromise,
2404
+ asObservable: function () {
2405
+ return rxjs.concat(resultSubject.asObservable(), rxjs.from(asPromise({
2406
+ isCrashedOnError: true,
2407
+ })));
2408
+ },
2409
+ };
2230
2410
  }
2231
2411
  /**
2232
- * TODO: [🔃][main] If the pipeline was prepared with different version or different set of models, prepare it once again
2233
- * TODO: [🐠] Maybe base this on `makeValidator`
2234
- * TODO: [🧊] Pipeline can be partially prepared, this should return true ONLY if fully prepared
2235
- * TODO: [🧿] Maybe do same process with same granularity and subfinctions as `preparePipeline`
2236
- * - [🏍] ? Is context in each task
2237
- * - [] Are examples prepared
2238
- * - [♨] Are tasks prepared
2412
+ * TODO: Maybe allow to terminate the task and add getter `isFinished` or `status`
2413
+ * TODO: [🐚] Split into more files and make `PrepareTask` & `RemoteTask` + split the function
2414
+ */
2415
+
2416
+ /**
2417
+ * Serializes an error into a [🚉] JSON-serializable object
2418
+ *
2419
+ * @public exported from `@promptbook/utils`
2239
2420
  */
2421
+ function serializeError(error) {
2422
+ var name = error.name, message = error.message, stack = error.stack;
2423
+ if (!Object.keys(ALL_ERRORS).includes(name)) {
2424
+ console.error(spaceTrim__default["default"](function (block) { return "\n \n Cannot serialize error with name \"".concat(name, "\"\n\n ").concat(block(stack || message), "\n \n "); }));
2425
+ }
2426
+ return {
2427
+ name: name,
2428
+ message: message,
2429
+ stack: stack,
2430
+ };
2431
+ }
2240
2432
 
2241
2433
  /**
2242
2434
  * Format either small or big number
@@ -2313,23 +2505,6 @@
2313
2505
  }
2314
2506
  }
2315
2507
 
2316
- /**
2317
- * Serializes an error into a [🚉] JSON-serializable object
2318
- *
2319
- * @public exported from `@promptbook/utils`
2320
- */
2321
- function serializeError(error) {
2322
- var name = error.name, message = error.message, stack = error.stack;
2323
- if (!Object.keys(ALL_ERRORS).includes(name)) {
2324
- console.error(spaceTrim__default["default"](function (block) { return "\n \n Cannot serialize error with name \"".concat(name, "\"\n\n ").concat(block(stack || message), "\n \n "); }));
2325
- }
2326
- return {
2327
- name: name,
2328
- message: message,
2329
- stack: stack,
2330
- };
2331
- }
2332
-
2333
2508
  /**
2334
2509
  * Represents the usage with no resources consumed
2335
2510
  *
@@ -3187,7 +3362,7 @@
3187
3362
  if (!(error_1 instanceof Error) || error_1 instanceof UnexpectedError) {
3188
3363
  throw error_1;
3189
3364
  }
3190
- errors.push(error_1);
3365
+ errors.push({ llmExecutionTools: llmExecutionTools, error: error_1 });
3191
3366
  return [3 /*break*/, 13];
3192
3367
  case 13:
3193
3368
  _b = _a.next();
@@ -3214,7 +3389,10 @@
3214
3389
  // 2) AnthropicClaude throw PipelineExecutionError: Parameter `{knowledge}` is not defined
3215
3390
  // 3) ...
3216
3391
  spaceTrim__default["default"](function (block) { return "\n All execution tools failed:\n\n ".concat(block(errors
3217
- .map(function (error, i) { return "".concat(i + 1, ") **").concat(error.name || 'Error', ":** ").concat(error.message); })
3392
+ .map(function (_a, i) {
3393
+ var error = _a.error, llmExecutionTools = _a.llmExecutionTools;
3394
+ return "".concat(i + 1, ") **").concat(llmExecutionTools.title, "** thrown **").concat(error.name || 'Error', ":** ").concat(error.message);
3395
+ })
3218
3396
  .join('\n')), "\n\n "); }));
3219
3397
  }
3220
3398
  else if (this.llmExecutionTools.length === 0) {
@@ -4496,27 +4674,20 @@
4496
4674
  */
4497
4675
  function executeTask(options) {
4498
4676
  return __awaiter(this, void 0, void 0, function () {
4499
- var currentTask, preparedPipeline, parametersToPass, tools, onProgress, $executionReport, pipelineIdentification, maxExecutionAttempts, maxParallelCount, csvSettings, isVerbose, rootDirname, cacheDirname, intermediateFilesStrategy, isAutoInstalled, isNotPreparedWarningSupressed, name, title, priority, usedParameterNames, dependentParameterNames, definedParameters, _a, _b, _c, definedParameterNames, parameters, _loop_1, _d, _e, parameterName, maxAttempts, jokerParameterNames, preparedContent, resultString;
4500
- var e_1, _f, _g;
4501
- return __generator(this, function (_h) {
4502
- switch (_h.label) {
4677
+ var currentTask, preparedPipeline, parametersToPass, tools, onProgress, $executionReport, pipelineIdentification, maxExecutionAttempts, maxParallelCount, csvSettings, isVerbose, rootDirname, cacheDirname, intermediateFilesStrategy, isAutoInstalled, isNotPreparedWarningSupressed, priority, usedParameterNames, dependentParameterNames, definedParameters, _a, _b, _c, definedParameterNames, parameters, _loop_1, _d, _e, parameterName, maxAttempts, jokerParameterNames, preparedContent, resultString;
4678
+ var _f, e_1, _g, _h, _j;
4679
+ return __generator(this, function (_k) {
4680
+ switch (_k.label) {
4503
4681
  case 0:
4504
4682
  currentTask = options.currentTask, preparedPipeline = options.preparedPipeline, parametersToPass = options.parametersToPass, tools = options.tools, onProgress = options.onProgress, $executionReport = options.$executionReport, pipelineIdentification = options.pipelineIdentification, maxExecutionAttempts = options.maxExecutionAttempts, maxParallelCount = options.maxParallelCount, csvSettings = options.csvSettings, isVerbose = options.isVerbose, rootDirname = options.rootDirname, cacheDirname = options.cacheDirname, intermediateFilesStrategy = options.intermediateFilesStrategy, isAutoInstalled = options.isAutoInstalled, isNotPreparedWarningSupressed = options.isNotPreparedWarningSupressed;
4505
- name = "pipeline-executor-frame-".concat(currentTask.name);
4506
- title = currentTask.title;
4507
4683
  priority = preparedPipeline.tasks.length - preparedPipeline.tasks.indexOf(currentTask);
4508
4684
  return [4 /*yield*/, onProgress({
4509
- name: name,
4510
- title: title,
4511
- isStarted: false,
4512
- isDone: false,
4513
- taskType: currentTask.taskType,
4514
- parameterName: currentTask.resultingParameterName,
4515
- parameterValue: null,
4516
- // <- [🍸]
4685
+ outputParameters: (_f = {},
4686
+ _f[currentTask.resultingParameterName] = '',
4687
+ _f),
4517
4688
  })];
4518
4689
  case 1:
4519
- _h.sent();
4690
+ _k.sent();
4520
4691
  usedParameterNames = extractParameterNamesFromTask(currentTask);
4521
4692
  dependentParameterNames = new Set(currentTask.dependentParameterNames);
4522
4693
  // TODO: [👩🏾‍🤝‍👩🏻] Use here `mapAvailableToExpectedParameters`
@@ -4535,7 +4706,7 @@
4535
4706
  pipelineIdentification: pipelineIdentification,
4536
4707
  })];
4537
4708
  case 2:
4538
- definedParameters = _b.apply(_a, [__assign.apply(void 0, [__assign.apply(void 0, _c.concat([(_h.sent())])), parametersToPass])]);
4709
+ definedParameters = _b.apply(_a, [__assign.apply(void 0, [__assign.apply(void 0, _c.concat([(_k.sent())])), parametersToPass])]);
4539
4710
  definedParameterNames = new Set(Object.keys(definedParameters));
4540
4711
  parameters = {};
4541
4712
  _loop_1 = function (parameterName) {
@@ -4563,7 +4734,7 @@
4563
4734
  catch (e_1_1) { e_1 = { error: e_1_1 }; }
4564
4735
  finally {
4565
4736
  try {
4566
- if (_e && !_e.done && (_f = _d.return)) _f.call(_d);
4737
+ if (_e && !_e.done && (_g = _d.return)) _g.call(_d);
4567
4738
  }
4568
4739
  finally { if (e_1) throw e_1.error; }
4569
4740
  }
@@ -4594,24 +4765,19 @@
4594
4765
  isNotPreparedWarningSupressed: isNotPreparedWarningSupressed,
4595
4766
  })];
4596
4767
  case 3:
4597
- resultString = _h.sent();
4768
+ resultString = _k.sent();
4598
4769
  return [4 /*yield*/, onProgress({
4599
- name: name,
4600
- title: title,
4601
- isStarted: true,
4602
- isDone: true,
4603
- taskType: currentTask.taskType,
4604
- parameterName: currentTask.resultingParameterName,
4605
- parameterValue: resultString,
4606
- // <- [🍸]
4770
+ outputParameters: (_h = {},
4771
+ _h[currentTask.resultingParameterName] = resultString,
4772
+ _h),
4607
4773
  })];
4608
4774
  case 4:
4609
- _h.sent();
4610
- return [2 /*return*/, Object.freeze((_g = {},
4611
- _g[currentTask.resultingParameterName] =
4775
+ _k.sent();
4776
+ return [2 /*return*/, Object.freeze((_j = {},
4777
+ _j[currentTask.resultingParameterName] =
4612
4778
  // <- Note: [👩‍👩‍👧] No need to detect parameter collision here because pipeline checks logic consistency during construction
4613
4779
  resultString,
4614
- _g))];
4780
+ _j))];
4615
4781
  }
4616
4782
  });
4617
4783
  });
@@ -4619,9 +4785,6 @@
4619
4785
  /**
4620
4786
  * TODO: [🤹‍♂️]
4621
4787
  */
4622
- /**
4623
- * TODO: [🐚] Change onProgress to object that represents the running execution, can be subscribed via RxJS to and also awaited
4624
- */
4625
4788
 
4626
4789
  /**
4627
4790
  * @@@
@@ -4883,15 +5046,15 @@
4883
5046
  return [3 /*break*/, 4];
4884
5047
  case 3:
4885
5048
  unresovedTasks_1 = unresovedTasks_1.filter(function (task) { return task !== currentTask; });
4886
- work_1 = executeTask(__assign(__assign({}, options), { currentTask: currentTask, preparedPipeline: preparedPipeline, parametersToPass: parametersToPass, tools: tools, onProgress: function (progress) {
5049
+ work_1 = executeTask(__assign(__assign({}, options), { currentTask: currentTask, preparedPipeline: preparedPipeline, parametersToPass: parametersToPass, tools: tools, onProgress: function (newOngoingResult) {
4887
5050
  if (isReturned) {
4888
- throw new UnexpectedError(spaceTrim.spaceTrim(function (block) { return "\n Can not call `onProgress` after pipeline execution is finished\n\n ".concat(block(pipelineIdentification), "\n\n ").concat(block(JSON.stringify(progress, null, 4)
5051
+ throw new UnexpectedError(spaceTrim.spaceTrim(function (block) { return "\n Can not call `onProgress` after pipeline execution is finished\n\n ".concat(block(pipelineIdentification), "\n\n ").concat(block(JSON.stringify(newOngoingResult, null, 4)
4889
5052
  .split('\n')
4890
5053
  .map(function (line) { return "> ".concat(line); })
4891
5054
  .join('\n')), "\n "); }));
4892
5055
  }
4893
5056
  if (onProgress) {
4894
- onProgress(progress);
5057
+ onProgress(newOngoingResult);
4895
5058
  }
4896
5059
  }, $executionReport: executionReport, pipelineIdentification: spaceTrim.spaceTrim(function (block) { return "\n ".concat(block(pipelineIdentification), "\n Task name: ").concat(currentTask.name, "\n Task title: ").concat(currentTask.title, "\n "); }) }))
4897
5060
  .then(function (newParametersToPass) {
@@ -4994,9 +5157,6 @@
4994
5157
  });
4995
5158
  });
4996
5159
  }
4997
- /**
4998
- * TODO: [🐚] Change onProgress to object that represents the running execution, can be subscribed via RxJS to and also awaited
4999
- */
5000
5160
 
5001
5161
  /**
5002
5162
  * Creates executor function from pipeline and execution tools.
@@ -5028,7 +5188,7 @@
5028
5188
  console.warn(spaceTrim.spaceTrim(function (block) { return "\n Pipeline is not prepared\n\n ".concat(block(pipelineIdentification), "\n\n It will be prepared ad-hoc before the first execution and **returned as `preparedPipeline` in `PipelineExecutorResult`**\n But it is recommended to prepare the pipeline during collection preparation\n\n @see more at https://ptbk.io/prepare-pipeline\n "); }));
5029
5189
  }
5030
5190
  var runCount = 0;
5031
- var pipelineExecutor = function (inputParameters, onProgress) { return __awaiter(_this, void 0, void 0, function () {
5191
+ var pipelineExecutorWithCallback = function (inputParameters, onProgress) { return __awaiter(_this, void 0, void 0, function () {
5032
5192
  return __generator(this, function (_a) {
5033
5193
  runCount++;
5034
5194
  return [2 /*return*/, /* not await */ executePipeline({
@@ -5053,11 +5213,23 @@
5053
5213
  })];
5054
5214
  });
5055
5215
  }); };
5216
+ var pipelineExecutor = function (inputParameters) {
5217
+ return createTask({
5218
+ taskType: 'EXECUTION',
5219
+ taskProcessCallback: function (updateOngoingResult) {
5220
+ var _this = this;
5221
+ return pipelineExecutorWithCallback(inputParameters, function (newOngoingResult) { return __awaiter(_this, void 0, void 0, function () {
5222
+ return __generator(this, function (_a) {
5223
+ updateOngoingResult(newOngoingResult);
5224
+ return [2 /*return*/];
5225
+ });
5226
+ }); });
5227
+ },
5228
+ });
5229
+ };
5230
+ // <- TODO: Make types such as there is no need to do `as` for `createTask`
5056
5231
  return pipelineExecutor;
5057
5232
  }
5058
- /**
5059
- * TODO: [🐚] Change onProgress to object that represents the running execution, can be subscribed via RxJS to and also awaited
5060
- */
5061
5233
 
5062
5234
  /**
5063
5235
  * Async version of Array.forEach
@@ -5262,10 +5434,9 @@
5262
5434
  return modelName;
5263
5435
  })
5264
5436
  .join(',');
5265
- return [4 /*yield*/, preparePersonaExecutor({ availableModelNames: availableModelNames, personaDescription: personaDescription })];
5437
+ return [4 /*yield*/, preparePersonaExecutor({ availableModelNames: availableModelNames, personaDescription: personaDescription }).asPromise()];
5266
5438
  case 3:
5267
5439
  result = _d.sent();
5268
- assertsExecutionSuccessful(result);
5269
5440
  outputParameters = result.outputParameters;
5270
5441
  modelRequirementsRaw = outputParameters.modelRequirements;
5271
5442
  modelRequirements = JSON.parse(modelRequirementsRaw);
@@ -5662,6 +5833,15 @@
5662
5833
  * TODO: [🐱‍🐉][🧠] Make some smart crop NOT source-i-m-pavol-a-develop-... BUT source-i-m-pavol-a-developer-...
5663
5834
  */
5664
5835
 
5836
+ /**
5837
+ * @@@
5838
+ *
5839
+ * @private for `FileCacheStorage`
5840
+ */
5841
+ function nameToSubfolderPath(name) {
5842
+ return [name.substr(0, 1).toLowerCase(), name.substr(1, 1).toLowerCase()];
5843
+ }
5844
+
5665
5845
  /**
5666
5846
  * Convert file extension to mime type
5667
5847
  *
@@ -5718,53 +5898,55 @@
5718
5898
  */
5719
5899
 
5720
5900
  /**
5721
- * Tests if given string is valid URL.
5901
+ * Convert mime type to file extension
5722
5902
  *
5723
- * Note: This does not check if the file exists only if the path is valid
5903
+ * Note: If the mime type is invalid, `null` is returned
5904
+ *
5905
+ * @private within the repository
5906
+ */
5907
+ function mimeTypeToExtension(value) {
5908
+ return mimeTypes.extension(value) || null;
5909
+ }
5910
+
5911
+ /**
5912
+ * Removes emojis from a string and fix whitespaces
5913
+ *
5914
+ * @param text with emojis
5915
+ * @returns text without emojis
5724
5916
  * @public exported from `@promptbook/utils`
5725
5917
  */
5726
- function isValidFilePath(filename) {
5727
- if (typeof filename !== 'string') {
5728
- return false;
5729
- }
5730
- if (filename.split('\n').length > 1) {
5731
- return false;
5732
- }
5733
- if (filename.split(' ').length >
5734
- 5 /* <- TODO: [🧠][🈷] Make some better non-arbitrary way how to distinct filenames from informational texts */) {
5735
- return false;
5736
- }
5737
- var filenameSlashes = filename.split('\\').join('/');
5738
- // Absolute Unix path: /hello.txt
5739
- if (/^(\/)/i.test(filenameSlashes)) {
5740
- // console.log(filename, 'Absolute Unix path: /hello.txt');
5741
- return true;
5742
- }
5743
- // Absolute Windows path: /hello.txt
5744
- if (/^([A-Z]{1,2}:\/?)\//i.test(filenameSlashes)) {
5745
- // console.log(filename, 'Absolute Windows path: /hello.txt');
5746
- return true;
5747
- }
5748
- // Relative path: ./hello.txt
5749
- if (/^(\.\.?\/)+/i.test(filenameSlashes)) {
5750
- // console.log(filename, 'Relative path: ./hello.txt');
5751
- return true;
5752
- }
5753
- // Allow paths like foo/hello
5754
- if (/^[^/]+\/[^/]+/i.test(filenameSlashes)) {
5755
- // console.log(filename, 'Allow paths like foo/hello');
5756
- return true;
5757
- }
5758
- // Allow paths like hello.book
5759
- if (/^[^/]+\.[^/]+$/i.test(filenameSlashes)) {
5760
- // console.log(filename, 'Allow paths like hello.book');
5761
- return true;
5762
- }
5763
- return false;
5918
+ function removeEmojis(text) {
5919
+ // Replace emojis (and also ZWJ sequence) with hyphens
5920
+ text = text.replace(/(\p{Extended_Pictographic})\p{Modifier_Symbol}/gu, '$1');
5921
+ text = text.replace(/(\p{Extended_Pictographic})[\u{FE00}-\u{FE0F}]/gu, '$1');
5922
+ text = text.replace(/(\p{Extended_Pictographic})(\u{200D}\p{Extended_Pictographic})*/gu, '$1');
5923
+ text = text.replace(/\p{Extended_Pictographic}/gu, '');
5924
+ return text;
5764
5925
  }
5926
+
5765
5927
  /**
5766
- * TODO: [🍏] Implement for MacOs
5928
+ * @@@
5929
+ *
5930
+ * @param value @@@
5931
+ * @returns @@@
5932
+ * @example @@@
5933
+ * @public exported from `@promptbook/utils`
5767
5934
  */
5935
+ function titleToName(value) {
5936
+ if (isValidUrl(value)) {
5937
+ value = value.replace(/^https?:\/\//, '');
5938
+ value = value.replace(/\.html$/, '');
5939
+ }
5940
+ else if (isValidFilePath(value)) {
5941
+ value = path.basename(value);
5942
+ // Note: Keeping extension in the name
5943
+ }
5944
+ value = value.split('/').join('-');
5945
+ value = removeEmojis(value);
5946
+ value = normalizeToKebabCase(value);
5947
+ // TODO: [🧠] Maybe warn or add some padding to short name which are not good identifiers
5948
+ return value;
5949
+ }
5768
5950
 
5769
5951
  /**
5770
5952
  * The built-in `fetch' function with a lightweight error handling wrapper as default fetch function used in Promptbook scrapers
@@ -5801,9 +5983,9 @@
5801
5983
  function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
5802
5984
  var _a;
5803
5985
  return __awaiter(this, void 0, void 0, function () {
5804
- var _b, fetch, knowledgeSourceContent, name, _c, _d, rootDirname, url, response_1, mimeType, filename_1, fileExtension, mimeType;
5805
- return __generator(this, function (_f) {
5806
- switch (_f.label) {
5986
+ var _b, fetch, knowledgeSourceContent, name, _c, _d, rootDirname, url, response_1, mimeType, basename, hash, rootDirname_1, filepath, fileContent, _f, _g, filename_1, fileExtension, mimeType;
5987
+ return __generator(this, function (_h) {
5988
+ switch (_h.label) {
5807
5989
  case 0:
5808
5990
  _b = tools.fetch, fetch = _b === void 0 ? scraperFetch : _b;
5809
5991
  knowledgeSourceContent = knowledgeSource.knowledgeSourceContent;
@@ -5812,54 +5994,76 @@
5812
5994
  if (!name) {
5813
5995
  name = knowledgeSourceContentToName(knowledgeSourceContent);
5814
5996
  }
5815
- if (!isValidUrl(knowledgeSourceContent)) return [3 /*break*/, 2];
5997
+ if (!isValidUrl(knowledgeSourceContent)) return [3 /*break*/, 5];
5816
5998
  url = knowledgeSourceContent;
5817
5999
  return [4 /*yield*/, fetch(url)];
5818
6000
  case 1:
5819
- response_1 = _f.sent();
6001
+ response_1 = _h.sent();
5820
6002
  mimeType = ((_a = response_1.headers.get('content-type')) === null || _a === void 0 ? void 0 : _a.split(';')[0]) || 'text/html';
5821
- return [2 /*return*/, {
5822
- source: name,
5823
- filename: null,
5824
- url: url,
5825
- mimeType: mimeType,
5826
- /*
5827
- TODO: [🥽]
5828
- > async asBlob() {
5829
- > // TODO: [👨🏻‍🤝‍👨🏻] This can be called multiple times BUT when called second time, response in already consumed
5830
- > const content = await response.blob();
5831
- > return content;
5832
- > },
5833
- */
5834
- asJson: function () {
5835
- return __awaiter(this, void 0, void 0, function () {
5836
- var content;
5837
- return __generator(this, function (_a) {
5838
- switch (_a.label) {
5839
- case 0: return [4 /*yield*/, response_1.json()];
5840
- case 1:
5841
- content = _a.sent();
5842
- return [2 /*return*/, content];
5843
- }
6003
+ if (tools.fs === undefined || !url.endsWith('.pdf' /* <- TODO: [💵] */)) {
6004
+ return [2 /*return*/, {
6005
+ source: name,
6006
+ filename: null,
6007
+ url: url,
6008
+ mimeType: mimeType,
6009
+ /*
6010
+ TODO: [🥽]
6011
+ > async asBlob() {
6012
+ > // TODO: [👨🏻‍🤝‍👨🏻] This can be called multiple times BUT when called second time, response in already consumed
6013
+ > const content = await response.blob();
6014
+ > return content;
6015
+ > },
6016
+ */
6017
+ asJson: function () {
6018
+ return __awaiter(this, void 0, void 0, function () {
6019
+ var content;
6020
+ return __generator(this, function (_a) {
6021
+ switch (_a.label) {
6022
+ case 0: return [4 /*yield*/, response_1.json()];
6023
+ case 1:
6024
+ content = _a.sent();
6025
+ return [2 /*return*/, content];
6026
+ }
6027
+ });
5844
6028
  });
5845
- });
5846
- },
5847
- asText: function () {
5848
- return __awaiter(this, void 0, void 0, function () {
5849
- var content;
5850
- return __generator(this, function (_a) {
5851
- switch (_a.label) {
5852
- case 0: return [4 /*yield*/, response_1.text()];
5853
- case 1:
5854
- content = _a.sent();
5855
- return [2 /*return*/, content];
5856
- }
6029
+ },
6030
+ asText: function () {
6031
+ return __awaiter(this, void 0, void 0, function () {
6032
+ var content;
6033
+ return __generator(this, function (_a) {
6034
+ switch (_a.label) {
6035
+ case 0: return [4 /*yield*/, response_1.text()];
6036
+ case 1:
6037
+ content = _a.sent();
6038
+ return [2 /*return*/, content];
6039
+ }
6040
+ });
5857
6041
  });
5858
- });
5859
- },
5860
- }];
6042
+ },
6043
+ }];
6044
+ }
6045
+ basename = url.split('/').pop() || titleToName(url);
6046
+ hash = sha256__default["default"](hexEncoder__default["default"].parse(url)).toString( /* hex */);
6047
+ rootDirname_1 = path.join(process.cwd(), DEFAULT_DOWNLOAD_CACHE_DIRNAME);
6048
+ filepath = path.join.apply(void 0, __spreadArray(__spreadArray([], __read(nameToSubfolderPath(hash /* <- TODO: [🎎] Maybe add some SHA256 prefix */)), false), ["".concat(basename.substring(0, MAX_FILENAME_LENGTH), ".").concat(mimeTypeToExtension(mimeType))], false));
6049
+ return [4 /*yield*/, tools.fs.mkdir(path.dirname(path.join(rootDirname_1, filepath)), { recursive: true })];
5861
6050
  case 2:
5862
- if (!isValidFilePath(knowledgeSourceContent)) return [3 /*break*/, 4];
6051
+ _h.sent();
6052
+ _g = (_f = Buffer).from;
6053
+ return [4 /*yield*/, response_1.arrayBuffer()];
6054
+ case 3:
6055
+ fileContent = _g.apply(_f, [_h.sent()]);
6056
+ if (fileContent.length > DEFAULT_MAX_FILE_SIZE /* <- TODO: Allow to pass different value to remote server */) {
6057
+ throw new LimitReachedError("File is too large (".concat(Math.round(fileContent.length / 1024 / 1024), "MB). Maximum allowed size is ").concat(Math.round(DEFAULT_MAX_FILE_SIZE / 1024 / 1024), "MB."));
6058
+ }
6059
+ return [4 /*yield*/, tools.fs.writeFile(path.join(rootDirname_1, filepath), fileContent)];
6060
+ case 4:
6061
+ _h.sent();
6062
+ // TODO: [💵] Check the file security
6063
+ // TODO: [🧹][🧠] Delete the file after the scraping is done
6064
+ return [2 /*return*/, makeKnowledgeSourceHandler({ name: name, knowledgeSourceContent: filepath }, tools, __assign(__assign({}, options), { rootDirname: rootDirname_1 }))];
6065
+ case 5:
6066
+ if (!isValidFilePath(knowledgeSourceContent)) return [3 /*break*/, 7];
5863
6067
  if (tools.fs === undefined) {
5864
6068
  throw new EnvironmentMismatchError('Can not import file knowledge without filesystem tools');
5865
6069
  // <- TODO: [🧠] What is the best error type here`
@@ -5872,8 +6076,8 @@
5872
6076
  fileExtension = getFileExtension(filename_1);
5873
6077
  mimeType = extensionToMimeType(fileExtension || '');
5874
6078
  return [4 /*yield*/, isFileExisting(filename_1, tools.fs)];
5875
- case 3:
5876
- if (!(_f.sent())) {
6079
+ case 6:
6080
+ if (!(_h.sent())) {
5877
6081
  throw new NotFoundError(spaceTrim__default["default"](function (block) { return "\n Can not make source handler for file which does not exist:\n\n File:\n ".concat(block(knowledgeSourceContent), "\n\n Full file path:\n ").concat(block(filename_1), "\n "); }));
5878
6082
  }
5879
6083
  // TODO: [🧠][😿] Test security file - file is scoped to the project (BUT maybe do this in `filesystemTools`)
@@ -5919,7 +6123,7 @@
5919
6123
  });
5920
6124
  },
5921
6125
  }];
5922
- case 4: return [2 /*return*/, {
6126
+ case 7: return [2 /*return*/, {
5923
6127
  source: name,
5924
6128
  filename: null,
5925
6129
  url: null,
@@ -6183,10 +6387,9 @@
6183
6387
  var content = _a.content;
6184
6388
  return content;
6185
6389
  }).join('\n\n'),
6186
- })];
6390
+ }).asPromise()];
6187
6391
  case 2:
6188
6392
  result = _e.sent();
6189
- assertsExecutionSuccessful(result);
6190
6393
  outputParameters = result.outputParameters;
6191
6394
  titleRaw = outputParameters.title;
6192
6395
  if (isVerbose) {
@@ -7030,22 +7233,6 @@
7030
7233
  * TODO: [🌺] Use some intermediate util splitWords
7031
7234
  */
7032
7235
 
7033
- /**
7034
- * Removes emojis from a string and fix whitespaces
7035
- *
7036
- * @param text with emojis
7037
- * @returns text without emojis
7038
- * @public exported from `@promptbook/utils`
7039
- */
7040
- function removeEmojis(text) {
7041
- // Replace emojis (and also ZWJ sequence) with hyphens
7042
- text = text.replace(/(\p{Extended_Pictographic})\p{Modifier_Symbol}/gu, '$1');
7043
- text = text.replace(/(\p{Extended_Pictographic})[\u{FE00}-\u{FE0F}]/gu, '$1');
7044
- text = text.replace(/(\p{Extended_Pictographic})(\u{200D}\p{Extended_Pictographic})*/gu, '$1');
7045
- text = text.replace(/\p{Extended_Pictographic}/gu, '');
7046
- return text;
7047
- }
7048
-
7049
7236
  /**
7050
7237
  * Removes quotes from a string
7051
7238
  *
@@ -8905,7 +9092,10 @@
8905
9092
  description: 'Chatbot response',
8906
9093
  isInput: false,
8907
9094
  isOutput: true,
8908
- exampleValues: ['Hello, I am a Pavol`s virtual avatar. How can I help you?'],
9095
+ exampleValues: [
9096
+ 'Hello, how can I help you?',
9097
+ // <- TODO: [🧠] Make this dynamic, async, prepare-phase HLAs
9098
+ ],
8909
9099
  });
8910
9100
  // TODO: Use spaceTrim in multiline strings
8911
9101
  $pipelineJson.tasks.push({
@@ -8993,12 +9183,15 @@
8993
9183
  pipelineString = removeMarkdownComments(pipelineString);
8994
9184
  pipelineString = spaceTrim__default["default"](pipelineString);
8995
9185
  var isMarkdownBeginningWithHeadline = pipelineString.startsWith('# ');
8996
- var isLastLineReturnStatement = pipelineString.split('\n').pop().split('`').join('').startsWith('->');
8997
- // TODO: Also (double)check
9186
+ //const isLastLineReturnStatement = pipelineString.split('\n').pop()!.split('`').join('').startsWith('->');
9187
+ var isBacktickBlockUsed = pipelineString.includes('```');
9188
+ var isQuoteBlocksUsed = /^>\s+/m.test(pipelineString);
9189
+ var isBlocksUsed = isBacktickBlockUsed || isQuoteBlocksUsed;
9190
+ // TODO: [🧉] Also (double)check
8998
9191
  // > const usedCommands
8999
9192
  // > const isBlocksUsed
9000
9193
  // > const returnStatementCount
9001
- var isFlat = !isMarkdownBeginningWithHeadline && isLastLineReturnStatement;
9194
+ var isFlat = !isMarkdownBeginningWithHeadline && !isBlocksUsed; /* && isLastLineReturnStatement */
9002
9195
  return isFlat;
9003
9196
  }
9004
9197
 
@@ -9012,9 +9205,26 @@
9012
9205
  return pipelineString;
9013
9206
  }
9014
9207
  var pipelineStringLines = pipelineString.split('\n');
9015
- var returnStatement = pipelineStringLines.pop();
9208
+ var potentialReturnStatement = pipelineStringLines.pop();
9209
+ var returnStatement;
9210
+ if (/(-|=)>\s*\{.*\}/.test(potentialReturnStatement)) {
9211
+ // Note: Last line is return statement
9212
+ returnStatement = potentialReturnStatement;
9213
+ }
9214
+ else {
9215
+ // Note: Last line is not a return statement
9216
+ returnStatement = "-> {".concat(DEFAULT_BOOK_OUTPUT_PARAMETER_NAME, "}");
9217
+ pipelineStringLines.push(potentialReturnStatement);
9218
+ }
9016
9219
  var prompt = spaceTrim__default["default"](pipelineStringLines.join('\n'));
9017
- pipelineString = validatePipelineString(spaceTrim__default["default"](function (block) { return "\n # ".concat(DEFAULT_BOOK_TITLE, "\n\n ## Prompt\n\n ```\n ").concat(block(prompt), "\n ```\n\n ").concat(returnStatement, "\n "); }));
9220
+ var quotedPrompt;
9221
+ if (prompt.split('\n').length <= 1) {
9222
+ quotedPrompt = "> ".concat(prompt);
9223
+ }
9224
+ else {
9225
+ quotedPrompt = spaceTrim__default["default"](function (block) { return "\n ```\n ".concat(block(prompt.split('`').join('\\`')), "\n ```\n "); });
9226
+ }
9227
+ pipelineString = validatePipelineString(spaceTrim__default["default"](function (block) { return "\n # ".concat(DEFAULT_BOOK_TITLE, "\n\n ## Prompt\n\n ").concat(block(quotedPrompt), "\n\n ").concat(returnStatement, "\n "); }));
9018
9228
  // <- TODO: Maybe use book` notation
9019
9229
  return pipelineString;
9020
9230
  }
@@ -9237,30 +9447,6 @@
9237
9447
  * NOW we are working just with markdown string and its good enough
9238
9448
  */
9239
9449
 
9240
- /**
9241
- * @@@
9242
- *
9243
- * @param value @@@
9244
- * @returns @@@
9245
- * @example @@@
9246
- * @public exported from `@promptbook/utils`
9247
- */
9248
- function titleToName(value) {
9249
- if (isValidUrl(value)) {
9250
- value = value.replace(/^https?:\/\//, '');
9251
- value = value.replace(/\.html$/, '');
9252
- }
9253
- else if (isValidFilePath(value)) {
9254
- value = path.basename(value);
9255
- // Note: Keeping extension in the name
9256
- }
9257
- value = value.split('/').join('-');
9258
- value = removeEmojis(value);
9259
- value = normalizeToKebabCase(value);
9260
- // TODO: [🧠] Maybe warn or add some padding to short name which are not good identifiers
9261
- return value;
9262
- }
9263
-
9264
9450
  /**
9265
9451
  * Compile pipeline from string (markdown) format to JSON format synchronously
9266
9452
  *
@@ -10382,6 +10568,9 @@
10382
10568
  * @public exported from `@promptbook/utils`
10383
10569
  */
10384
10570
  var $isRunningInNode = new Function("\n try {\n return this === global;\n } catch (e) {\n return false;\n }\n");
10571
+ /**
10572
+ * TODO: [🎺]
10573
+ */
10385
10574
 
10386
10575
  /**
10387
10576
  * Creates a message with all registered LLM tools
@@ -10461,10 +10650,11 @@
10461
10650
  // <- Note: [🗨]
10462
10651
  return __assign(__assign({}, metadata), { isMetadataAviailable: isMetadataAviailable, isInstalled: isInstalled, isFullyConfigured: isFullyConfigured, isPartiallyConfigured: isPartiallyConfigured });
10463
10652
  });
10653
+ var usedEnvMessage = "Unknown `.env` file" ;
10464
10654
  if (metadata.length === 0) {
10465
- return "No LLM providers are available.";
10655
+ return spaceTrim__default["default"](function (block) { return "\n No LLM providers are available.\n\n ".concat(block(usedEnvMessage), "\n "); });
10466
10656
  }
10467
- return spaceTrim__default["default"](function (block) { return "\n Relevant environment variables:\n ".concat(block(Object.keys(env)
10657
+ return spaceTrim__default["default"](function (block) { return "\n\n ".concat(block(usedEnvMessage), "\n\n Relevant environment variables:\n ").concat(block(Object.keys(env)
10468
10658
  .filter(function (envVariableName) {
10469
10659
  return metadata.some(function (_a) {
10470
10660
  var envVariables = _a.envVariables;
@@ -10912,6 +11102,58 @@
10912
11102
  * @public exported from `@promptbook/utils`
10913
11103
  */
10914
11104
  var $isRunningInJest = new Function("\n try {\n return process.env.JEST_WORKER_ID !== undefined;\n } catch (e) {\n return false;\n }\n");
11105
+ /**
11106
+ * TODO: [🎺]
11107
+ */
11108
+
11109
+ /**
11110
+ * Registration of LLM provider metadata
11111
+ *
11112
+ * Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available LLM tools
11113
+ *
11114
+ * @public exported from `@promptbook/core`
11115
+ * @public exported from `@promptbook/wizzard`
11116
+ * @public exported from `@promptbook/cli`
11117
+ */
11118
+ var _DeepseekMetadataRegistration = $llmToolsMetadataRegister.register({
11119
+ title: 'Deepseek',
11120
+ packageName: '@promptbook/deepseek',
11121
+ className: 'DeepseekExecutionTools',
11122
+ envVariables: ['DEEPSEEK_GENERATIVE_AI_API_KEY'],
11123
+ getBoilerplateConfiguration: function () {
11124
+ return {
11125
+ title: 'Deepseek (boilerplate)',
11126
+ packageName: '@promptbook/deepseek',
11127
+ className: 'DeepseekExecutionTools',
11128
+ options: {
11129
+ apiKey: 'AI',
11130
+ },
11131
+ };
11132
+ },
11133
+ createConfigurationFromEnv: function (env) {
11134
+ if ($isRunningInJest()
11135
+ // <- TODO: Maybe check `env.JEST_WORKER_ID` directly here or pass `env` into `$isRunningInJest`
11136
+ ) {
11137
+ // Note: [🔘] Maybe same problem as Gemini
11138
+ return null;
11139
+ }
11140
+ // Note: Note using `process.env` BUT `env` to pass in the environment variables dynamically
11141
+ if (typeof env.DEEPSEEK_GENERATIVE_AI_API_KEY === 'string') {
11142
+ return {
11143
+ title: 'Deepseek (from env)',
11144
+ packageName: '@promptbook/deepseek',
11145
+ className: 'DeepseekExecutionTools',
11146
+ options: {
11147
+ apiKey: env.DEEPSEEK_GENERATIVE_AI_API_KEY,
11148
+ },
11149
+ };
11150
+ }
11151
+ return null;
11152
+ },
11153
+ });
11154
+ /**
11155
+ * Note: [💞] Ignore a discrepancy between file name and entity name
11156
+ */
10915
11157
 
10916
11158
  /**
10917
11159
  * Registration of LLM provider metadata
@@ -11077,13 +11319,14 @@
11077
11319
  /**
11078
11320
  * Tag function for notating a prompt as template literal
11079
11321
  *
11080
- * Note: There are 2 similar functions:
11322
+ * Note: There are 3 similar functions:
11081
11323
  * 1) `prompt` for notating single prompt exported from `@promptbook/utils`
11082
- * 1) `book` for notating and validating entire books exported from `@promptbook/utils`
11324
+ * 2) `promptTemplate` alias for `prompt`
11325
+ * 3) `book` for notating and validating entire books exported from `@promptbook/utils`
11083
11326
  *
11084
- * @param strings @@@
11085
- * @param values @@@
11086
- * @returns the pipeline string
11327
+ * @param strings
11328
+ * @param values
11329
+ * @returns the prompt string
11087
11330
  * @public exported from `@promptbook/utils`
11088
11331
  */
11089
11332
  function prompt(strings) {
@@ -11094,10 +11337,14 @@
11094
11337
  if (values.length === 0) {
11095
11338
  return spaceTrim__default["default"](strings.join(''));
11096
11339
  }
11340
+ var stringsWithHiddenParameters = strings.map(function (stringsItem) {
11341
+ // TODO: [0] DRY
11342
+ return stringsItem.split('{').join("".concat(REPLACING_NONCE, "beginbracket")).split('}').join("".concat(REPLACING_NONCE, "endbracket"));
11343
+ });
11097
11344
  var placeholderParameterNames = values.map(function (value, i) { return "".concat(REPLACING_NONCE).concat(i); });
11098
11345
  var parameters = Object.fromEntries(values.map(function (value, i) { return [placeholderParameterNames[i], value]; }));
11099
11346
  // Combine strings and values
11100
- var pipelineString = strings.reduce(function (result, stringsItem, i) {
11347
+ var pipelineString = stringsWithHiddenParameters.reduce(function (result, stringsItem, i) {
11101
11348
  return placeholderParameterNames[i] === undefined
11102
11349
  ? "".concat(result).concat(stringsItem)
11103
11350
  : "".concat(result).concat(stringsItem, "{").concat(placeholderParameterNames[i], "}");
@@ -11113,6 +11360,12 @@
11113
11360
  console.error({ pipelineString: pipelineString, parameters: parameters, placeholderParameterNames: placeholderParameterNames, error: error });
11114
11361
  throw new UnexpectedError(spaceTrim__default["default"](function (block) { return "\n Internal error in prompt template literal\n \n ".concat(block(JSON.stringify({ strings: strings, values: values }, null, 4)), "}\n \n "); }));
11115
11362
  }
11363
+ // TODO: [0] DRY
11364
+ pipelineString = pipelineString
11365
+ .split("".concat(REPLACING_NONCE, "beginbracket"))
11366
+ .join('{')
11367
+ .split("".concat(REPLACING_NONCE, "endbracket"))
11368
+ .join('}');
11116
11369
  return pipelineString;
11117
11370
  }
11118
11371
  /**
@@ -11123,9 +11376,10 @@
11123
11376
  /**
11124
11377
  * Tag function for notating a pipeline with a book\`...\ notation as template literal
11125
11378
  *
11126
- * Note: There are 2 similar functions:
11379
+ * Note: There are 3 similar functions:
11127
11380
  * 1) `prompt` for notating single prompt exported from `@promptbook/utils`
11128
- * 1) `book` for notating and validating entire books exported from `@promptbook/utils`
11381
+ * 2) `promptTemplate` alias for `prompt`
11382
+ * 3) `book` for notating and validating entire books exported from `@promptbook/utils`
11129
11383
  *
11130
11384
  * @param strings @@@
11131
11385
  * @param values @@@
@@ -11285,7 +11539,7 @@
11285
11539
  className: 'MarkitdownScraper',
11286
11540
  mimeTypes: [
11287
11541
  'application/pdf',
11288
- // TODO: Make priority for scrapers and than allow all mime types here:
11542
+ // TODO: [💵] Make priority for scrapers and than analyze which mime-types can Markitdown scrape and allow all mime types here:
11289
11543
  // 'text/html',
11290
11544
  // 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
11291
11545
  ],
@@ -11465,17 +11719,21 @@
11465
11719
  exports.CsvFormatDefinition = CsvFormatDefinition;
11466
11720
  exports.CsvFormatError = CsvFormatError;
11467
11721
  exports.DEFAULT_BOOKS_DIRNAME = DEFAULT_BOOKS_DIRNAME;
11722
+ exports.DEFAULT_BOOK_OUTPUT_PARAMETER_NAME = DEFAULT_BOOK_OUTPUT_PARAMETER_NAME;
11468
11723
  exports.DEFAULT_BOOK_TITLE = DEFAULT_BOOK_TITLE;
11469
11724
  exports.DEFAULT_CSV_SETTINGS = DEFAULT_CSV_SETTINGS;
11470
- exports.DEFAULT_EXECUTIONS_CACHE_DIRNAME = DEFAULT_EXECUTIONS_CACHE_DIRNAME;
11725
+ exports.DEFAULT_DOWNLOAD_CACHE_DIRNAME = DEFAULT_DOWNLOAD_CACHE_DIRNAME;
11726
+ exports.DEFAULT_EXECUTION_CACHE_DIRNAME = DEFAULT_EXECUTION_CACHE_DIRNAME;
11471
11727
  exports.DEFAULT_GET_PIPELINE_COLLECTION_FUNCTION_NAME = DEFAULT_GET_PIPELINE_COLLECTION_FUNCTION_NAME;
11472
11728
  exports.DEFAULT_INTERMEDIATE_FILES_STRATEGY = DEFAULT_INTERMEDIATE_FILES_STRATEGY;
11473
11729
  exports.DEFAULT_IS_AUTO_INSTALLED = DEFAULT_IS_AUTO_INSTALLED;
11474
11730
  exports.DEFAULT_MAX_EXECUTION_ATTEMPTS = DEFAULT_MAX_EXECUTION_ATTEMPTS;
11731
+ exports.DEFAULT_MAX_FILE_SIZE = DEFAULT_MAX_FILE_SIZE;
11475
11732
  exports.DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_DEPTH = DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_DEPTH;
11476
11733
  exports.DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_TOTAL = DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_TOTAL;
11477
11734
  exports.DEFAULT_MAX_PARALLEL_COUNT = DEFAULT_MAX_PARALLEL_COUNT;
11478
11735
  exports.DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME = DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME;
11736
+ exports.DEFAULT_PROMPT_TASK_TITLE = DEFAULT_PROMPT_TASK_TITLE;
11479
11737
  exports.DEFAULT_REMOTE_URL = DEFAULT_REMOTE_URL;
11480
11738
  exports.DEFAULT_REMOTE_URL_PATH = DEFAULT_REMOTE_URL_PATH;
11481
11739
  exports.DEFAULT_SCRAPE_CACHE_DIRNAME = DEFAULT_SCRAPE_CACHE_DIRNAME;
@@ -11525,6 +11783,7 @@
11525
11783
  exports._AnthropicClaudeMetadataRegistration = _AnthropicClaudeMetadataRegistration;
11526
11784
  exports._AzureOpenAiMetadataRegistration = _AzureOpenAiMetadataRegistration;
11527
11785
  exports._BoilerplateScraperMetadataRegistration = _BoilerplateScraperMetadataRegistration;
11786
+ exports._DeepseekMetadataRegistration = _DeepseekMetadataRegistration;
11528
11787
  exports._DocumentScraperMetadataRegistration = _DocumentScraperMetadataRegistration;
11529
11788
  exports._GoogleMetadataRegistration = _GoogleMetadataRegistration;
11530
11789
  exports._LegacyDocumentScraperMetadataRegistration = _LegacyDocumentScraperMetadataRegistration;
@@ -11535,7 +11794,6 @@
11535
11794
  exports._PdfScraperMetadataRegistration = _PdfScraperMetadataRegistration;
11536
11795
  exports._WebsiteScraperMetadataRegistration = _WebsiteScraperMetadataRegistration;
11537
11796
  exports.addUsage = addUsage;
11538
- exports.assertsExecutionSuccessful = assertsExecutionSuccessful;
11539
11797
  exports.book = book;
11540
11798
  exports.cacheLlmTools = cacheLlmTools;
11541
11799
  exports.collectionToJson = collectionToJson;