@promptbook/markdown-utils 0.75.3 → 0.75.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -24,6 +24,7 @@ import { DEFAULT_REMOTE_URL } from '../config';
24
24
  import { DEFAULT_REMOTE_URL_PATH } from '../config';
25
25
  import { DEFAULT_CSV_SETTINGS } from '../config';
26
26
  import { DEFAULT_IS_VERBOSE } from '../config';
27
+ import { SET_IS_VERBOSE } from '../config';
27
28
  import { DEFAULT_IS_AUTO_INSTALLED } from '../config';
28
29
  import { pipelineJsonToString } from '../conversion/pipelineJsonToString';
29
30
  import { pipelineStringToJson } from '../conversion/pipelineStringToJson';
@@ -33,11 +34,12 @@ import { stringifyPipelineJson } from '../conversion/utils/stringifyPipelineJson
33
34
  import { validatePipeline } from '../conversion/validation/validatePipeline';
34
35
  import { CallbackInterfaceTools } from '../dialogs/callback/CallbackInterfaceTools';
35
36
  import type { CallbackInterfaceToolsOptions } from '../dialogs/callback/CallbackInterfaceToolsOptions';
37
+ import { BoilerplateError } from '../errors/0-BoilerplateError';
38
+ import { ERRORS } from '../errors/0-index';
36
39
  import { AbstractFormatError } from '../errors/AbstractFormatError';
37
40
  import { CollectionError } from '../errors/CollectionError';
38
41
  import { EnvironmentMismatchError } from '../errors/EnvironmentMismatchError';
39
42
  import { ExpectError } from '../errors/ExpectError';
40
- import { ERRORS } from '../errors/index';
41
43
  import { KnowledgeScrapeError } from '../errors/KnowledgeScrapeError';
42
44
  import { LimitReachedError } from '../errors/LimitReachedError';
43
45
  import { MissingToolsError } from '../errors/MissingToolsError';
@@ -136,6 +138,7 @@ export { DEFAULT_REMOTE_URL };
136
138
  export { DEFAULT_REMOTE_URL_PATH };
137
139
  export { DEFAULT_CSV_SETTINGS };
138
140
  export { DEFAULT_IS_VERBOSE };
141
+ export { SET_IS_VERBOSE };
139
142
  export { DEFAULT_IS_AUTO_INSTALLED };
140
143
  export { pipelineJsonToString };
141
144
  export { pipelineStringToJson };
@@ -145,11 +148,12 @@ export { stringifyPipelineJson };
145
148
  export { validatePipeline };
146
149
  export { CallbackInterfaceTools };
147
150
  export type { CallbackInterfaceToolsOptions };
151
+ export { BoilerplateError };
152
+ export { ERRORS };
148
153
  export { AbstractFormatError };
149
154
  export { CollectionError };
150
155
  export { EnvironmentMismatchError };
151
156
  export { ExpectError };
152
- export { ERRORS };
153
157
  export { KnowledgeScrapeError };
154
158
  export { LimitReachedError };
155
159
  export { MissingToolsError };
@@ -204,7 +204,15 @@ export declare const DEFAULT_CSV_SETTINGS: CsvSettings;
204
204
  *
205
205
  * @public exported from `@promptbook/core`
206
206
  */
207
- export declare const DEFAULT_IS_VERBOSE = false;
207
+ export declare let DEFAULT_IS_VERBOSE: boolean;
208
+ /**
209
+ * @@@
210
+ *
211
+ * Note: This is experimental feature
212
+ *
213
+ * @public exported from `@promptbook/core`
214
+ */
215
+ export declare function SET_IS_VERBOSE(isVerbose: boolean): void;
208
216
  /**
209
217
  * @@@
210
218
  *
@@ -0,0 +1,12 @@
1
+ /**
2
+ * This error indicates @@@
3
+ *
4
+ * @public exported from `@promptbook/core`
5
+ */
6
+ export declare class BoilerplateError extends Error {
7
+ readonly name = "BoilerplateError";
8
+ constructor(message: string);
9
+ }
10
+ /**
11
+ * TODO: @@@ Do not forget to add the error into `0-index.ts` ERRORS
12
+ */
@@ -1,7 +1,11 @@
1
+ import { CsvFormatError } from '../formats/csv/CsvFormatError';
2
+ import { AbstractFormatError } from './AbstractFormatError';
1
3
  import { CollectionError } from './CollectionError';
2
4
  import { EnvironmentMismatchError } from './EnvironmentMismatchError';
3
5
  import { ExpectError } from './ExpectError';
6
+ import { KnowledgeScrapeError } from './KnowledgeScrapeError';
4
7
  import { LimitReachedError } from './LimitReachedError';
8
+ import { MissingToolsError } from './MissingToolsError';
5
9
  import { NotFoundError } from './NotFoundError';
6
10
  import { NotYetImplementedError } from './NotYetImplementedError';
7
11
  import { ParseError } from './ParseError';
@@ -15,10 +19,14 @@ import { UnexpectedError } from './UnexpectedError';
15
19
  * @public exported from `@promptbook/core`
16
20
  */
17
21
  export declare const ERRORS: {
18
- readonly ExpectError: typeof ExpectError;
22
+ readonly AbstractFormatError: typeof AbstractFormatError;
23
+ readonly CsvFormatError: typeof CsvFormatError;
19
24
  readonly CollectionError: typeof CollectionError;
20
25
  readonly EnvironmentMismatchError: typeof EnvironmentMismatchError;
26
+ readonly ExpectError: typeof ExpectError;
27
+ readonly KnowledgeScrapeError: typeof KnowledgeScrapeError;
21
28
  readonly LimitReachedError: typeof LimitReachedError;
29
+ readonly MissingToolsError: typeof MissingToolsError;
22
30
  readonly NotFoundError: typeof NotFoundError;
23
31
  readonly NotYetImplementedError: typeof NotYetImplementedError;
24
32
  readonly ParseError: typeof ParseError;
@@ -29,4 +37,4 @@ export declare const ERRORS: {
29
37
  };
30
38
  /**
31
39
  * Note: [💞] Ignore a discrepancy between file name and entity name
32
- */
40
+ */
@@ -1,4 +1,4 @@
1
- import type { ERRORS } from '../index';
1
+ import type { ERRORS } from '../0-index';
2
2
  /**
3
3
  * Represents a serialized error or custom Promptbook error
4
4
  *
@@ -1,4 +1,5 @@
1
1
  import type { string_markdown } from '../../../types/typeAliases';
2
+ import type { Scraper } from '../Scraper';
2
3
  /**
3
4
  * Creates a message with all registered scrapers
4
5
  *
@@ -6,7 +7,7 @@ import type { string_markdown } from '../../../types/typeAliases';
6
7
  *
7
8
  * @private internal function of `createScrapersFromConfiguration` and `createScrapersFromEnv`
8
9
  */
9
- export declare function $registeredScrapersMessage(): string_markdown;
10
+ export declare function $registeredScrapersMessage(availableScrapers: ReadonlyArray<Scraper>): string_markdown;
10
11
  /**
11
12
  * TODO: [®] DRY Register logic
12
13
  */
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@promptbook/markdown-utils",
3
- "version": "0.75.3",
3
+ "version": "0.75.5",
4
4
  "description": "It's time for a paradigm shift. The future of software in plain English, French or Latin",
5
5
  "--note-0": " <- [🐊]",
6
6
  "private": false,
package/umd/index.umd.js CHANGED
@@ -22,7 +22,7 @@
22
22
  *
23
23
  * @see https://github.com/webgptorg/promptbook
24
24
  */
25
- var PROMPTBOOK_ENGINE_VERSION = '0.75.2';
25
+ var PROMPTBOOK_ENGINE_VERSION = '0.75.4';
26
26
  /**
27
27
  * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
28
28
  * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -1965,6 +1965,42 @@
1965
1965
  return PipelineExecutionError;
1966
1966
  }(Error));
1967
1967
 
1968
+ /**
1969
+ * This error indicates problems parsing the format value
1970
+ *
1971
+ * For example, when the format value is not a valid JSON or CSV
1972
+ * This is not thrown directly but in extended classes
1973
+ *
1974
+ * @public exported from `@promptbook/core`
1975
+ */
1976
+ var AbstractFormatError = /** @class */ (function (_super) {
1977
+ __extends(AbstractFormatError, _super);
1978
+ // Note: To allow instanceof do not put here error `name`
1979
+ // public readonly name = 'AbstractFormatError';
1980
+ function AbstractFormatError(message) {
1981
+ var _this = _super.call(this, message) || this;
1982
+ Object.setPrototypeOf(_this, AbstractFormatError.prototype);
1983
+ return _this;
1984
+ }
1985
+ return AbstractFormatError;
1986
+ }(Error));
1987
+
1988
+ /**
1989
+ * This error indicates problem with parsing of CSV
1990
+ *
1991
+ * @public exported from `@promptbook/core`
1992
+ */
1993
+ var CsvFormatError = /** @class */ (function (_super) {
1994
+ __extends(CsvFormatError, _super);
1995
+ function CsvFormatError(message) {
1996
+ var _this = _super.call(this, message) || this;
1997
+ _this.name = 'CsvFormatError';
1998
+ Object.setPrototypeOf(_this, CsvFormatError.prototype);
1999
+ return _this;
2000
+ }
2001
+ return CsvFormatError;
2002
+ }(AbstractFormatError));
2003
+
1968
2004
  /**
1969
2005
  * This error indicates that the pipeline collection cannot be propperly loaded
1970
2006
  *
@@ -2016,6 +2052,22 @@
2016
2052
  return ExpectError;
2017
2053
  }(Error));
2018
2054
 
2055
+ /**
2056
+ * This error indicates that the promptbook can not retrieve knowledge from external sources
2057
+ *
2058
+ * @public exported from `@promptbook/core`
2059
+ */
2060
+ var KnowledgeScrapeError = /** @class */ (function (_super) {
2061
+ __extends(KnowledgeScrapeError, _super);
2062
+ function KnowledgeScrapeError(message) {
2063
+ var _this = _super.call(this, message) || this;
2064
+ _this.name = 'KnowledgeScrapeError';
2065
+ Object.setPrototypeOf(_this, KnowledgeScrapeError.prototype);
2066
+ return _this;
2067
+ }
2068
+ return KnowledgeScrapeError;
2069
+ }(Error));
2070
+
2019
2071
  /**
2020
2072
  * This error type indicates that some limit was reached
2021
2073
  *
@@ -2054,10 +2106,14 @@
2054
2106
  * @public exported from `@promptbook/core`
2055
2107
  */
2056
2108
  var ERRORS = {
2057
- ExpectError: ExpectError,
2109
+ AbstractFormatError: AbstractFormatError,
2110
+ CsvFormatError: CsvFormatError,
2058
2111
  CollectionError: CollectionError,
2059
2112
  EnvironmentMismatchError: EnvironmentMismatchError,
2113
+ ExpectError: ExpectError,
2114
+ KnowledgeScrapeError: KnowledgeScrapeError,
2060
2115
  LimitReachedError: LimitReachedError,
2116
+ MissingToolsError: MissingToolsError,
2061
2117
  NotFoundError: NotFoundError,
2062
2118
  NotYetImplementedError: NotYetImplementedError,
2063
2119
  ParseError: ParseError,
@@ -2858,22 +2914,6 @@
2858
2914
  * TODO: [🏢] !! Check validity of `temperature` in pipeline
2859
2915
  */
2860
2916
 
2861
- /**
2862
- * This error indicates that the promptbook can not retrieve knowledge from external sources
2863
- *
2864
- * @public exported from `@promptbook/core`
2865
- */
2866
- var KnowledgeScrapeError = /** @class */ (function (_super) {
2867
- __extends(KnowledgeScrapeError, _super);
2868
- function KnowledgeScrapeError(message) {
2869
- var _this = _super.call(this, message) || this;
2870
- _this.name = 'KnowledgeScrapeError';
2871
- Object.setPrototypeOf(_this, KnowledgeScrapeError.prototype);
2872
- return _this;
2873
- }
2874
- return KnowledgeScrapeError;
2875
- }(Error));
2876
-
2877
2917
  /**
2878
2918
  * @@@
2879
2919
  *
@@ -3044,8 +3084,8 @@
3044
3084
  *
3045
3085
  * @private internal function of `createScrapersFromConfiguration` and `createScrapersFromEnv`
3046
3086
  */
3047
- function $registeredScrapersMessage() {
3048
- var e_1, _a, e_2, _b;
3087
+ function $registeredScrapersMessage(availableScrapers) {
3088
+ var e_1, _a, e_2, _b, e_3, _c;
3049
3089
  /**
3050
3090
  * Mixes registered scrapers from $scrapersMetadataRegister and $scrapersRegister
3051
3091
  */
@@ -3057,15 +3097,15 @@
3057
3097
  all.push({ packageName: packageName, className: className, mimeTypes: mimeTypes, documentationUrl: documentationUrl, isAvilableInBrowser: isAvilableInBrowser });
3058
3098
  };
3059
3099
  try {
3060
- for (var _c = __values($scrapersMetadataRegister.list()), _d = _c.next(); !_d.done; _d = _c.next()) {
3061
- var _e = _d.value, packageName = _e.packageName, className = _e.className, mimeTypes = _e.mimeTypes, documentationUrl = _e.documentationUrl, isAvilableInBrowser = _e.isAvilableInBrowser;
3100
+ for (var _d = __values($scrapersMetadataRegister.list()), _e = _d.next(); !_e.done; _e = _d.next()) {
3101
+ var _f = _e.value, packageName = _f.packageName, className = _f.className, mimeTypes = _f.mimeTypes, documentationUrl = _f.documentationUrl, isAvilableInBrowser = _f.isAvilableInBrowser;
3062
3102
  _loop_1(packageName, className, mimeTypes, documentationUrl, isAvilableInBrowser);
3063
3103
  }
3064
3104
  }
3065
3105
  catch (e_1_1) { e_1 = { error: e_1_1 }; }
3066
3106
  finally {
3067
3107
  try {
3068
- if (_d && !_d.done && (_a = _c.return)) _a.call(_c);
3108
+ if (_e && !_e.done && (_a = _d.return)) _a.call(_d);
3069
3109
  }
3070
3110
  finally { if (e_1) throw e_1.error; }
3071
3111
  }
@@ -3076,18 +3116,31 @@
3076
3116
  all.push({ packageName: packageName, className: className, mimeTypes: mimeTypes, documentationUrl: documentationUrl, isAvilableInBrowser: isAvilableInBrowser });
3077
3117
  };
3078
3118
  try {
3079
- for (var _f = __values($scrapersRegister.list()), _g = _f.next(); !_g.done; _g = _f.next()) {
3080
- var _h = _g.value, packageName = _h.packageName, className = _h.className, mimeTypes = _h.mimeTypes, documentationUrl = _h.documentationUrl, isAvilableInBrowser = _h.isAvilableInBrowser;
3119
+ for (var _g = __values($scrapersRegister.list()), _h = _g.next(); !_h.done; _h = _g.next()) {
3120
+ var _j = _h.value, packageName = _j.packageName, className = _j.className, mimeTypes = _j.mimeTypes, documentationUrl = _j.documentationUrl, isAvilableInBrowser = _j.isAvilableInBrowser;
3081
3121
  _loop_2(packageName, className, mimeTypes, documentationUrl, isAvilableInBrowser);
3082
3122
  }
3083
3123
  }
3084
3124
  catch (e_2_1) { e_2 = { error: e_2_1 }; }
3085
3125
  finally {
3086
3126
  try {
3087
- if (_g && !_g.done && (_b = _f.return)) _b.call(_f);
3127
+ if (_h && !_h.done && (_b = _g.return)) _b.call(_g);
3088
3128
  }
3089
3129
  finally { if (e_2) throw e_2.error; }
3090
3130
  }
3131
+ try {
3132
+ for (var availableScrapers_1 = __values(availableScrapers), availableScrapers_1_1 = availableScrapers_1.next(); !availableScrapers_1_1.done; availableScrapers_1_1 = availableScrapers_1.next()) {
3133
+ var metadata_1 = availableScrapers_1_1.value.metadata;
3134
+ all.push(metadata_1);
3135
+ }
3136
+ }
3137
+ catch (e_3_1) { e_3 = { error: e_3_1 }; }
3138
+ finally {
3139
+ try {
3140
+ if (availableScrapers_1_1 && !availableScrapers_1_1.done && (_c = availableScrapers_1.return)) _c.call(availableScrapers_1);
3141
+ }
3142
+ finally { if (e_3) throw e_3.error; }
3143
+ }
3091
3144
  var metadata = all.map(function (metadata) {
3092
3145
  var isMetadataAviailable = $scrapersMetadataRegister
3093
3146
  .list()
@@ -3101,42 +3154,44 @@
3101
3154
  var packageName = _a.packageName, className = _a.className;
3102
3155
  return metadata.packageName === packageName && metadata.className === className;
3103
3156
  });
3104
- return __assign(__assign({}, metadata), { isMetadataAviailable: isMetadataAviailable, isInstalled: isInstalled });
3157
+ var isAvilableInTools = availableScrapers.some(function (_a) {
3158
+ var _b = _a.metadata, packageName = _b.packageName, className = _b.className;
3159
+ return metadata.packageName === packageName && metadata.className === className;
3160
+ });
3161
+ return __assign(__assign({}, metadata), { isMetadataAviailable: isMetadataAviailable, isInstalled: isInstalled, isAvilableInTools: isAvilableInTools });
3105
3162
  });
3106
3163
  if (metadata.length === 0) {
3107
- return "No scrapers are available";
3164
+ return spaceTrim__default["default"]("\n **No scrapers are available**\n\n This is a unexpected behavior, you are probably using some broken version of Promptbook\n At least there should be available the metadata of the scrapers\n ");
3108
3165
  }
3109
3166
  return spaceTrim__default["default"](function (block) { return "\n Available scrapers are:\n ".concat(block(metadata
3110
3167
  .map(function (_a, i) {
3111
- var packageName = _a.packageName, className = _a.className, isMetadataAviailable = _a.isMetadataAviailable, isInstalled = _a.isInstalled, mimeTypes = _a.mimeTypes, isAvilableInBrowser = _a.isAvilableInBrowser;
3112
- var more;
3113
- // TODO: Use documentationUrl
3114
- if (just(false)) {
3115
- more = '';
3116
- }
3117
- else if (!isMetadataAviailable && !isInstalled) {
3118
- // TODO: [�][�] Maybe do allow to do auto-install if package not registered and not found
3119
- more = "*(not installed and no metadata, looks like a unexpected behavior)*";
3120
- }
3121
- else if (isMetadataAviailable && !isInstalled) {
3122
- // TODO: [�][�]
3123
- more = "*(not installed)*";
3124
- }
3125
- else if (!isMetadataAviailable && isInstalled) {
3126
- more = "*(no metadata, looks like a unexpected behavior)*";
3127
- }
3128
- else if (isMetadataAviailable && isInstalled) {
3129
- more = "(installed)";
3130
- }
3131
- else {
3132
- more = "*(unknown state, looks like a unexpected behavior)*";
3133
- }
3168
+ var packageName = _a.packageName, className = _a.className, isMetadataAviailable = _a.isMetadataAviailable, isInstalled = _a.isInstalled, mimeTypes = _a.mimeTypes, isAvilableInBrowser = _a.isAvilableInBrowser, isAvilableInTools = _a.isAvilableInTools;
3169
+ var more = [];
3170
+ // TODO: [🧠] Maybe use `documentationUrl`
3171
+ if (isMetadataAviailable) {
3172
+ more.push("\u2B1C Metadata registered");
3173
+ } // not else
3174
+ if (isInstalled) {
3175
+ more.push("\uD83D\uDFE9 Installed");
3176
+ } // not else
3177
+ if (isAvilableInTools) {
3178
+ more.push("\uD83D\uDFE6 Available in tools");
3179
+ } // not else
3180
+ if (!isMetadataAviailable && isInstalled) {
3181
+ more.push("When no metadata registered but scraper is installed, it is an unexpected behavior");
3182
+ } // not else
3183
+ if (!isInstalled && isAvilableInTools) {
3184
+ more.push("When the scraper is not installed but available in tools, it is an unexpected compatibility behavior");
3185
+ } // not else
3134
3186
  if (!isAvilableInBrowser) {
3135
- more += " *(not available in browser)*";
3187
+ more.push("Not usable in browser");
3136
3188
  }
3137
- return "".concat(i + 1, ") `").concat(className, "` from `").concat(packageName, "` compatible to scrape ").concat(mimeTypes.join(', '), " ").concat(more);
3189
+ var moreText = more.length === 0 ? '' : " *(".concat(more.join('; '), ")*");
3190
+ return "".concat(i + 1, ") `").concat(className, "` from `").concat(packageName, "` compatible to scrape ").concat(mimeTypes
3191
+ .map(function (mimeType) { return "\"".concat(mimeType, "\""); })
3192
+ .join(', ')).concat(moreText);
3138
3193
  })
3139
- .join('\n')), "\n "); });
3194
+ .join('\n')), "\n\n Legend:\n - \u2B1C **Metadata registered** means that Promptbook knows about the scraper, it is similar to registration in some registry\n - \uD83D\uDFE9 **Installed** means that you have imported package with particular scraper\n - \uD83D\uDFE6 **Available in tools** means that you have passed scraper as dependency into prepare or execution process\n\n "); });
3140
3195
  }
3141
3196
  /**
3142
3197
  * TODO: [®] DRY Register logic
@@ -3384,57 +3439,75 @@
3384
3439
  _a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a, rootDirname = options.rootDirname, _b = options.isVerbose, isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b;
3385
3440
  knowledgePreparedUnflatten = new Array(knowledgeSources.length);
3386
3441
  return [4 /*yield*/, forEachAsync(knowledgeSources, { maxParallelCount: maxParallelCount }, function (knowledgeSource, index) { return __awaiter(_this, void 0, void 0, function () {
3387
- var partialPieces, sourceHandler, _a, _b, scraper, partialPiecesUnchecked, e_1_1, pieces;
3388
- var e_1, _c;
3389
- return __generator(this, function (_d) {
3390
- switch (_d.label) {
3442
+ var partialPieces, sourceHandler, scrapers, _loop_1, scrapers_1, scrapers_1_1, scraper, state_1, e_1_1, pieces;
3443
+ var e_1, _a;
3444
+ return __generator(this, function (_b) {
3445
+ switch (_b.label) {
3391
3446
  case 0:
3392
3447
  partialPieces = null;
3393
3448
  return [4 /*yield*/, makeKnowledgeSourceHandler(knowledgeSource, tools, { rootDirname: rootDirname, isVerbose: isVerbose })];
3394
3449
  case 1:
3395
- sourceHandler = _d.sent();
3396
- _d.label = 2;
3450
+ sourceHandler = _b.sent();
3451
+ scrapers = arrayableToArray(tools.scrapers);
3452
+ _loop_1 = function (scraper) {
3453
+ var partialPiecesUnchecked;
3454
+ return __generator(this, function (_c) {
3455
+ switch (_c.label) {
3456
+ case 0:
3457
+ if (!scraper.metadata.mimeTypes.includes(sourceHandler.mimeType)
3458
+ // <- TODO: [🦔] Implement mime-type wildcards
3459
+ ) {
3460
+ return [2 /*return*/, "continue"];
3461
+ }
3462
+ return [4 /*yield*/, scraper.scrape(sourceHandler)];
3463
+ case 1:
3464
+ partialPiecesUnchecked = _c.sent();
3465
+ if (partialPiecesUnchecked !== null) {
3466
+ partialPieces = __spreadArray([], __read(partialPiecesUnchecked), false);
3467
+ return [2 /*return*/, "break"];
3468
+ }
3469
+ console.warn(spaceTrim__default["default"](function (block) { return "\n Cannot scrape knowledge from source despite the scraper `".concat(scraper.metadata.className, "` supports the mime type \"").concat(sourceHandler.mimeType, "\".\n \n The source:\n > ").concat(block(knowledgeSource.sourceContent
3470
+ .split('\n')
3471
+ .map(function (line) { return "> ".concat(line); })
3472
+ .join('\n')), "\n\n ").concat(block($registeredScrapersMessage(scrapers)), "\n\n\n "); }));
3473
+ return [2 /*return*/];
3474
+ }
3475
+ });
3476
+ };
3477
+ _b.label = 2;
3397
3478
  case 2:
3398
- _d.trys.push([2, 7, 8, 9]);
3399
- _a = __values(arrayableToArray(tools.scrapers)), _b = _a.next();
3400
- _d.label = 3;
3479
+ _b.trys.push([2, 7, 8, 9]);
3480
+ scrapers_1 = __values(scrapers), scrapers_1_1 = scrapers_1.next();
3481
+ _b.label = 3;
3401
3482
  case 3:
3402
- if (!!_b.done) return [3 /*break*/, 6];
3403
- scraper = _b.value;
3404
- if (!scraper.metadata.mimeTypes.includes(sourceHandler.mimeType)
3405
- // <- TODO: [🦔] Implement mime-type wildcards
3406
- ) {
3407
- return [3 /*break*/, 5];
3408
- }
3409
- return [4 /*yield*/, scraper.scrape(sourceHandler)];
3483
+ if (!!scrapers_1_1.done) return [3 /*break*/, 6];
3484
+ scraper = scrapers_1_1.value;
3485
+ return [5 /*yield**/, _loop_1(scraper)];
3410
3486
  case 4:
3411
- partialPiecesUnchecked = _d.sent();
3412
- if (partialPiecesUnchecked !== null) {
3413
- partialPieces = __spreadArray([], __read(partialPiecesUnchecked), false);
3414
- // <- TODO: [🪓] Here should be no need for spreading new array, just `partialPieces = partialPiecesUnchecked`
3487
+ state_1 = _b.sent();
3488
+ if (state_1 === "break")
3415
3489
  return [3 /*break*/, 6];
3416
- }
3417
- _d.label = 5;
3490
+ _b.label = 5;
3418
3491
  case 5:
3419
- _b = _a.next();
3492
+ scrapers_1_1 = scrapers_1.next();
3420
3493
  return [3 /*break*/, 3];
3421
3494
  case 6: return [3 /*break*/, 9];
3422
3495
  case 7:
3423
- e_1_1 = _d.sent();
3496
+ e_1_1 = _b.sent();
3424
3497
  e_1 = { error: e_1_1 };
3425
3498
  return [3 /*break*/, 9];
3426
3499
  case 8:
3427
3500
  try {
3428
- if (_b && !_b.done && (_c = _a.return)) _c.call(_a);
3501
+ if (scrapers_1_1 && !scrapers_1_1.done && (_a = scrapers_1.return)) _a.call(scrapers_1);
3429
3502
  }
3430
3503
  finally { if (e_1) throw e_1.error; }
3431
3504
  return [7 /*endfinally*/];
3432
3505
  case 9:
3433
3506
  if (partialPieces === null) {
3434
- throw new KnowledgeScrapeError(spaceTrim__default["default"](function (block) { return "\n Cannot scrape knowledge from source:\n \n > ".concat(block(knowledgeSource.sourceContent
3507
+ throw new KnowledgeScrapeError(spaceTrim__default["default"](function (block) { return "\n Cannot scrape knowledge\n \n The source:\n > ".concat(block(knowledgeSource.sourceContent
3435
3508
  .split('\n')
3436
3509
  .map(function (line) { return "> ".concat(line); })
3437
- .join('\n')), "\n\n No scraper found for the mime type \"").concat(sourceHandler.mimeType, "\"\n\n ").concat(block($registeredScrapersMessage()), "\n\n\n "); }));
3510
+ .join('\n')), "\n\n No scraper found for the mime type \"").concat(sourceHandler.mimeType, "\"\n\n ").concat(block($registeredScrapersMessage(scrapers)), "\n\n\n "); }));
3438
3511
  }
3439
3512
  pieces = partialPieces.map(function (partialPiece) { return (__assign(__assign({}, partialPiece), { sources: [
3440
3513
  {
@@ -3860,42 +3933,6 @@
3860
3933
  return union;
3861
3934
  }
3862
3935
 
3863
- /**
3864
- * This error indicates problems parsing the format value
3865
- *
3866
- * For example, when the format value is not a valid JSON or CSV
3867
- * This is not thrown directly but in extended classes
3868
- *
3869
- * @public exported from `@promptbook/core`
3870
- */
3871
- var AbstractFormatError = /** @class */ (function (_super) {
3872
- __extends(AbstractFormatError, _super);
3873
- // Note: To allow instanceof do not put here error `name`
3874
- // public readonly name = 'AbstractFormatError';
3875
- function AbstractFormatError(message) {
3876
- var _this = _super.call(this, message) || this;
3877
- Object.setPrototypeOf(_this, AbstractFormatError.prototype);
3878
- return _this;
3879
- }
3880
- return AbstractFormatError;
3881
- }(Error));
3882
-
3883
- /**
3884
- * This error indicates problem with parsing of CSV
3885
- *
3886
- * @public exported from `@promptbook/core`
3887
- */
3888
- var CsvFormatError = /** @class */ (function (_super) {
3889
- __extends(CsvFormatError, _super);
3890
- function CsvFormatError(message) {
3891
- var _this = _super.call(this, message) || this;
3892
- _this.name = 'CsvFormatError';
3893
- Object.setPrototypeOf(_this, CsvFormatError.prototype);
3894
- return _this;
3895
- }
3896
- return CsvFormatError;
3897
- }(AbstractFormatError));
3898
-
3899
3936
  /**
3900
3937
  * @@@
3901
3938
  *
@@ -3936,7 +3973,7 @@
3936
3973
  case 0:
3937
3974
  csv = papaparse.parse(value, __assign(__assign({}, settings), MANDATORY_CSV_SETTINGS));
3938
3975
  if (csv.errors.length !== 0) {
3939
- throw new CsvFormatError(spaceTrim__default["default"](function (block) { return "\n CSV parsing error\n\n ".concat(block(csv.errors.map(function (error) { return error.message; }).join('\n\n')), "\n "); }));
3976
+ throw new CsvFormatError(spaceTrim__default["default"](function (block) { return "\n CSV parsing error\n\n Error(s) from CSV parsing:\n ".concat(block(csv.errors.map(function (error) { return error.message; }).join('\n\n')), "\n\n The CSV data:\n ").concat(block(value), "\n "); }));
3940
3977
  }
3941
3978
  return [4 /*yield*/, Promise.all(csv.data.map(function (row, index) { return __awaiter(_this, void 0, void 0, function () {
3942
3979
  var _a, _b;
@@ -3974,7 +4011,7 @@
3974
4011
  case 0:
3975
4012
  csv = papaparse.parse(value, __assign(__assign({}, settings), MANDATORY_CSV_SETTINGS));
3976
4013
  if (csv.errors.length !== 0) {
3977
- throw new CsvFormatError(spaceTrim__default["default"](function (block) { return "\n CSV parsing error\n\n ".concat(block(csv.errors.map(function (error) { return error.message; }).join('\n\n')), "\n "); }));
4014
+ throw new CsvFormatError(spaceTrim__default["default"](function (block) { return "\n CSV parsing error\n\n Error(s) from CSV parsing:\n ".concat(block(csv.errors.map(function (error) { return error.message; }).join('\n\n')), "\n\n The CSV data:\n ").concat(block(value), "\n "); }));
3978
4015
  }
3979
4016
  return [4 /*yield*/, Promise.all(csv.data.map(function (row, rowIndex) { return __awaiter(_this, void 0, void 0, function () {
3980
4017
  var _this = this;