@promptbook/legacy-documents 0.75.3 → 0.75.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/esm/index.es.js +141 -104
- package/esm/index.es.js.map +1 -1
- package/esm/typings/src/_packages/core.index.d.ts +6 -2
- package/esm/typings/src/config.d.ts +9 -1
- package/esm/typings/src/errors/0-BoilerplateError.d.ts +12 -0
- package/esm/typings/src/errors/{index.d.ts → 0-index.d.ts} +10 -2
- package/esm/typings/src/errors/utils/ErrorJson.d.ts +1 -1
- package/esm/typings/src/scrapers/_common/register/$registeredScrapersMessage.d.ts +2 -1
- package/package.json +2 -2
- package/umd/index.umd.js +141 -104
- package/umd/index.umd.js.map +1 -1
|
@@ -24,6 +24,7 @@ import { DEFAULT_REMOTE_URL } from '../config';
|
|
|
24
24
|
import { DEFAULT_REMOTE_URL_PATH } from '../config';
|
|
25
25
|
import { DEFAULT_CSV_SETTINGS } from '../config';
|
|
26
26
|
import { DEFAULT_IS_VERBOSE } from '../config';
|
|
27
|
+
import { SET_IS_VERBOSE } from '../config';
|
|
27
28
|
import { DEFAULT_IS_AUTO_INSTALLED } from '../config';
|
|
28
29
|
import { pipelineJsonToString } from '../conversion/pipelineJsonToString';
|
|
29
30
|
import { pipelineStringToJson } from '../conversion/pipelineStringToJson';
|
|
@@ -33,11 +34,12 @@ import { stringifyPipelineJson } from '../conversion/utils/stringifyPipelineJson
|
|
|
33
34
|
import { validatePipeline } from '../conversion/validation/validatePipeline';
|
|
34
35
|
import { CallbackInterfaceTools } from '../dialogs/callback/CallbackInterfaceTools';
|
|
35
36
|
import type { CallbackInterfaceToolsOptions } from '../dialogs/callback/CallbackInterfaceToolsOptions';
|
|
37
|
+
import { BoilerplateError } from '../errors/0-BoilerplateError';
|
|
38
|
+
import { ERRORS } from '../errors/0-index';
|
|
36
39
|
import { AbstractFormatError } from '../errors/AbstractFormatError';
|
|
37
40
|
import { CollectionError } from '../errors/CollectionError';
|
|
38
41
|
import { EnvironmentMismatchError } from '../errors/EnvironmentMismatchError';
|
|
39
42
|
import { ExpectError } from '../errors/ExpectError';
|
|
40
|
-
import { ERRORS } from '../errors/index';
|
|
41
43
|
import { KnowledgeScrapeError } from '../errors/KnowledgeScrapeError';
|
|
42
44
|
import { LimitReachedError } from '../errors/LimitReachedError';
|
|
43
45
|
import { MissingToolsError } from '../errors/MissingToolsError';
|
|
@@ -136,6 +138,7 @@ export { DEFAULT_REMOTE_URL };
|
|
|
136
138
|
export { DEFAULT_REMOTE_URL_PATH };
|
|
137
139
|
export { DEFAULT_CSV_SETTINGS };
|
|
138
140
|
export { DEFAULT_IS_VERBOSE };
|
|
141
|
+
export { SET_IS_VERBOSE };
|
|
139
142
|
export { DEFAULT_IS_AUTO_INSTALLED };
|
|
140
143
|
export { pipelineJsonToString };
|
|
141
144
|
export { pipelineStringToJson };
|
|
@@ -145,11 +148,12 @@ export { stringifyPipelineJson };
|
|
|
145
148
|
export { validatePipeline };
|
|
146
149
|
export { CallbackInterfaceTools };
|
|
147
150
|
export type { CallbackInterfaceToolsOptions };
|
|
151
|
+
export { BoilerplateError };
|
|
152
|
+
export { ERRORS };
|
|
148
153
|
export { AbstractFormatError };
|
|
149
154
|
export { CollectionError };
|
|
150
155
|
export { EnvironmentMismatchError };
|
|
151
156
|
export { ExpectError };
|
|
152
|
-
export { ERRORS };
|
|
153
157
|
export { KnowledgeScrapeError };
|
|
154
158
|
export { LimitReachedError };
|
|
155
159
|
export { MissingToolsError };
|
|
@@ -204,7 +204,15 @@ export declare const DEFAULT_CSV_SETTINGS: CsvSettings;
|
|
|
204
204
|
*
|
|
205
205
|
* @public exported from `@promptbook/core`
|
|
206
206
|
*/
|
|
207
|
-
export declare
|
|
207
|
+
export declare let DEFAULT_IS_VERBOSE: boolean;
|
|
208
|
+
/**
|
|
209
|
+
* @@@
|
|
210
|
+
*
|
|
211
|
+
* Note: This is experimental feature
|
|
212
|
+
*
|
|
213
|
+
* @public exported from `@promptbook/core`
|
|
214
|
+
*/
|
|
215
|
+
export declare function SET_IS_VERBOSE(isVerbose: boolean): void;
|
|
208
216
|
/**
|
|
209
217
|
* @@@
|
|
210
218
|
*
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* This error indicates @@@
|
|
3
|
+
*
|
|
4
|
+
* @public exported from `@promptbook/core`
|
|
5
|
+
*/
|
|
6
|
+
export declare class BoilerplateError extends Error {
|
|
7
|
+
readonly name = "BoilerplateError";
|
|
8
|
+
constructor(message: string);
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* TODO: @@@ Do not forget to add the error into `0-index.ts` ERRORS
|
|
12
|
+
*/
|
|
@@ -1,7 +1,11 @@
|
|
|
1
|
+
import { CsvFormatError } from '../formats/csv/CsvFormatError';
|
|
2
|
+
import { AbstractFormatError } from './AbstractFormatError';
|
|
1
3
|
import { CollectionError } from './CollectionError';
|
|
2
4
|
import { EnvironmentMismatchError } from './EnvironmentMismatchError';
|
|
3
5
|
import { ExpectError } from './ExpectError';
|
|
6
|
+
import { KnowledgeScrapeError } from './KnowledgeScrapeError';
|
|
4
7
|
import { LimitReachedError } from './LimitReachedError';
|
|
8
|
+
import { MissingToolsError } from './MissingToolsError';
|
|
5
9
|
import { NotFoundError } from './NotFoundError';
|
|
6
10
|
import { NotYetImplementedError } from './NotYetImplementedError';
|
|
7
11
|
import { ParseError } from './ParseError';
|
|
@@ -15,10 +19,14 @@ import { UnexpectedError } from './UnexpectedError';
|
|
|
15
19
|
* @public exported from `@promptbook/core`
|
|
16
20
|
*/
|
|
17
21
|
export declare const ERRORS: {
|
|
18
|
-
readonly
|
|
22
|
+
readonly AbstractFormatError: typeof AbstractFormatError;
|
|
23
|
+
readonly CsvFormatError: typeof CsvFormatError;
|
|
19
24
|
readonly CollectionError: typeof CollectionError;
|
|
20
25
|
readonly EnvironmentMismatchError: typeof EnvironmentMismatchError;
|
|
26
|
+
readonly ExpectError: typeof ExpectError;
|
|
27
|
+
readonly KnowledgeScrapeError: typeof KnowledgeScrapeError;
|
|
21
28
|
readonly LimitReachedError: typeof LimitReachedError;
|
|
29
|
+
readonly MissingToolsError: typeof MissingToolsError;
|
|
22
30
|
readonly NotFoundError: typeof NotFoundError;
|
|
23
31
|
readonly NotYetImplementedError: typeof NotYetImplementedError;
|
|
24
32
|
readonly ParseError: typeof ParseError;
|
|
@@ -29,4 +37,4 @@ export declare const ERRORS: {
|
|
|
29
37
|
};
|
|
30
38
|
/**
|
|
31
39
|
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
32
|
-
*/
|
|
40
|
+
*/
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { string_markdown } from '../../../types/typeAliases';
|
|
2
|
+
import type { Scraper } from '../Scraper';
|
|
2
3
|
/**
|
|
3
4
|
* Creates a message with all registered scrapers
|
|
4
5
|
*
|
|
@@ -6,7 +7,7 @@ import type { string_markdown } from '../../../types/typeAliases';
|
|
|
6
7
|
*
|
|
7
8
|
* @private internal function of `createScrapersFromConfiguration` and `createScrapersFromEnv`
|
|
8
9
|
*/
|
|
9
|
-
export declare function $registeredScrapersMessage(): string_markdown;
|
|
10
|
+
export declare function $registeredScrapersMessage(availableScrapers: ReadonlyArray<Scraper>): string_markdown;
|
|
10
11
|
/**
|
|
11
12
|
* TODO: [®] DRY Register logic
|
|
12
13
|
*/
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@promptbook/legacy-documents",
|
|
3
|
-
"version": "0.75.
|
|
3
|
+
"version": "0.75.5",
|
|
4
4
|
"description": "It's time for a paradigm shift. The future of software in plain English, French or Latin",
|
|
5
5
|
"--note-0": " <- [🐊]",
|
|
6
6
|
"private": false,
|
|
@@ -54,7 +54,7 @@
|
|
|
54
54
|
"module": "./esm/index.es.js",
|
|
55
55
|
"typings": "./esm/typings/src/_packages/legacy-documents.index.d.ts",
|
|
56
56
|
"peerDependencies": {
|
|
57
|
-
"@promptbook/core": "0.75.
|
|
57
|
+
"@promptbook/core": "0.75.5"
|
|
58
58
|
},
|
|
59
59
|
"dependencies": {
|
|
60
60
|
"colors": "1.4.0",
|
package/umd/index.umd.js
CHANGED
|
@@ -23,7 +23,7 @@
|
|
|
23
23
|
*
|
|
24
24
|
* @see https://github.com/webgptorg/promptbook
|
|
25
25
|
*/
|
|
26
|
-
var PROMPTBOOK_ENGINE_VERSION = '0.75.
|
|
26
|
+
var PROMPTBOOK_ENGINE_VERSION = '0.75.4';
|
|
27
27
|
/**
|
|
28
28
|
* TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
|
|
29
29
|
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
@@ -2114,6 +2114,42 @@
|
|
|
2114
2114
|
return PipelineExecutionError;
|
|
2115
2115
|
}(Error));
|
|
2116
2116
|
|
|
2117
|
+
/**
|
|
2118
|
+
* This error indicates problems parsing the format value
|
|
2119
|
+
*
|
|
2120
|
+
* For example, when the format value is not a valid JSON or CSV
|
|
2121
|
+
* This is not thrown directly but in extended classes
|
|
2122
|
+
*
|
|
2123
|
+
* @public exported from `@promptbook/core`
|
|
2124
|
+
*/
|
|
2125
|
+
var AbstractFormatError = /** @class */ (function (_super) {
|
|
2126
|
+
__extends(AbstractFormatError, _super);
|
|
2127
|
+
// Note: To allow instanceof do not put here error `name`
|
|
2128
|
+
// public readonly name = 'AbstractFormatError';
|
|
2129
|
+
function AbstractFormatError(message) {
|
|
2130
|
+
var _this = _super.call(this, message) || this;
|
|
2131
|
+
Object.setPrototypeOf(_this, AbstractFormatError.prototype);
|
|
2132
|
+
return _this;
|
|
2133
|
+
}
|
|
2134
|
+
return AbstractFormatError;
|
|
2135
|
+
}(Error));
|
|
2136
|
+
|
|
2137
|
+
/**
|
|
2138
|
+
* This error indicates problem with parsing of CSV
|
|
2139
|
+
*
|
|
2140
|
+
* @public exported from `@promptbook/core`
|
|
2141
|
+
*/
|
|
2142
|
+
var CsvFormatError = /** @class */ (function (_super) {
|
|
2143
|
+
__extends(CsvFormatError, _super);
|
|
2144
|
+
function CsvFormatError(message) {
|
|
2145
|
+
var _this = _super.call(this, message) || this;
|
|
2146
|
+
_this.name = 'CsvFormatError';
|
|
2147
|
+
Object.setPrototypeOf(_this, CsvFormatError.prototype);
|
|
2148
|
+
return _this;
|
|
2149
|
+
}
|
|
2150
|
+
return CsvFormatError;
|
|
2151
|
+
}(AbstractFormatError));
|
|
2152
|
+
|
|
2117
2153
|
/**
|
|
2118
2154
|
* This error indicates that the pipeline collection cannot be propperly loaded
|
|
2119
2155
|
*
|
|
@@ -2187,10 +2223,14 @@
|
|
|
2187
2223
|
* @public exported from `@promptbook/core`
|
|
2188
2224
|
*/
|
|
2189
2225
|
var ERRORS = {
|
|
2190
|
-
|
|
2226
|
+
AbstractFormatError: AbstractFormatError,
|
|
2227
|
+
CsvFormatError: CsvFormatError,
|
|
2191
2228
|
CollectionError: CollectionError,
|
|
2192
2229
|
EnvironmentMismatchError: EnvironmentMismatchError,
|
|
2230
|
+
ExpectError: ExpectError,
|
|
2231
|
+
KnowledgeScrapeError: KnowledgeScrapeError,
|
|
2193
2232
|
LimitReachedError: LimitReachedError,
|
|
2233
|
+
MissingToolsError: MissingToolsError,
|
|
2194
2234
|
NotFoundError: NotFoundError,
|
|
2195
2235
|
NotYetImplementedError: NotYetImplementedError,
|
|
2196
2236
|
ParseError: ParseError,
|
|
@@ -3161,8 +3201,8 @@
|
|
|
3161
3201
|
*
|
|
3162
3202
|
* @private internal function of `createScrapersFromConfiguration` and `createScrapersFromEnv`
|
|
3163
3203
|
*/
|
|
3164
|
-
function $registeredScrapersMessage() {
|
|
3165
|
-
var e_1, _a, e_2, _b;
|
|
3204
|
+
function $registeredScrapersMessage(availableScrapers) {
|
|
3205
|
+
var e_1, _a, e_2, _b, e_3, _c;
|
|
3166
3206
|
/**
|
|
3167
3207
|
* Mixes registered scrapers from $scrapersMetadataRegister and $scrapersRegister
|
|
3168
3208
|
*/
|
|
@@ -3174,15 +3214,15 @@
|
|
|
3174
3214
|
all.push({ packageName: packageName, className: className, mimeTypes: mimeTypes, documentationUrl: documentationUrl, isAvilableInBrowser: isAvilableInBrowser });
|
|
3175
3215
|
};
|
|
3176
3216
|
try {
|
|
3177
|
-
for (var
|
|
3178
|
-
var
|
|
3217
|
+
for (var _d = __values($scrapersMetadataRegister.list()), _e = _d.next(); !_e.done; _e = _d.next()) {
|
|
3218
|
+
var _f = _e.value, packageName = _f.packageName, className = _f.className, mimeTypes = _f.mimeTypes, documentationUrl = _f.documentationUrl, isAvilableInBrowser = _f.isAvilableInBrowser;
|
|
3179
3219
|
_loop_1(packageName, className, mimeTypes, documentationUrl, isAvilableInBrowser);
|
|
3180
3220
|
}
|
|
3181
3221
|
}
|
|
3182
3222
|
catch (e_1_1) { e_1 = { error: e_1_1 }; }
|
|
3183
3223
|
finally {
|
|
3184
3224
|
try {
|
|
3185
|
-
if (
|
|
3225
|
+
if (_e && !_e.done && (_a = _d.return)) _a.call(_d);
|
|
3186
3226
|
}
|
|
3187
3227
|
finally { if (e_1) throw e_1.error; }
|
|
3188
3228
|
}
|
|
@@ -3193,18 +3233,31 @@
|
|
|
3193
3233
|
all.push({ packageName: packageName, className: className, mimeTypes: mimeTypes, documentationUrl: documentationUrl, isAvilableInBrowser: isAvilableInBrowser });
|
|
3194
3234
|
};
|
|
3195
3235
|
try {
|
|
3196
|
-
for (var
|
|
3197
|
-
var
|
|
3236
|
+
for (var _g = __values($scrapersRegister.list()), _h = _g.next(); !_h.done; _h = _g.next()) {
|
|
3237
|
+
var _j = _h.value, packageName = _j.packageName, className = _j.className, mimeTypes = _j.mimeTypes, documentationUrl = _j.documentationUrl, isAvilableInBrowser = _j.isAvilableInBrowser;
|
|
3198
3238
|
_loop_2(packageName, className, mimeTypes, documentationUrl, isAvilableInBrowser);
|
|
3199
3239
|
}
|
|
3200
3240
|
}
|
|
3201
3241
|
catch (e_2_1) { e_2 = { error: e_2_1 }; }
|
|
3202
3242
|
finally {
|
|
3203
3243
|
try {
|
|
3204
|
-
if (
|
|
3244
|
+
if (_h && !_h.done && (_b = _g.return)) _b.call(_g);
|
|
3205
3245
|
}
|
|
3206
3246
|
finally { if (e_2) throw e_2.error; }
|
|
3207
3247
|
}
|
|
3248
|
+
try {
|
|
3249
|
+
for (var availableScrapers_1 = __values(availableScrapers), availableScrapers_1_1 = availableScrapers_1.next(); !availableScrapers_1_1.done; availableScrapers_1_1 = availableScrapers_1.next()) {
|
|
3250
|
+
var metadata_1 = availableScrapers_1_1.value.metadata;
|
|
3251
|
+
all.push(metadata_1);
|
|
3252
|
+
}
|
|
3253
|
+
}
|
|
3254
|
+
catch (e_3_1) { e_3 = { error: e_3_1 }; }
|
|
3255
|
+
finally {
|
|
3256
|
+
try {
|
|
3257
|
+
if (availableScrapers_1_1 && !availableScrapers_1_1.done && (_c = availableScrapers_1.return)) _c.call(availableScrapers_1);
|
|
3258
|
+
}
|
|
3259
|
+
finally { if (e_3) throw e_3.error; }
|
|
3260
|
+
}
|
|
3208
3261
|
var metadata = all.map(function (metadata) {
|
|
3209
3262
|
var isMetadataAviailable = $scrapersMetadataRegister
|
|
3210
3263
|
.list()
|
|
@@ -3218,42 +3271,44 @@
|
|
|
3218
3271
|
var packageName = _a.packageName, className = _a.className;
|
|
3219
3272
|
return metadata.packageName === packageName && metadata.className === className;
|
|
3220
3273
|
});
|
|
3221
|
-
|
|
3274
|
+
var isAvilableInTools = availableScrapers.some(function (_a) {
|
|
3275
|
+
var _b = _a.metadata, packageName = _b.packageName, className = _b.className;
|
|
3276
|
+
return metadata.packageName === packageName && metadata.className === className;
|
|
3277
|
+
});
|
|
3278
|
+
return __assign(__assign({}, metadata), { isMetadataAviailable: isMetadataAviailable, isInstalled: isInstalled, isAvilableInTools: isAvilableInTools });
|
|
3222
3279
|
});
|
|
3223
3280
|
if (metadata.length === 0) {
|
|
3224
|
-
return "No scrapers are available";
|
|
3281
|
+
return spaceTrim__default["default"]("\n **No scrapers are available**\n\n This is a unexpected behavior, you are probably using some broken version of Promptbook\n At least there should be available the metadata of the scrapers\n ");
|
|
3225
3282
|
}
|
|
3226
3283
|
return spaceTrim__default["default"](function (block) { return "\n Available scrapers are:\n ".concat(block(metadata
|
|
3227
3284
|
.map(function (_a, i) {
|
|
3228
|
-
var packageName = _a.packageName, className = _a.className, isMetadataAviailable = _a.isMetadataAviailable, isInstalled = _a.isInstalled, mimeTypes = _a.mimeTypes, isAvilableInBrowser = _a.isAvilableInBrowser;
|
|
3229
|
-
var more;
|
|
3230
|
-
// TODO:
|
|
3231
|
-
if (
|
|
3232
|
-
more
|
|
3233
|
-
}
|
|
3234
|
-
|
|
3235
|
-
|
|
3236
|
-
|
|
3237
|
-
|
|
3238
|
-
|
|
3239
|
-
|
|
3240
|
-
|
|
3241
|
-
|
|
3242
|
-
|
|
3243
|
-
|
|
3244
|
-
|
|
3245
|
-
|
|
3246
|
-
more = "(installed)";
|
|
3247
|
-
}
|
|
3248
|
-
else {
|
|
3249
|
-
more = "*(unknown state, looks like a unexpected behavior)*";
|
|
3250
|
-
}
|
|
3285
|
+
var packageName = _a.packageName, className = _a.className, isMetadataAviailable = _a.isMetadataAviailable, isInstalled = _a.isInstalled, mimeTypes = _a.mimeTypes, isAvilableInBrowser = _a.isAvilableInBrowser, isAvilableInTools = _a.isAvilableInTools;
|
|
3286
|
+
var more = [];
|
|
3287
|
+
// TODO: [🧠] Maybe use `documentationUrl`
|
|
3288
|
+
if (isMetadataAviailable) {
|
|
3289
|
+
more.push("\u2B1C Metadata registered");
|
|
3290
|
+
} // not else
|
|
3291
|
+
if (isInstalled) {
|
|
3292
|
+
more.push("\uD83D\uDFE9 Installed");
|
|
3293
|
+
} // not else
|
|
3294
|
+
if (isAvilableInTools) {
|
|
3295
|
+
more.push("\uD83D\uDFE6 Available in tools");
|
|
3296
|
+
} // not else
|
|
3297
|
+
if (!isMetadataAviailable && isInstalled) {
|
|
3298
|
+
more.push("When no metadata registered but scraper is installed, it is an unexpected behavior");
|
|
3299
|
+
} // not else
|
|
3300
|
+
if (!isInstalled && isAvilableInTools) {
|
|
3301
|
+
more.push("When the scraper is not installed but available in tools, it is an unexpected compatibility behavior");
|
|
3302
|
+
} // not else
|
|
3251
3303
|
if (!isAvilableInBrowser) {
|
|
3252
|
-
more
|
|
3304
|
+
more.push("Not usable in browser");
|
|
3253
3305
|
}
|
|
3254
|
-
|
|
3306
|
+
var moreText = more.length === 0 ? '' : " *(".concat(more.join('; '), ")*");
|
|
3307
|
+
return "".concat(i + 1, ") `").concat(className, "` from `").concat(packageName, "` compatible to scrape ").concat(mimeTypes
|
|
3308
|
+
.map(function (mimeType) { return "\"".concat(mimeType, "\""); })
|
|
3309
|
+
.join(', ')).concat(moreText);
|
|
3255
3310
|
})
|
|
3256
|
-
.join('\n')), "\n "); });
|
|
3311
|
+
.join('\n')), "\n\n Legend:\n - \u2B1C **Metadata registered** means that Promptbook knows about the scraper, it is similar to registration in some registry\n - \uD83D\uDFE9 **Installed** means that you have imported package with particular scraper\n - \uD83D\uDFE6 **Available in tools** means that you have passed scraper as dependency into prepare or execution process\n\n "); });
|
|
3257
3312
|
}
|
|
3258
3313
|
/**
|
|
3259
3314
|
* TODO: [®] DRY Register logic
|
|
@@ -3455,57 +3510,75 @@
|
|
|
3455
3510
|
_a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a, rootDirname = options.rootDirname, _b = options.isVerbose, isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b;
|
|
3456
3511
|
knowledgePreparedUnflatten = new Array(knowledgeSources.length);
|
|
3457
3512
|
return [4 /*yield*/, forEachAsync(knowledgeSources, { maxParallelCount: maxParallelCount }, function (knowledgeSource, index) { return __awaiter(_this, void 0, void 0, function () {
|
|
3458
|
-
var partialPieces, sourceHandler,
|
|
3459
|
-
var e_1,
|
|
3460
|
-
return __generator(this, function (
|
|
3461
|
-
switch (
|
|
3513
|
+
var partialPieces, sourceHandler, scrapers, _loop_1, scrapers_1, scrapers_1_1, scraper, state_1, e_1_1, pieces;
|
|
3514
|
+
var e_1, _a;
|
|
3515
|
+
return __generator(this, function (_b) {
|
|
3516
|
+
switch (_b.label) {
|
|
3462
3517
|
case 0:
|
|
3463
3518
|
partialPieces = null;
|
|
3464
3519
|
return [4 /*yield*/, makeKnowledgeSourceHandler(knowledgeSource, tools, { rootDirname: rootDirname, isVerbose: isVerbose })];
|
|
3465
3520
|
case 1:
|
|
3466
|
-
sourceHandler =
|
|
3467
|
-
|
|
3521
|
+
sourceHandler = _b.sent();
|
|
3522
|
+
scrapers = arrayableToArray(tools.scrapers);
|
|
3523
|
+
_loop_1 = function (scraper) {
|
|
3524
|
+
var partialPiecesUnchecked;
|
|
3525
|
+
return __generator(this, function (_c) {
|
|
3526
|
+
switch (_c.label) {
|
|
3527
|
+
case 0:
|
|
3528
|
+
if (!scraper.metadata.mimeTypes.includes(sourceHandler.mimeType)
|
|
3529
|
+
// <- TODO: [🦔] Implement mime-type wildcards
|
|
3530
|
+
) {
|
|
3531
|
+
return [2 /*return*/, "continue"];
|
|
3532
|
+
}
|
|
3533
|
+
return [4 /*yield*/, scraper.scrape(sourceHandler)];
|
|
3534
|
+
case 1:
|
|
3535
|
+
partialPiecesUnchecked = _c.sent();
|
|
3536
|
+
if (partialPiecesUnchecked !== null) {
|
|
3537
|
+
partialPieces = __spreadArray([], __read(partialPiecesUnchecked), false);
|
|
3538
|
+
return [2 /*return*/, "break"];
|
|
3539
|
+
}
|
|
3540
|
+
console.warn(spaceTrim__default["default"](function (block) { return "\n Cannot scrape knowledge from source despite the scraper `".concat(scraper.metadata.className, "` supports the mime type \"").concat(sourceHandler.mimeType, "\".\n \n The source:\n > ").concat(block(knowledgeSource.sourceContent
|
|
3541
|
+
.split('\n')
|
|
3542
|
+
.map(function (line) { return "> ".concat(line); })
|
|
3543
|
+
.join('\n')), "\n\n ").concat(block($registeredScrapersMessage(scrapers)), "\n\n\n "); }));
|
|
3544
|
+
return [2 /*return*/];
|
|
3545
|
+
}
|
|
3546
|
+
});
|
|
3547
|
+
};
|
|
3548
|
+
_b.label = 2;
|
|
3468
3549
|
case 2:
|
|
3469
|
-
|
|
3470
|
-
|
|
3471
|
-
|
|
3550
|
+
_b.trys.push([2, 7, 8, 9]);
|
|
3551
|
+
scrapers_1 = __values(scrapers), scrapers_1_1 = scrapers_1.next();
|
|
3552
|
+
_b.label = 3;
|
|
3472
3553
|
case 3:
|
|
3473
|
-
if (!!
|
|
3474
|
-
scraper =
|
|
3475
|
-
|
|
3476
|
-
// <- TODO: [🦔] Implement mime-type wildcards
|
|
3477
|
-
) {
|
|
3478
|
-
return [3 /*break*/, 5];
|
|
3479
|
-
}
|
|
3480
|
-
return [4 /*yield*/, scraper.scrape(sourceHandler)];
|
|
3554
|
+
if (!!scrapers_1_1.done) return [3 /*break*/, 6];
|
|
3555
|
+
scraper = scrapers_1_1.value;
|
|
3556
|
+
return [5 /*yield**/, _loop_1(scraper)];
|
|
3481
3557
|
case 4:
|
|
3482
|
-
|
|
3483
|
-
if (
|
|
3484
|
-
partialPieces = __spreadArray([], __read(partialPiecesUnchecked), false);
|
|
3485
|
-
// <- TODO: [🪓] Here should be no need for spreading new array, just `partialPieces = partialPiecesUnchecked`
|
|
3558
|
+
state_1 = _b.sent();
|
|
3559
|
+
if (state_1 === "break")
|
|
3486
3560
|
return [3 /*break*/, 6];
|
|
3487
|
-
|
|
3488
|
-
_d.label = 5;
|
|
3561
|
+
_b.label = 5;
|
|
3489
3562
|
case 5:
|
|
3490
|
-
|
|
3563
|
+
scrapers_1_1 = scrapers_1.next();
|
|
3491
3564
|
return [3 /*break*/, 3];
|
|
3492
3565
|
case 6: return [3 /*break*/, 9];
|
|
3493
3566
|
case 7:
|
|
3494
|
-
e_1_1 =
|
|
3567
|
+
e_1_1 = _b.sent();
|
|
3495
3568
|
e_1 = { error: e_1_1 };
|
|
3496
3569
|
return [3 /*break*/, 9];
|
|
3497
3570
|
case 8:
|
|
3498
3571
|
try {
|
|
3499
|
-
if (
|
|
3572
|
+
if (scrapers_1_1 && !scrapers_1_1.done && (_a = scrapers_1.return)) _a.call(scrapers_1);
|
|
3500
3573
|
}
|
|
3501
3574
|
finally { if (e_1) throw e_1.error; }
|
|
3502
3575
|
return [7 /*endfinally*/];
|
|
3503
3576
|
case 9:
|
|
3504
3577
|
if (partialPieces === null) {
|
|
3505
|
-
throw new KnowledgeScrapeError(spaceTrim__default["default"](function (block) { return "\n Cannot scrape knowledge
|
|
3578
|
+
throw new KnowledgeScrapeError(spaceTrim__default["default"](function (block) { return "\n Cannot scrape knowledge\n \n The source:\n > ".concat(block(knowledgeSource.sourceContent
|
|
3506
3579
|
.split('\n')
|
|
3507
3580
|
.map(function (line) { return "> ".concat(line); })
|
|
3508
|
-
.join('\n')), "\n\n No scraper found for the mime type \"").concat(sourceHandler.mimeType, "\"\n\n ").concat(block($registeredScrapersMessage()), "\n\n\n "); }));
|
|
3581
|
+
.join('\n')), "\n\n No scraper found for the mime type \"").concat(sourceHandler.mimeType, "\"\n\n ").concat(block($registeredScrapersMessage(scrapers)), "\n\n\n "); }));
|
|
3509
3582
|
}
|
|
3510
3583
|
pieces = partialPieces.map(function (partialPiece) { return (__assign(__assign({}, partialPiece), { sources: [
|
|
3511
3584
|
{
|
|
@@ -3931,42 +4004,6 @@
|
|
|
3931
4004
|
return union;
|
|
3932
4005
|
}
|
|
3933
4006
|
|
|
3934
|
-
/**
|
|
3935
|
-
* This error indicates problems parsing the format value
|
|
3936
|
-
*
|
|
3937
|
-
* For example, when the format value is not a valid JSON or CSV
|
|
3938
|
-
* This is not thrown directly but in extended classes
|
|
3939
|
-
*
|
|
3940
|
-
* @public exported from `@promptbook/core`
|
|
3941
|
-
*/
|
|
3942
|
-
var AbstractFormatError = /** @class */ (function (_super) {
|
|
3943
|
-
__extends(AbstractFormatError, _super);
|
|
3944
|
-
// Note: To allow instanceof do not put here error `name`
|
|
3945
|
-
// public readonly name = 'AbstractFormatError';
|
|
3946
|
-
function AbstractFormatError(message) {
|
|
3947
|
-
var _this = _super.call(this, message) || this;
|
|
3948
|
-
Object.setPrototypeOf(_this, AbstractFormatError.prototype);
|
|
3949
|
-
return _this;
|
|
3950
|
-
}
|
|
3951
|
-
return AbstractFormatError;
|
|
3952
|
-
}(Error));
|
|
3953
|
-
|
|
3954
|
-
/**
|
|
3955
|
-
* This error indicates problem with parsing of CSV
|
|
3956
|
-
*
|
|
3957
|
-
* @public exported from `@promptbook/core`
|
|
3958
|
-
*/
|
|
3959
|
-
var CsvFormatError = /** @class */ (function (_super) {
|
|
3960
|
-
__extends(CsvFormatError, _super);
|
|
3961
|
-
function CsvFormatError(message) {
|
|
3962
|
-
var _this = _super.call(this, message) || this;
|
|
3963
|
-
_this.name = 'CsvFormatError';
|
|
3964
|
-
Object.setPrototypeOf(_this, CsvFormatError.prototype);
|
|
3965
|
-
return _this;
|
|
3966
|
-
}
|
|
3967
|
-
return CsvFormatError;
|
|
3968
|
-
}(AbstractFormatError));
|
|
3969
|
-
|
|
3970
4007
|
/**
|
|
3971
4008
|
* @@@
|
|
3972
4009
|
*
|
|
@@ -4007,7 +4044,7 @@
|
|
|
4007
4044
|
case 0:
|
|
4008
4045
|
csv = papaparse.parse(value, __assign(__assign({}, settings), MANDATORY_CSV_SETTINGS));
|
|
4009
4046
|
if (csv.errors.length !== 0) {
|
|
4010
|
-
throw new CsvFormatError(spaceTrim__default["default"](function (block) { return "\n CSV parsing error\n\n ".concat(block(csv.errors.map(function (error) { return error.message; }).join('\n\n')), "\n "); }));
|
|
4047
|
+
throw new CsvFormatError(spaceTrim__default["default"](function (block) { return "\n CSV parsing error\n\n Error(s) from CSV parsing:\n ".concat(block(csv.errors.map(function (error) { return error.message; }).join('\n\n')), "\n\n The CSV data:\n ").concat(block(value), "\n "); }));
|
|
4011
4048
|
}
|
|
4012
4049
|
return [4 /*yield*/, Promise.all(csv.data.map(function (row, index) { return __awaiter(_this, void 0, void 0, function () {
|
|
4013
4050
|
var _a, _b;
|
|
@@ -4045,7 +4082,7 @@
|
|
|
4045
4082
|
case 0:
|
|
4046
4083
|
csv = papaparse.parse(value, __assign(__assign({}, settings), MANDATORY_CSV_SETTINGS));
|
|
4047
4084
|
if (csv.errors.length !== 0) {
|
|
4048
|
-
throw new CsvFormatError(spaceTrim__default["default"](function (block) { return "\n CSV parsing error\n\n ".concat(block(csv.errors.map(function (error) { return error.message; }).join('\n\n')), "\n "); }));
|
|
4085
|
+
throw new CsvFormatError(spaceTrim__default["default"](function (block) { return "\n CSV parsing error\n\n Error(s) from CSV parsing:\n ".concat(block(csv.errors.map(function (error) { return error.message; }).join('\n\n')), "\n\n The CSV data:\n ").concat(block(value), "\n "); }));
|
|
4049
4086
|
}
|
|
4050
4087
|
return [4 /*yield*/, Promise.all(csv.data.map(function (row, rowIndex) { return __awaiter(_this, void 0, void 0, function () {
|
|
4051
4088
|
var _this = this;
|