@promptbook/markdown-utils 0.75.3 → 0.75.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/esm/index.es.js +157 -120
- package/esm/index.es.js.map +1 -1
- package/esm/typings/src/_packages/core.index.d.ts +6 -2
- package/esm/typings/src/config.d.ts +9 -1
- package/esm/typings/src/errors/0-BoilerplateError.d.ts +12 -0
- package/esm/typings/src/errors/{index.d.ts → 0-index.d.ts} +10 -2
- package/esm/typings/src/errors/utils/ErrorJson.d.ts +1 -1
- package/esm/typings/src/scrapers/_common/register/$registeredScrapersMessage.d.ts +2 -1
- package/package.json +1 -1
- package/umd/index.umd.js +157 -120
- package/umd/index.umd.js.map +1 -1
|
@@ -24,6 +24,7 @@ import { DEFAULT_REMOTE_URL } from '../config';
|
|
|
24
24
|
import { DEFAULT_REMOTE_URL_PATH } from '../config';
|
|
25
25
|
import { DEFAULT_CSV_SETTINGS } from '../config';
|
|
26
26
|
import { DEFAULT_IS_VERBOSE } from '../config';
|
|
27
|
+
import { SET_IS_VERBOSE } from '../config';
|
|
27
28
|
import { DEFAULT_IS_AUTO_INSTALLED } from '../config';
|
|
28
29
|
import { pipelineJsonToString } from '../conversion/pipelineJsonToString';
|
|
29
30
|
import { pipelineStringToJson } from '../conversion/pipelineStringToJson';
|
|
@@ -33,11 +34,12 @@ import { stringifyPipelineJson } from '../conversion/utils/stringifyPipelineJson
|
|
|
33
34
|
import { validatePipeline } from '../conversion/validation/validatePipeline';
|
|
34
35
|
import { CallbackInterfaceTools } from '../dialogs/callback/CallbackInterfaceTools';
|
|
35
36
|
import type { CallbackInterfaceToolsOptions } from '../dialogs/callback/CallbackInterfaceToolsOptions';
|
|
37
|
+
import { BoilerplateError } from '../errors/0-BoilerplateError';
|
|
38
|
+
import { ERRORS } from '../errors/0-index';
|
|
36
39
|
import { AbstractFormatError } from '../errors/AbstractFormatError';
|
|
37
40
|
import { CollectionError } from '../errors/CollectionError';
|
|
38
41
|
import { EnvironmentMismatchError } from '../errors/EnvironmentMismatchError';
|
|
39
42
|
import { ExpectError } from '../errors/ExpectError';
|
|
40
|
-
import { ERRORS } from '../errors/index';
|
|
41
43
|
import { KnowledgeScrapeError } from '../errors/KnowledgeScrapeError';
|
|
42
44
|
import { LimitReachedError } from '../errors/LimitReachedError';
|
|
43
45
|
import { MissingToolsError } from '../errors/MissingToolsError';
|
|
@@ -136,6 +138,7 @@ export { DEFAULT_REMOTE_URL };
|
|
|
136
138
|
export { DEFAULT_REMOTE_URL_PATH };
|
|
137
139
|
export { DEFAULT_CSV_SETTINGS };
|
|
138
140
|
export { DEFAULT_IS_VERBOSE };
|
|
141
|
+
export { SET_IS_VERBOSE };
|
|
139
142
|
export { DEFAULT_IS_AUTO_INSTALLED };
|
|
140
143
|
export { pipelineJsonToString };
|
|
141
144
|
export { pipelineStringToJson };
|
|
@@ -145,11 +148,12 @@ export { stringifyPipelineJson };
|
|
|
145
148
|
export { validatePipeline };
|
|
146
149
|
export { CallbackInterfaceTools };
|
|
147
150
|
export type { CallbackInterfaceToolsOptions };
|
|
151
|
+
export { BoilerplateError };
|
|
152
|
+
export { ERRORS };
|
|
148
153
|
export { AbstractFormatError };
|
|
149
154
|
export { CollectionError };
|
|
150
155
|
export { EnvironmentMismatchError };
|
|
151
156
|
export { ExpectError };
|
|
152
|
-
export { ERRORS };
|
|
153
157
|
export { KnowledgeScrapeError };
|
|
154
158
|
export { LimitReachedError };
|
|
155
159
|
export { MissingToolsError };
|
|
@@ -204,7 +204,15 @@ export declare const DEFAULT_CSV_SETTINGS: CsvSettings;
|
|
|
204
204
|
*
|
|
205
205
|
* @public exported from `@promptbook/core`
|
|
206
206
|
*/
|
|
207
|
-
export declare
|
|
207
|
+
export declare let DEFAULT_IS_VERBOSE: boolean;
|
|
208
|
+
/**
|
|
209
|
+
* @@@
|
|
210
|
+
*
|
|
211
|
+
* Note: This is experimental feature
|
|
212
|
+
*
|
|
213
|
+
* @public exported from `@promptbook/core`
|
|
214
|
+
*/
|
|
215
|
+
export declare function SET_IS_VERBOSE(isVerbose: boolean): void;
|
|
208
216
|
/**
|
|
209
217
|
* @@@
|
|
210
218
|
*
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* This error indicates @@@
|
|
3
|
+
*
|
|
4
|
+
* @public exported from `@promptbook/core`
|
|
5
|
+
*/
|
|
6
|
+
export declare class BoilerplateError extends Error {
|
|
7
|
+
readonly name = "BoilerplateError";
|
|
8
|
+
constructor(message: string);
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* TODO: @@@ Do not forget to add the error into `0-index.ts` ERRORS
|
|
12
|
+
*/
|
|
@@ -1,7 +1,11 @@
|
|
|
1
|
+
import { CsvFormatError } from '../formats/csv/CsvFormatError';
|
|
2
|
+
import { AbstractFormatError } from './AbstractFormatError';
|
|
1
3
|
import { CollectionError } from './CollectionError';
|
|
2
4
|
import { EnvironmentMismatchError } from './EnvironmentMismatchError';
|
|
3
5
|
import { ExpectError } from './ExpectError';
|
|
6
|
+
import { KnowledgeScrapeError } from './KnowledgeScrapeError';
|
|
4
7
|
import { LimitReachedError } from './LimitReachedError';
|
|
8
|
+
import { MissingToolsError } from './MissingToolsError';
|
|
5
9
|
import { NotFoundError } from './NotFoundError';
|
|
6
10
|
import { NotYetImplementedError } from './NotYetImplementedError';
|
|
7
11
|
import { ParseError } from './ParseError';
|
|
@@ -15,10 +19,14 @@ import { UnexpectedError } from './UnexpectedError';
|
|
|
15
19
|
* @public exported from `@promptbook/core`
|
|
16
20
|
*/
|
|
17
21
|
export declare const ERRORS: {
|
|
18
|
-
readonly
|
|
22
|
+
readonly AbstractFormatError: typeof AbstractFormatError;
|
|
23
|
+
readonly CsvFormatError: typeof CsvFormatError;
|
|
19
24
|
readonly CollectionError: typeof CollectionError;
|
|
20
25
|
readonly EnvironmentMismatchError: typeof EnvironmentMismatchError;
|
|
26
|
+
readonly ExpectError: typeof ExpectError;
|
|
27
|
+
readonly KnowledgeScrapeError: typeof KnowledgeScrapeError;
|
|
21
28
|
readonly LimitReachedError: typeof LimitReachedError;
|
|
29
|
+
readonly MissingToolsError: typeof MissingToolsError;
|
|
22
30
|
readonly NotFoundError: typeof NotFoundError;
|
|
23
31
|
readonly NotYetImplementedError: typeof NotYetImplementedError;
|
|
24
32
|
readonly ParseError: typeof ParseError;
|
|
@@ -29,4 +37,4 @@ export declare const ERRORS: {
|
|
|
29
37
|
};
|
|
30
38
|
/**
|
|
31
39
|
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
32
|
-
*/
|
|
40
|
+
*/
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { string_markdown } from '../../../types/typeAliases';
|
|
2
|
+
import type { Scraper } from '../Scraper';
|
|
2
3
|
/**
|
|
3
4
|
* Creates a message with all registered scrapers
|
|
4
5
|
*
|
|
@@ -6,7 +7,7 @@ import type { string_markdown } from '../../../types/typeAliases';
|
|
|
6
7
|
*
|
|
7
8
|
* @private internal function of `createScrapersFromConfiguration` and `createScrapersFromEnv`
|
|
8
9
|
*/
|
|
9
|
-
export declare function $registeredScrapersMessage(): string_markdown;
|
|
10
|
+
export declare function $registeredScrapersMessage(availableScrapers: ReadonlyArray<Scraper>): string_markdown;
|
|
10
11
|
/**
|
|
11
12
|
* TODO: [®] DRY Register logic
|
|
12
13
|
*/
|
package/package.json
CHANGED
package/umd/index.umd.js
CHANGED
|
@@ -22,7 +22,7 @@
|
|
|
22
22
|
*
|
|
23
23
|
* @see https://github.com/webgptorg/promptbook
|
|
24
24
|
*/
|
|
25
|
-
var PROMPTBOOK_ENGINE_VERSION = '0.75.
|
|
25
|
+
var PROMPTBOOK_ENGINE_VERSION = '0.75.4';
|
|
26
26
|
/**
|
|
27
27
|
* TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
|
|
28
28
|
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
@@ -1965,6 +1965,42 @@
|
|
|
1965
1965
|
return PipelineExecutionError;
|
|
1966
1966
|
}(Error));
|
|
1967
1967
|
|
|
1968
|
+
/**
|
|
1969
|
+
* This error indicates problems parsing the format value
|
|
1970
|
+
*
|
|
1971
|
+
* For example, when the format value is not a valid JSON or CSV
|
|
1972
|
+
* This is not thrown directly but in extended classes
|
|
1973
|
+
*
|
|
1974
|
+
* @public exported from `@promptbook/core`
|
|
1975
|
+
*/
|
|
1976
|
+
var AbstractFormatError = /** @class */ (function (_super) {
|
|
1977
|
+
__extends(AbstractFormatError, _super);
|
|
1978
|
+
// Note: To allow instanceof do not put here error `name`
|
|
1979
|
+
// public readonly name = 'AbstractFormatError';
|
|
1980
|
+
function AbstractFormatError(message) {
|
|
1981
|
+
var _this = _super.call(this, message) || this;
|
|
1982
|
+
Object.setPrototypeOf(_this, AbstractFormatError.prototype);
|
|
1983
|
+
return _this;
|
|
1984
|
+
}
|
|
1985
|
+
return AbstractFormatError;
|
|
1986
|
+
}(Error));
|
|
1987
|
+
|
|
1988
|
+
/**
|
|
1989
|
+
* This error indicates problem with parsing of CSV
|
|
1990
|
+
*
|
|
1991
|
+
* @public exported from `@promptbook/core`
|
|
1992
|
+
*/
|
|
1993
|
+
var CsvFormatError = /** @class */ (function (_super) {
|
|
1994
|
+
__extends(CsvFormatError, _super);
|
|
1995
|
+
function CsvFormatError(message) {
|
|
1996
|
+
var _this = _super.call(this, message) || this;
|
|
1997
|
+
_this.name = 'CsvFormatError';
|
|
1998
|
+
Object.setPrototypeOf(_this, CsvFormatError.prototype);
|
|
1999
|
+
return _this;
|
|
2000
|
+
}
|
|
2001
|
+
return CsvFormatError;
|
|
2002
|
+
}(AbstractFormatError));
|
|
2003
|
+
|
|
1968
2004
|
/**
|
|
1969
2005
|
* This error indicates that the pipeline collection cannot be propperly loaded
|
|
1970
2006
|
*
|
|
@@ -2016,6 +2052,22 @@
|
|
|
2016
2052
|
return ExpectError;
|
|
2017
2053
|
}(Error));
|
|
2018
2054
|
|
|
2055
|
+
/**
|
|
2056
|
+
* This error indicates that the promptbook can not retrieve knowledge from external sources
|
|
2057
|
+
*
|
|
2058
|
+
* @public exported from `@promptbook/core`
|
|
2059
|
+
*/
|
|
2060
|
+
var KnowledgeScrapeError = /** @class */ (function (_super) {
|
|
2061
|
+
__extends(KnowledgeScrapeError, _super);
|
|
2062
|
+
function KnowledgeScrapeError(message) {
|
|
2063
|
+
var _this = _super.call(this, message) || this;
|
|
2064
|
+
_this.name = 'KnowledgeScrapeError';
|
|
2065
|
+
Object.setPrototypeOf(_this, KnowledgeScrapeError.prototype);
|
|
2066
|
+
return _this;
|
|
2067
|
+
}
|
|
2068
|
+
return KnowledgeScrapeError;
|
|
2069
|
+
}(Error));
|
|
2070
|
+
|
|
2019
2071
|
/**
|
|
2020
2072
|
* This error type indicates that some limit was reached
|
|
2021
2073
|
*
|
|
@@ -2054,10 +2106,14 @@
|
|
|
2054
2106
|
* @public exported from `@promptbook/core`
|
|
2055
2107
|
*/
|
|
2056
2108
|
var ERRORS = {
|
|
2057
|
-
|
|
2109
|
+
AbstractFormatError: AbstractFormatError,
|
|
2110
|
+
CsvFormatError: CsvFormatError,
|
|
2058
2111
|
CollectionError: CollectionError,
|
|
2059
2112
|
EnvironmentMismatchError: EnvironmentMismatchError,
|
|
2113
|
+
ExpectError: ExpectError,
|
|
2114
|
+
KnowledgeScrapeError: KnowledgeScrapeError,
|
|
2060
2115
|
LimitReachedError: LimitReachedError,
|
|
2116
|
+
MissingToolsError: MissingToolsError,
|
|
2061
2117
|
NotFoundError: NotFoundError,
|
|
2062
2118
|
NotYetImplementedError: NotYetImplementedError,
|
|
2063
2119
|
ParseError: ParseError,
|
|
@@ -2858,22 +2914,6 @@
|
|
|
2858
2914
|
* TODO: [🏢] !! Check validity of `temperature` in pipeline
|
|
2859
2915
|
*/
|
|
2860
2916
|
|
|
2861
|
-
/**
|
|
2862
|
-
* This error indicates that the promptbook can not retrieve knowledge from external sources
|
|
2863
|
-
*
|
|
2864
|
-
* @public exported from `@promptbook/core`
|
|
2865
|
-
*/
|
|
2866
|
-
var KnowledgeScrapeError = /** @class */ (function (_super) {
|
|
2867
|
-
__extends(KnowledgeScrapeError, _super);
|
|
2868
|
-
function KnowledgeScrapeError(message) {
|
|
2869
|
-
var _this = _super.call(this, message) || this;
|
|
2870
|
-
_this.name = 'KnowledgeScrapeError';
|
|
2871
|
-
Object.setPrototypeOf(_this, KnowledgeScrapeError.prototype);
|
|
2872
|
-
return _this;
|
|
2873
|
-
}
|
|
2874
|
-
return KnowledgeScrapeError;
|
|
2875
|
-
}(Error));
|
|
2876
|
-
|
|
2877
2917
|
/**
|
|
2878
2918
|
* @@@
|
|
2879
2919
|
*
|
|
@@ -3044,8 +3084,8 @@
|
|
|
3044
3084
|
*
|
|
3045
3085
|
* @private internal function of `createScrapersFromConfiguration` and `createScrapersFromEnv`
|
|
3046
3086
|
*/
|
|
3047
|
-
function $registeredScrapersMessage() {
|
|
3048
|
-
var e_1, _a, e_2, _b;
|
|
3087
|
+
function $registeredScrapersMessage(availableScrapers) {
|
|
3088
|
+
var e_1, _a, e_2, _b, e_3, _c;
|
|
3049
3089
|
/**
|
|
3050
3090
|
* Mixes registered scrapers from $scrapersMetadataRegister and $scrapersRegister
|
|
3051
3091
|
*/
|
|
@@ -3057,15 +3097,15 @@
|
|
|
3057
3097
|
all.push({ packageName: packageName, className: className, mimeTypes: mimeTypes, documentationUrl: documentationUrl, isAvilableInBrowser: isAvilableInBrowser });
|
|
3058
3098
|
};
|
|
3059
3099
|
try {
|
|
3060
|
-
for (var
|
|
3061
|
-
var
|
|
3100
|
+
for (var _d = __values($scrapersMetadataRegister.list()), _e = _d.next(); !_e.done; _e = _d.next()) {
|
|
3101
|
+
var _f = _e.value, packageName = _f.packageName, className = _f.className, mimeTypes = _f.mimeTypes, documentationUrl = _f.documentationUrl, isAvilableInBrowser = _f.isAvilableInBrowser;
|
|
3062
3102
|
_loop_1(packageName, className, mimeTypes, documentationUrl, isAvilableInBrowser);
|
|
3063
3103
|
}
|
|
3064
3104
|
}
|
|
3065
3105
|
catch (e_1_1) { e_1 = { error: e_1_1 }; }
|
|
3066
3106
|
finally {
|
|
3067
3107
|
try {
|
|
3068
|
-
if (
|
|
3108
|
+
if (_e && !_e.done && (_a = _d.return)) _a.call(_d);
|
|
3069
3109
|
}
|
|
3070
3110
|
finally { if (e_1) throw e_1.error; }
|
|
3071
3111
|
}
|
|
@@ -3076,18 +3116,31 @@
|
|
|
3076
3116
|
all.push({ packageName: packageName, className: className, mimeTypes: mimeTypes, documentationUrl: documentationUrl, isAvilableInBrowser: isAvilableInBrowser });
|
|
3077
3117
|
};
|
|
3078
3118
|
try {
|
|
3079
|
-
for (var
|
|
3080
|
-
var
|
|
3119
|
+
for (var _g = __values($scrapersRegister.list()), _h = _g.next(); !_h.done; _h = _g.next()) {
|
|
3120
|
+
var _j = _h.value, packageName = _j.packageName, className = _j.className, mimeTypes = _j.mimeTypes, documentationUrl = _j.documentationUrl, isAvilableInBrowser = _j.isAvilableInBrowser;
|
|
3081
3121
|
_loop_2(packageName, className, mimeTypes, documentationUrl, isAvilableInBrowser);
|
|
3082
3122
|
}
|
|
3083
3123
|
}
|
|
3084
3124
|
catch (e_2_1) { e_2 = { error: e_2_1 }; }
|
|
3085
3125
|
finally {
|
|
3086
3126
|
try {
|
|
3087
|
-
if (
|
|
3127
|
+
if (_h && !_h.done && (_b = _g.return)) _b.call(_g);
|
|
3088
3128
|
}
|
|
3089
3129
|
finally { if (e_2) throw e_2.error; }
|
|
3090
3130
|
}
|
|
3131
|
+
try {
|
|
3132
|
+
for (var availableScrapers_1 = __values(availableScrapers), availableScrapers_1_1 = availableScrapers_1.next(); !availableScrapers_1_1.done; availableScrapers_1_1 = availableScrapers_1.next()) {
|
|
3133
|
+
var metadata_1 = availableScrapers_1_1.value.metadata;
|
|
3134
|
+
all.push(metadata_1);
|
|
3135
|
+
}
|
|
3136
|
+
}
|
|
3137
|
+
catch (e_3_1) { e_3 = { error: e_3_1 }; }
|
|
3138
|
+
finally {
|
|
3139
|
+
try {
|
|
3140
|
+
if (availableScrapers_1_1 && !availableScrapers_1_1.done && (_c = availableScrapers_1.return)) _c.call(availableScrapers_1);
|
|
3141
|
+
}
|
|
3142
|
+
finally { if (e_3) throw e_3.error; }
|
|
3143
|
+
}
|
|
3091
3144
|
var metadata = all.map(function (metadata) {
|
|
3092
3145
|
var isMetadataAviailable = $scrapersMetadataRegister
|
|
3093
3146
|
.list()
|
|
@@ -3101,42 +3154,44 @@
|
|
|
3101
3154
|
var packageName = _a.packageName, className = _a.className;
|
|
3102
3155
|
return metadata.packageName === packageName && metadata.className === className;
|
|
3103
3156
|
});
|
|
3104
|
-
|
|
3157
|
+
var isAvilableInTools = availableScrapers.some(function (_a) {
|
|
3158
|
+
var _b = _a.metadata, packageName = _b.packageName, className = _b.className;
|
|
3159
|
+
return metadata.packageName === packageName && metadata.className === className;
|
|
3160
|
+
});
|
|
3161
|
+
return __assign(__assign({}, metadata), { isMetadataAviailable: isMetadataAviailable, isInstalled: isInstalled, isAvilableInTools: isAvilableInTools });
|
|
3105
3162
|
});
|
|
3106
3163
|
if (metadata.length === 0) {
|
|
3107
|
-
return "No scrapers are available";
|
|
3164
|
+
return spaceTrim__default["default"]("\n **No scrapers are available**\n\n This is a unexpected behavior, you are probably using some broken version of Promptbook\n At least there should be available the metadata of the scrapers\n ");
|
|
3108
3165
|
}
|
|
3109
3166
|
return spaceTrim__default["default"](function (block) { return "\n Available scrapers are:\n ".concat(block(metadata
|
|
3110
3167
|
.map(function (_a, i) {
|
|
3111
|
-
var packageName = _a.packageName, className = _a.className, isMetadataAviailable = _a.isMetadataAviailable, isInstalled = _a.isInstalled, mimeTypes = _a.mimeTypes, isAvilableInBrowser = _a.isAvilableInBrowser;
|
|
3112
|
-
var more;
|
|
3113
|
-
// TODO:
|
|
3114
|
-
if (
|
|
3115
|
-
more
|
|
3116
|
-
}
|
|
3117
|
-
|
|
3118
|
-
|
|
3119
|
-
|
|
3120
|
-
|
|
3121
|
-
|
|
3122
|
-
|
|
3123
|
-
|
|
3124
|
-
|
|
3125
|
-
|
|
3126
|
-
|
|
3127
|
-
|
|
3128
|
-
|
|
3129
|
-
more = "(installed)";
|
|
3130
|
-
}
|
|
3131
|
-
else {
|
|
3132
|
-
more = "*(unknown state, looks like a unexpected behavior)*";
|
|
3133
|
-
}
|
|
3168
|
+
var packageName = _a.packageName, className = _a.className, isMetadataAviailable = _a.isMetadataAviailable, isInstalled = _a.isInstalled, mimeTypes = _a.mimeTypes, isAvilableInBrowser = _a.isAvilableInBrowser, isAvilableInTools = _a.isAvilableInTools;
|
|
3169
|
+
var more = [];
|
|
3170
|
+
// TODO: [🧠] Maybe use `documentationUrl`
|
|
3171
|
+
if (isMetadataAviailable) {
|
|
3172
|
+
more.push("\u2B1C Metadata registered");
|
|
3173
|
+
} // not else
|
|
3174
|
+
if (isInstalled) {
|
|
3175
|
+
more.push("\uD83D\uDFE9 Installed");
|
|
3176
|
+
} // not else
|
|
3177
|
+
if (isAvilableInTools) {
|
|
3178
|
+
more.push("\uD83D\uDFE6 Available in tools");
|
|
3179
|
+
} // not else
|
|
3180
|
+
if (!isMetadataAviailable && isInstalled) {
|
|
3181
|
+
more.push("When no metadata registered but scraper is installed, it is an unexpected behavior");
|
|
3182
|
+
} // not else
|
|
3183
|
+
if (!isInstalled && isAvilableInTools) {
|
|
3184
|
+
more.push("When the scraper is not installed but available in tools, it is an unexpected compatibility behavior");
|
|
3185
|
+
} // not else
|
|
3134
3186
|
if (!isAvilableInBrowser) {
|
|
3135
|
-
more
|
|
3187
|
+
more.push("Not usable in browser");
|
|
3136
3188
|
}
|
|
3137
|
-
|
|
3189
|
+
var moreText = more.length === 0 ? '' : " *(".concat(more.join('; '), ")*");
|
|
3190
|
+
return "".concat(i + 1, ") `").concat(className, "` from `").concat(packageName, "` compatible to scrape ").concat(mimeTypes
|
|
3191
|
+
.map(function (mimeType) { return "\"".concat(mimeType, "\""); })
|
|
3192
|
+
.join(', ')).concat(moreText);
|
|
3138
3193
|
})
|
|
3139
|
-
.join('\n')), "\n "); });
|
|
3194
|
+
.join('\n')), "\n\n Legend:\n - \u2B1C **Metadata registered** means that Promptbook knows about the scraper, it is similar to registration in some registry\n - \uD83D\uDFE9 **Installed** means that you have imported package with particular scraper\n - \uD83D\uDFE6 **Available in tools** means that you have passed scraper as dependency into prepare or execution process\n\n "); });
|
|
3140
3195
|
}
|
|
3141
3196
|
/**
|
|
3142
3197
|
* TODO: [®] DRY Register logic
|
|
@@ -3384,57 +3439,75 @@
|
|
|
3384
3439
|
_a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a, rootDirname = options.rootDirname, _b = options.isVerbose, isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b;
|
|
3385
3440
|
knowledgePreparedUnflatten = new Array(knowledgeSources.length);
|
|
3386
3441
|
return [4 /*yield*/, forEachAsync(knowledgeSources, { maxParallelCount: maxParallelCount }, function (knowledgeSource, index) { return __awaiter(_this, void 0, void 0, function () {
|
|
3387
|
-
var partialPieces, sourceHandler,
|
|
3388
|
-
var e_1,
|
|
3389
|
-
return __generator(this, function (
|
|
3390
|
-
switch (
|
|
3442
|
+
var partialPieces, sourceHandler, scrapers, _loop_1, scrapers_1, scrapers_1_1, scraper, state_1, e_1_1, pieces;
|
|
3443
|
+
var e_1, _a;
|
|
3444
|
+
return __generator(this, function (_b) {
|
|
3445
|
+
switch (_b.label) {
|
|
3391
3446
|
case 0:
|
|
3392
3447
|
partialPieces = null;
|
|
3393
3448
|
return [4 /*yield*/, makeKnowledgeSourceHandler(knowledgeSource, tools, { rootDirname: rootDirname, isVerbose: isVerbose })];
|
|
3394
3449
|
case 1:
|
|
3395
|
-
sourceHandler =
|
|
3396
|
-
|
|
3450
|
+
sourceHandler = _b.sent();
|
|
3451
|
+
scrapers = arrayableToArray(tools.scrapers);
|
|
3452
|
+
_loop_1 = function (scraper) {
|
|
3453
|
+
var partialPiecesUnchecked;
|
|
3454
|
+
return __generator(this, function (_c) {
|
|
3455
|
+
switch (_c.label) {
|
|
3456
|
+
case 0:
|
|
3457
|
+
if (!scraper.metadata.mimeTypes.includes(sourceHandler.mimeType)
|
|
3458
|
+
// <- TODO: [🦔] Implement mime-type wildcards
|
|
3459
|
+
) {
|
|
3460
|
+
return [2 /*return*/, "continue"];
|
|
3461
|
+
}
|
|
3462
|
+
return [4 /*yield*/, scraper.scrape(sourceHandler)];
|
|
3463
|
+
case 1:
|
|
3464
|
+
partialPiecesUnchecked = _c.sent();
|
|
3465
|
+
if (partialPiecesUnchecked !== null) {
|
|
3466
|
+
partialPieces = __spreadArray([], __read(partialPiecesUnchecked), false);
|
|
3467
|
+
return [2 /*return*/, "break"];
|
|
3468
|
+
}
|
|
3469
|
+
console.warn(spaceTrim__default["default"](function (block) { return "\n Cannot scrape knowledge from source despite the scraper `".concat(scraper.metadata.className, "` supports the mime type \"").concat(sourceHandler.mimeType, "\".\n \n The source:\n > ").concat(block(knowledgeSource.sourceContent
|
|
3470
|
+
.split('\n')
|
|
3471
|
+
.map(function (line) { return "> ".concat(line); })
|
|
3472
|
+
.join('\n')), "\n\n ").concat(block($registeredScrapersMessage(scrapers)), "\n\n\n "); }));
|
|
3473
|
+
return [2 /*return*/];
|
|
3474
|
+
}
|
|
3475
|
+
});
|
|
3476
|
+
};
|
|
3477
|
+
_b.label = 2;
|
|
3397
3478
|
case 2:
|
|
3398
|
-
|
|
3399
|
-
|
|
3400
|
-
|
|
3479
|
+
_b.trys.push([2, 7, 8, 9]);
|
|
3480
|
+
scrapers_1 = __values(scrapers), scrapers_1_1 = scrapers_1.next();
|
|
3481
|
+
_b.label = 3;
|
|
3401
3482
|
case 3:
|
|
3402
|
-
if (!!
|
|
3403
|
-
scraper =
|
|
3404
|
-
|
|
3405
|
-
// <- TODO: [🦔] Implement mime-type wildcards
|
|
3406
|
-
) {
|
|
3407
|
-
return [3 /*break*/, 5];
|
|
3408
|
-
}
|
|
3409
|
-
return [4 /*yield*/, scraper.scrape(sourceHandler)];
|
|
3483
|
+
if (!!scrapers_1_1.done) return [3 /*break*/, 6];
|
|
3484
|
+
scraper = scrapers_1_1.value;
|
|
3485
|
+
return [5 /*yield**/, _loop_1(scraper)];
|
|
3410
3486
|
case 4:
|
|
3411
|
-
|
|
3412
|
-
if (
|
|
3413
|
-
partialPieces = __spreadArray([], __read(partialPiecesUnchecked), false);
|
|
3414
|
-
// <- TODO: [🪓] Here should be no need for spreading new array, just `partialPieces = partialPiecesUnchecked`
|
|
3487
|
+
state_1 = _b.sent();
|
|
3488
|
+
if (state_1 === "break")
|
|
3415
3489
|
return [3 /*break*/, 6];
|
|
3416
|
-
|
|
3417
|
-
_d.label = 5;
|
|
3490
|
+
_b.label = 5;
|
|
3418
3491
|
case 5:
|
|
3419
|
-
|
|
3492
|
+
scrapers_1_1 = scrapers_1.next();
|
|
3420
3493
|
return [3 /*break*/, 3];
|
|
3421
3494
|
case 6: return [3 /*break*/, 9];
|
|
3422
3495
|
case 7:
|
|
3423
|
-
e_1_1 =
|
|
3496
|
+
e_1_1 = _b.sent();
|
|
3424
3497
|
e_1 = { error: e_1_1 };
|
|
3425
3498
|
return [3 /*break*/, 9];
|
|
3426
3499
|
case 8:
|
|
3427
3500
|
try {
|
|
3428
|
-
if (
|
|
3501
|
+
if (scrapers_1_1 && !scrapers_1_1.done && (_a = scrapers_1.return)) _a.call(scrapers_1);
|
|
3429
3502
|
}
|
|
3430
3503
|
finally { if (e_1) throw e_1.error; }
|
|
3431
3504
|
return [7 /*endfinally*/];
|
|
3432
3505
|
case 9:
|
|
3433
3506
|
if (partialPieces === null) {
|
|
3434
|
-
throw new KnowledgeScrapeError(spaceTrim__default["default"](function (block) { return "\n Cannot scrape knowledge
|
|
3507
|
+
throw new KnowledgeScrapeError(spaceTrim__default["default"](function (block) { return "\n Cannot scrape knowledge\n \n The source:\n > ".concat(block(knowledgeSource.sourceContent
|
|
3435
3508
|
.split('\n')
|
|
3436
3509
|
.map(function (line) { return "> ".concat(line); })
|
|
3437
|
-
.join('\n')), "\n\n No scraper found for the mime type \"").concat(sourceHandler.mimeType, "\"\n\n ").concat(block($registeredScrapersMessage()), "\n\n\n "); }));
|
|
3510
|
+
.join('\n')), "\n\n No scraper found for the mime type \"").concat(sourceHandler.mimeType, "\"\n\n ").concat(block($registeredScrapersMessage(scrapers)), "\n\n\n "); }));
|
|
3438
3511
|
}
|
|
3439
3512
|
pieces = partialPieces.map(function (partialPiece) { return (__assign(__assign({}, partialPiece), { sources: [
|
|
3440
3513
|
{
|
|
@@ -3860,42 +3933,6 @@
|
|
|
3860
3933
|
return union;
|
|
3861
3934
|
}
|
|
3862
3935
|
|
|
3863
|
-
/**
|
|
3864
|
-
* This error indicates problems parsing the format value
|
|
3865
|
-
*
|
|
3866
|
-
* For example, when the format value is not a valid JSON or CSV
|
|
3867
|
-
* This is not thrown directly but in extended classes
|
|
3868
|
-
*
|
|
3869
|
-
* @public exported from `@promptbook/core`
|
|
3870
|
-
*/
|
|
3871
|
-
var AbstractFormatError = /** @class */ (function (_super) {
|
|
3872
|
-
__extends(AbstractFormatError, _super);
|
|
3873
|
-
// Note: To allow instanceof do not put here error `name`
|
|
3874
|
-
// public readonly name = 'AbstractFormatError';
|
|
3875
|
-
function AbstractFormatError(message) {
|
|
3876
|
-
var _this = _super.call(this, message) || this;
|
|
3877
|
-
Object.setPrototypeOf(_this, AbstractFormatError.prototype);
|
|
3878
|
-
return _this;
|
|
3879
|
-
}
|
|
3880
|
-
return AbstractFormatError;
|
|
3881
|
-
}(Error));
|
|
3882
|
-
|
|
3883
|
-
/**
|
|
3884
|
-
* This error indicates problem with parsing of CSV
|
|
3885
|
-
*
|
|
3886
|
-
* @public exported from `@promptbook/core`
|
|
3887
|
-
*/
|
|
3888
|
-
var CsvFormatError = /** @class */ (function (_super) {
|
|
3889
|
-
__extends(CsvFormatError, _super);
|
|
3890
|
-
function CsvFormatError(message) {
|
|
3891
|
-
var _this = _super.call(this, message) || this;
|
|
3892
|
-
_this.name = 'CsvFormatError';
|
|
3893
|
-
Object.setPrototypeOf(_this, CsvFormatError.prototype);
|
|
3894
|
-
return _this;
|
|
3895
|
-
}
|
|
3896
|
-
return CsvFormatError;
|
|
3897
|
-
}(AbstractFormatError));
|
|
3898
|
-
|
|
3899
3936
|
/**
|
|
3900
3937
|
* @@@
|
|
3901
3938
|
*
|
|
@@ -3936,7 +3973,7 @@
|
|
|
3936
3973
|
case 0:
|
|
3937
3974
|
csv = papaparse.parse(value, __assign(__assign({}, settings), MANDATORY_CSV_SETTINGS));
|
|
3938
3975
|
if (csv.errors.length !== 0) {
|
|
3939
|
-
throw new CsvFormatError(spaceTrim__default["default"](function (block) { return "\n CSV parsing error\n\n ".concat(block(csv.errors.map(function (error) { return error.message; }).join('\n\n')), "\n "); }));
|
|
3976
|
+
throw new CsvFormatError(spaceTrim__default["default"](function (block) { return "\n CSV parsing error\n\n Error(s) from CSV parsing:\n ".concat(block(csv.errors.map(function (error) { return error.message; }).join('\n\n')), "\n\n The CSV data:\n ").concat(block(value), "\n "); }));
|
|
3940
3977
|
}
|
|
3941
3978
|
return [4 /*yield*/, Promise.all(csv.data.map(function (row, index) { return __awaiter(_this, void 0, void 0, function () {
|
|
3942
3979
|
var _a, _b;
|
|
@@ -3974,7 +4011,7 @@
|
|
|
3974
4011
|
case 0:
|
|
3975
4012
|
csv = papaparse.parse(value, __assign(__assign({}, settings), MANDATORY_CSV_SETTINGS));
|
|
3976
4013
|
if (csv.errors.length !== 0) {
|
|
3977
|
-
throw new CsvFormatError(spaceTrim__default["default"](function (block) { return "\n CSV parsing error\n\n ".concat(block(csv.errors.map(function (error) { return error.message; }).join('\n\n')), "\n "); }));
|
|
4014
|
+
throw new CsvFormatError(spaceTrim__default["default"](function (block) { return "\n CSV parsing error\n\n Error(s) from CSV parsing:\n ".concat(block(csv.errors.map(function (error) { return error.message; }).join('\n\n')), "\n\n The CSV data:\n ").concat(block(value), "\n "); }));
|
|
3978
4015
|
}
|
|
3979
4016
|
return [4 /*yield*/, Promise.all(csv.data.map(function (row, rowIndex) { return __awaiter(_this, void 0, void 0, function () {
|
|
3980
4017
|
var _this = this;
|