@promptbook/pdf 0.75.3 → 0.75.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/esm/index.es.js +157 -120
- package/esm/index.es.js.map +1 -1
- package/esm/typings/src/_packages/core.index.d.ts +6 -2
- package/esm/typings/src/config.d.ts +9 -1
- package/esm/typings/src/errors/0-BoilerplateError.d.ts +12 -0
- package/esm/typings/src/errors/{index.d.ts → 0-index.d.ts} +10 -2
- package/esm/typings/src/errors/utils/ErrorJson.d.ts +1 -1
- package/esm/typings/src/scrapers/_common/register/$registeredScrapersMessage.d.ts +2 -1
- package/package.json +2 -2
- package/umd/index.umd.js +157 -120
- package/umd/index.umd.js.map +1 -1
|
@@ -24,6 +24,7 @@ import { DEFAULT_REMOTE_URL } from '../config';
|
|
|
24
24
|
import { DEFAULT_REMOTE_URL_PATH } from '../config';
|
|
25
25
|
import { DEFAULT_CSV_SETTINGS } from '../config';
|
|
26
26
|
import { DEFAULT_IS_VERBOSE } from '../config';
|
|
27
|
+
import { SET_IS_VERBOSE } from '../config';
|
|
27
28
|
import { DEFAULT_IS_AUTO_INSTALLED } from '../config';
|
|
28
29
|
import { pipelineJsonToString } from '../conversion/pipelineJsonToString';
|
|
29
30
|
import { pipelineStringToJson } from '../conversion/pipelineStringToJson';
|
|
@@ -33,11 +34,12 @@ import { stringifyPipelineJson } from '../conversion/utils/stringifyPipelineJson
|
|
|
33
34
|
import { validatePipeline } from '../conversion/validation/validatePipeline';
|
|
34
35
|
import { CallbackInterfaceTools } from '../dialogs/callback/CallbackInterfaceTools';
|
|
35
36
|
import type { CallbackInterfaceToolsOptions } from '../dialogs/callback/CallbackInterfaceToolsOptions';
|
|
37
|
+
import { BoilerplateError } from '../errors/0-BoilerplateError';
|
|
38
|
+
import { ERRORS } from '../errors/0-index';
|
|
36
39
|
import { AbstractFormatError } from '../errors/AbstractFormatError';
|
|
37
40
|
import { CollectionError } from '../errors/CollectionError';
|
|
38
41
|
import { EnvironmentMismatchError } from '../errors/EnvironmentMismatchError';
|
|
39
42
|
import { ExpectError } from '../errors/ExpectError';
|
|
40
|
-
import { ERRORS } from '../errors/index';
|
|
41
43
|
import { KnowledgeScrapeError } from '../errors/KnowledgeScrapeError';
|
|
42
44
|
import { LimitReachedError } from '../errors/LimitReachedError';
|
|
43
45
|
import { MissingToolsError } from '../errors/MissingToolsError';
|
|
@@ -136,6 +138,7 @@ export { DEFAULT_REMOTE_URL };
|
|
|
136
138
|
export { DEFAULT_REMOTE_URL_PATH };
|
|
137
139
|
export { DEFAULT_CSV_SETTINGS };
|
|
138
140
|
export { DEFAULT_IS_VERBOSE };
|
|
141
|
+
export { SET_IS_VERBOSE };
|
|
139
142
|
export { DEFAULT_IS_AUTO_INSTALLED };
|
|
140
143
|
export { pipelineJsonToString };
|
|
141
144
|
export { pipelineStringToJson };
|
|
@@ -145,11 +148,12 @@ export { stringifyPipelineJson };
|
|
|
145
148
|
export { validatePipeline };
|
|
146
149
|
export { CallbackInterfaceTools };
|
|
147
150
|
export type { CallbackInterfaceToolsOptions };
|
|
151
|
+
export { BoilerplateError };
|
|
152
|
+
export { ERRORS };
|
|
148
153
|
export { AbstractFormatError };
|
|
149
154
|
export { CollectionError };
|
|
150
155
|
export { EnvironmentMismatchError };
|
|
151
156
|
export { ExpectError };
|
|
152
|
-
export { ERRORS };
|
|
153
157
|
export { KnowledgeScrapeError };
|
|
154
158
|
export { LimitReachedError };
|
|
155
159
|
export { MissingToolsError };
|
|
@@ -204,7 +204,15 @@ export declare const DEFAULT_CSV_SETTINGS: CsvSettings;
|
|
|
204
204
|
*
|
|
205
205
|
* @public exported from `@promptbook/core`
|
|
206
206
|
*/
|
|
207
|
-
export declare
|
|
207
|
+
export declare let DEFAULT_IS_VERBOSE: boolean;
|
|
208
|
+
/**
|
|
209
|
+
* @@@
|
|
210
|
+
*
|
|
211
|
+
* Note: This is experimental feature
|
|
212
|
+
*
|
|
213
|
+
* @public exported from `@promptbook/core`
|
|
214
|
+
*/
|
|
215
|
+
export declare function SET_IS_VERBOSE(isVerbose: boolean): void;
|
|
208
216
|
/**
|
|
209
217
|
* @@@
|
|
210
218
|
*
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* This error indicates @@@
|
|
3
|
+
*
|
|
4
|
+
* @public exported from `@promptbook/core`
|
|
5
|
+
*/
|
|
6
|
+
export declare class BoilerplateError extends Error {
|
|
7
|
+
readonly name = "BoilerplateError";
|
|
8
|
+
constructor(message: string);
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* TODO: @@@ Do not forget to add the error into `0-index.ts` ERRORS
|
|
12
|
+
*/
|
|
@@ -1,7 +1,11 @@
|
|
|
1
|
+
import { CsvFormatError } from '../formats/csv/CsvFormatError';
|
|
2
|
+
import { AbstractFormatError } from './AbstractFormatError';
|
|
1
3
|
import { CollectionError } from './CollectionError';
|
|
2
4
|
import { EnvironmentMismatchError } from './EnvironmentMismatchError';
|
|
3
5
|
import { ExpectError } from './ExpectError';
|
|
6
|
+
import { KnowledgeScrapeError } from './KnowledgeScrapeError';
|
|
4
7
|
import { LimitReachedError } from './LimitReachedError';
|
|
8
|
+
import { MissingToolsError } from './MissingToolsError';
|
|
5
9
|
import { NotFoundError } from './NotFoundError';
|
|
6
10
|
import { NotYetImplementedError } from './NotYetImplementedError';
|
|
7
11
|
import { ParseError } from './ParseError';
|
|
@@ -15,10 +19,14 @@ import { UnexpectedError } from './UnexpectedError';
|
|
|
15
19
|
* @public exported from `@promptbook/core`
|
|
16
20
|
*/
|
|
17
21
|
export declare const ERRORS: {
|
|
18
|
-
readonly
|
|
22
|
+
readonly AbstractFormatError: typeof AbstractFormatError;
|
|
23
|
+
readonly CsvFormatError: typeof CsvFormatError;
|
|
19
24
|
readonly CollectionError: typeof CollectionError;
|
|
20
25
|
readonly EnvironmentMismatchError: typeof EnvironmentMismatchError;
|
|
26
|
+
readonly ExpectError: typeof ExpectError;
|
|
27
|
+
readonly KnowledgeScrapeError: typeof KnowledgeScrapeError;
|
|
21
28
|
readonly LimitReachedError: typeof LimitReachedError;
|
|
29
|
+
readonly MissingToolsError: typeof MissingToolsError;
|
|
22
30
|
readonly NotFoundError: typeof NotFoundError;
|
|
23
31
|
readonly NotYetImplementedError: typeof NotYetImplementedError;
|
|
24
32
|
readonly ParseError: typeof ParseError;
|
|
@@ -29,4 +37,4 @@ export declare const ERRORS: {
|
|
|
29
37
|
};
|
|
30
38
|
/**
|
|
31
39
|
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
32
|
-
*/
|
|
40
|
+
*/
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { string_markdown } from '../../../types/typeAliases';
|
|
2
|
+
import type { Scraper } from '../Scraper';
|
|
2
3
|
/**
|
|
3
4
|
* Creates a message with all registered scrapers
|
|
4
5
|
*
|
|
@@ -6,7 +7,7 @@ import type { string_markdown } from '../../../types/typeAliases';
|
|
|
6
7
|
*
|
|
7
8
|
* @private internal function of `createScrapersFromConfiguration` and `createScrapersFromEnv`
|
|
8
9
|
*/
|
|
9
|
-
export declare function $registeredScrapersMessage(): string_markdown;
|
|
10
|
+
export declare function $registeredScrapersMessage(availableScrapers: ReadonlyArray<Scraper>): string_markdown;
|
|
10
11
|
/**
|
|
11
12
|
* TODO: [®] DRY Register logic
|
|
12
13
|
*/
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@promptbook/pdf",
|
|
3
|
-
"version": "0.75.
|
|
3
|
+
"version": "0.75.5",
|
|
4
4
|
"description": "It's time for a paradigm shift. The future of software in plain English, French or Latin",
|
|
5
5
|
"--note-0": " <- [🐊]",
|
|
6
6
|
"private": false,
|
|
@@ -54,7 +54,7 @@
|
|
|
54
54
|
"module": "./esm/index.es.js",
|
|
55
55
|
"typings": "./esm/typings/src/_packages/pdf.index.d.ts",
|
|
56
56
|
"peerDependencies": {
|
|
57
|
-
"@promptbook/core": "0.75.
|
|
57
|
+
"@promptbook/core": "0.75.5"
|
|
58
58
|
},
|
|
59
59
|
"dependencies": {
|
|
60
60
|
"crypto-js": "4.2.0",
|
package/umd/index.umd.js
CHANGED
|
@@ -22,7 +22,7 @@
|
|
|
22
22
|
*
|
|
23
23
|
* @see https://github.com/webgptorg/promptbook
|
|
24
24
|
*/
|
|
25
|
-
var PROMPTBOOK_ENGINE_VERSION = '0.75.
|
|
25
|
+
var PROMPTBOOK_ENGINE_VERSION = '0.75.4';
|
|
26
26
|
/**
|
|
27
27
|
* TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
|
|
28
28
|
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
@@ -1799,6 +1799,42 @@
|
|
|
1799
1799
|
return PipelineExecutionError;
|
|
1800
1800
|
}(Error));
|
|
1801
1801
|
|
|
1802
|
+
/**
|
|
1803
|
+
* This error indicates problems parsing the format value
|
|
1804
|
+
*
|
|
1805
|
+
* For example, when the format value is not a valid JSON or CSV
|
|
1806
|
+
* This is not thrown directly but in extended classes
|
|
1807
|
+
*
|
|
1808
|
+
* @public exported from `@promptbook/core`
|
|
1809
|
+
*/
|
|
1810
|
+
var AbstractFormatError = /** @class */ (function (_super) {
|
|
1811
|
+
__extends(AbstractFormatError, _super);
|
|
1812
|
+
// Note: To allow instanceof do not put here error `name`
|
|
1813
|
+
// public readonly name = 'AbstractFormatError';
|
|
1814
|
+
function AbstractFormatError(message) {
|
|
1815
|
+
var _this = _super.call(this, message) || this;
|
|
1816
|
+
Object.setPrototypeOf(_this, AbstractFormatError.prototype);
|
|
1817
|
+
return _this;
|
|
1818
|
+
}
|
|
1819
|
+
return AbstractFormatError;
|
|
1820
|
+
}(Error));
|
|
1821
|
+
|
|
1822
|
+
/**
|
|
1823
|
+
* This error indicates problem with parsing of CSV
|
|
1824
|
+
*
|
|
1825
|
+
* @public exported from `@promptbook/core`
|
|
1826
|
+
*/
|
|
1827
|
+
var CsvFormatError = /** @class */ (function (_super) {
|
|
1828
|
+
__extends(CsvFormatError, _super);
|
|
1829
|
+
function CsvFormatError(message) {
|
|
1830
|
+
var _this = _super.call(this, message) || this;
|
|
1831
|
+
_this.name = 'CsvFormatError';
|
|
1832
|
+
Object.setPrototypeOf(_this, CsvFormatError.prototype);
|
|
1833
|
+
return _this;
|
|
1834
|
+
}
|
|
1835
|
+
return CsvFormatError;
|
|
1836
|
+
}(AbstractFormatError));
|
|
1837
|
+
|
|
1802
1838
|
/**
|
|
1803
1839
|
* This error indicates that the pipeline collection cannot be propperly loaded
|
|
1804
1840
|
*
|
|
@@ -1850,6 +1886,22 @@
|
|
|
1850
1886
|
return ExpectError;
|
|
1851
1887
|
}(Error));
|
|
1852
1888
|
|
|
1889
|
+
/**
|
|
1890
|
+
* This error indicates that the promptbook can not retrieve knowledge from external sources
|
|
1891
|
+
*
|
|
1892
|
+
* @public exported from `@promptbook/core`
|
|
1893
|
+
*/
|
|
1894
|
+
var KnowledgeScrapeError = /** @class */ (function (_super) {
|
|
1895
|
+
__extends(KnowledgeScrapeError, _super);
|
|
1896
|
+
function KnowledgeScrapeError(message) {
|
|
1897
|
+
var _this = _super.call(this, message) || this;
|
|
1898
|
+
_this.name = 'KnowledgeScrapeError';
|
|
1899
|
+
Object.setPrototypeOf(_this, KnowledgeScrapeError.prototype);
|
|
1900
|
+
return _this;
|
|
1901
|
+
}
|
|
1902
|
+
return KnowledgeScrapeError;
|
|
1903
|
+
}(Error));
|
|
1904
|
+
|
|
1853
1905
|
/**
|
|
1854
1906
|
* This error type indicates that some limit was reached
|
|
1855
1907
|
*
|
|
@@ -1872,10 +1924,14 @@
|
|
|
1872
1924
|
* @public exported from `@promptbook/core`
|
|
1873
1925
|
*/
|
|
1874
1926
|
var ERRORS = {
|
|
1875
|
-
|
|
1927
|
+
AbstractFormatError: AbstractFormatError,
|
|
1928
|
+
CsvFormatError: CsvFormatError,
|
|
1876
1929
|
CollectionError: CollectionError,
|
|
1877
1930
|
EnvironmentMismatchError: EnvironmentMismatchError,
|
|
1931
|
+
ExpectError: ExpectError,
|
|
1932
|
+
KnowledgeScrapeError: KnowledgeScrapeError,
|
|
1878
1933
|
LimitReachedError: LimitReachedError,
|
|
1934
|
+
MissingToolsError: MissingToolsError,
|
|
1879
1935
|
NotFoundError: NotFoundError,
|
|
1880
1936
|
NotYetImplementedError: NotYetImplementedError,
|
|
1881
1937
|
ParseError: ParseError,
|
|
@@ -2676,22 +2732,6 @@
|
|
|
2676
2732
|
* TODO: [🏢] !! Check validity of `temperature` in pipeline
|
|
2677
2733
|
*/
|
|
2678
2734
|
|
|
2679
|
-
/**
|
|
2680
|
-
* This error indicates that the promptbook can not retrieve knowledge from external sources
|
|
2681
|
-
*
|
|
2682
|
-
* @public exported from `@promptbook/core`
|
|
2683
|
-
*/
|
|
2684
|
-
var KnowledgeScrapeError = /** @class */ (function (_super) {
|
|
2685
|
-
__extends(KnowledgeScrapeError, _super);
|
|
2686
|
-
function KnowledgeScrapeError(message) {
|
|
2687
|
-
var _this = _super.call(this, message) || this;
|
|
2688
|
-
_this.name = 'KnowledgeScrapeError';
|
|
2689
|
-
Object.setPrototypeOf(_this, KnowledgeScrapeError.prototype);
|
|
2690
|
-
return _this;
|
|
2691
|
-
}
|
|
2692
|
-
return KnowledgeScrapeError;
|
|
2693
|
-
}(Error));
|
|
2694
|
-
|
|
2695
2735
|
/**
|
|
2696
2736
|
* @@@
|
|
2697
2737
|
*
|
|
@@ -2862,8 +2902,8 @@
|
|
|
2862
2902
|
*
|
|
2863
2903
|
* @private internal function of `createScrapersFromConfiguration` and `createScrapersFromEnv`
|
|
2864
2904
|
*/
|
|
2865
|
-
function $registeredScrapersMessage() {
|
|
2866
|
-
var e_1, _a, e_2, _b;
|
|
2905
|
+
function $registeredScrapersMessage(availableScrapers) {
|
|
2906
|
+
var e_1, _a, e_2, _b, e_3, _c;
|
|
2867
2907
|
/**
|
|
2868
2908
|
* Mixes registered scrapers from $scrapersMetadataRegister and $scrapersRegister
|
|
2869
2909
|
*/
|
|
@@ -2875,15 +2915,15 @@
|
|
|
2875
2915
|
all.push({ packageName: packageName, className: className, mimeTypes: mimeTypes, documentationUrl: documentationUrl, isAvilableInBrowser: isAvilableInBrowser });
|
|
2876
2916
|
};
|
|
2877
2917
|
try {
|
|
2878
|
-
for (var
|
|
2879
|
-
var
|
|
2918
|
+
for (var _d = __values($scrapersMetadataRegister.list()), _e = _d.next(); !_e.done; _e = _d.next()) {
|
|
2919
|
+
var _f = _e.value, packageName = _f.packageName, className = _f.className, mimeTypes = _f.mimeTypes, documentationUrl = _f.documentationUrl, isAvilableInBrowser = _f.isAvilableInBrowser;
|
|
2880
2920
|
_loop_1(packageName, className, mimeTypes, documentationUrl, isAvilableInBrowser);
|
|
2881
2921
|
}
|
|
2882
2922
|
}
|
|
2883
2923
|
catch (e_1_1) { e_1 = { error: e_1_1 }; }
|
|
2884
2924
|
finally {
|
|
2885
2925
|
try {
|
|
2886
|
-
if (
|
|
2926
|
+
if (_e && !_e.done && (_a = _d.return)) _a.call(_d);
|
|
2887
2927
|
}
|
|
2888
2928
|
finally { if (e_1) throw e_1.error; }
|
|
2889
2929
|
}
|
|
@@ -2894,18 +2934,31 @@
|
|
|
2894
2934
|
all.push({ packageName: packageName, className: className, mimeTypes: mimeTypes, documentationUrl: documentationUrl, isAvilableInBrowser: isAvilableInBrowser });
|
|
2895
2935
|
};
|
|
2896
2936
|
try {
|
|
2897
|
-
for (var
|
|
2898
|
-
var
|
|
2937
|
+
for (var _g = __values($scrapersRegister.list()), _h = _g.next(); !_h.done; _h = _g.next()) {
|
|
2938
|
+
var _j = _h.value, packageName = _j.packageName, className = _j.className, mimeTypes = _j.mimeTypes, documentationUrl = _j.documentationUrl, isAvilableInBrowser = _j.isAvilableInBrowser;
|
|
2899
2939
|
_loop_2(packageName, className, mimeTypes, documentationUrl, isAvilableInBrowser);
|
|
2900
2940
|
}
|
|
2901
2941
|
}
|
|
2902
2942
|
catch (e_2_1) { e_2 = { error: e_2_1 }; }
|
|
2903
2943
|
finally {
|
|
2904
2944
|
try {
|
|
2905
|
-
if (
|
|
2945
|
+
if (_h && !_h.done && (_b = _g.return)) _b.call(_g);
|
|
2906
2946
|
}
|
|
2907
2947
|
finally { if (e_2) throw e_2.error; }
|
|
2908
2948
|
}
|
|
2949
|
+
try {
|
|
2950
|
+
for (var availableScrapers_1 = __values(availableScrapers), availableScrapers_1_1 = availableScrapers_1.next(); !availableScrapers_1_1.done; availableScrapers_1_1 = availableScrapers_1.next()) {
|
|
2951
|
+
var metadata_1 = availableScrapers_1_1.value.metadata;
|
|
2952
|
+
all.push(metadata_1);
|
|
2953
|
+
}
|
|
2954
|
+
}
|
|
2955
|
+
catch (e_3_1) { e_3 = { error: e_3_1 }; }
|
|
2956
|
+
finally {
|
|
2957
|
+
try {
|
|
2958
|
+
if (availableScrapers_1_1 && !availableScrapers_1_1.done && (_c = availableScrapers_1.return)) _c.call(availableScrapers_1);
|
|
2959
|
+
}
|
|
2960
|
+
finally { if (e_3) throw e_3.error; }
|
|
2961
|
+
}
|
|
2909
2962
|
var metadata = all.map(function (metadata) {
|
|
2910
2963
|
var isMetadataAviailable = $scrapersMetadataRegister
|
|
2911
2964
|
.list()
|
|
@@ -2919,42 +2972,44 @@
|
|
|
2919
2972
|
var packageName = _a.packageName, className = _a.className;
|
|
2920
2973
|
return metadata.packageName === packageName && metadata.className === className;
|
|
2921
2974
|
});
|
|
2922
|
-
|
|
2975
|
+
var isAvilableInTools = availableScrapers.some(function (_a) {
|
|
2976
|
+
var _b = _a.metadata, packageName = _b.packageName, className = _b.className;
|
|
2977
|
+
return metadata.packageName === packageName && metadata.className === className;
|
|
2978
|
+
});
|
|
2979
|
+
return __assign(__assign({}, metadata), { isMetadataAviailable: isMetadataAviailable, isInstalled: isInstalled, isAvilableInTools: isAvilableInTools });
|
|
2923
2980
|
});
|
|
2924
2981
|
if (metadata.length === 0) {
|
|
2925
|
-
return "No scrapers are available";
|
|
2982
|
+
return spaceTrim__default["default"]("\n **No scrapers are available**\n\n This is a unexpected behavior, you are probably using some broken version of Promptbook\n At least there should be available the metadata of the scrapers\n ");
|
|
2926
2983
|
}
|
|
2927
2984
|
return spaceTrim__default["default"](function (block) { return "\n Available scrapers are:\n ".concat(block(metadata
|
|
2928
2985
|
.map(function (_a, i) {
|
|
2929
|
-
var packageName = _a.packageName, className = _a.className, isMetadataAviailable = _a.isMetadataAviailable, isInstalled = _a.isInstalled, mimeTypes = _a.mimeTypes, isAvilableInBrowser = _a.isAvilableInBrowser;
|
|
2930
|
-
var more;
|
|
2931
|
-
// TODO:
|
|
2932
|
-
if (
|
|
2933
|
-
more
|
|
2934
|
-
}
|
|
2935
|
-
|
|
2936
|
-
|
|
2937
|
-
|
|
2938
|
-
|
|
2939
|
-
|
|
2940
|
-
|
|
2941
|
-
|
|
2942
|
-
|
|
2943
|
-
|
|
2944
|
-
|
|
2945
|
-
|
|
2946
|
-
|
|
2947
|
-
more = "(installed)";
|
|
2948
|
-
}
|
|
2949
|
-
else {
|
|
2950
|
-
more = "*(unknown state, looks like a unexpected behavior)*";
|
|
2951
|
-
}
|
|
2986
|
+
var packageName = _a.packageName, className = _a.className, isMetadataAviailable = _a.isMetadataAviailable, isInstalled = _a.isInstalled, mimeTypes = _a.mimeTypes, isAvilableInBrowser = _a.isAvilableInBrowser, isAvilableInTools = _a.isAvilableInTools;
|
|
2987
|
+
var more = [];
|
|
2988
|
+
// TODO: [🧠] Maybe use `documentationUrl`
|
|
2989
|
+
if (isMetadataAviailable) {
|
|
2990
|
+
more.push("\u2B1C Metadata registered");
|
|
2991
|
+
} // not else
|
|
2992
|
+
if (isInstalled) {
|
|
2993
|
+
more.push("\uD83D\uDFE9 Installed");
|
|
2994
|
+
} // not else
|
|
2995
|
+
if (isAvilableInTools) {
|
|
2996
|
+
more.push("\uD83D\uDFE6 Available in tools");
|
|
2997
|
+
} // not else
|
|
2998
|
+
if (!isMetadataAviailable && isInstalled) {
|
|
2999
|
+
more.push("When no metadata registered but scraper is installed, it is an unexpected behavior");
|
|
3000
|
+
} // not else
|
|
3001
|
+
if (!isInstalled && isAvilableInTools) {
|
|
3002
|
+
more.push("When the scraper is not installed but available in tools, it is an unexpected compatibility behavior");
|
|
3003
|
+
} // not else
|
|
2952
3004
|
if (!isAvilableInBrowser) {
|
|
2953
|
-
more
|
|
3005
|
+
more.push("Not usable in browser");
|
|
2954
3006
|
}
|
|
2955
|
-
|
|
3007
|
+
var moreText = more.length === 0 ? '' : " *(".concat(more.join('; '), ")*");
|
|
3008
|
+
return "".concat(i + 1, ") `").concat(className, "` from `").concat(packageName, "` compatible to scrape ").concat(mimeTypes
|
|
3009
|
+
.map(function (mimeType) { return "\"".concat(mimeType, "\""); })
|
|
3010
|
+
.join(', ')).concat(moreText);
|
|
2956
3011
|
})
|
|
2957
|
-
.join('\n')), "\n "); });
|
|
3012
|
+
.join('\n')), "\n\n Legend:\n - \u2B1C **Metadata registered** means that Promptbook knows about the scraper, it is similar to registration in some registry\n - \uD83D\uDFE9 **Installed** means that you have imported package with particular scraper\n - \uD83D\uDFE6 **Available in tools** means that you have passed scraper as dependency into prepare or execution process\n\n "); });
|
|
2958
3013
|
}
|
|
2959
3014
|
/**
|
|
2960
3015
|
* TODO: [®] DRY Register logic
|
|
@@ -3202,57 +3257,75 @@
|
|
|
3202
3257
|
_a = options.maxParallelCount, maxParallelCount = _a === void 0 ? DEFAULT_MAX_PARALLEL_COUNT : _a, rootDirname = options.rootDirname, _b = options.isVerbose, isVerbose = _b === void 0 ? DEFAULT_IS_VERBOSE : _b;
|
|
3203
3258
|
knowledgePreparedUnflatten = new Array(knowledgeSources.length);
|
|
3204
3259
|
return [4 /*yield*/, forEachAsync(knowledgeSources, { maxParallelCount: maxParallelCount }, function (knowledgeSource, index) { return __awaiter(_this, void 0, void 0, function () {
|
|
3205
|
-
var partialPieces, sourceHandler,
|
|
3206
|
-
var e_1,
|
|
3207
|
-
return __generator(this, function (
|
|
3208
|
-
switch (
|
|
3260
|
+
var partialPieces, sourceHandler, scrapers, _loop_1, scrapers_1, scrapers_1_1, scraper, state_1, e_1_1, pieces;
|
|
3261
|
+
var e_1, _a;
|
|
3262
|
+
return __generator(this, function (_b) {
|
|
3263
|
+
switch (_b.label) {
|
|
3209
3264
|
case 0:
|
|
3210
3265
|
partialPieces = null;
|
|
3211
3266
|
return [4 /*yield*/, makeKnowledgeSourceHandler(knowledgeSource, tools, { rootDirname: rootDirname, isVerbose: isVerbose })];
|
|
3212
3267
|
case 1:
|
|
3213
|
-
sourceHandler =
|
|
3214
|
-
|
|
3268
|
+
sourceHandler = _b.sent();
|
|
3269
|
+
scrapers = arrayableToArray(tools.scrapers);
|
|
3270
|
+
_loop_1 = function (scraper) {
|
|
3271
|
+
var partialPiecesUnchecked;
|
|
3272
|
+
return __generator(this, function (_c) {
|
|
3273
|
+
switch (_c.label) {
|
|
3274
|
+
case 0:
|
|
3275
|
+
if (!scraper.metadata.mimeTypes.includes(sourceHandler.mimeType)
|
|
3276
|
+
// <- TODO: [🦔] Implement mime-type wildcards
|
|
3277
|
+
) {
|
|
3278
|
+
return [2 /*return*/, "continue"];
|
|
3279
|
+
}
|
|
3280
|
+
return [4 /*yield*/, scraper.scrape(sourceHandler)];
|
|
3281
|
+
case 1:
|
|
3282
|
+
partialPiecesUnchecked = _c.sent();
|
|
3283
|
+
if (partialPiecesUnchecked !== null) {
|
|
3284
|
+
partialPieces = __spreadArray([], __read(partialPiecesUnchecked), false);
|
|
3285
|
+
return [2 /*return*/, "break"];
|
|
3286
|
+
}
|
|
3287
|
+
console.warn(spaceTrim__default["default"](function (block) { return "\n Cannot scrape knowledge from source despite the scraper `".concat(scraper.metadata.className, "` supports the mime type \"").concat(sourceHandler.mimeType, "\".\n \n The source:\n > ").concat(block(knowledgeSource.sourceContent
|
|
3288
|
+
.split('\n')
|
|
3289
|
+
.map(function (line) { return "> ".concat(line); })
|
|
3290
|
+
.join('\n')), "\n\n ").concat(block($registeredScrapersMessage(scrapers)), "\n\n\n "); }));
|
|
3291
|
+
return [2 /*return*/];
|
|
3292
|
+
}
|
|
3293
|
+
});
|
|
3294
|
+
};
|
|
3295
|
+
_b.label = 2;
|
|
3215
3296
|
case 2:
|
|
3216
|
-
|
|
3217
|
-
|
|
3218
|
-
|
|
3297
|
+
_b.trys.push([2, 7, 8, 9]);
|
|
3298
|
+
scrapers_1 = __values(scrapers), scrapers_1_1 = scrapers_1.next();
|
|
3299
|
+
_b.label = 3;
|
|
3219
3300
|
case 3:
|
|
3220
|
-
if (!!
|
|
3221
|
-
scraper =
|
|
3222
|
-
|
|
3223
|
-
// <- TODO: [🦔] Implement mime-type wildcards
|
|
3224
|
-
) {
|
|
3225
|
-
return [3 /*break*/, 5];
|
|
3226
|
-
}
|
|
3227
|
-
return [4 /*yield*/, scraper.scrape(sourceHandler)];
|
|
3301
|
+
if (!!scrapers_1_1.done) return [3 /*break*/, 6];
|
|
3302
|
+
scraper = scrapers_1_1.value;
|
|
3303
|
+
return [5 /*yield**/, _loop_1(scraper)];
|
|
3228
3304
|
case 4:
|
|
3229
|
-
|
|
3230
|
-
if (
|
|
3231
|
-
partialPieces = __spreadArray([], __read(partialPiecesUnchecked), false);
|
|
3232
|
-
// <- TODO: [🪓] Here should be no need for spreading new array, just `partialPieces = partialPiecesUnchecked`
|
|
3305
|
+
state_1 = _b.sent();
|
|
3306
|
+
if (state_1 === "break")
|
|
3233
3307
|
return [3 /*break*/, 6];
|
|
3234
|
-
|
|
3235
|
-
_d.label = 5;
|
|
3308
|
+
_b.label = 5;
|
|
3236
3309
|
case 5:
|
|
3237
|
-
|
|
3310
|
+
scrapers_1_1 = scrapers_1.next();
|
|
3238
3311
|
return [3 /*break*/, 3];
|
|
3239
3312
|
case 6: return [3 /*break*/, 9];
|
|
3240
3313
|
case 7:
|
|
3241
|
-
e_1_1 =
|
|
3314
|
+
e_1_1 = _b.sent();
|
|
3242
3315
|
e_1 = { error: e_1_1 };
|
|
3243
3316
|
return [3 /*break*/, 9];
|
|
3244
3317
|
case 8:
|
|
3245
3318
|
try {
|
|
3246
|
-
if (
|
|
3319
|
+
if (scrapers_1_1 && !scrapers_1_1.done && (_a = scrapers_1.return)) _a.call(scrapers_1);
|
|
3247
3320
|
}
|
|
3248
3321
|
finally { if (e_1) throw e_1.error; }
|
|
3249
3322
|
return [7 /*endfinally*/];
|
|
3250
3323
|
case 9:
|
|
3251
3324
|
if (partialPieces === null) {
|
|
3252
|
-
throw new KnowledgeScrapeError(spaceTrim__default["default"](function (block) { return "\n Cannot scrape knowledge
|
|
3325
|
+
throw new KnowledgeScrapeError(spaceTrim__default["default"](function (block) { return "\n Cannot scrape knowledge\n \n The source:\n > ".concat(block(knowledgeSource.sourceContent
|
|
3253
3326
|
.split('\n')
|
|
3254
3327
|
.map(function (line) { return "> ".concat(line); })
|
|
3255
|
-
.join('\n')), "\n\n No scraper found for the mime type \"").concat(sourceHandler.mimeType, "\"\n\n ").concat(block($registeredScrapersMessage()), "\n\n\n "); }));
|
|
3328
|
+
.join('\n')), "\n\n No scraper found for the mime type \"").concat(sourceHandler.mimeType, "\"\n\n ").concat(block($registeredScrapersMessage(scrapers)), "\n\n\n "); }));
|
|
3256
3329
|
}
|
|
3257
3330
|
pieces = partialPieces.map(function (partialPiece) { return (__assign(__assign({}, partialPiece), { sources: [
|
|
3258
3331
|
{
|
|
@@ -3678,42 +3751,6 @@
|
|
|
3678
3751
|
return union;
|
|
3679
3752
|
}
|
|
3680
3753
|
|
|
3681
|
-
/**
|
|
3682
|
-
* This error indicates problems parsing the format value
|
|
3683
|
-
*
|
|
3684
|
-
* For example, when the format value is not a valid JSON or CSV
|
|
3685
|
-
* This is not thrown directly but in extended classes
|
|
3686
|
-
*
|
|
3687
|
-
* @public exported from `@promptbook/core`
|
|
3688
|
-
*/
|
|
3689
|
-
var AbstractFormatError = /** @class */ (function (_super) {
|
|
3690
|
-
__extends(AbstractFormatError, _super);
|
|
3691
|
-
// Note: To allow instanceof do not put here error `name`
|
|
3692
|
-
// public readonly name = 'AbstractFormatError';
|
|
3693
|
-
function AbstractFormatError(message) {
|
|
3694
|
-
var _this = _super.call(this, message) || this;
|
|
3695
|
-
Object.setPrototypeOf(_this, AbstractFormatError.prototype);
|
|
3696
|
-
return _this;
|
|
3697
|
-
}
|
|
3698
|
-
return AbstractFormatError;
|
|
3699
|
-
}(Error));
|
|
3700
|
-
|
|
3701
|
-
/**
|
|
3702
|
-
* This error indicates problem with parsing of CSV
|
|
3703
|
-
*
|
|
3704
|
-
* @public exported from `@promptbook/core`
|
|
3705
|
-
*/
|
|
3706
|
-
var CsvFormatError = /** @class */ (function (_super) {
|
|
3707
|
-
__extends(CsvFormatError, _super);
|
|
3708
|
-
function CsvFormatError(message) {
|
|
3709
|
-
var _this = _super.call(this, message) || this;
|
|
3710
|
-
_this.name = 'CsvFormatError';
|
|
3711
|
-
Object.setPrototypeOf(_this, CsvFormatError.prototype);
|
|
3712
|
-
return _this;
|
|
3713
|
-
}
|
|
3714
|
-
return CsvFormatError;
|
|
3715
|
-
}(AbstractFormatError));
|
|
3716
|
-
|
|
3717
3754
|
/**
|
|
3718
3755
|
* @@@
|
|
3719
3756
|
*
|
|
@@ -3754,7 +3791,7 @@
|
|
|
3754
3791
|
case 0:
|
|
3755
3792
|
csv = papaparse.parse(value, __assign(__assign({}, settings), MANDATORY_CSV_SETTINGS));
|
|
3756
3793
|
if (csv.errors.length !== 0) {
|
|
3757
|
-
throw new CsvFormatError(spaceTrim__default["default"](function (block) { return "\n CSV parsing error\n\n ".concat(block(csv.errors.map(function (error) { return error.message; }).join('\n\n')), "\n "); }));
|
|
3794
|
+
throw new CsvFormatError(spaceTrim__default["default"](function (block) { return "\n CSV parsing error\n\n Error(s) from CSV parsing:\n ".concat(block(csv.errors.map(function (error) { return error.message; }).join('\n\n')), "\n\n The CSV data:\n ").concat(block(value), "\n "); }));
|
|
3758
3795
|
}
|
|
3759
3796
|
return [4 /*yield*/, Promise.all(csv.data.map(function (row, index) { return __awaiter(_this, void 0, void 0, function () {
|
|
3760
3797
|
var _a, _b;
|
|
@@ -3792,7 +3829,7 @@
|
|
|
3792
3829
|
case 0:
|
|
3793
3830
|
csv = papaparse.parse(value, __assign(__assign({}, settings), MANDATORY_CSV_SETTINGS));
|
|
3794
3831
|
if (csv.errors.length !== 0) {
|
|
3795
|
-
throw new CsvFormatError(spaceTrim__default["default"](function (block) { return "\n CSV parsing error\n\n ".concat(block(csv.errors.map(function (error) { return error.message; }).join('\n\n')), "\n "); }));
|
|
3832
|
+
throw new CsvFormatError(spaceTrim__default["default"](function (block) { return "\n CSV parsing error\n\n Error(s) from CSV parsing:\n ".concat(block(csv.errors.map(function (error) { return error.message; }).join('\n\n')), "\n\n The CSV data:\n ").concat(block(value), "\n "); }));
|
|
3796
3833
|
}
|
|
3797
3834
|
return [4 /*yield*/, Promise.all(csv.data.map(function (row, rowIndex) { return __awaiter(_this, void 0, void 0, function () {
|
|
3798
3835
|
var _this = this;
|