@promptbook/core 0.84.0-10 → 0.84.0-12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/esm/index.es.js +239 -191
- package/esm/index.es.js.map +1 -1
- package/esm/typings/src/_packages/core.index.d.ts +4 -2
- package/esm/typings/src/_packages/utils.index.d.ts +2 -0
- package/esm/typings/src/config.d.ts +9 -1
- package/esm/typings/src/execution/FilesystemTools.d.ts +1 -1
- package/esm/typings/src/execution/assertsExecutionSuccessful.d.ts +3 -1
- package/esm/typings/src/pipeline/book-notation.d.ts +3 -2
- package/esm/typings/src/pipeline/prompt-notation.d.ts +18 -5
- package/esm/typings/src/wizzard/wizzard.d.ts +7 -1
- package/package.json +1 -1
- package/umd/index.umd.js +242 -193
- package/umd/index.umd.js.map +1 -1
package/esm/index.es.js
CHANGED
|
@@ -3,13 +3,13 @@ import { format } from 'prettier';
|
|
|
3
3
|
import parserHtml from 'prettier/parser-html';
|
|
4
4
|
import { forTime } from 'waitasecond';
|
|
5
5
|
import { unparse, parse } from 'papaparse';
|
|
6
|
-
import { join, basename } from 'path';
|
|
7
|
-
import { SHA256 } from 'crypto-js';
|
|
8
6
|
import hexEncoder from 'crypto-js/enc-hex';
|
|
7
|
+
import sha256 from 'crypto-js/sha256';
|
|
8
|
+
import { basename, join, dirname } from 'path';
|
|
9
|
+
import { SHA256 } from 'crypto-js';
|
|
9
10
|
import { lookup } from 'mime-types';
|
|
10
11
|
import moment from 'moment';
|
|
11
12
|
import colors from 'colors';
|
|
12
|
-
import sha256 from 'crypto-js/sha256';
|
|
13
13
|
|
|
14
14
|
// ⚠️ WARNING: This code has been generated so that any manual changes will be overwritten
|
|
15
15
|
/**
|
|
@@ -25,7 +25,7 @@ var BOOK_LANGUAGE_VERSION = '1.0.0';
|
|
|
25
25
|
* @generated
|
|
26
26
|
* @see https://github.com/webgptorg/promptbook
|
|
27
27
|
*/
|
|
28
|
-
var PROMPTBOOK_ENGINE_VERSION = '0.84.0-
|
|
28
|
+
var PROMPTBOOK_ENGINE_VERSION = '0.84.0-11';
|
|
29
29
|
/**
|
|
30
30
|
* TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
|
|
31
31
|
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
@@ -178,22 +178,94 @@ function collectionToJson(collection) {
|
|
|
178
178
|
*/
|
|
179
179
|
|
|
180
180
|
/**
|
|
181
|
-
*
|
|
181
|
+
* Checks if value is valid email
|
|
182
182
|
*
|
|
183
183
|
* @public exported from `@promptbook/utils`
|
|
184
184
|
*/
|
|
185
|
-
function
|
|
186
|
-
|
|
187
|
-
|
|
185
|
+
function isValidEmail(email) {
|
|
186
|
+
if (typeof email !== 'string') {
|
|
187
|
+
return false;
|
|
188
|
+
}
|
|
189
|
+
if (email.split('\n').length > 1) {
|
|
190
|
+
return false;
|
|
191
|
+
}
|
|
192
|
+
return /^.+@.+\..+$/.test(email);
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
/**
|
|
196
|
+
* Tests if given string is valid URL.
|
|
197
|
+
*
|
|
198
|
+
* Note: This does not check if the file exists only if the path is valid
|
|
199
|
+
* @public exported from `@promptbook/utils`
|
|
200
|
+
*/
|
|
201
|
+
function isValidFilePath(filename) {
|
|
202
|
+
if (typeof filename !== 'string') {
|
|
203
|
+
return false;
|
|
204
|
+
}
|
|
205
|
+
if (filename.split('\n').length > 1) {
|
|
206
|
+
return false;
|
|
207
|
+
}
|
|
208
|
+
if (filename.split(' ').length >
|
|
209
|
+
5 /* <- TODO: [🧠][🈷] Make some better non-arbitrary way how to distinct filenames from informational texts */) {
|
|
210
|
+
return false;
|
|
211
|
+
}
|
|
212
|
+
var filenameSlashes = filename.split('\\').join('/');
|
|
213
|
+
// Absolute Unix path: /hello.txt
|
|
214
|
+
if (/^(\/)/i.test(filenameSlashes)) {
|
|
215
|
+
// console.log(filename, 'Absolute Unix path: /hello.txt');
|
|
188
216
|
return true;
|
|
189
217
|
}
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
218
|
+
// Absolute Windows path: /hello.txt
|
|
219
|
+
if (/^([A-Z]{1,2}:\/?)\//i.test(filenameSlashes)) {
|
|
220
|
+
// console.log(filename, 'Absolute Windows path: /hello.txt');
|
|
221
|
+
return true;
|
|
222
|
+
}
|
|
223
|
+
// Relative path: ./hello.txt
|
|
224
|
+
if (/^(\.\.?\/)+/i.test(filenameSlashes)) {
|
|
225
|
+
// console.log(filename, 'Relative path: ./hello.txt');
|
|
226
|
+
return true;
|
|
227
|
+
}
|
|
228
|
+
// Allow paths like foo/hello
|
|
229
|
+
if (/^[^/]+\/[^/]+/i.test(filenameSlashes)) {
|
|
230
|
+
// console.log(filename, 'Allow paths like foo/hello');
|
|
231
|
+
return true;
|
|
232
|
+
}
|
|
233
|
+
// Allow paths like hello.book
|
|
234
|
+
if (/^[^/]+\.[^/]+$/i.test(filenameSlashes)) {
|
|
235
|
+
// console.log(filename, 'Allow paths like hello.book');
|
|
236
|
+
return true;
|
|
237
|
+
}
|
|
238
|
+
return false;
|
|
239
|
+
}
|
|
240
|
+
/**
|
|
241
|
+
* TODO: [🍏] Implement for MacOs
|
|
242
|
+
*/
|
|
243
|
+
|
|
244
|
+
/**
|
|
245
|
+
* Tests if given string is valid URL.
|
|
246
|
+
*
|
|
247
|
+
* Note: Dataurl are considered perfectly valid.
|
|
248
|
+
* Note: There are two simmilar functions:
|
|
249
|
+
* - `isValidUrl` which tests any URL
|
|
250
|
+
* - `isValidPipelineUrl` *(this one)* which tests just promptbook URL
|
|
251
|
+
*
|
|
252
|
+
* @public exported from `@promptbook/utils`
|
|
253
|
+
*/
|
|
254
|
+
function isValidUrl(url) {
|
|
255
|
+
if (typeof url !== 'string') {
|
|
256
|
+
return false;
|
|
257
|
+
}
|
|
258
|
+
try {
|
|
259
|
+
if (url.startsWith('blob:')) {
|
|
260
|
+
url = url.replace(/^blob:/, '');
|
|
193
261
|
}
|
|
194
|
-
|
|
262
|
+
var urlObject = new URL(url /* because fail is handled */);
|
|
263
|
+
if (!['http:', 'https:', 'data:'].includes(urlObject.protocol)) {
|
|
195
264
|
return false;
|
|
196
265
|
}
|
|
266
|
+
return true;
|
|
267
|
+
}
|
|
268
|
+
catch (error) {
|
|
197
269
|
return false;
|
|
198
270
|
}
|
|
199
271
|
}
|
|
@@ -217,6 +289,27 @@ var ParseError = /** @class */ (function (_super) {
|
|
|
217
289
|
* TODO: Maybe split `ParseError` and `ApplyError`
|
|
218
290
|
*/
|
|
219
291
|
|
|
292
|
+
/**
|
|
293
|
+
* Function isValidJsonString will tell you if the string is valid JSON or not
|
|
294
|
+
*
|
|
295
|
+
* @public exported from `@promptbook/utils`
|
|
296
|
+
*/
|
|
297
|
+
function isValidJsonString(value /* <- [👨⚖️] */) {
|
|
298
|
+
try {
|
|
299
|
+
JSON.parse(value);
|
|
300
|
+
return true;
|
|
301
|
+
}
|
|
302
|
+
catch (error) {
|
|
303
|
+
if (!(error instanceof Error)) {
|
|
304
|
+
throw error;
|
|
305
|
+
}
|
|
306
|
+
if (error.message.includes('Unexpected token')) {
|
|
307
|
+
return false;
|
|
308
|
+
}
|
|
309
|
+
return false;
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
|
|
220
313
|
/**
|
|
221
314
|
* Function `validatePipelineString` will validate the if the string is a valid pipeline string
|
|
222
315
|
* It does not check if the string is fully logically correct, but if it is a string that can be a pipeline string or the string looks completely different.
|
|
@@ -230,6 +323,15 @@ function validatePipelineString(pipelineString) {
|
|
|
230
323
|
if (isValidJsonString(pipelineString)) {
|
|
231
324
|
throw new ParseError('Expected a book, but got a JSON string');
|
|
232
325
|
}
|
|
326
|
+
else if (isValidUrl(pipelineString)) {
|
|
327
|
+
throw new ParseError("Expected a book, but got just the URL \"".concat(pipelineString, "\""));
|
|
328
|
+
}
|
|
329
|
+
else if (isValidFilePath(pipelineString)) {
|
|
330
|
+
throw new ParseError("Expected a book, but got just the file path \"".concat(pipelineString, "\""));
|
|
331
|
+
}
|
|
332
|
+
else if (isValidEmail(pipelineString)) {
|
|
333
|
+
throw new ParseError("Expected a book, but got just the email \"".concat(pipelineString, "\""));
|
|
334
|
+
}
|
|
233
335
|
// <- TODO: Implement the validation + add tests when the pipeline logic considered as invalid
|
|
234
336
|
return pipelineString;
|
|
235
337
|
}
|
|
@@ -650,6 +752,14 @@ var DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_TOTAL = 200;
|
|
|
650
752
|
*/
|
|
651
753
|
var DEFAULT_BOOKS_DIRNAME = './books';
|
|
652
754
|
// <- TODO: [🕝] Make also `BOOKS_DIRNAME_ALTERNATIVES`
|
|
755
|
+
/**
|
|
756
|
+
* Where to store the temporary downloads
|
|
757
|
+
*
|
|
758
|
+
* Note: When the folder does not exist, it is created recursively
|
|
759
|
+
*
|
|
760
|
+
* @public exported from `@promptbook/core`
|
|
761
|
+
*/
|
|
762
|
+
var DEFAULT_DOWNLOAD_CACHE_DIRNAME = './.promptbook/download-cache';
|
|
653
763
|
/**
|
|
654
764
|
* Where to store the cache of executions for promptbook CLI
|
|
655
765
|
*
|
|
@@ -657,7 +767,7 @@ var DEFAULT_BOOKS_DIRNAME = './books';
|
|
|
657
767
|
*
|
|
658
768
|
* @public exported from `@promptbook/core`
|
|
659
769
|
*/
|
|
660
|
-
var
|
|
770
|
+
var DEFAULT_EXECUTION_CACHE_DIRNAME = './.promptbook/execution-cache';
|
|
661
771
|
/**
|
|
662
772
|
* Where to store the scrape cache
|
|
663
773
|
*
|
|
@@ -1183,35 +1293,6 @@ function isUrlOnPrivateNetwork(url) {
|
|
|
1183
1293
|
return isHostnameOnPrivateNetwork(url.hostname);
|
|
1184
1294
|
}
|
|
1185
1295
|
|
|
1186
|
-
/**
|
|
1187
|
-
* Tests if given string is valid URL.
|
|
1188
|
-
*
|
|
1189
|
-
* Note: Dataurl are considered perfectly valid.
|
|
1190
|
-
* Note: There are two simmilar functions:
|
|
1191
|
-
* - `isValidUrl` which tests any URL
|
|
1192
|
-
* - `isValidPipelineUrl` *(this one)* which tests just promptbook URL
|
|
1193
|
-
*
|
|
1194
|
-
* @public exported from `@promptbook/utils`
|
|
1195
|
-
*/
|
|
1196
|
-
function isValidUrl(url) {
|
|
1197
|
-
if (typeof url !== 'string') {
|
|
1198
|
-
return false;
|
|
1199
|
-
}
|
|
1200
|
-
try {
|
|
1201
|
-
if (url.startsWith('blob:')) {
|
|
1202
|
-
url = url.replace(/^blob:/, '');
|
|
1203
|
-
}
|
|
1204
|
-
var urlObject = new URL(url /* because fail is handled */);
|
|
1205
|
-
if (!['http:', 'https:', 'data:'].includes(urlObject.protocol)) {
|
|
1206
|
-
return false;
|
|
1207
|
-
}
|
|
1208
|
-
return true;
|
|
1209
|
-
}
|
|
1210
|
-
catch (error) {
|
|
1211
|
-
return false;
|
|
1212
|
-
}
|
|
1213
|
-
}
|
|
1214
|
-
|
|
1215
1296
|
/**
|
|
1216
1297
|
* Tests if given string is valid pipeline URL URL.
|
|
1217
1298
|
*
|
|
@@ -2170,12 +2251,28 @@ function deserializeError(error) {
|
|
|
2170
2251
|
/**
|
|
2171
2252
|
* Asserts that the execution of a Promptbook is successful
|
|
2172
2253
|
*
|
|
2254
|
+
* Note: If there are only warnings, the execution is still successful but the warnings are logged in the console
|
|
2255
|
+
*
|
|
2173
2256
|
* @param executionResult - The partial result of the Promptbook execution
|
|
2174
2257
|
* @throws {PipelineExecutionError} If the execution is not successful or if multiple errors occurred
|
|
2175
2258
|
* @public exported from `@promptbook/core`
|
|
2176
2259
|
*/
|
|
2177
2260
|
function assertsExecutionSuccessful(executionResult) {
|
|
2178
|
-
var
|
|
2261
|
+
var e_1, _a;
|
|
2262
|
+
var isSuccessful = executionResult.isSuccessful, errors = executionResult.errors, warnings = executionResult.warnings;
|
|
2263
|
+
try {
|
|
2264
|
+
for (var warnings_1 = __values(warnings), warnings_1_1 = warnings_1.next(); !warnings_1_1.done; warnings_1_1 = warnings_1.next()) {
|
|
2265
|
+
var warning = warnings_1_1.value;
|
|
2266
|
+
console.warn(warning.message);
|
|
2267
|
+
}
|
|
2268
|
+
}
|
|
2269
|
+
catch (e_1_1) { e_1 = { error: e_1_1 }; }
|
|
2270
|
+
finally {
|
|
2271
|
+
try {
|
|
2272
|
+
if (warnings_1_1 && !warnings_1_1.done && (_a = warnings_1.return)) _a.call(warnings_1);
|
|
2273
|
+
}
|
|
2274
|
+
finally { if (e_1) throw e_1.error; }
|
|
2275
|
+
}
|
|
2179
2276
|
if (isSuccessful === true) {
|
|
2180
2277
|
return;
|
|
2181
2278
|
}
|
|
@@ -5577,6 +5674,22 @@ function $registeredScrapersMessage(availableScrapers) {
|
|
|
5577
5674
|
* TODO: [®] DRY Register logic
|
|
5578
5675
|
*/
|
|
5579
5676
|
|
|
5677
|
+
/**
|
|
5678
|
+
* Removes emojis from a string and fix whitespaces
|
|
5679
|
+
*
|
|
5680
|
+
* @param text with emojis
|
|
5681
|
+
* @returns text without emojis
|
|
5682
|
+
* @public exported from `@promptbook/utils`
|
|
5683
|
+
*/
|
|
5684
|
+
function removeEmojis(text) {
|
|
5685
|
+
// Replace emojis (and also ZWJ sequence) with hyphens
|
|
5686
|
+
text = text.replace(/(\p{Extended_Pictographic})\p{Modifier_Symbol}/gu, '$1');
|
|
5687
|
+
text = text.replace(/(\p{Extended_Pictographic})[\u{FE00}-\u{FE0F}]/gu, '$1');
|
|
5688
|
+
text = text.replace(/(\p{Extended_Pictographic})(\u{200D}\p{Extended_Pictographic})*/gu, '$1');
|
|
5689
|
+
text = text.replace(/\p{Extended_Pictographic}/gu, '');
|
|
5690
|
+
return text;
|
|
5691
|
+
}
|
|
5692
|
+
|
|
5580
5693
|
/**
|
|
5581
5694
|
* @@@
|
|
5582
5695
|
*
|
|
@@ -5639,6 +5752,30 @@ function normalizeToKebabCase(text) {
|
|
|
5639
5752
|
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
5640
5753
|
*/
|
|
5641
5754
|
|
|
5755
|
+
/**
|
|
5756
|
+
* @@@
|
|
5757
|
+
*
|
|
5758
|
+
* @param value @@@
|
|
5759
|
+
* @returns @@@
|
|
5760
|
+
* @example @@@
|
|
5761
|
+
* @public exported from `@promptbook/utils`
|
|
5762
|
+
*/
|
|
5763
|
+
function titleToName(value) {
|
|
5764
|
+
if (isValidUrl(value)) {
|
|
5765
|
+
value = value.replace(/^https?:\/\//, '');
|
|
5766
|
+
value = value.replace(/\.html$/, '');
|
|
5767
|
+
}
|
|
5768
|
+
else if (isValidFilePath(value)) {
|
|
5769
|
+
value = basename(value);
|
|
5770
|
+
// Note: Keeping extension in the name
|
|
5771
|
+
}
|
|
5772
|
+
value = value.split('/').join('-');
|
|
5773
|
+
value = removeEmojis(value);
|
|
5774
|
+
value = normalizeToKebabCase(value);
|
|
5775
|
+
// TODO: [🧠] Maybe warn or add some padding to short name which are not good identifiers
|
|
5776
|
+
return value;
|
|
5777
|
+
}
|
|
5778
|
+
|
|
5642
5779
|
/**
|
|
5643
5780
|
* Creates unique name for the source
|
|
5644
5781
|
*
|
|
@@ -5660,6 +5797,15 @@ function knowledgeSourceContentToName(knowledgeSourceContent) {
|
|
|
5660
5797
|
* TODO: [🐱🐉][🧠] Make some smart crop NOT source-i-m-pavol-a-develop-... BUT source-i-m-pavol-a-developer-...
|
|
5661
5798
|
*/
|
|
5662
5799
|
|
|
5800
|
+
/**
|
|
5801
|
+
* @@@
|
|
5802
|
+
*
|
|
5803
|
+
* @private for `FileCacheStorage`
|
|
5804
|
+
*/
|
|
5805
|
+
function nameToSubfolderPath(name) {
|
|
5806
|
+
return [name.substr(0, 1).toLowerCase(), name.substr(1, 1).toLowerCase()];
|
|
5807
|
+
}
|
|
5808
|
+
|
|
5663
5809
|
/**
|
|
5664
5810
|
* Convert file extension to mime type
|
|
5665
5811
|
*
|
|
@@ -5715,55 +5861,6 @@ function isFileExisting(filename, fs) {
|
|
|
5715
5861
|
* TODO: [🖇] What about symlinks?
|
|
5716
5862
|
*/
|
|
5717
5863
|
|
|
5718
|
-
/**
|
|
5719
|
-
* Tests if given string is valid URL.
|
|
5720
|
-
*
|
|
5721
|
-
* Note: This does not check if the file exists only if the path is valid
|
|
5722
|
-
* @public exported from `@promptbook/utils`
|
|
5723
|
-
*/
|
|
5724
|
-
function isValidFilePath(filename) {
|
|
5725
|
-
if (typeof filename !== 'string') {
|
|
5726
|
-
return false;
|
|
5727
|
-
}
|
|
5728
|
-
if (filename.split('\n').length > 1) {
|
|
5729
|
-
return false;
|
|
5730
|
-
}
|
|
5731
|
-
if (filename.split(' ').length >
|
|
5732
|
-
5 /* <- TODO: [🧠][🈷] Make some better non-arbitrary way how to distinct filenames from informational texts */) {
|
|
5733
|
-
return false;
|
|
5734
|
-
}
|
|
5735
|
-
var filenameSlashes = filename.split('\\').join('/');
|
|
5736
|
-
// Absolute Unix path: /hello.txt
|
|
5737
|
-
if (/^(\/)/i.test(filenameSlashes)) {
|
|
5738
|
-
// console.log(filename, 'Absolute Unix path: /hello.txt');
|
|
5739
|
-
return true;
|
|
5740
|
-
}
|
|
5741
|
-
// Absolute Windows path: /hello.txt
|
|
5742
|
-
if (/^([A-Z]{1,2}:\/?)\//i.test(filenameSlashes)) {
|
|
5743
|
-
// console.log(filename, 'Absolute Windows path: /hello.txt');
|
|
5744
|
-
return true;
|
|
5745
|
-
}
|
|
5746
|
-
// Relative path: ./hello.txt
|
|
5747
|
-
if (/^(\.\.?\/)+/i.test(filenameSlashes)) {
|
|
5748
|
-
// console.log(filename, 'Relative path: ./hello.txt');
|
|
5749
|
-
return true;
|
|
5750
|
-
}
|
|
5751
|
-
// Allow paths like foo/hello
|
|
5752
|
-
if (/^[^/]+\/[^/]+/i.test(filenameSlashes)) {
|
|
5753
|
-
// console.log(filename, 'Allow paths like foo/hello');
|
|
5754
|
-
return true;
|
|
5755
|
-
}
|
|
5756
|
-
// Allow paths like hello.book
|
|
5757
|
-
if (/^[^/]+\.[^/]+$/i.test(filenameSlashes)) {
|
|
5758
|
-
// console.log(filename, 'Allow paths like hello.book');
|
|
5759
|
-
return true;
|
|
5760
|
-
}
|
|
5761
|
-
return false;
|
|
5762
|
-
}
|
|
5763
|
-
/**
|
|
5764
|
-
* TODO: [🍏] Implement for MacOs
|
|
5765
|
-
*/
|
|
5766
|
-
|
|
5767
5864
|
/**
|
|
5768
5865
|
* The built-in `fetch' function with a lightweight error handling wrapper as default fetch function used in Promptbook scrapers
|
|
5769
5866
|
*
|
|
@@ -5799,10 +5896,11 @@ var scraperFetch = function (url, init) { return __awaiter(void 0, void 0, void
|
|
|
5799
5896
|
function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
|
|
5800
5897
|
var _a;
|
|
5801
5898
|
return __awaiter(this, void 0, void 0, function () {
|
|
5802
|
-
var _b, fetch, knowledgeSourceContent, name, _c, _d, rootDirname, url,
|
|
5803
|
-
return __generator(this, function (
|
|
5804
|
-
switch (
|
|
5899
|
+
var _b, fetch, knowledgeSourceContent, name, _c, _d, rootDirname, url, response, mimeType, filename, hash, rootDirname_1, filepath, _f, _g, _h, _j, _k, filename_1, fileExtension, mimeType;
|
|
5900
|
+
return __generator(this, function (_l) {
|
|
5901
|
+
switch (_l.label) {
|
|
5805
5902
|
case 0:
|
|
5903
|
+
console.log('!!! makeKnowledgeSourceHandler', knowledgeSource);
|
|
5806
5904
|
_b = tools.fetch, fetch = _b === void 0 ? scraperFetch : _b;
|
|
5807
5905
|
knowledgeSourceContent = knowledgeSource.knowledgeSourceContent;
|
|
5808
5906
|
name = knowledgeSource.name;
|
|
@@ -5810,54 +5908,32 @@ function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
|
|
|
5810
5908
|
if (!name) {
|
|
5811
5909
|
name = knowledgeSourceContentToName(knowledgeSourceContent);
|
|
5812
5910
|
}
|
|
5813
|
-
if (!isValidUrl(knowledgeSourceContent)) return [3 /*break*/,
|
|
5911
|
+
if (!isValidUrl(knowledgeSourceContent)) return [3 /*break*/, 5];
|
|
5814
5912
|
url = knowledgeSourceContent;
|
|
5815
5913
|
return [4 /*yield*/, fetch(url)];
|
|
5816
5914
|
case 1:
|
|
5817
|
-
|
|
5818
|
-
mimeType = ((_a =
|
|
5819
|
-
|
|
5820
|
-
|
|
5821
|
-
|
|
5822
|
-
|
|
5823
|
-
|
|
5824
|
-
/*
|
|
5825
|
-
TODO: [🥽]
|
|
5826
|
-
> async asBlob() {
|
|
5827
|
-
> // TODO: [👨🏻🤝👨🏻] This can be called multiple times BUT when called second time, response in already consumed
|
|
5828
|
-
> const content = await response.blob();
|
|
5829
|
-
> return content;
|
|
5830
|
-
> },
|
|
5831
|
-
*/
|
|
5832
|
-
asJson: function () {
|
|
5833
|
-
return __awaiter(this, void 0, void 0, function () {
|
|
5834
|
-
var content;
|
|
5835
|
-
return __generator(this, function (_a) {
|
|
5836
|
-
switch (_a.label) {
|
|
5837
|
-
case 0: return [4 /*yield*/, response_1.json()];
|
|
5838
|
-
case 1:
|
|
5839
|
-
content = _a.sent();
|
|
5840
|
-
return [2 /*return*/, content];
|
|
5841
|
-
}
|
|
5842
|
-
});
|
|
5843
|
-
});
|
|
5844
|
-
},
|
|
5845
|
-
asText: function () {
|
|
5846
|
-
return __awaiter(this, void 0, void 0, function () {
|
|
5847
|
-
var content;
|
|
5848
|
-
return __generator(this, function (_a) {
|
|
5849
|
-
switch (_a.label) {
|
|
5850
|
-
case 0: return [4 /*yield*/, response_1.text()];
|
|
5851
|
-
case 1:
|
|
5852
|
-
content = _a.sent();
|
|
5853
|
-
return [2 /*return*/, content];
|
|
5854
|
-
}
|
|
5855
|
-
});
|
|
5856
|
-
});
|
|
5857
|
-
},
|
|
5858
|
-
}];
|
|
5915
|
+
response = _l.sent();
|
|
5916
|
+
mimeType = ((_a = response.headers.get('content-type')) === null || _a === void 0 ? void 0 : _a.split(';')[0]) || 'text/html';
|
|
5917
|
+
filename = url.split('/').pop() || titleToName(url);
|
|
5918
|
+
hash = sha256(hexEncoder.parse(url)).toString( /* hex */);
|
|
5919
|
+
rootDirname_1 = join(process.cwd(), DEFAULT_DOWNLOAD_CACHE_DIRNAME);
|
|
5920
|
+
filepath = join.apply(void 0, __spreadArray(__spreadArray([], __read(nameToSubfolderPath(hash /* <- TODO: [🎎] Maybe add some SHA256 prefix */)), false), ["".concat(filename.substring(0, MAX_FILENAME_LENGTH), ".pdf")], false));
|
|
5921
|
+
return [4 /*yield*/, tools.fs.mkdir(dirname(join(rootDirname_1, filepath)), { recursive: true })];
|
|
5859
5922
|
case 2:
|
|
5860
|
-
|
|
5923
|
+
_l.sent();
|
|
5924
|
+
_g = (_f = tools.fs).writeFile;
|
|
5925
|
+
_h = [join(rootDirname_1, filepath)];
|
|
5926
|
+
_k = (_j = Buffer).from;
|
|
5927
|
+
return [4 /*yield*/, response.arrayBuffer()];
|
|
5928
|
+
case 3: return [4 /*yield*/, _g.apply(_f, _h.concat([_k.apply(_j, [_l.sent()])]))];
|
|
5929
|
+
case 4:
|
|
5930
|
+
_l.sent();
|
|
5931
|
+
// TODO: !!!!!!!! Check the file security
|
|
5932
|
+
// TODO: !!!!!!!! Check the file size (if it is not too big)
|
|
5933
|
+
// TODO: !!!!!!!! Delete the file
|
|
5934
|
+
return [2 /*return*/, makeKnowledgeSourceHandler({ name: name, knowledgeSourceContent: filepath }, tools, __assign(__assign({}, options), { rootDirname: rootDirname_1 }))];
|
|
5935
|
+
case 5:
|
|
5936
|
+
if (!isValidFilePath(knowledgeSourceContent)) return [3 /*break*/, 7];
|
|
5861
5937
|
if (tools.fs === undefined) {
|
|
5862
5938
|
throw new EnvironmentMismatchError('Can not import file knowledge without filesystem tools');
|
|
5863
5939
|
// <- TODO: [🧠] What is the best error type here`
|
|
@@ -5870,8 +5946,8 @@ function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
|
|
|
5870
5946
|
fileExtension = getFileExtension(filename_1);
|
|
5871
5947
|
mimeType = extensionToMimeType(fileExtension || '');
|
|
5872
5948
|
return [4 /*yield*/, isFileExisting(filename_1, tools.fs)];
|
|
5873
|
-
case
|
|
5874
|
-
if (!(
|
|
5949
|
+
case 6:
|
|
5950
|
+
if (!(_l.sent())) {
|
|
5875
5951
|
throw new NotFoundError(spaceTrim(function (block) { return "\n Can not make source handler for file which does not exist:\n\n File:\n ".concat(block(knowledgeSourceContent), "\n\n Full file path:\n ").concat(block(filename_1), "\n "); }));
|
|
5876
5952
|
}
|
|
5877
5953
|
// TODO: [🧠][😿] Test security file - file is scoped to the project (BUT maybe do this in `filesystemTools`)
|
|
@@ -5917,7 +5993,7 @@ function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
|
|
|
5917
5993
|
});
|
|
5918
5994
|
},
|
|
5919
5995
|
}];
|
|
5920
|
-
case
|
|
5996
|
+
case 7: return [2 /*return*/, {
|
|
5921
5997
|
source: name,
|
|
5922
5998
|
filename: null,
|
|
5923
5999
|
url: null,
|
|
@@ -7028,22 +7104,6 @@ function normalizeTo_camelCase(text, _isFirstLetterCapital) {
|
|
|
7028
7104
|
* TODO: [🌺] Use some intermediate util splitWords
|
|
7029
7105
|
*/
|
|
7030
7106
|
|
|
7031
|
-
/**
|
|
7032
|
-
* Removes emojis from a string and fix whitespaces
|
|
7033
|
-
*
|
|
7034
|
-
* @param text with emojis
|
|
7035
|
-
* @returns text without emojis
|
|
7036
|
-
* @public exported from `@promptbook/utils`
|
|
7037
|
-
*/
|
|
7038
|
-
function removeEmojis(text) {
|
|
7039
|
-
// Replace emojis (and also ZWJ sequence) with hyphens
|
|
7040
|
-
text = text.replace(/(\p{Extended_Pictographic})\p{Modifier_Symbol}/gu, '$1');
|
|
7041
|
-
text = text.replace(/(\p{Extended_Pictographic})[\u{FE00}-\u{FE0F}]/gu, '$1');
|
|
7042
|
-
text = text.replace(/(\p{Extended_Pictographic})(\u{200D}\p{Extended_Pictographic})*/gu, '$1');
|
|
7043
|
-
text = text.replace(/\p{Extended_Pictographic}/gu, '');
|
|
7044
|
-
return text;
|
|
7045
|
-
}
|
|
7046
|
-
|
|
7047
7107
|
/**
|
|
7048
7108
|
* Removes quotes from a string
|
|
7049
7109
|
*
|
|
@@ -9235,30 +9295,6 @@ function flattenMarkdown(markdown) {
|
|
|
9235
9295
|
* NOW we are working just with markdown string and its good enough
|
|
9236
9296
|
*/
|
|
9237
9297
|
|
|
9238
|
-
/**
|
|
9239
|
-
* @@@
|
|
9240
|
-
*
|
|
9241
|
-
* @param value @@@
|
|
9242
|
-
* @returns @@@
|
|
9243
|
-
* @example @@@
|
|
9244
|
-
* @public exported from `@promptbook/utils`
|
|
9245
|
-
*/
|
|
9246
|
-
function titleToName(value) {
|
|
9247
|
-
if (isValidUrl(value)) {
|
|
9248
|
-
value = value.replace(/^https?:\/\//, '');
|
|
9249
|
-
value = value.replace(/\.html$/, '');
|
|
9250
|
-
}
|
|
9251
|
-
else if (isValidFilePath(value)) {
|
|
9252
|
-
value = basename(value);
|
|
9253
|
-
// Note: Keeping extension in the name
|
|
9254
|
-
}
|
|
9255
|
-
value = value.split('/').join('-');
|
|
9256
|
-
value = removeEmojis(value);
|
|
9257
|
-
value = normalizeToKebabCase(value);
|
|
9258
|
-
// TODO: [🧠] Maybe warn or add some padding to short name which are not good identifiers
|
|
9259
|
-
return value;
|
|
9260
|
-
}
|
|
9261
|
-
|
|
9262
9298
|
/**
|
|
9263
9299
|
* Compile pipeline from string (markdown) format to JSON format synchronously
|
|
9264
9300
|
*
|
|
@@ -11075,13 +11111,14 @@ function isValidPipelineString(pipelineString) {
|
|
|
11075
11111
|
/**
|
|
11076
11112
|
* Tag function for notating a prompt as template literal
|
|
11077
11113
|
*
|
|
11078
|
-
* Note: There are
|
|
11114
|
+
* Note: There are 3 similar functions:
|
|
11079
11115
|
* 1) `prompt` for notating single prompt exported from `@promptbook/utils`
|
|
11080
|
-
*
|
|
11116
|
+
* 2) `promptTemplate` alias for `prompt`
|
|
11117
|
+
* 3) `book` for notating and validating entire books exported from `@promptbook/utils`
|
|
11081
11118
|
*
|
|
11082
|
-
* @param strings
|
|
11083
|
-
* @param values
|
|
11084
|
-
* @returns the
|
|
11119
|
+
* @param strings
|
|
11120
|
+
* @param values
|
|
11121
|
+
* @returns the prompt string
|
|
11085
11122
|
* @public exported from `@promptbook/utils`
|
|
11086
11123
|
*/
|
|
11087
11124
|
function prompt(strings) {
|
|
@@ -11092,10 +11129,14 @@ function prompt(strings) {
|
|
|
11092
11129
|
if (values.length === 0) {
|
|
11093
11130
|
return spaceTrim(strings.join(''));
|
|
11094
11131
|
}
|
|
11132
|
+
var stringsWithHiddenParameters = strings.map(function (stringsItem) {
|
|
11133
|
+
// TODO: [0] DRY
|
|
11134
|
+
return stringsItem.split('{').join("".concat(REPLACING_NONCE, "beginbracket")).split('}').join("".concat(REPLACING_NONCE, "endbracket"));
|
|
11135
|
+
});
|
|
11095
11136
|
var placeholderParameterNames = values.map(function (value, i) { return "".concat(REPLACING_NONCE).concat(i); });
|
|
11096
11137
|
var parameters = Object.fromEntries(values.map(function (value, i) { return [placeholderParameterNames[i], value]; }));
|
|
11097
11138
|
// Combine strings and values
|
|
11098
|
-
var pipelineString =
|
|
11139
|
+
var pipelineString = stringsWithHiddenParameters.reduce(function (result, stringsItem, i) {
|
|
11099
11140
|
return placeholderParameterNames[i] === undefined
|
|
11100
11141
|
? "".concat(result).concat(stringsItem)
|
|
11101
11142
|
: "".concat(result).concat(stringsItem, "{").concat(placeholderParameterNames[i], "}");
|
|
@@ -11111,6 +11152,12 @@ function prompt(strings) {
|
|
|
11111
11152
|
console.error({ pipelineString: pipelineString, parameters: parameters, placeholderParameterNames: placeholderParameterNames, error: error });
|
|
11112
11153
|
throw new UnexpectedError(spaceTrim(function (block) { return "\n Internal error in prompt template literal\n \n ".concat(block(JSON.stringify({ strings: strings, values: values }, null, 4)), "}\n \n "); }));
|
|
11113
11154
|
}
|
|
11155
|
+
// TODO: [0] DRY
|
|
11156
|
+
pipelineString = pipelineString
|
|
11157
|
+
.split("".concat(REPLACING_NONCE, "beginbracket"))
|
|
11158
|
+
.join('{')
|
|
11159
|
+
.split("".concat(REPLACING_NONCE, "endbracket"))
|
|
11160
|
+
.join('}');
|
|
11114
11161
|
return pipelineString;
|
|
11115
11162
|
}
|
|
11116
11163
|
/**
|
|
@@ -11121,9 +11168,10 @@ function prompt(strings) {
|
|
|
11121
11168
|
/**
|
|
11122
11169
|
* Tag function for notating a pipeline with a book\`...\ notation as template literal
|
|
11123
11170
|
*
|
|
11124
|
-
* Note: There are
|
|
11171
|
+
* Note: There are 3 similar functions:
|
|
11125
11172
|
* 1) `prompt` for notating single prompt exported from `@promptbook/utils`
|
|
11126
|
-
*
|
|
11173
|
+
* 2) `promptTemplate` alias for `prompt`
|
|
11174
|
+
* 3) `book` for notating and validating entire books exported from `@promptbook/utils`
|
|
11127
11175
|
*
|
|
11128
11176
|
* @param strings @@@
|
|
11129
11177
|
* @param values @@@
|
|
@@ -11445,5 +11493,5 @@ var PrefixStorage = /** @class */ (function () {
|
|
|
11445
11493
|
return PrefixStorage;
|
|
11446
11494
|
}());
|
|
11447
11495
|
|
|
11448
|
-
export { $llmToolsMetadataRegister, $llmToolsRegister, $scrapersMetadataRegister, $scrapersRegister, ADMIN_EMAIL, ADMIN_GITHUB_NAME, AbstractFormatError, BOOK_LANGUAGE_VERSION, BlackholeStorage, BoilerplateError, BoilerplateFormfactorDefinition, CLAIM, CallbackInterfaceTools, ChatbotFormfactorDefinition, CollectionError, CsvFormatDefinition, CsvFormatError, DEFAULT_BOOKS_DIRNAME, DEFAULT_BOOK_TITLE, DEFAULT_CSV_SETTINGS,
|
|
11496
|
+
export { $llmToolsMetadataRegister, $llmToolsRegister, $scrapersMetadataRegister, $scrapersRegister, ADMIN_EMAIL, ADMIN_GITHUB_NAME, AbstractFormatError, BOOK_LANGUAGE_VERSION, BlackholeStorage, BoilerplateError, BoilerplateFormfactorDefinition, CLAIM, CallbackInterfaceTools, ChatbotFormfactorDefinition, CollectionError, CsvFormatDefinition, CsvFormatError, DEFAULT_BOOKS_DIRNAME, DEFAULT_BOOK_TITLE, DEFAULT_CSV_SETTINGS, DEFAULT_DOWNLOAD_CACHE_DIRNAME, DEFAULT_EXECUTION_CACHE_DIRNAME, DEFAULT_GET_PIPELINE_COLLECTION_FUNCTION_NAME, DEFAULT_INTERMEDIATE_FILES_STRATEGY, DEFAULT_IS_AUTO_INSTALLED, DEFAULT_IS_VERBOSE, DEFAULT_MAX_EXECUTION_ATTEMPTS, DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_DEPTH, DEFAULT_MAX_KNOWLEDGE_SOURCES_SCRAPING_TOTAL, DEFAULT_MAX_PARALLEL_COUNT, DEFAULT_PIPELINE_COLLECTION_BASE_FILENAME, DEFAULT_REMOTE_URL, DEFAULT_REMOTE_URL_PATH, DEFAULT_SCRAPE_CACHE_DIRNAME, DEFAULT_TASK_TITLE, EXPECTATION_UNITS, EnvironmentMismatchError, ExecutionReportStringOptionsDefaults, ExpectError, FORMFACTOR_DEFINITIONS, GENERIC_PIPELINE_INTERFACE, GeneratorFormfactorDefinition, GenericFormfactorDefinition, ImageGeneratorFormfactorDefinition, KnowledgeScrapeError, LOGO_DARK_SRC, LOGO_LIGHT_SRC, LimitReachedError, MANDATORY_CSV_SETTINGS, MAX_FILENAME_LENGTH, MODEL_VARIANTS, MatcherFormfactorDefinition, MemoryStorage, MissingToolsError, MultipleLlmExecutionTools, NAME, NonTaskSectionTypes, NotFoundError, NotYetImplementedError, ORDER_OF_PIPELINE_JSON, PROMPTBOOK_ENGINE_VERSION, PROMPTBOOK_ERRORS, ParseError, PipelineExecutionError, PipelineLogicError, PipelineUrlError, PrefixStorage, RESERVED_PARAMETER_NAMES, SET_IS_VERBOSE, SectionTypes, SheetsFormfactorDefinition, TaskTypes, TextFormatDefinition, TranslatorFormfactorDefinition, UNCERTAIN_USAGE, UnexpectedError, ZERO_USAGE, _AnthropicClaudeMetadataRegistration, _AzureOpenAiMetadataRegistration, _BoilerplateScraperMetadataRegistration, _DocumentScraperMetadataRegistration, _GoogleMetadataRegistration, _LegacyDocumentScraperMetadataRegistration, _MarkdownScraperMetadataRegistration, _MarkitdownScraperMetadataRegistration, _OpenAiAssistantMetadataRegistration, _OpenAiMetadataRegistration, _PdfScraperMetadataRegistration, _WebsiteScraperMetadataRegistration, addUsage, assertsExecutionSuccessful, book, cacheLlmTools, collectionToJson, compilePipeline, countTotalUsage, createCollectionFromJson, createCollectionFromPromise, createCollectionFromUrl, createLlmToolsFromConfiguration, createPipelineExecutor, createSubcollection, embeddingVectorToString, executionReportJsonToString, extractParameterNamesFromTask, getPipelineInterface, isPassingExpectations, isPipelineImplementingInterface, isPipelineInterfacesEqual, isPipelinePrepared, isValidPipelineString, joinLlmExecutionTools, limitTotalUsage, makeKnowledgeSourceHandler, parsePipeline, pipelineJsonToString, prepareKnowledgePieces, preparePersona, preparePipeline, prepareTasks, prettifyPipelineString, unpreparePipeline, usageToHuman, usageToWorktime, validatePipeline, validatePipelineString };
|
|
11449
11497
|
//# sourceMappingURL=index.es.js.map
|