@promptbook/core 0.71.0-12 → 0.71.0-14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/esm/index.es.js +31 -46
- package/esm/index.es.js.map +1 -1
- package/esm/typings/src/_packages/node.index.d.ts +2 -0
- package/esm/typings/src/_packages/types.index.d.ts +2 -0
- package/esm/typings/src/collection/constructors/createCollectionFromDirectory.d.ts +2 -2
- package/esm/typings/src/conversion/pipelineStringToJson.d.ts +1 -1
- package/esm/typings/src/execution/ExecutionTools.d.ts +12 -2
- package/esm/typings/src/execution/FilesystemTools.d.ts +9 -0
- package/esm/typings/src/execution/translation/automatic-translate/translateMessages.d.ts +1 -0
- package/esm/typings/src/prepare/preparePipeline.d.ts +1 -1
- package/esm/typings/src/prepare/prepareTemplates.d.ts +1 -1
- package/esm/typings/src/scrapers/_common/prepareKnowledgePieces.d.ts +1 -1
- package/esm/typings/src/scrapers/_common/register/$provideFilesystemForNode.d.ts +11 -0
- package/esm/typings/src/scrapers/_common/utils/getScraperIntermediateSource.d.ts +1 -0
- package/esm/typings/src/scrapers/_common/utils/makeKnowledgeSourceHandler.d.ts +2 -4
- package/esm/typings/src/scrapers/document/DocumentScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/document-legacy/LegacyDocumentScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/website/WebsiteScraper.d.ts +1 -1
- package/esm/typings/src/storage/file-cache-storage/FileCacheStorage.d.ts +3 -1
- package/esm/typings/src/utils/files/{$isDirectoryExisting.d.ts → isDirectoryExisting.d.ts} +3 -4
- package/esm/typings/src/utils/files/isFileExisting.d.ts +13 -0
- package/esm/typings/src/utils/files/{$listAllFiles.d.ts → listAllFiles.d.ts} +3 -4
- package/package.json +1 -1
- package/umd/index.umd.js +30 -45
- package/umd/index.umd.js.map +1 -1
- package/esm/typings/src/utils/files/$isFileExisting.d.ts +0 -14
- /package/esm/typings/src/utils/files/{$isDirectoryExisting.test.d.ts → isDirectoryExisting.test.d.ts} +0 -0
- /package/esm/typings/src/utils/files/{$isFileExisting.test.d.ts → isFileExisting.test.d.ts} +0 -0
- /package/esm/typings/src/utils/files/{$listAllFiles.test.d.ts → listAllFiles.test.d.ts} +0 -0
package/esm/index.es.js
CHANGED
|
@@ -3,7 +3,7 @@ import { format } from 'prettier';
|
|
|
3
3
|
import parserHtml from 'prettier/parser-html';
|
|
4
4
|
import { forTime } from 'waitasecond';
|
|
5
5
|
import { unparse, parse } from 'papaparse';
|
|
6
|
-
import {
|
|
6
|
+
import { readFile } from 'fs/promises';
|
|
7
7
|
import { join, basename } from 'path';
|
|
8
8
|
import { SHA256 } from 'crypto-js';
|
|
9
9
|
import hexEncoder from 'crypto-js/enc-hex';
|
|
@@ -15,7 +15,7 @@ import moment from 'moment';
|
|
|
15
15
|
/**
|
|
16
16
|
* The version of the Promptbook library
|
|
17
17
|
*/
|
|
18
|
-
var PROMPTBOOK_VERSION = '0.71.0-
|
|
18
|
+
var PROMPTBOOK_VERSION = '0.71.0-13';
|
|
19
19
|
// TODO: [main] !!!! List here all the versions and annotate + put into script
|
|
20
20
|
|
|
21
21
|
/*! *****************************************************************************
|
|
@@ -5262,39 +5262,45 @@ function sourceContentToName(sourceContent) {
|
|
|
5262
5262
|
*/
|
|
5263
5263
|
|
|
5264
5264
|
/**
|
|
5265
|
-
*
|
|
5265
|
+
* Convert file extension to mime type
|
|
5266
5266
|
*
|
|
5267
|
-
*
|
|
5267
|
+
* @private within the repository
|
|
5268
|
+
*/
|
|
5269
|
+
function extensionToMimeType(value) {
|
|
5270
|
+
return lookup(value) || 'application/octet-stream';
|
|
5271
|
+
}
|
|
5272
|
+
|
|
5273
|
+
/**
|
|
5274
|
+
* Get the file extension from a file name
|
|
5268
5275
|
*
|
|
5269
|
-
* @
|
|
5276
|
+
* @private within the repository
|
|
5270
5277
|
*/
|
|
5271
|
-
|
|
5278
|
+
function getFileExtension(value) {
|
|
5279
|
+
var match = value.match(/\.([0-9a-z]+)(?:[?#]|$)/i);
|
|
5280
|
+
return match ? match[1].toLowerCase() : null;
|
|
5281
|
+
}
|
|
5272
5282
|
|
|
5273
5283
|
/**
|
|
5274
5284
|
* Checks if the file exists
|
|
5275
5285
|
*
|
|
5276
|
-
* Note: `$` is used to indicate that this function is not a pure function - it looks at the filesystem
|
|
5277
|
-
*
|
|
5278
5286
|
* @private within the repository
|
|
5279
5287
|
*/
|
|
5280
|
-
function
|
|
5288
|
+
function isFileExisting(filename, fs) {
|
|
5281
5289
|
return __awaiter(this, void 0, void 0, function () {
|
|
5282
5290
|
var isReadAccessAllowed, isFile;
|
|
5283
5291
|
return __generator(this, function (_a) {
|
|
5284
5292
|
switch (_a.label) {
|
|
5285
|
-
case 0:
|
|
5286
|
-
|
|
5287
|
-
|
|
5288
|
-
|
|
5289
|
-
return [4 /*yield*/, access(filename, constants.R_OK)
|
|
5290
|
-
.then(function () { return true; })
|
|
5291
|
-
.catch(function () { return false; })];
|
|
5293
|
+
case 0: return [4 /*yield*/, fs
|
|
5294
|
+
.access(filename, fs.constants.R_OK)
|
|
5295
|
+
.then(function () { return true; })
|
|
5296
|
+
.catch(function () { return false; })];
|
|
5292
5297
|
case 1:
|
|
5293
5298
|
isReadAccessAllowed = _a.sent();
|
|
5294
5299
|
if (!isReadAccessAllowed) {
|
|
5295
5300
|
return [2 /*return*/, false];
|
|
5296
5301
|
}
|
|
5297
|
-
return [4 /*yield*/,
|
|
5302
|
+
return [4 /*yield*/, fs
|
|
5303
|
+
.stat(filename)
|
|
5298
5304
|
.then(function (fileStat) { return fileStat.isFile(); })
|
|
5299
5305
|
.catch(function () { return false; })];
|
|
5300
5306
|
case 2:
|
|
@@ -5305,30 +5311,11 @@ function $isFileExisting(filename) {
|
|
|
5305
5311
|
});
|
|
5306
5312
|
}
|
|
5307
5313
|
/**
|
|
5308
|
-
* Note: [
|
|
5314
|
+
* Note: Not [~🟢~] because it is not directly dependent on `fs
|
|
5309
5315
|
* TODO: [🐠] This can be a validator - with variants that return true/false and variants that throw errors with meaningless messages
|
|
5310
5316
|
* TODO: [🖇] What about symlinks?
|
|
5311
5317
|
*/
|
|
5312
5318
|
|
|
5313
|
-
/**
|
|
5314
|
-
* Convert file extension to mime type
|
|
5315
|
-
*
|
|
5316
|
-
* @private within the repository
|
|
5317
|
-
*/
|
|
5318
|
-
function extensionToMimeType(value) {
|
|
5319
|
-
return lookup(value) || 'application/octet-stream';
|
|
5320
|
-
}
|
|
5321
|
-
|
|
5322
|
-
/**
|
|
5323
|
-
* Get the file extension from a file name
|
|
5324
|
-
*
|
|
5325
|
-
* @private within the repository
|
|
5326
|
-
*/
|
|
5327
|
-
function getFileExtension(value) {
|
|
5328
|
-
var match = value.match(/\.([0-9a-z]+)(?:[?#]|$)/i);
|
|
5329
|
-
return match ? match[1].toLowerCase() : null;
|
|
5330
|
-
}
|
|
5331
|
-
|
|
5332
5319
|
/**
|
|
5333
5320
|
* Tests if given string is valid URL.
|
|
5334
5321
|
*
|
|
@@ -5360,7 +5347,7 @@ function isValidFilePath(filename) {
|
|
|
5360
5347
|
*
|
|
5361
5348
|
* @private for scraper utilities
|
|
5362
5349
|
*/
|
|
5363
|
-
function makeKnowledgeSourceHandler(knowledgeSource, options) {
|
|
5350
|
+
function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
|
|
5364
5351
|
var _a;
|
|
5365
5352
|
return __awaiter(this, void 0, void 0, function () {
|
|
5366
5353
|
var sourceContent, name, _b, _c, rootDirname, _d,
|
|
@@ -5429,8 +5416,9 @@ function makeKnowledgeSourceHandler(knowledgeSource, options) {
|
|
|
5429
5416
|
}];
|
|
5430
5417
|
case 2:
|
|
5431
5418
|
if (!(isValidFilePath(sourceContent) || /\.[a-z]{1,10}$/i.exec(sourceContent))) return [3 /*break*/, 4];
|
|
5432
|
-
if (
|
|
5433
|
-
throw new EnvironmentMismatchError('
|
|
5419
|
+
if (tools.fs === undefined) {
|
|
5420
|
+
throw new EnvironmentMismatchError('Can not import file knowledge without filesystem tools');
|
|
5421
|
+
// <- TODO: [🧠] What is the best error type here`
|
|
5434
5422
|
}
|
|
5435
5423
|
if (rootDirname === null) {
|
|
5436
5424
|
throw new EnvironmentMismatchError('Can not import file knowledge in non-file pipeline');
|
|
@@ -5439,7 +5427,7 @@ function makeKnowledgeSourceHandler(knowledgeSource, options) {
|
|
|
5439
5427
|
filename_1 = join(rootDirname, sourceContent).split('\\').join('/');
|
|
5440
5428
|
fileExtension = getFileExtension(filename_1);
|
|
5441
5429
|
mimeType_1 = extensionToMimeType(fileExtension || '');
|
|
5442
|
-
return [4 /*yield*/,
|
|
5430
|
+
return [4 /*yield*/, isFileExisting(filename_1, tools.fs)];
|
|
5443
5431
|
case 3:
|
|
5444
5432
|
if (!(_e.sent())) {
|
|
5445
5433
|
throw new NotFoundError(spaceTrim(function (block) { return "\n Can not make source handler for file which does not exist:\n\n File:\n ".concat(block(filename_1), "\n "); }));
|
|
@@ -5455,7 +5443,7 @@ function makeKnowledgeSourceHandler(knowledgeSource, options) {
|
|
|
5455
5443
|
var content;
|
|
5456
5444
|
return __generator(this, function (_a) {
|
|
5457
5445
|
switch (_a.label) {
|
|
5458
|
-
case 0: return [4 /*yield*/, readFile(filename_1)];
|
|
5446
|
+
case 0: return [4 /*yield*/, tools.fs.readFile(filename_1)];
|
|
5459
5447
|
case 1:
|
|
5460
5448
|
content = _a.sent();
|
|
5461
5449
|
return [2 /*return*/, new Blob([
|
|
@@ -5509,9 +5497,6 @@ function makeKnowledgeSourceHandler(knowledgeSource, options) {
|
|
|
5509
5497
|
});
|
|
5510
5498
|
});
|
|
5511
5499
|
}
|
|
5512
|
-
/**
|
|
5513
|
-
* TODO: !!!!!!! Maybe constrain to @promptbook/node bundle
|
|
5514
|
-
*/
|
|
5515
5500
|
|
|
5516
5501
|
/**
|
|
5517
5502
|
* Prepares the knowle
|
|
@@ -5535,7 +5520,7 @@ function prepareKnowledgePieces(knowledgeSources, tools, options) {
|
|
|
5535
5520
|
switch (_d.label) {
|
|
5536
5521
|
case 0:
|
|
5537
5522
|
partialPieces = null;
|
|
5538
|
-
return [4 /*yield*/, makeKnowledgeSourceHandler(knowledgeSource, { rootDirname: rootDirname, isVerbose: isVerbose })];
|
|
5523
|
+
return [4 /*yield*/, makeKnowledgeSourceHandler(knowledgeSource, tools, { rootDirname: rootDirname, isVerbose: isVerbose })];
|
|
5539
5524
|
case 1:
|
|
5540
5525
|
sourceHandler = _d.sent();
|
|
5541
5526
|
_d.label = 2;
|