@promptbook/markdown-utils 0.71.0-13 → 0.71.0-15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/esm/index.es.js +31 -46
- package/esm/index.es.js.map +1 -1
- package/esm/typings/src/_packages/node.index.d.ts +2 -0
- package/esm/typings/src/_packages/types.index.d.ts +2 -0
- package/esm/typings/src/collection/constructors/createCollectionFromDirectory.d.ts +2 -2
- package/esm/typings/src/conversion/pipelineStringToJson.d.ts +1 -1
- package/esm/typings/src/execution/ExecutionTools.d.ts +12 -4
- package/esm/typings/src/execution/FilesystemTools.d.ts +9 -0
- package/esm/typings/src/execution/translation/automatic-translate/translateMessages.d.ts +1 -0
- package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsFromEnv.d.ts +0 -1
- package/esm/typings/src/prepare/preparePipeline.d.ts +1 -1
- package/esm/typings/src/prepare/prepareTemplates.d.ts +1 -1
- package/esm/typings/src/scrapers/_common/prepareKnowledgePieces.d.ts +1 -1
- package/esm/typings/src/scrapers/_common/register/$provideFilesystemForNode.d.ts +11 -0
- package/esm/typings/src/scrapers/_common/register/$provideScrapersForNode.d.ts +1 -1
- package/esm/typings/src/scrapers/_common/utils/getScraperIntermediateSource.d.ts +1 -0
- package/esm/typings/src/scrapers/_common/utils/makeKnowledgeSourceHandler.d.ts +2 -4
- package/esm/typings/src/scrapers/document/DocumentScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/document-legacy/LegacyDocumentScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/website/WebsiteScraper.d.ts +1 -1
- package/esm/typings/src/storage/file-cache-storage/FileCacheStorage.d.ts +3 -1
- package/esm/typings/src/utils/files/{$isDirectoryExisting.d.ts → isDirectoryExisting.d.ts} +3 -4
- package/esm/typings/src/utils/files/isFileExisting.d.ts +13 -0
- package/esm/typings/src/utils/files/{$listAllFiles.d.ts → listAllFiles.d.ts} +3 -4
- package/package.json +1 -1
- package/umd/index.umd.js +30 -45
- package/umd/index.umd.js.map +1 -1
- package/esm/typings/src/utils/files/$isFileExisting.d.ts +0 -14
- /package/esm/typings/src/utils/files/{$isDirectoryExisting.test.d.ts → isDirectoryExisting.test.d.ts} +0 -0
- /package/esm/typings/src/utils/files/{$isFileExisting.test.d.ts → isFileExisting.test.d.ts} +0 -0
- /package/esm/typings/src/utils/files/{$listAllFiles.test.d.ts → listAllFiles.test.d.ts} +0 -0
package/esm/index.es.js
CHANGED
|
@@ -3,7 +3,7 @@ import { format } from 'prettier';
|
|
|
3
3
|
import parserHtml from 'prettier/parser-html';
|
|
4
4
|
import { basename, join } from 'path';
|
|
5
5
|
import { forTime } from 'waitasecond';
|
|
6
|
-
import {
|
|
6
|
+
import { readFile } from 'fs/promises';
|
|
7
7
|
import { SHA256 } from 'crypto-js';
|
|
8
8
|
import hexEncoder from 'crypto-js/enc-hex';
|
|
9
9
|
import { lookup } from 'mime-types';
|
|
@@ -13,7 +13,7 @@ import { unparse, parse } from 'papaparse';
|
|
|
13
13
|
/**
|
|
14
14
|
* The version of the Promptbook library
|
|
15
15
|
*/
|
|
16
|
-
var PROMPTBOOK_VERSION = '0.71.0-
|
|
16
|
+
var PROMPTBOOK_VERSION = '0.71.0-14';
|
|
17
17
|
// TODO: [main] !!!! List here all the versions and annotate + put into script
|
|
18
18
|
|
|
19
19
|
/*! *****************************************************************************
|
|
@@ -3112,39 +3112,45 @@ function sourceContentToName(sourceContent) {
|
|
|
3112
3112
|
*/
|
|
3113
3113
|
|
|
3114
3114
|
/**
|
|
3115
|
-
*
|
|
3115
|
+
* Convert file extension to mime type
|
|
3116
3116
|
*
|
|
3117
|
-
*
|
|
3117
|
+
* @private within the repository
|
|
3118
|
+
*/
|
|
3119
|
+
function extensionToMimeType(value) {
|
|
3120
|
+
return lookup(value) || 'application/octet-stream';
|
|
3121
|
+
}
|
|
3122
|
+
|
|
3123
|
+
/**
|
|
3124
|
+
* Get the file extension from a file name
|
|
3118
3125
|
*
|
|
3119
|
-
* @
|
|
3126
|
+
* @private within the repository
|
|
3120
3127
|
*/
|
|
3121
|
-
|
|
3128
|
+
function getFileExtension(value) {
|
|
3129
|
+
var match = value.match(/\.([0-9a-z]+)(?:[?#]|$)/i);
|
|
3130
|
+
return match ? match[1].toLowerCase() : null;
|
|
3131
|
+
}
|
|
3122
3132
|
|
|
3123
3133
|
/**
|
|
3124
3134
|
* Checks if the file exists
|
|
3125
3135
|
*
|
|
3126
|
-
* Note: `$` is used to indicate that this function is not a pure function - it looks at the filesystem
|
|
3127
|
-
*
|
|
3128
3136
|
* @private within the repository
|
|
3129
3137
|
*/
|
|
3130
|
-
function
|
|
3138
|
+
function isFileExisting(filename, fs) {
|
|
3131
3139
|
return __awaiter(this, void 0, void 0, function () {
|
|
3132
3140
|
var isReadAccessAllowed, isFile;
|
|
3133
3141
|
return __generator(this, function (_a) {
|
|
3134
3142
|
switch (_a.label) {
|
|
3135
|
-
case 0:
|
|
3136
|
-
|
|
3137
|
-
|
|
3138
|
-
|
|
3139
|
-
return [4 /*yield*/, access(filename, constants.R_OK)
|
|
3140
|
-
.then(function () { return true; })
|
|
3141
|
-
.catch(function () { return false; })];
|
|
3143
|
+
case 0: return [4 /*yield*/, fs
|
|
3144
|
+
.access(filename, fs.constants.R_OK)
|
|
3145
|
+
.then(function () { return true; })
|
|
3146
|
+
.catch(function () { return false; })];
|
|
3142
3147
|
case 1:
|
|
3143
3148
|
isReadAccessAllowed = _a.sent();
|
|
3144
3149
|
if (!isReadAccessAllowed) {
|
|
3145
3150
|
return [2 /*return*/, false];
|
|
3146
3151
|
}
|
|
3147
|
-
return [4 /*yield*/,
|
|
3152
|
+
return [4 /*yield*/, fs
|
|
3153
|
+
.stat(filename)
|
|
3148
3154
|
.then(function (fileStat) { return fileStat.isFile(); })
|
|
3149
3155
|
.catch(function () { return false; })];
|
|
3150
3156
|
case 2:
|
|
@@ -3155,36 +3161,17 @@ function $isFileExisting(filename) {
|
|
|
3155
3161
|
});
|
|
3156
3162
|
}
|
|
3157
3163
|
/**
|
|
3158
|
-
* Note: [
|
|
3164
|
+
* Note: Not [~🟢~] because it is not directly dependent on `fs
|
|
3159
3165
|
* TODO: [🐠] This can be a validator - with variants that return true/false and variants that throw errors with meaningless messages
|
|
3160
3166
|
* TODO: [🖇] What about symlinks?
|
|
3161
3167
|
*/
|
|
3162
3168
|
|
|
3163
|
-
/**
|
|
3164
|
-
* Convert file extension to mime type
|
|
3165
|
-
*
|
|
3166
|
-
* @private within the repository
|
|
3167
|
-
*/
|
|
3168
|
-
function extensionToMimeType(value) {
|
|
3169
|
-
return lookup(value) || 'application/octet-stream';
|
|
3170
|
-
}
|
|
3171
|
-
|
|
3172
|
-
/**
|
|
3173
|
-
* Get the file extension from a file name
|
|
3174
|
-
*
|
|
3175
|
-
* @private within the repository
|
|
3176
|
-
*/
|
|
3177
|
-
function getFileExtension(value) {
|
|
3178
|
-
var match = value.match(/\.([0-9a-z]+)(?:[?#]|$)/i);
|
|
3179
|
-
return match ? match[1].toLowerCase() : null;
|
|
3180
|
-
}
|
|
3181
|
-
|
|
3182
3169
|
/**
|
|
3183
3170
|
* @@@
|
|
3184
3171
|
*
|
|
3185
3172
|
* @private for scraper utilities
|
|
3186
3173
|
*/
|
|
3187
|
-
function makeKnowledgeSourceHandler(knowledgeSource, options) {
|
|
3174
|
+
function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
|
|
3188
3175
|
var _a;
|
|
3189
3176
|
return __awaiter(this, void 0, void 0, function () {
|
|
3190
3177
|
var sourceContent, name, _b, _c, rootDirname, _d,
|
|
@@ -3253,8 +3240,9 @@ function makeKnowledgeSourceHandler(knowledgeSource, options) {
|
|
|
3253
3240
|
}];
|
|
3254
3241
|
case 2:
|
|
3255
3242
|
if (!(isValidFilePath(sourceContent) || /\.[a-z]{1,10}$/i.exec(sourceContent))) return [3 /*break*/, 4];
|
|
3256
|
-
if (
|
|
3257
|
-
throw new EnvironmentMismatchError('
|
|
3243
|
+
if (tools.fs === undefined) {
|
|
3244
|
+
throw new EnvironmentMismatchError('Can not import file knowledge without filesystem tools');
|
|
3245
|
+
// <- TODO: [🧠] What is the best error type here`
|
|
3258
3246
|
}
|
|
3259
3247
|
if (rootDirname === null) {
|
|
3260
3248
|
throw new EnvironmentMismatchError('Can not import file knowledge in non-file pipeline');
|
|
@@ -3263,7 +3251,7 @@ function makeKnowledgeSourceHandler(knowledgeSource, options) {
|
|
|
3263
3251
|
filename_1 = join(rootDirname, sourceContent).split('\\').join('/');
|
|
3264
3252
|
fileExtension = getFileExtension(filename_1);
|
|
3265
3253
|
mimeType_1 = extensionToMimeType(fileExtension || '');
|
|
3266
|
-
return [4 /*yield*/,
|
|
3254
|
+
return [4 /*yield*/, isFileExisting(filename_1, tools.fs)];
|
|
3267
3255
|
case 3:
|
|
3268
3256
|
if (!(_e.sent())) {
|
|
3269
3257
|
throw new NotFoundError(spaceTrim(function (block) { return "\n Can not make source handler for file which does not exist:\n\n File:\n ".concat(block(filename_1), "\n "); }));
|
|
@@ -3279,7 +3267,7 @@ function makeKnowledgeSourceHandler(knowledgeSource, options) {
|
|
|
3279
3267
|
var content;
|
|
3280
3268
|
return __generator(this, function (_a) {
|
|
3281
3269
|
switch (_a.label) {
|
|
3282
|
-
case 0: return [4 /*yield*/, readFile(filename_1)];
|
|
3270
|
+
case 0: return [4 /*yield*/, tools.fs.readFile(filename_1)];
|
|
3283
3271
|
case 1:
|
|
3284
3272
|
content = _a.sent();
|
|
3285
3273
|
return [2 /*return*/, new Blob([
|
|
@@ -3333,9 +3321,6 @@ function makeKnowledgeSourceHandler(knowledgeSource, options) {
|
|
|
3333
3321
|
});
|
|
3334
3322
|
});
|
|
3335
3323
|
}
|
|
3336
|
-
/**
|
|
3337
|
-
* TODO: !!!!!!! Maybe constrain to @promptbook/node bundle
|
|
3338
|
-
*/
|
|
3339
3324
|
|
|
3340
3325
|
/**
|
|
3341
3326
|
* Prepares the knowle
|
|
@@ -3359,7 +3344,7 @@ function prepareKnowledgePieces(knowledgeSources, tools, options) {
|
|
|
3359
3344
|
switch (_d.label) {
|
|
3360
3345
|
case 0:
|
|
3361
3346
|
partialPieces = null;
|
|
3362
|
-
return [4 /*yield*/, makeKnowledgeSourceHandler(knowledgeSource, { rootDirname: rootDirname, isVerbose: isVerbose })];
|
|
3347
|
+
return [4 /*yield*/, makeKnowledgeSourceHandler(knowledgeSource, tools, { rootDirname: rootDirname, isVerbose: isVerbose })];
|
|
3363
3348
|
case 1:
|
|
3364
3349
|
sourceHandler = _d.sent();
|
|
3365
3350
|
_d.label = 2;
|