@promptbook/pdf 0.71.0-13 → 0.71.0-15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/esm/index.es.js +31 -46
- package/esm/index.es.js.map +1 -1
- package/esm/typings/src/_packages/node.index.d.ts +2 -0
- package/esm/typings/src/_packages/types.index.d.ts +2 -0
- package/esm/typings/src/collection/constructors/createCollectionFromDirectory.d.ts +2 -2
- package/esm/typings/src/conversion/pipelineStringToJson.d.ts +1 -1
- package/esm/typings/src/execution/ExecutionTools.d.ts +12 -4
- package/esm/typings/src/execution/FilesystemTools.d.ts +9 -0
- package/esm/typings/src/execution/translation/automatic-translate/translateMessages.d.ts +1 -0
- package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsFromEnv.d.ts +0 -1
- package/esm/typings/src/prepare/preparePipeline.d.ts +1 -1
- package/esm/typings/src/prepare/prepareTemplates.d.ts +1 -1
- package/esm/typings/src/scrapers/_common/prepareKnowledgePieces.d.ts +1 -1
- package/esm/typings/src/scrapers/_common/register/$provideFilesystemForNode.d.ts +11 -0
- package/esm/typings/src/scrapers/_common/register/$provideScrapersForNode.d.ts +1 -1
- package/esm/typings/src/scrapers/_common/utils/getScraperIntermediateSource.d.ts +1 -0
- package/esm/typings/src/scrapers/_common/utils/makeKnowledgeSourceHandler.d.ts +2 -4
- package/esm/typings/src/scrapers/document/DocumentScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/document-legacy/LegacyDocumentScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/website/WebsiteScraper.d.ts +1 -1
- package/esm/typings/src/storage/file-cache-storage/FileCacheStorage.d.ts +3 -1
- package/esm/typings/src/utils/files/{$isDirectoryExisting.d.ts → isDirectoryExisting.d.ts} +3 -4
- package/esm/typings/src/utils/files/isFileExisting.d.ts +13 -0
- package/esm/typings/src/utils/files/{$listAllFiles.d.ts → listAllFiles.d.ts} +3 -4
- package/package.json +2 -2
- package/umd/index.umd.js +30 -45
- package/umd/index.umd.js.map +1 -1
- package/esm/typings/src/utils/files/$isFileExisting.d.ts +0 -14
- /package/esm/typings/src/utils/files/{$isDirectoryExisting.test.d.ts → isDirectoryExisting.test.d.ts} +0 -0
- /package/esm/typings/src/utils/files/{$isFileExisting.test.d.ts → isFileExisting.test.d.ts} +0 -0
- /package/esm/typings/src/utils/files/{$listAllFiles.test.d.ts → listAllFiles.test.d.ts} +0 -0
package/esm/index.es.js
CHANGED
|
@@ -3,7 +3,7 @@ import { format } from 'prettier';
|
|
|
3
3
|
import parserHtml from 'prettier/parser-html';
|
|
4
4
|
import { basename, join } from 'path';
|
|
5
5
|
import { forTime } from 'waitasecond';
|
|
6
|
-
import {
|
|
6
|
+
import { readFile } from 'fs/promises';
|
|
7
7
|
import { SHA256 } from 'crypto-js';
|
|
8
8
|
import hexEncoder from 'crypto-js/enc-hex';
|
|
9
9
|
import { lookup } from 'mime-types';
|
|
@@ -13,7 +13,7 @@ import { unparse, parse } from 'papaparse';
|
|
|
13
13
|
/**
|
|
14
14
|
* The version of the Promptbook library
|
|
15
15
|
*/
|
|
16
|
-
var PROMPTBOOK_VERSION = '0.71.0-
|
|
16
|
+
var PROMPTBOOK_VERSION = '0.71.0-14';
|
|
17
17
|
// TODO: [main] !!!! List here all the versions and annotate + put into script
|
|
18
18
|
|
|
19
19
|
/*! *****************************************************************************
|
|
@@ -2923,39 +2923,45 @@ function sourceContentToName(sourceContent) {
|
|
|
2923
2923
|
*/
|
|
2924
2924
|
|
|
2925
2925
|
/**
|
|
2926
|
-
*
|
|
2926
|
+
* Convert file extension to mime type
|
|
2927
2927
|
*
|
|
2928
|
-
*
|
|
2928
|
+
* @private within the repository
|
|
2929
|
+
*/
|
|
2930
|
+
function extensionToMimeType(value) {
|
|
2931
|
+
return lookup(value) || 'application/octet-stream';
|
|
2932
|
+
}
|
|
2933
|
+
|
|
2934
|
+
/**
|
|
2935
|
+
* Get the file extension from a file name
|
|
2929
2936
|
*
|
|
2930
|
-
* @
|
|
2937
|
+
* @private within the repository
|
|
2931
2938
|
*/
|
|
2932
|
-
|
|
2939
|
+
function getFileExtension(value) {
|
|
2940
|
+
var match = value.match(/\.([0-9a-z]+)(?:[?#]|$)/i);
|
|
2941
|
+
return match ? match[1].toLowerCase() : null;
|
|
2942
|
+
}
|
|
2933
2943
|
|
|
2934
2944
|
/**
|
|
2935
2945
|
* Checks if the file exists
|
|
2936
2946
|
*
|
|
2937
|
-
* Note: `$` is used to indicate that this function is not a pure function - it looks at the filesystem
|
|
2938
|
-
*
|
|
2939
2947
|
* @private within the repository
|
|
2940
2948
|
*/
|
|
2941
|
-
function
|
|
2949
|
+
function isFileExisting(filename, fs) {
|
|
2942
2950
|
return __awaiter(this, void 0, void 0, function () {
|
|
2943
2951
|
var isReadAccessAllowed, isFile;
|
|
2944
2952
|
return __generator(this, function (_a) {
|
|
2945
2953
|
switch (_a.label) {
|
|
2946
|
-
case 0:
|
|
2947
|
-
|
|
2948
|
-
|
|
2949
|
-
|
|
2950
|
-
return [4 /*yield*/, access(filename, constants.R_OK)
|
|
2951
|
-
.then(function () { return true; })
|
|
2952
|
-
.catch(function () { return false; })];
|
|
2954
|
+
case 0: return [4 /*yield*/, fs
|
|
2955
|
+
.access(filename, fs.constants.R_OK)
|
|
2956
|
+
.then(function () { return true; })
|
|
2957
|
+
.catch(function () { return false; })];
|
|
2953
2958
|
case 1:
|
|
2954
2959
|
isReadAccessAllowed = _a.sent();
|
|
2955
2960
|
if (!isReadAccessAllowed) {
|
|
2956
2961
|
return [2 /*return*/, false];
|
|
2957
2962
|
}
|
|
2958
|
-
return [4 /*yield*/,
|
|
2963
|
+
return [4 /*yield*/, fs
|
|
2964
|
+
.stat(filename)
|
|
2959
2965
|
.then(function (fileStat) { return fileStat.isFile(); })
|
|
2960
2966
|
.catch(function () { return false; })];
|
|
2961
2967
|
case 2:
|
|
@@ -2966,36 +2972,17 @@ function $isFileExisting(filename) {
|
|
|
2966
2972
|
});
|
|
2967
2973
|
}
|
|
2968
2974
|
/**
|
|
2969
|
-
* Note: [
|
|
2975
|
+
* Note: Not [~🟢~] because it is not directly dependent on `fs
|
|
2970
2976
|
* TODO: [🐠] This can be a validator - with variants that return true/false and variants that throw errors with meaningless messages
|
|
2971
2977
|
* TODO: [🖇] What about symlinks?
|
|
2972
2978
|
*/
|
|
2973
2979
|
|
|
2974
|
-
/**
|
|
2975
|
-
* Convert file extension to mime type
|
|
2976
|
-
*
|
|
2977
|
-
* @private within the repository
|
|
2978
|
-
*/
|
|
2979
|
-
function extensionToMimeType(value) {
|
|
2980
|
-
return lookup(value) || 'application/octet-stream';
|
|
2981
|
-
}
|
|
2982
|
-
|
|
2983
|
-
/**
|
|
2984
|
-
* Get the file extension from a file name
|
|
2985
|
-
*
|
|
2986
|
-
* @private within the repository
|
|
2987
|
-
*/
|
|
2988
|
-
function getFileExtension(value) {
|
|
2989
|
-
var match = value.match(/\.([0-9a-z]+)(?:[?#]|$)/i);
|
|
2990
|
-
return match ? match[1].toLowerCase() : null;
|
|
2991
|
-
}
|
|
2992
|
-
|
|
2993
2980
|
/**
|
|
2994
2981
|
* @@@
|
|
2995
2982
|
*
|
|
2996
2983
|
* @private for scraper utilities
|
|
2997
2984
|
*/
|
|
2998
|
-
function makeKnowledgeSourceHandler(knowledgeSource, options) {
|
|
2985
|
+
function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
|
|
2999
2986
|
var _a;
|
|
3000
2987
|
return __awaiter(this, void 0, void 0, function () {
|
|
3001
2988
|
var sourceContent, name, _b, _c, rootDirname, _d,
|
|
@@ -3064,8 +3051,9 @@ function makeKnowledgeSourceHandler(knowledgeSource, options) {
|
|
|
3064
3051
|
}];
|
|
3065
3052
|
case 2:
|
|
3066
3053
|
if (!(isValidFilePath(sourceContent) || /\.[a-z]{1,10}$/i.exec(sourceContent))) return [3 /*break*/, 4];
|
|
3067
|
-
if (
|
|
3068
|
-
throw new EnvironmentMismatchError('
|
|
3054
|
+
if (tools.fs === undefined) {
|
|
3055
|
+
throw new EnvironmentMismatchError('Can not import file knowledge without filesystem tools');
|
|
3056
|
+
// <- TODO: [🧠] What is the best error type here`
|
|
3069
3057
|
}
|
|
3070
3058
|
if (rootDirname === null) {
|
|
3071
3059
|
throw new EnvironmentMismatchError('Can not import file knowledge in non-file pipeline');
|
|
@@ -3074,7 +3062,7 @@ function makeKnowledgeSourceHandler(knowledgeSource, options) {
|
|
|
3074
3062
|
filename_1 = join(rootDirname, sourceContent).split('\\').join('/');
|
|
3075
3063
|
fileExtension = getFileExtension(filename_1);
|
|
3076
3064
|
mimeType_1 = extensionToMimeType(fileExtension || '');
|
|
3077
|
-
return [4 /*yield*/,
|
|
3065
|
+
return [4 /*yield*/, isFileExisting(filename_1, tools.fs)];
|
|
3078
3066
|
case 3:
|
|
3079
3067
|
if (!(_e.sent())) {
|
|
3080
3068
|
throw new NotFoundError(spaceTrim$1(function (block) { return "\n Can not make source handler for file which does not exist:\n\n File:\n ".concat(block(filename_1), "\n "); }));
|
|
@@ -3090,7 +3078,7 @@ function makeKnowledgeSourceHandler(knowledgeSource, options) {
|
|
|
3090
3078
|
var content;
|
|
3091
3079
|
return __generator(this, function (_a) {
|
|
3092
3080
|
switch (_a.label) {
|
|
3093
|
-
case 0: return [4 /*yield*/, readFile(filename_1)];
|
|
3081
|
+
case 0: return [4 /*yield*/, tools.fs.readFile(filename_1)];
|
|
3094
3082
|
case 1:
|
|
3095
3083
|
content = _a.sent();
|
|
3096
3084
|
return [2 /*return*/, new Blob([
|
|
@@ -3144,9 +3132,6 @@ function makeKnowledgeSourceHandler(knowledgeSource, options) {
|
|
|
3144
3132
|
});
|
|
3145
3133
|
});
|
|
3146
3134
|
}
|
|
3147
|
-
/**
|
|
3148
|
-
* TODO: !!!!!!! Maybe constrain to @promptbook/node bundle
|
|
3149
|
-
*/
|
|
3150
3135
|
|
|
3151
3136
|
/**
|
|
3152
3137
|
* Prepares the knowle
|
|
@@ -3170,7 +3155,7 @@ function prepareKnowledgePieces(knowledgeSources, tools, options) {
|
|
|
3170
3155
|
switch (_d.label) {
|
|
3171
3156
|
case 0:
|
|
3172
3157
|
partialPieces = null;
|
|
3173
|
-
return [4 /*yield*/, makeKnowledgeSourceHandler(knowledgeSource, { rootDirname: rootDirname, isVerbose: isVerbose })];
|
|
3158
|
+
return [4 /*yield*/, makeKnowledgeSourceHandler(knowledgeSource, tools, { rootDirname: rootDirname, isVerbose: isVerbose })];
|
|
3174
3159
|
case 1:
|
|
3175
3160
|
sourceHandler = _d.sent();
|
|
3176
3161
|
_d.label = 2;
|