@promptbook/cli 0.84.0-20 → 0.84.0-9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -11
- package/bin/promptbook-cli.js +0 -1
- package/esm/index.es.js +91 -322
- package/esm/index.es.js.map +1 -1
- package/esm/typings/src/_packages/cli.index.d.ts +0 -4
- package/esm/typings/src/_packages/core.index.d.ts +2 -12
- package/esm/typings/src/_packages/types.index.d.ts +0 -2
- package/esm/typings/src/_packages/utils.index.d.ts +0 -2
- package/esm/typings/src/_packages/wizzard.index.d.ts +0 -4
- package/esm/typings/src/config.d.ts +1 -27
- package/esm/typings/src/conversion/compilePipelineOnRemoteServer.d.ts +1 -1
- package/esm/typings/src/execution/FilesystemTools.d.ts +1 -1
- package/esm/typings/src/execution/assertsExecutionSuccessful.d.ts +1 -3
- package/esm/typings/src/pipeline/book-notation.d.ts +2 -3
- package/esm/typings/src/pipeline/prompt-notation.d.ts +5 -18
- package/esm/typings/src/prepare/preparePipelineOnRemoteServer.d.ts +1 -1
- package/esm/typings/src/remote-server/socket-types/_subtypes/PromptbookServer_Identification.d.ts +2 -5
- package/esm/typings/src/wizzard/wizzard.d.ts +1 -7
- package/package.json +16 -12
- package/umd/index.umd.js +89 -322
- package/umd/index.umd.js.map +1 -1
- package/esm/typings/src/_packages/deepseek.index.d.ts +0 -8
- package/esm/typings/src/llm-providers/deepseek/DeepseekExecutionToolsOptions.d.ts +0 -9
- package/esm/typings/src/llm-providers/deepseek/createDeepseekExecutionTools.d.ts +0 -14
- package/esm/typings/src/llm-providers/deepseek/register-configuration.d.ts +0 -14
- package/esm/typings/src/llm-providers/deepseek/register-constructor.d.ts +0 -15
- package/esm/typings/src/utils/editable/edit-pipeline-string/deflatePipeline.test.d.ts +0 -1
- package/esm/typings/src/utils/editable/utils/isFlatPipeline.test.d.ts +0 -1
- package/esm/typings/src/utils/files/mimeTypeToExtension.d.ts +0 -10
- package/esm/typings/src/utils/files/mimeTypeToExtension.test.d.ts +0 -1
package/umd/index.umd.js
CHANGED
|
@@ -51,7 +51,7 @@
|
|
|
51
51
|
* @generated
|
|
52
52
|
* @see https://github.com/webgptorg/promptbook
|
|
53
53
|
*/
|
|
54
|
-
var PROMPTBOOK_ENGINE_VERSION = '0.84.0-
|
|
54
|
+
var PROMPTBOOK_ENGINE_VERSION = '0.84.0-8';
|
|
55
55
|
/**
|
|
56
56
|
* TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
|
|
57
57
|
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
@@ -242,18 +242,6 @@
|
|
|
242
242
|
* @public exported from `@promptbook/core`
|
|
243
243
|
*/
|
|
244
244
|
var DEFAULT_TASK_TITLE = "Task";
|
|
245
|
-
/**
|
|
246
|
-
* When the pipeline is flat and no name of return parameter is provided, this name is used
|
|
247
|
-
*
|
|
248
|
-
* @public exported from `@promptbook/core`
|
|
249
|
-
*/
|
|
250
|
-
var DEFAULT_BOOK_OUTPUT_PARAMETER_NAME = 'result';
|
|
251
|
-
/**
|
|
252
|
-
* Maximum file size limit
|
|
253
|
-
*
|
|
254
|
-
* @public exported from `@promptbook/core`
|
|
255
|
-
*/
|
|
256
|
-
var DEFAULT_MAX_FILE_SIZE = 100 * 1024 * 1024; // 100MB
|
|
257
245
|
/**
|
|
258
246
|
* Warning message for the generated sections and files files
|
|
259
247
|
*
|
|
@@ -339,14 +327,6 @@
|
|
|
339
327
|
*/
|
|
340
328
|
var DEFAULT_BOOKS_DIRNAME = './books';
|
|
341
329
|
// <- TODO: [🕝] Make also `BOOKS_DIRNAME_ALTERNATIVES`
|
|
342
|
-
/**
|
|
343
|
-
* Where to store the temporary downloads
|
|
344
|
-
*
|
|
345
|
-
* Note: When the folder does not exist, it is created recursively
|
|
346
|
-
*
|
|
347
|
-
* @public exported from `@promptbook/core`
|
|
348
|
-
*/
|
|
349
|
-
var DEFAULT_DOWNLOAD_CACHE_DIRNAME = './.promptbook/download-cache';
|
|
350
330
|
/**
|
|
351
331
|
* Where to store the cache of executions for promptbook CLI
|
|
352
332
|
*
|
|
@@ -354,7 +334,7 @@
|
|
|
354
334
|
*
|
|
355
335
|
* @public exported from `@promptbook/core`
|
|
356
336
|
*/
|
|
357
|
-
var
|
|
337
|
+
var DEFAULT_EXECUTIONS_CACHE_DIRNAME = './.promptbook/executions-cache';
|
|
358
338
|
/**
|
|
359
339
|
* Where to store the scrape cache
|
|
360
340
|
*
|
|
@@ -549,7 +529,6 @@
|
|
|
549
529
|
readFile: promises.readFile,
|
|
550
530
|
writeFile: promises.writeFile,
|
|
551
531
|
readdir: promises.readdir,
|
|
552
|
-
mkdir: promises.mkdir,
|
|
553
532
|
};
|
|
554
533
|
}
|
|
555
534
|
/**
|
|
@@ -2751,7 +2730,7 @@
|
|
|
2751
2730
|
_c.sent()]),
|
|
2752
2731
|
{
|
|
2753
2732
|
storage: new FileCacheStorage({ fs: $provideFilesystemForNode() }, {
|
|
2754
|
-
rootFolderPath: path.join(process.cwd(),
|
|
2733
|
+
rootFolderPath: path.join(process.cwd(), DEFAULT_EXECUTIONS_CACHE_DIRNAME),
|
|
2755
2734
|
}),
|
|
2756
2735
|
isCacheReloaded: isCacheReloaded,
|
|
2757
2736
|
}])];
|
|
@@ -2824,18 +2803,24 @@
|
|
|
2824
2803
|
var PipelineCollection = [{title:"Prepare Knowledge from Markdown",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.book.md",formfactorName:"GENERIC",parameters:[{name:"knowledgeContent",description:"Markdown document content",isInput:true,isOutput:false},{name:"knowledgePieces",description:"The knowledge JSON object",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, extract the important knowledge from the document.\n\n# Rules\n\n- Make pieces of information concise, clear, and easy to understand\n- One piece of information should be approximately 1 paragraph\n- Divide the paragraphs by markdown horizontal lines ---\n- Omit irrelevant information\n- Group redundant information\n- Write just extracted information, nothing else\n\n# The document\n\nTake information from this document:\n\n> {knowledgeContent}",resultingParameterName:"knowledgePieces",dependentParameterNames:["knowledgeContent"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Knowledge from Markdown\n\n- PIPELINE URL `https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.book.md`\n- INPUT PARAMETER `{knowledgeContent}` Markdown document content\n- OUTPUT PARAMETER `{knowledgePieces}` The knowledge JSON object\n\n## Knowledge\n\n<!-- TODO: [🍆] -FORMAT JSON -->\n\n```markdown\nYou are experienced data researcher, extract the important knowledge from the document.\n\n# Rules\n\n- Make pieces of information concise, clear, and easy to understand\n- One piece of information should be approximately 1 paragraph\n- Divide the paragraphs by markdown horizontal lines ---\n- Omit irrelevant information\n- Group redundant information\n- Write just extracted information, nothing else\n\n# The document\n\nTake information from this document:\n\n> {knowledgeContent}\n```\n\n`-> {knowledgePieces}`\n"}],sourceFile:"./books/prepare-knowledge-from-markdown.book.md"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-keywords.book.md",formfactorName:"GENERIC",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"keywords",description:"Keywords separated by comma",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, detect the important keywords in the document.\n\n# Rules\n\n- Write just keywords separated by comma\n\n# The document\n\nTake information from this document:\n\n> {knowledgePieceContent}",resultingParameterName:"keywords",dependentParameterNames:["knowledgePieceContent"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Keywords\n\n- PIPELINE URL `https://promptbook.studio/promptbook/prepare-knowledge-keywords.book.md`\n- INPUT PARAMETER `{knowledgePieceContent}` The content\n- OUTPUT PARAMETER `{keywords}` Keywords separated by comma\n\n## Knowledge\n\n<!-- TODO: [🍆] -FORMAT JSON -->\n\n```markdown\nYou are experienced data researcher, detect the important keywords in the document.\n\n# Rules\n\n- Write just keywords separated by comma\n\n# The document\n\nTake information from this document:\n\n> {knowledgePieceContent}\n```\n\n`-> {keywords}`\n"}],sourceFile:"./books/prepare-knowledge-keywords.book.md"},{title:"Prepare Knowledge-piece Title",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-title.book.md",formfactorName:"GENERIC",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"title",description:"The title of the document",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"knowledge",title:"Knowledge",content:"You are experienced content creator, write best title for the document.\n\n# Rules\n\n- Write just title, nothing else\n- Write maximum 5 words for the title\n\n# The document\n\n> {knowledgePieceContent}",resultingParameterName:"title",expectations:{words:{min:1,max:8}},dependentParameterNames:["knowledgePieceContent"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Knowledge-piece Title\n\n- PIPELINE URL `https://promptbook.studio/promptbook/prepare-knowledge-title.book.md`\n- INPUT PARAMETER `{knowledgePieceContent}` The content\n- OUTPUT PARAMETER `{title}` The title of the document\n\n## Knowledge\n\n- EXPECT MIN 1 WORD\n- EXPECT MAX 8 WORDS\n\n```markdown\nYou are experienced content creator, write best title for the document.\n\n# Rules\n\n- Write just title, nothing else\n- Write maximum 5 words for the title\n\n# The document\n\n> {knowledgePieceContent}\n```\n\n`-> {title}`\n"}],sourceFile:"./books/prepare-knowledge-title.book.md"},{title:"Prepare Persona",pipelineUrl:"https://promptbook.studio/promptbook/prepare-persona.book.md",formfactorName:"GENERIC",parameters:[{name:"availableModelNames",description:"List of available model names separated by comma (,)",isInput:true,isOutput:false},{name:"personaDescription",description:"Description of the persona",isInput:true,isOutput:false},{name:"modelRequirements",description:"Specific requirements for the model",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"make-model-requirements",title:"Make modelRequirements",content:"You are experienced AI engineer, you need to create virtual assistant.\nWrite\n\n## Example\n\n```json\n{\n\"modelName\": \"gpt-4o\",\n\"systemMessage\": \"You are experienced AI engineer and helpfull assistant.\",\n\"temperature\": 0.7\n}\n```\n\n## Instructions\n\n- Your output format is JSON object\n- Write just the JSON object, no other text should be present\n- It contains the following keys:\n - `modelName`: The name of the model to use\n - `systemMessage`: The system message to provide context to the model\n - `temperature`: The sampling temperature to use\n\n### Key `modelName`\n\nPick from the following models:\n\n- {availableModelNames}\n\n### Key `systemMessage`\n\nThe system message is used to communicate instructions or provide context to the model at the beginning of a conversation. It is displayed in a different format compared to user messages, helping the model understand its role in the conversation. The system message typically guides the model's behavior, sets the tone, or specifies desired output from the model. By utilizing the system message effectively, users can steer the model towards generating more accurate and relevant responses.\n\nFor example:\n\n> You are an experienced AI engineer and helpful assistant.\n\n> You are a friendly and knowledgeable chatbot.\n\n### Key `temperature`\n\nThe sampling temperature, between 0 and 1. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. If set to 0, the model will use log probability to automatically increase the temperature until certain thresholds are hit.\n\nYou can pick a value between 0 and 2. For example:\n\n- `0.1`: Low temperature, extremely conservative and deterministic\n- `0.5`: Medium temperature, balanced between conservative and creative\n- `1.0`: High temperature, creative and bit random\n- `1.5`: Very high temperature, extremely creative and often chaotic and unpredictable\n- `2.0`: Maximum temperature, completely random and unpredictable, for some extreme creative use cases\n\n# The assistant\n\nTake this description of the persona:\n\n> {personaDescription}",resultingParameterName:"modelRequirements",format:"JSON",dependentParameterNames:["availableModelNames","personaDescription"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Persona\n\n- PIPELINE URL `https://promptbook.studio/promptbook/prepare-persona.book.md`\n- INPUT PARAMETER `{availableModelNames}` List of available model names separated by comma (,)\n- INPUT PARAMETER `{personaDescription}` Description of the persona\n- OUTPUT PARAMETER `{modelRequirements}` Specific requirements for the model\n\n## Make modelRequirements\n\n- FORMAT JSON\n\n```markdown\nYou are experienced AI engineer, you need to create virtual assistant.\nWrite\n\n## Example\n\n\\`\\`\\`json\n{\n\"modelName\": \"gpt-4o\",\n\"systemMessage\": \"You are experienced AI engineer and helpfull assistant.\",\n\"temperature\": 0.7\n}\n\\`\\`\\`\n\n## Instructions\n\n- Your output format is JSON object\n- Write just the JSON object, no other text should be present\n- It contains the following keys:\n - `modelName`: The name of the model to use\n - `systemMessage`: The system message to provide context to the model\n - `temperature`: The sampling temperature to use\n\n### Key `modelName`\n\nPick from the following models:\n\n- {availableModelNames}\n\n### Key `systemMessage`\n\nThe system message is used to communicate instructions or provide context to the model at the beginning of a conversation. It is displayed in a different format compared to user messages, helping the model understand its role in the conversation. The system message typically guides the model's behavior, sets the tone, or specifies desired output from the model. By utilizing the system message effectively, users can steer the model towards generating more accurate and relevant responses.\n\nFor example:\n\n> You are an experienced AI engineer and helpful assistant.\n\n> You are a friendly and knowledgeable chatbot.\n\n### Key `temperature`\n\nThe sampling temperature, between 0 and 1. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. If set to 0, the model will use log probability to automatically increase the temperature until certain thresholds are hit.\n\nYou can pick a value between 0 and 2. For example:\n\n- `0.1`: Low temperature, extremely conservative and deterministic\n- `0.5`: Medium temperature, balanced between conservative and creative\n- `1.0`: High temperature, creative and bit random\n- `1.5`: Very high temperature, extremely creative and often chaotic and unpredictable\n- `2.0`: Maximum temperature, completely random and unpredictable, for some extreme creative use cases\n\n# The assistant\n\nTake this description of the persona:\n\n> {personaDescription}\n```\n\n`-> {modelRequirements}`\n"}],sourceFile:"./books/prepare-persona.book.md"},{title:"Prepare Title",pipelineUrl:"https://promptbook.studio/promptbook/prepare-title.book.md",formfactorName:"GENERIC",parameters:[{name:"book",description:"The book to prepare the title for",isInput:true,isOutput:false},{name:"title",description:"Best title for the book",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"make-title",title:"Make title",content:"Make best title for given text which describes the workflow:\n\n## Rules\n\n- Write just title, nothing else\n- Title should be concise and clear - Write maximum ideally 2 words, maximum 5 words\n- Title starts with emoticon\n- Title should not mention the input and output of the workflow but the main purpose of the workflow\n _For example, not \"✍ Convert Knowledge-piece to title\" but \"✍ Title\"_\n\n## The workflow\n\n> {book}",resultingParameterName:"title",expectations:{words:{min:1,max:8},lines:{min:1,max:1}},dependentParameterNames:["book"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Title\n\n- PIPELINE URL `https://promptbook.studio/promptbook/prepare-title.book.md`\n- INPUT PARAMETER `{book}` The book to prepare the title for\n- OUTPUT PARAMETER `{title}` Best title for the book\n\n## Make title\n\n- EXPECT MIN 1 Word\n- EXPECT MAX 8 Words\n- EXPECT EXACTLY 1 Line\n\n```markdown\nMake best title for given text which describes the workflow:\n\n## Rules\n\n- Write just title, nothing else\n- Title should be concise and clear - Write maximum ideally 2 words, maximum 5 words\n- Title starts with emoticon\n- Title should not mention the input and output of the workflow but the main purpose of the workflow\n _For example, not \"✍ Convert Knowledge-piece to title\" but \"✍ Title\"_\n\n## The workflow\n\n> {book}\n```\n\n`-> {title}`\n"}],sourceFile:"./books/prepare-title.book.md"}];
|
|
2825
2804
|
|
|
2826
2805
|
/**
|
|
2827
|
-
*
|
|
2806
|
+
* Function isValidJsonString will tell you if the string is valid JSON or not
|
|
2828
2807
|
*
|
|
2829
2808
|
* @public exported from `@promptbook/utils`
|
|
2830
2809
|
*/
|
|
2831
|
-
function
|
|
2832
|
-
|
|
2833
|
-
|
|
2810
|
+
function isValidJsonString(value /* <- [👨⚖️] */) {
|
|
2811
|
+
try {
|
|
2812
|
+
JSON.parse(value);
|
|
2813
|
+
return true;
|
|
2834
2814
|
}
|
|
2835
|
-
|
|
2815
|
+
catch (error) {
|
|
2816
|
+
if (!(error instanceof Error)) {
|
|
2817
|
+
throw error;
|
|
2818
|
+
}
|
|
2819
|
+
if (error.message.includes('Unexpected token')) {
|
|
2820
|
+
return false;
|
|
2821
|
+
}
|
|
2836
2822
|
return false;
|
|
2837
2823
|
}
|
|
2838
|
-
return /^.+@.+\..+$/.test(email);
|
|
2839
2824
|
}
|
|
2840
2825
|
|
|
2841
2826
|
/**
|
|
@@ -2857,27 +2842,6 @@
|
|
|
2857
2842
|
* TODO: Maybe split `ParseError` and `ApplyError`
|
|
2858
2843
|
*/
|
|
2859
2844
|
|
|
2860
|
-
/**
|
|
2861
|
-
* Function isValidJsonString will tell you if the string is valid JSON or not
|
|
2862
|
-
*
|
|
2863
|
-
* @public exported from `@promptbook/utils`
|
|
2864
|
-
*/
|
|
2865
|
-
function isValidJsonString(value /* <- [👨⚖️] */) {
|
|
2866
|
-
try {
|
|
2867
|
-
JSON.parse(value);
|
|
2868
|
-
return true;
|
|
2869
|
-
}
|
|
2870
|
-
catch (error) {
|
|
2871
|
-
if (!(error instanceof Error)) {
|
|
2872
|
-
throw error;
|
|
2873
|
-
}
|
|
2874
|
-
if (error.message.includes('Unexpected token')) {
|
|
2875
|
-
return false;
|
|
2876
|
-
}
|
|
2877
|
-
return false;
|
|
2878
|
-
}
|
|
2879
|
-
}
|
|
2880
|
-
|
|
2881
2845
|
/**
|
|
2882
2846
|
* Function `validatePipelineString` will validate the if the string is a valid pipeline string
|
|
2883
2847
|
* It does not check if the string is fully logically correct, but if it is a string that can be a pipeline string or the string looks completely different.
|
|
@@ -2891,15 +2855,6 @@
|
|
|
2891
2855
|
if (isValidJsonString(pipelineString)) {
|
|
2892
2856
|
throw new ParseError('Expected a book, but got a JSON string');
|
|
2893
2857
|
}
|
|
2894
|
-
else if (isValidUrl(pipelineString)) {
|
|
2895
|
-
throw new ParseError("Expected a book, but got just the URL \"".concat(pipelineString, "\""));
|
|
2896
|
-
}
|
|
2897
|
-
else if (isValidFilePath(pipelineString)) {
|
|
2898
|
-
throw new ParseError("Expected a book, but got just the file path \"".concat(pipelineString, "\""));
|
|
2899
|
-
}
|
|
2900
|
-
else if (isValidEmail(pipelineString)) {
|
|
2901
|
-
throw new ParseError("Expected a book, but got just the email \"".concat(pipelineString, "\""));
|
|
2902
|
-
}
|
|
2903
2858
|
// <- TODO: Implement the validation + add tests when the pipeline logic considered as invalid
|
|
2904
2859
|
return pipelineString;
|
|
2905
2860
|
}
|
|
@@ -3972,28 +3927,12 @@
|
|
|
3972
3927
|
/**
|
|
3973
3928
|
* Asserts that the execution of a Promptbook is successful
|
|
3974
3929
|
*
|
|
3975
|
-
* Note: If there are only warnings, the execution is still successful but the warnings are logged in the console
|
|
3976
|
-
*
|
|
3977
3930
|
* @param executionResult - The partial result of the Promptbook execution
|
|
3978
3931
|
* @throws {PipelineExecutionError} If the execution is not successful or if multiple errors occurred
|
|
3979
3932
|
* @public exported from `@promptbook/core`
|
|
3980
3933
|
*/
|
|
3981
3934
|
function assertsExecutionSuccessful(executionResult) {
|
|
3982
|
-
var
|
|
3983
|
-
var isSuccessful = executionResult.isSuccessful, errors = executionResult.errors, warnings = executionResult.warnings;
|
|
3984
|
-
try {
|
|
3985
|
-
for (var warnings_1 = __values(warnings), warnings_1_1 = warnings_1.next(); !warnings_1_1.done; warnings_1_1 = warnings_1.next()) {
|
|
3986
|
-
var warning = warnings_1_1.value;
|
|
3987
|
-
console.warn(warning.message);
|
|
3988
|
-
}
|
|
3989
|
-
}
|
|
3990
|
-
catch (e_1_1) { e_1 = { error: e_1_1 }; }
|
|
3991
|
-
finally {
|
|
3992
|
-
try {
|
|
3993
|
-
if (warnings_1_1 && !warnings_1_1.done && (_a = warnings_1.return)) _a.call(warnings_1);
|
|
3994
|
-
}
|
|
3995
|
-
finally { if (e_1) throw e_1.error; }
|
|
3996
|
-
}
|
|
3935
|
+
var isSuccessful = executionResult.isSuccessful, errors = executionResult.errors;
|
|
3997
3936
|
if (isSuccessful === true) {
|
|
3998
3937
|
return;
|
|
3999
3938
|
}
|
|
@@ -6500,17 +6439,6 @@
|
|
|
6500
6439
|
return match ? match[1].toLowerCase() : null;
|
|
6501
6440
|
}
|
|
6502
6441
|
|
|
6503
|
-
/**
|
|
6504
|
-
* Convert mime type to file extension
|
|
6505
|
-
*
|
|
6506
|
-
* Note: If the mime type is invalid, `null` is returned
|
|
6507
|
-
*
|
|
6508
|
-
* @private within the repository
|
|
6509
|
-
*/
|
|
6510
|
-
function mimeTypeToExtension(value) {
|
|
6511
|
-
return mimeTypes.extension(value) || null;
|
|
6512
|
-
}
|
|
6513
|
-
|
|
6514
6442
|
/**
|
|
6515
6443
|
* The built-in `fetch' function with a lightweight error handling wrapper as default fetch function used in Promptbook scrapers
|
|
6516
6444
|
*
|
|
@@ -6546,9 +6474,9 @@
|
|
|
6546
6474
|
function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
|
|
6547
6475
|
var _a;
|
|
6548
6476
|
return __awaiter(this, void 0, void 0, function () {
|
|
6549
|
-
var _b, fetch, knowledgeSourceContent, name, _c, _d, rootDirname, url, response_1, mimeType,
|
|
6550
|
-
return __generator(this, function (
|
|
6551
|
-
switch (
|
|
6477
|
+
var _b, fetch, knowledgeSourceContent, name, _c, _d, rootDirname, url, response_1, mimeType, filename_1, fileExtension, mimeType;
|
|
6478
|
+
return __generator(this, function (_f) {
|
|
6479
|
+
switch (_f.label) {
|
|
6552
6480
|
case 0:
|
|
6553
6481
|
_b = tools.fetch, fetch = _b === void 0 ? scraperFetch : _b;
|
|
6554
6482
|
knowledgeSourceContent = knowledgeSource.knowledgeSourceContent;
|
|
@@ -6557,76 +6485,54 @@
|
|
|
6557
6485
|
if (!name) {
|
|
6558
6486
|
name = knowledgeSourceContentToName(knowledgeSourceContent);
|
|
6559
6487
|
}
|
|
6560
|
-
if (!isValidUrl(knowledgeSourceContent)) return [3 /*break*/,
|
|
6488
|
+
if (!isValidUrl(knowledgeSourceContent)) return [3 /*break*/, 2];
|
|
6561
6489
|
url = knowledgeSourceContent;
|
|
6562
6490
|
return [4 /*yield*/, fetch(url)];
|
|
6563
6491
|
case 1:
|
|
6564
|
-
response_1 =
|
|
6492
|
+
response_1 = _f.sent();
|
|
6565
6493
|
mimeType = ((_a = response_1.headers.get('content-type')) === null || _a === void 0 ? void 0 : _a.split(';')[0]) || 'text/html';
|
|
6566
|
-
|
|
6567
|
-
|
|
6568
|
-
|
|
6569
|
-
|
|
6570
|
-
|
|
6571
|
-
|
|
6572
|
-
|
|
6573
|
-
|
|
6574
|
-
|
|
6575
|
-
|
|
6576
|
-
|
|
6577
|
-
|
|
6578
|
-
|
|
6579
|
-
|
|
6580
|
-
|
|
6581
|
-
|
|
6582
|
-
|
|
6583
|
-
|
|
6584
|
-
|
|
6585
|
-
|
|
6586
|
-
|
|
6587
|
-
|
|
6588
|
-
|
|
6589
|
-
}
|
|
6590
|
-
});
|
|
6494
|
+
return [2 /*return*/, {
|
|
6495
|
+
source: name,
|
|
6496
|
+
filename: null,
|
|
6497
|
+
url: url,
|
|
6498
|
+
mimeType: mimeType,
|
|
6499
|
+
/*
|
|
6500
|
+
TODO: [🥽]
|
|
6501
|
+
> async asBlob() {
|
|
6502
|
+
> // TODO: [👨🏻🤝👨🏻] This can be called multiple times BUT when called second time, response in already consumed
|
|
6503
|
+
> const content = await response.blob();
|
|
6504
|
+
> return content;
|
|
6505
|
+
> },
|
|
6506
|
+
*/
|
|
6507
|
+
asJson: function () {
|
|
6508
|
+
return __awaiter(this, void 0, void 0, function () {
|
|
6509
|
+
var content;
|
|
6510
|
+
return __generator(this, function (_a) {
|
|
6511
|
+
switch (_a.label) {
|
|
6512
|
+
case 0: return [4 /*yield*/, response_1.json()];
|
|
6513
|
+
case 1:
|
|
6514
|
+
content = _a.sent();
|
|
6515
|
+
return [2 /*return*/, content];
|
|
6516
|
+
}
|
|
6591
6517
|
});
|
|
6592
|
-
}
|
|
6593
|
-
|
|
6594
|
-
|
|
6595
|
-
|
|
6596
|
-
|
|
6597
|
-
|
|
6598
|
-
|
|
6599
|
-
|
|
6600
|
-
|
|
6601
|
-
|
|
6602
|
-
|
|
6603
|
-
}
|
|
6518
|
+
});
|
|
6519
|
+
},
|
|
6520
|
+
asText: function () {
|
|
6521
|
+
return __awaiter(this, void 0, void 0, function () {
|
|
6522
|
+
var content;
|
|
6523
|
+
return __generator(this, function (_a) {
|
|
6524
|
+
switch (_a.label) {
|
|
6525
|
+
case 0: return [4 /*yield*/, response_1.text()];
|
|
6526
|
+
case 1:
|
|
6527
|
+
content = _a.sent();
|
|
6528
|
+
return [2 /*return*/, content];
|
|
6529
|
+
}
|
|
6604
6530
|
});
|
|
6605
|
-
}
|
|
6606
|
-
}
|
|
6607
|
-
|
|
6608
|
-
basename = url.split('/').pop() || titleToName(url);
|
|
6609
|
-
hash = sha256__default["default"](hexEncoder__default["default"].parse(url)).toString( /* hex */);
|
|
6610
|
-
rootDirname_1 = path.join(process.cwd(), DEFAULT_DOWNLOAD_CACHE_DIRNAME);
|
|
6611
|
-
filepath = path.join.apply(void 0, __spreadArray(__spreadArray([], __read(nameToSubfolderPath(hash /* <- TODO: [🎎] Maybe add some SHA256 prefix */)), false), ["".concat(basename.substring(0, MAX_FILENAME_LENGTH), ".").concat(mimeTypeToExtension(mimeType))], false));
|
|
6612
|
-
return [4 /*yield*/, tools.fs.mkdir(path.dirname(path.join(rootDirname_1, filepath)), { recursive: true })];
|
|
6531
|
+
});
|
|
6532
|
+
},
|
|
6533
|
+
}];
|
|
6613
6534
|
case 2:
|
|
6614
|
-
|
|
6615
|
-
_g = (_f = Buffer).from;
|
|
6616
|
-
return [4 /*yield*/, response_1.arrayBuffer()];
|
|
6617
|
-
case 3:
|
|
6618
|
-
fileContent = _g.apply(_f, [_h.sent()]);
|
|
6619
|
-
if (fileContent.length > DEFAULT_MAX_FILE_SIZE /* <- TODO: Allow to pass different value to remote server */) {
|
|
6620
|
-
throw new LimitReachedError("File is too large (".concat(Math.round(fileContent.length / 1024 / 1024), "MB). Maximum allowed size is ").concat(Math.round(DEFAULT_MAX_FILE_SIZE / 1024 / 1024), "MB."));
|
|
6621
|
-
}
|
|
6622
|
-
return [4 /*yield*/, tools.fs.writeFile(path.join(rootDirname_1, filepath), fileContent)];
|
|
6623
|
-
case 4:
|
|
6624
|
-
_h.sent();
|
|
6625
|
-
// TODO: [💵] Check the file security
|
|
6626
|
-
// TODO: [🧹][🧠] Delete the file after the scraping is done
|
|
6627
|
-
return [2 /*return*/, makeKnowledgeSourceHandler({ name: name, knowledgeSourceContent: filepath }, tools, __assign(__assign({}, options), { rootDirname: rootDirname_1 }))];
|
|
6628
|
-
case 5:
|
|
6629
|
-
if (!isValidFilePath(knowledgeSourceContent)) return [3 /*break*/, 7];
|
|
6535
|
+
if (!isValidFilePath(knowledgeSourceContent)) return [3 /*break*/, 4];
|
|
6630
6536
|
if (tools.fs === undefined) {
|
|
6631
6537
|
throw new EnvironmentMismatchError('Can not import file knowledge without filesystem tools');
|
|
6632
6538
|
// <- TODO: [🧠] What is the best error type here`
|
|
@@ -6639,8 +6545,8 @@
|
|
|
6639
6545
|
fileExtension = getFileExtension(filename_1);
|
|
6640
6546
|
mimeType = extensionToMimeType(fileExtension || '');
|
|
6641
6547
|
return [4 /*yield*/, isFileExisting(filename_1, tools.fs)];
|
|
6642
|
-
case
|
|
6643
|
-
if (!(
|
|
6548
|
+
case 3:
|
|
6549
|
+
if (!(_f.sent())) {
|
|
6644
6550
|
throw new NotFoundError(spaceTrim__default["default"](function (block) { return "\n Can not make source handler for file which does not exist:\n\n File:\n ".concat(block(knowledgeSourceContent), "\n\n Full file path:\n ").concat(block(filename_1), "\n "); }));
|
|
6645
6551
|
}
|
|
6646
6552
|
// TODO: [🧠][😿] Test security file - file is scoped to the project (BUT maybe do this in `filesystemTools`)
|
|
@@ -6686,7 +6592,7 @@
|
|
|
6686
6592
|
});
|
|
6687
6593
|
},
|
|
6688
6594
|
}];
|
|
6689
|
-
case
|
|
6595
|
+
case 4: return [2 /*return*/, {
|
|
6690
6596
|
source: name,
|
|
6691
6597
|
filename: null,
|
|
6692
6598
|
url: null,
|
|
@@ -9744,15 +9650,12 @@
|
|
|
9744
9650
|
pipelineString = removeMarkdownComments(pipelineString);
|
|
9745
9651
|
pipelineString = spaceTrim__default["default"](pipelineString);
|
|
9746
9652
|
var isMarkdownBeginningWithHeadline = pipelineString.startsWith('# ');
|
|
9747
|
-
|
|
9748
|
-
|
|
9749
|
-
var isQuoteBlocksUsed = /^>\s+/m.test(pipelineString);
|
|
9750
|
-
var isBlocksUsed = isBacktickBlockUsed || isQuoteBlocksUsed;
|
|
9751
|
-
// TODO: [🧉] Also (double)check
|
|
9653
|
+
var isLastLineReturnStatement = pipelineString.split('\n').pop().split('`').join('').startsWith('->');
|
|
9654
|
+
// TODO: Also (double)check
|
|
9752
9655
|
// > const usedCommands
|
|
9753
9656
|
// > const isBlocksUsed
|
|
9754
9657
|
// > const returnStatementCount
|
|
9755
|
-
var isFlat = !isMarkdownBeginningWithHeadline &&
|
|
9658
|
+
var isFlat = !isMarkdownBeginningWithHeadline && isLastLineReturnStatement;
|
|
9756
9659
|
return isFlat;
|
|
9757
9660
|
}
|
|
9758
9661
|
|
|
@@ -9766,26 +9669,9 @@
|
|
|
9766
9669
|
return pipelineString;
|
|
9767
9670
|
}
|
|
9768
9671
|
var pipelineStringLines = pipelineString.split('\n');
|
|
9769
|
-
var
|
|
9770
|
-
var returnStatement;
|
|
9771
|
-
if (/(-|=)>\s*\{.*\}/.test(potentialReturnStatement)) {
|
|
9772
|
-
// Note: Last line is return statement
|
|
9773
|
-
returnStatement = potentialReturnStatement;
|
|
9774
|
-
}
|
|
9775
|
-
else {
|
|
9776
|
-
// Note: Last line is not a return statement
|
|
9777
|
-
returnStatement = "-> {".concat(DEFAULT_BOOK_OUTPUT_PARAMETER_NAME, "}");
|
|
9778
|
-
pipelineStringLines.push(potentialReturnStatement);
|
|
9779
|
-
}
|
|
9672
|
+
var returnStatement = pipelineStringLines.pop();
|
|
9780
9673
|
var prompt = spaceTrim__default["default"](pipelineStringLines.join('\n'));
|
|
9781
|
-
|
|
9782
|
-
if (prompt.split('\n').length <= 1) {
|
|
9783
|
-
quotedPrompt = "> ".concat(prompt);
|
|
9784
|
-
}
|
|
9785
|
-
else {
|
|
9786
|
-
quotedPrompt = spaceTrim__default["default"](function (block) { return "\n ```\n ".concat(block(prompt.split('`').join('\\`')), "\n ```\n "); });
|
|
9787
|
-
}
|
|
9788
|
-
pipelineString = validatePipelineString(spaceTrim__default["default"](function (block) { return "\n # ".concat(DEFAULT_BOOK_TITLE, "\n\n ## Prompt\n\n ").concat(block(quotedPrompt), "\n\n ").concat(returnStatement, "\n "); }));
|
|
9674
|
+
pipelineString = validatePipelineString(spaceTrim__default["default"](function (block) { return "\n # ".concat(DEFAULT_BOOK_TITLE, "\n\n ## Prompt\n\n ```\n ").concat(block(prompt), "\n ```\n\n ").concat(returnStatement, "\n "); }));
|
|
9789
9675
|
// <- TODO: Maybe use book` notation
|
|
9790
9676
|
return pipelineString;
|
|
9791
9677
|
}
|
|
@@ -14678,30 +14564,6 @@
|
|
|
14678
14564
|
},
|
|
14679
14565
|
/**/
|
|
14680
14566
|
/**/
|
|
14681
|
-
{
|
|
14682
|
-
modelVariant: 'CHAT',
|
|
14683
|
-
modelTitle: 'o1',
|
|
14684
|
-
modelName: 'o1',
|
|
14685
|
-
pricing: {
|
|
14686
|
-
prompt: computeUsage("$3.00 / 1M tokens"),
|
|
14687
|
-
output: computeUsage("$12.00 / 1M tokens"),
|
|
14688
|
-
// <- TODO: !!! Unsure, check the pricing
|
|
14689
|
-
},
|
|
14690
|
-
},
|
|
14691
|
-
/**/
|
|
14692
|
-
/**/
|
|
14693
|
-
{
|
|
14694
|
-
modelVariant: 'CHAT',
|
|
14695
|
-
modelTitle: 'o3-mini',
|
|
14696
|
-
modelName: 'o3-mini',
|
|
14697
|
-
pricing: {
|
|
14698
|
-
prompt: computeUsage("$3.00 / 1M tokens"),
|
|
14699
|
-
output: computeUsage("$12.00 / 1M tokens"),
|
|
14700
|
-
// <- TODO: !!! Unsure, check the pricing
|
|
14701
|
-
},
|
|
14702
|
-
},
|
|
14703
|
-
/**/
|
|
14704
|
-
/**/
|
|
14705
14567
|
{
|
|
14706
14568
|
modelVariant: 'CHAT',
|
|
14707
14569
|
modelTitle: 'o1-mini-2024-09-12',
|
|
@@ -15102,16 +14964,16 @@
|
|
|
15102
14964
|
* @public exported from `@promptbook/wizzard`
|
|
15103
14965
|
* @public exported from `@promptbook/cli`
|
|
15104
14966
|
*/
|
|
15105
|
-
var
|
|
15106
|
-
title: '
|
|
15107
|
-
packageName: '@promptbook/
|
|
15108
|
-
className: '
|
|
15109
|
-
envVariables: ['
|
|
14967
|
+
var _GoogleMetadataRegistration = $llmToolsMetadataRegister.register({
|
|
14968
|
+
title: 'Google Gemini',
|
|
14969
|
+
packageName: '@promptbook/google',
|
|
14970
|
+
className: 'GoogleExecutionTools',
|
|
14971
|
+
envVariables: ['GOOGLE_GENERATIVE_AI_API_KEY'],
|
|
15110
14972
|
getBoilerplateConfiguration: function () {
|
|
15111
14973
|
return {
|
|
15112
|
-
title: '
|
|
15113
|
-
packageName: '@promptbook/
|
|
15114
|
-
className: '
|
|
14974
|
+
title: 'Google Gemini (boilerplate)',
|
|
14975
|
+
packageName: '@promptbook/google',
|
|
14976
|
+
className: 'GoogleExecutionTools',
|
|
15115
14977
|
options: {
|
|
15116
14978
|
apiKey: 'AI',
|
|
15117
14979
|
},
|
|
@@ -15121,17 +14983,17 @@
|
|
|
15121
14983
|
if ($isRunningInJest()
|
|
15122
14984
|
// <- TODO: Maybe check `env.JEST_WORKER_ID` directly here or pass `env` into `$isRunningInJest`
|
|
15123
14985
|
) {
|
|
15124
|
-
// Note: [🔘]
|
|
14986
|
+
// Note: [🔘] Gemini makes problems in Jest environment
|
|
15125
14987
|
return null;
|
|
15126
14988
|
}
|
|
15127
14989
|
// Note: Note using `process.env` BUT `env` to pass in the environment variables dynamically
|
|
15128
|
-
if (typeof env.
|
|
14990
|
+
if (typeof env.GOOGLE_GENERATIVE_AI_API_KEY === 'string') {
|
|
15129
14991
|
return {
|
|
15130
|
-
title: '
|
|
15131
|
-
packageName: '@promptbook/
|
|
15132
|
-
className: '
|
|
14992
|
+
title: 'Google Gemini (from env)',
|
|
14993
|
+
packageName: '@promptbook/google',
|
|
14994
|
+
className: 'GoogleExecutionTools',
|
|
15133
14995
|
options: {
|
|
15134
|
-
apiKey: env.
|
|
14996
|
+
apiKey: env.GOOGLE_GENERATIVE_AI_API_KEY,
|
|
15135
14997
|
},
|
|
15136
14998
|
};
|
|
15137
14999
|
}
|
|
@@ -15312,98 +15174,6 @@
|
|
|
15312
15174
|
};
|
|
15313
15175
|
}
|
|
15314
15176
|
|
|
15315
|
-
/**
|
|
15316
|
-
* Execution Tools for calling Deepseek API.
|
|
15317
|
-
*
|
|
15318
|
-
* @public exported from `@promptbook/deepseek`
|
|
15319
|
-
*/
|
|
15320
|
-
var createDeepseekExecutionTools = Object.assign(function (options) {
|
|
15321
|
-
if ($isRunningInJest()) {
|
|
15322
|
-
// Note: [🔘]
|
|
15323
|
-
throw new Error('DeepseekExecutionTools are not supported in Jest environment');
|
|
15324
|
-
}
|
|
15325
|
-
// Note: [🔘] Maybe there is same compatibility problem as in '@ai-sdk/deepseek'
|
|
15326
|
-
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
|
15327
|
-
var createDeepSeek = require('@ai-sdk/deepseek').createDeepSeek;
|
|
15328
|
-
var deepseekVercelProvider = createDeepSeek(__assign({}, options));
|
|
15329
|
-
return createExecutionToolsFromVercelProvider(__assign({ title: 'Deepseek', description: 'Implementation of Deepseek models', vercelProvider: deepseekVercelProvider, availableModels: [
|
|
15330
|
-
// TODO: [🕘] Maybe list models in same way as in other providers - in separate file with metadata
|
|
15331
|
-
'deepseek-chat',
|
|
15332
|
-
'deepseek-reasoner',
|
|
15333
|
-
// <- TODO: How picking of the default model looks like in `createExecutionToolsFromVercelProvider`
|
|
15334
|
-
].map(function (modelName) { return ({ modelName: modelName, modelVariant: 'CHAT' }); }) }, options));
|
|
15335
|
-
}, {
|
|
15336
|
-
packageName: '@promptbook/deepseek',
|
|
15337
|
-
className: 'DeepseekExecutionTools',
|
|
15338
|
-
});
|
|
15339
|
-
/**
|
|
15340
|
-
* TODO: [🎶] Naming "constructor" vs "creator" vs "factory"
|
|
15341
|
-
*/
|
|
15342
|
-
|
|
15343
|
-
/**
|
|
15344
|
-
* Registration of LLM provider
|
|
15345
|
-
*
|
|
15346
|
-
* Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available LLM tools
|
|
15347
|
-
*
|
|
15348
|
-
* @public exported from `@promptbook/deepseek`
|
|
15349
|
-
* @public exported from `@promptbook/wizzard`
|
|
15350
|
-
* @public exported from `@promptbook/cli`
|
|
15351
|
-
*/
|
|
15352
|
-
var _DeepseekRegistration = $llmToolsRegister.register(createDeepseekExecutionTools);
|
|
15353
|
-
/**
|
|
15354
|
-
* TODO: [🎶] Naming "constructor" vs "creator" vs "factory"
|
|
15355
|
-
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
15356
|
-
*/
|
|
15357
|
-
|
|
15358
|
-
/**
|
|
15359
|
-
* Registration of LLM provider metadata
|
|
15360
|
-
*
|
|
15361
|
-
* Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available LLM tools
|
|
15362
|
-
*
|
|
15363
|
-
* @public exported from `@promptbook/core`
|
|
15364
|
-
* @public exported from `@promptbook/wizzard`
|
|
15365
|
-
* @public exported from `@promptbook/cli`
|
|
15366
|
-
*/
|
|
15367
|
-
var _GoogleMetadataRegistration = $llmToolsMetadataRegister.register({
|
|
15368
|
-
title: 'Google Gemini',
|
|
15369
|
-
packageName: '@promptbook/google',
|
|
15370
|
-
className: 'GoogleExecutionTools',
|
|
15371
|
-
envVariables: ['GOOGLE_GENERATIVE_AI_API_KEY'],
|
|
15372
|
-
getBoilerplateConfiguration: function () {
|
|
15373
|
-
return {
|
|
15374
|
-
title: 'Google Gemini (boilerplate)',
|
|
15375
|
-
packageName: '@promptbook/google',
|
|
15376
|
-
className: 'GoogleExecutionTools',
|
|
15377
|
-
options: {
|
|
15378
|
-
apiKey: 'AI',
|
|
15379
|
-
},
|
|
15380
|
-
};
|
|
15381
|
-
},
|
|
15382
|
-
createConfigurationFromEnv: function (env) {
|
|
15383
|
-
if ($isRunningInJest()
|
|
15384
|
-
// <- TODO: Maybe check `env.JEST_WORKER_ID` directly here or pass `env` into `$isRunningInJest`
|
|
15385
|
-
) {
|
|
15386
|
-
// Note: [🔘] Gemini makes problems in Jest environment
|
|
15387
|
-
return null;
|
|
15388
|
-
}
|
|
15389
|
-
// Note: Note using `process.env` BUT `env` to pass in the environment variables dynamically
|
|
15390
|
-
if (typeof env.GOOGLE_GENERATIVE_AI_API_KEY === 'string') {
|
|
15391
|
-
return {
|
|
15392
|
-
title: 'Google Gemini (from env)',
|
|
15393
|
-
packageName: '@promptbook/google',
|
|
15394
|
-
className: 'GoogleExecutionTools',
|
|
15395
|
-
options: {
|
|
15396
|
-
apiKey: env.GOOGLE_GENERATIVE_AI_API_KEY,
|
|
15397
|
-
},
|
|
15398
|
-
};
|
|
15399
|
-
}
|
|
15400
|
-
return null;
|
|
15401
|
-
},
|
|
15402
|
-
});
|
|
15403
|
-
/**
|
|
15404
|
-
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
15405
|
-
*/
|
|
15406
|
-
|
|
15407
15177
|
/**
|
|
15408
15178
|
* Execution Tools for calling Google Gemini API.
|
|
15409
15179
|
*
|
|
@@ -17117,7 +16887,7 @@
|
|
|
17117
16887
|
className: 'MarkitdownScraper',
|
|
17118
16888
|
mimeTypes: [
|
|
17119
16889
|
'application/pdf',
|
|
17120
|
-
// TODO:
|
|
16890
|
+
// TODO: Make priority for scrapers and than allow all mime types here:
|
|
17121
16891
|
// 'text/html',
|
|
17122
16892
|
// 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
|
|
17123
16893
|
],
|
|
@@ -17153,10 +16923,10 @@
|
|
|
17153
16923
|
this.tools = tools;
|
|
17154
16924
|
this.options = options;
|
|
17155
16925
|
this.markdownScraper = new MarkdownScraper(tools, options);
|
|
17156
|
-
// Note: Module `markitdown-ts` has no types available, so it is imported using `require`
|
|
17157
16926
|
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
|
17158
16927
|
var MarkItDown = require('markitdown-ts').MarkItDown;
|
|
17159
|
-
// <- TODO:
|
|
16928
|
+
// <- TODO: !!! Use Markitdown directly not through this package
|
|
16929
|
+
// <- Note: !!!!!!!
|
|
17160
16930
|
this.markitdown = new MarkItDown();
|
|
17161
16931
|
}
|
|
17162
16932
|
Object.defineProperty(MarkitdownScraper.prototype, "metadata", {
|
|
@@ -17201,12 +16971,12 @@
|
|
|
17201
16971
|
case 2:
|
|
17202
16972
|
if (!!(_f.sent())) return [3 /*break*/, 5];
|
|
17203
16973
|
src = source.filename || source.url || null;
|
|
17204
|
-
|
|
16974
|
+
console.log('!!!', { src: src, source: source, cacheFilehandler: cacheFilehandler });
|
|
17205
16975
|
if (src === null) {
|
|
17206
16976
|
throw new UnexpectedError('Source has no filename or url');
|
|
17207
16977
|
}
|
|
17208
16978
|
return [4 /*yield*/, this.markitdown.convert(src, {
|
|
17209
|
-
// TODO: Pass when sacraping Youtube
|
|
16979
|
+
// TODO: !!!!!! Pass when sacraping Youtube
|
|
17210
16980
|
// enableYoutubeTranscript: true,
|
|
17211
16981
|
// youtubeTranscriptLanguage: 'en',
|
|
17212
16982
|
})];
|
|
@@ -17214,12 +16984,11 @@
|
|
|
17214
16984
|
result = _f.sent();
|
|
17215
16985
|
if (result === null || result === undefined) {
|
|
17216
16986
|
throw new Error("Markitdown could not convert the \"".concat(source.source, "\""));
|
|
17217
|
-
// <- TODO:
|
|
16987
|
+
// <- TODO: !!! Make MarkitdownError
|
|
17218
16988
|
}
|
|
17219
|
-
|
|
16989
|
+
console.log('!!!', { result: result, cacheFilehandler: cacheFilehandler });
|
|
17220
16990
|
return [4 /*yield*/, this.tools.fs.writeFile(cacheFilehandler.filename, result.text_content)];
|
|
17221
16991
|
case 4:
|
|
17222
|
-
// console.log('!!', { result, cacheFilehandler });
|
|
17223
16992
|
_f.sent();
|
|
17224
16993
|
_f.label = 5;
|
|
17225
16994
|
case 5: return [2 /*return*/, cacheFilehandler];
|
|
@@ -17646,8 +17415,6 @@
|
|
|
17646
17415
|
exports._BoilerplateScraperMetadataRegistration = _BoilerplateScraperMetadataRegistration;
|
|
17647
17416
|
exports._BoilerplateScraperRegistration = _BoilerplateScraperRegistration;
|
|
17648
17417
|
exports._CLI = _CLI;
|
|
17649
|
-
exports._DeepseekMetadataRegistration = _DeepseekMetadataRegistration;
|
|
17650
|
-
exports._DeepseekRegistration = _DeepseekRegistration;
|
|
17651
17418
|
exports._DocumentScraperMetadataRegistration = _DocumentScraperMetadataRegistration;
|
|
17652
17419
|
exports._DocumentScraperRegistration = _DocumentScraperRegistration;
|
|
17653
17420
|
exports._GoogleMetadataRegistration = _GoogleMetadataRegistration;
|