@promptbook/website-crawler 0.94.0 → 0.95.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -10
- package/esm/index.es.js +43 -43
- package/esm/index.es.js.map +1 -1
- package/esm/typings/src/_packages/types.index.d.ts +2 -2
- package/esm/typings/src/_packages/{wizzard.index.d.ts → wizard.index.d.ts} +2 -2
- package/esm/typings/src/cli/cli-commands/prettify.d.ts +1 -1
- package/esm/typings/src/cli/cli-commands/test-command.d.ts +1 -1
- package/esm/typings/src/conversion/archive/loadArchive.d.ts +1 -1
- package/esm/typings/src/conversion/archive/saveArchive.d.ts +2 -2
- package/esm/typings/src/conversion/prettify/renderPipelineMermaidOptions.d.ts +1 -1
- package/esm/typings/src/dialogs/callback/CallbackInterfaceTools.d.ts +1 -1
- package/esm/typings/src/execution/AbstractTaskResult.d.ts +2 -2
- package/esm/typings/src/execution/createPipelineExecutor/00-CreatePipelineExecutorOptions.d.ts +1 -1
- package/esm/typings/src/execution/execution-report/ExecutionPromptReportJson.d.ts +2 -2
- package/esm/typings/src/execution/translation/automatic-translate/translateMessages.d.ts +1 -1
- package/esm/typings/src/llm-providers/_common/register/{$provideLlmToolsForWizzardOrCli.d.ts → $provideLlmToolsForWizardOrCli.d.ts} +2 -2
- package/esm/typings/src/llm-providers/anthropic-claude/register-configuration.d.ts +1 -1
- package/esm/typings/src/llm-providers/anthropic-claude/register-constructor.d.ts +1 -1
- package/esm/typings/src/llm-providers/azure-openai/register-configuration.d.ts +1 -1
- package/esm/typings/src/llm-providers/azure-openai/register-constructor.d.ts +1 -1
- package/esm/typings/src/llm-providers/deepseek/register-configuration.d.ts +1 -1
- package/esm/typings/src/llm-providers/deepseek/register-constructor.d.ts +1 -1
- package/esm/typings/src/llm-providers/google/register-configuration.d.ts +1 -1
- package/esm/typings/src/llm-providers/google/register-constructor.d.ts +1 -1
- package/esm/typings/src/llm-providers/ollama/register-configuration.d.ts +1 -1
- package/esm/typings/src/llm-providers/ollama/register-constructor.d.ts +1 -1
- package/esm/typings/src/llm-providers/openai/OpenAiAssistantExecutionTools.d.ts +1 -1
- package/esm/typings/src/llm-providers/openai/register-configuration.d.ts +2 -2
- package/esm/typings/src/llm-providers/openai/register-constructor.d.ts +2 -2
- package/esm/typings/src/remote-server/socket-types/listModels/PromptbookServer_ListModels_Request.d.ts +1 -1
- package/esm/typings/src/scrapers/_boilerplate/createBoilerplateScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/_boilerplate/register-constructor.d.ts +1 -1
- package/esm/typings/src/scrapers/_boilerplate/register-metadata.d.ts +2 -2
- package/esm/typings/src/scrapers/_common/prepareKnowledgePieces.d.ts +1 -1
- package/esm/typings/src/scrapers/_common/register/ScraperAndConverterMetadata.d.ts +1 -1
- package/esm/typings/src/scrapers/document/createDocumentScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/document/register-constructor.d.ts +1 -1
- package/esm/typings/src/scrapers/document/register-metadata.d.ts +2 -2
- package/esm/typings/src/scrapers/document-legacy/createLegacyDocumentScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/document-legacy/register-constructor.d.ts +1 -1
- package/esm/typings/src/scrapers/document-legacy/register-metadata.d.ts +2 -2
- package/esm/typings/src/scrapers/markdown/createMarkdownScraper.d.ts +1 -4
- package/esm/typings/src/scrapers/markdown/register-constructor.d.ts +1 -1
- package/esm/typings/src/scrapers/markdown/register-metadata.d.ts +2 -2
- package/esm/typings/src/scrapers/markitdown/createMarkitdownScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/markitdown/register-constructor.d.ts +1 -1
- package/esm/typings/src/scrapers/markitdown/register-metadata.d.ts +2 -2
- package/esm/typings/src/scrapers/pdf/createPdfScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/pdf/register-constructor.d.ts +1 -1
- package/esm/typings/src/scrapers/pdf/register-metadata.d.ts +2 -2
- package/esm/typings/src/scrapers/website/createWebsiteScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/website/register-constructor.d.ts +1 -1
- package/esm/typings/src/scrapers/website/register-metadata.d.ts +2 -2
- package/esm/typings/src/types/typeAliases.d.ts +1 -1
- package/esm/typings/src/utils/files/listAllFiles.d.ts +1 -1
- package/esm/typings/src/version.d.ts +1 -1
- package/esm/typings/src/{wizzard → wizard}/$getCompiledBook.d.ts +2 -2
- package/esm/typings/src/{wizzard/wizzard.d.ts → wizard/wizard.d.ts} +6 -6
- package/package.json +2 -14
- package/umd/index.umd.js +43 -43
- package/umd/index.umd.js.map +1 -1
|
@@ -10,7 +10,7 @@ export declare const websiteScraperMetadata: import("type-fest/source/readonly-d
|
|
|
10
10
|
className: string;
|
|
11
11
|
mimeTypes: string[];
|
|
12
12
|
documentationUrl: "https://github.com/webgptorg/promptbook/discussions/@@";
|
|
13
|
-
|
|
13
|
+
isAvailableInBrowser: false;
|
|
14
14
|
requiredExecutables: never[];
|
|
15
15
|
}>;
|
|
16
16
|
/**
|
|
@@ -19,7 +19,7 @@ export declare const websiteScraperMetadata: import("type-fest/source/readonly-d
|
|
|
19
19
|
* Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available known scrapers
|
|
20
20
|
*
|
|
21
21
|
* @public exported from `@promptbook/core`
|
|
22
|
-
* @public exported from `@promptbook/
|
|
22
|
+
* @public exported from `@promptbook/wizard`
|
|
23
23
|
* @public exported from `@promptbook/cli`
|
|
24
24
|
*/
|
|
25
25
|
export declare const _WebsiteScraperMetadataRegistration: Registration;
|
|
@@ -658,7 +658,7 @@ export type number_seed = number_percent;
|
|
|
658
658
|
* - ❤ is equivalent to more than 1
|
|
659
659
|
*/
|
|
660
660
|
export type number_likeness = number;
|
|
661
|
-
export type
|
|
661
|
+
export type number_milliseconds = number_integer;
|
|
662
662
|
export type number_seconds = number;
|
|
663
663
|
export type number_minutes = number;
|
|
664
664
|
export type number_hours = number;
|
|
@@ -11,7 +11,7 @@ import type { string_filename } from '../../types/typeAliases';
|
|
|
11
11
|
*/
|
|
12
12
|
export declare function listAllFiles(path: string_dirname, isRecursive: boolean, fs: FilesystemTools): Promise<Array<string_filename>>;
|
|
13
13
|
/**
|
|
14
|
-
* TODO: [😶] Unite
|
|
14
|
+
* TODO: [😶] Unite folder listing
|
|
15
15
|
* Note: Not [~🟢~] because it is not directly dependent on `fs
|
|
16
16
|
* TODO: [🖇] What about symlinks?
|
|
17
17
|
*/
|
|
@@ -15,7 +15,7 @@ export declare const BOOK_LANGUAGE_VERSION: string_semantic_version;
|
|
|
15
15
|
export declare const PROMPTBOOK_ENGINE_VERSION: string_promptbook_version;
|
|
16
16
|
/**
|
|
17
17
|
* Represents the version string of the Promptbook engine.
|
|
18
|
-
* It follows semantic versioning (e.g., `0.94.0
|
|
18
|
+
* It follows semantic versioning (e.g., `0.94.0`).
|
|
19
19
|
*
|
|
20
20
|
* @generated
|
|
21
21
|
*/
|
|
@@ -5,9 +5,9 @@ import type { PrepareAndScrapeOptions } from '../prepare/PrepareAndScrapeOptions
|
|
|
5
5
|
import type { string_filename } from '../types/typeAliases';
|
|
6
6
|
import type { string_pipeline_url } from '../types/typeAliases';
|
|
7
7
|
/**
|
|
8
|
-
* @see ./
|
|
8
|
+
* @see ./wizard.ts `getPipeline` method
|
|
9
9
|
*
|
|
10
|
-
* @private usable through `ptbk run` and `@
|
|
10
|
+
* @private usable through `ptbk run` and `@promptbook/wizard`
|
|
11
11
|
*/
|
|
12
12
|
export declare function $getCompiledBook(tools: Required<Pick<ExecutionTools, 'fs' | 'fetch'>>, pipelineSource: string_filename | string_pipeline_url | PipelineString, options?: PrepareAndScrapeOptions): Promise<PipelineJson>;
|
|
13
13
|
/**
|
|
@@ -7,14 +7,14 @@ import type { string_filename } from '../types/typeAliases';
|
|
|
7
7
|
import type { string_parameter_value } from '../types/typeAliases';
|
|
8
8
|
import type { string_pipeline_url } from '../types/typeAliases';
|
|
9
9
|
/**
|
|
10
|
-
*
|
|
11
|
-
* Look at `
|
|
10
|
+
* Wizard for simple usage of the Promptbook
|
|
11
|
+
* Look at `wizard` for more details
|
|
12
12
|
*
|
|
13
13
|
* Note: This works only in Node.js environment and looks for the configuration, environment, tools and cache in the Node.js environment
|
|
14
14
|
*
|
|
15
15
|
* @private just for single instance
|
|
16
16
|
*/
|
|
17
|
-
declare class
|
|
17
|
+
declare class Wizard {
|
|
18
18
|
/**
|
|
19
19
|
* Run the book
|
|
20
20
|
*
|
|
@@ -53,14 +53,14 @@ declare class Wizzard {
|
|
|
53
53
|
getCompiledBook(pipelineSource: string_filename | string_pipeline_url | PipelineString): Promise<PipelineJson>;
|
|
54
54
|
}
|
|
55
55
|
/**
|
|
56
|
-
*
|
|
56
|
+
* Wizard for simple usage of the Promptbook
|
|
57
57
|
*
|
|
58
58
|
* Note: This works only in Node.js environment and looks for the configuration, environment, tools and cache in the Node.js environment
|
|
59
59
|
*
|
|
60
60
|
* @singleton
|
|
61
|
-
* @public exported from `@promptbook/
|
|
61
|
+
* @public exported from `@promptbook/wizard`
|
|
62
62
|
*/
|
|
63
|
-
export declare const
|
|
63
|
+
export declare const wizard: Wizard;
|
|
64
64
|
export {};
|
|
65
65
|
/**
|
|
66
66
|
* TODO: [🧠] Maybe some way how to handle the progress and streaming?
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@promptbook/website-crawler",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.95.0",
|
|
4
4
|
"description": "Promptbook: Run AI apps in plain human language across multiple models and platforms",
|
|
5
5
|
"private": false,
|
|
6
6
|
"sideEffects": false,
|
|
@@ -70,23 +70,11 @@
|
|
|
70
70
|
"node": ">=16.0.0",
|
|
71
71
|
"npm": ">=8.0.0"
|
|
72
72
|
},
|
|
73
|
-
"cspell": {
|
|
74
|
-
"version": "0.2",
|
|
75
|
-
"language": "en",
|
|
76
|
-
"ignorePaths": [
|
|
77
|
-
"node_modules",
|
|
78
|
-
".next",
|
|
79
|
-
"coverage",
|
|
80
|
-
"dist",
|
|
81
|
-
".git"
|
|
82
|
-
],
|
|
83
|
-
"words": []
|
|
84
|
-
},
|
|
85
73
|
"main": "./umd/index.umd.js",
|
|
86
74
|
"module": "./esm/index.es.js",
|
|
87
75
|
"typings": "./esm/typings/src/_packages/website-crawler.index.d.ts",
|
|
88
76
|
"peerDependencies": {
|
|
89
|
-
"@promptbook/core": "0.
|
|
77
|
+
"@promptbook/core": "0.95.0"
|
|
90
78
|
},
|
|
91
79
|
"dependencies": {
|
|
92
80
|
"@mozilla/readability": "0.6.0",
|
package/umd/index.umd.js
CHANGED
|
@@ -25,7 +25,7 @@
|
|
|
25
25
|
* @generated
|
|
26
26
|
* @see https://github.com/webgptorg/promptbook
|
|
27
27
|
*/
|
|
28
|
-
const PROMPTBOOK_ENGINE_VERSION = '0.
|
|
28
|
+
const PROMPTBOOK_ENGINE_VERSION = '0.95.0';
|
|
29
29
|
/**
|
|
30
30
|
* TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
|
|
31
31
|
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
@@ -500,7 +500,7 @@
|
|
|
500
500
|
className: 'WebsiteScraper',
|
|
501
501
|
mimeTypes: ['text/html'],
|
|
502
502
|
documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
|
|
503
|
-
|
|
503
|
+
isAvailableInBrowser: false,
|
|
504
504
|
// <- Note: [🌏] Only `MarkdownScraper` makes sense to be available in the browser, for scraping non-markdown sources in the browser use a remote server
|
|
505
505
|
requiredExecutables: [],
|
|
506
506
|
}); /* <- Note: [🤛] */
|
|
@@ -510,7 +510,7 @@
|
|
|
510
510
|
* Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available known scrapers
|
|
511
511
|
*
|
|
512
512
|
* @public exported from `@promptbook/core`
|
|
513
|
-
* @public exported from `@promptbook/
|
|
513
|
+
* @public exported from `@promptbook/wizard`
|
|
514
514
|
* @public exported from `@promptbook/cli`
|
|
515
515
|
*/
|
|
516
516
|
$scrapersMetadataRegister.register(websiteScraperMetadata);
|
|
@@ -1037,7 +1037,7 @@
|
|
|
1037
1037
|
* Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
|
|
1038
1038
|
*/
|
|
1039
1039
|
|
|
1040
|
-
var PipelineCollection = [{title:"Prepare Knowledge from Markdown",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.book",formfactorName:"GENERIC",parameters:[{name:"knowledgeContent",description:"Markdown document content",isInput:true,isOutput:false},{name:"knowledgePieces",description:"The knowledge JSON object",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, extract the important knowledge from the document.\n\n# Rules\n\n- Make pieces of information concise, clear, and easy to understand\n- One piece of information should be approximately 1 paragraph\n- Divide the paragraphs by markdown horizontal lines ---\n- Omit irrelevant information\n- Group redundant information\n- Write just extracted information, nothing else\n\n# The document\n\nTake information from this document:\n\n> {knowledgeContent}",resultingParameterName:"knowledgePieces",dependentParameterNames:["knowledgeContent"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Knowledge from Markdown\n\n- PIPELINE URL `https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.book`\n- INPUT PARAMETER `{knowledgeContent}` Markdown document content\n- OUTPUT PARAMETER `{knowledgePieces}` The knowledge JSON object\n\n## Knowledge\n\n<!-- TODO: [🍆] -FORMAT JSON -->\n\n```markdown\nYou are experienced data researcher, extract the important knowledge from the document.\n\n# Rules\n\n- Make pieces of information concise, clear, and easy to understand\n- One piece of information should be approximately 1 paragraph\n- Divide the paragraphs by markdown horizontal lines ---\n- Omit irrelevant information\n- Group redundant information\n- Write just extracted information, nothing else\n\n# The document\n\nTake information from this document:\n\n> {knowledgeContent}\n```\n\n`-> {knowledgePieces}`\n"}],sourceFile:"./books/prepare-knowledge-from-markdown.book"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-keywords.book",formfactorName:"GENERIC",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"keywords",description:"Keywords separated by comma",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, detect the important keywords in the document.\n\n# Rules\n\n- Write just keywords separated by comma\n\n# The document\n\nTake information from this document:\n\n> {knowledgePieceContent}",resultingParameterName:"keywords",dependentParameterNames:["knowledgePieceContent"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Keywords\n\n- PIPELINE URL `https://promptbook.studio/promptbook/prepare-knowledge-keywords.book`\n- INPUT PARAMETER `{knowledgePieceContent}` The content\n- OUTPUT PARAMETER `{keywords}` Keywords separated by comma\n\n## Knowledge\n\n<!-- TODO: [🍆] -FORMAT JSON -->\n\n```markdown\nYou are experienced data researcher, detect the important keywords in the document.\n\n# Rules\n\n- Write just keywords separated by comma\n\n# The document\n\nTake information from this document:\n\n> {knowledgePieceContent}\n```\n\n`-> {keywords}`\n"}],sourceFile:"./books/prepare-knowledge-keywords.book"},{title:"Prepare Knowledge-piece Title",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-title.book",formfactorName:"GENERIC",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"title",description:"The title of the document",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"knowledge",title:"Knowledge",content:"You are experienced content creator, write best title for the document.\n\n# Rules\n\n- Write just title, nothing else\n- Write maximum 5 words for the title\n\n# The document\n\n> {knowledgePieceContent}",resultingParameterName:"title",expectations:{words:{min:1,max:8}},dependentParameterNames:["knowledgePieceContent"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Knowledge-piece Title\n\n- PIPELINE URL `https://promptbook.studio/promptbook/prepare-knowledge-title.book`\n- INPUT PARAMETER `{knowledgePieceContent}` The content\n- OUTPUT PARAMETER `{title}` The title of the document\n\n## Knowledge\n\n- EXPECT MIN 1 WORD\n- EXPECT MAX 8 WORDS\n\n```markdown\nYou are experienced content creator, write best title for the document.\n\n# Rules\n\n- Write just title, nothing else\n- Write maximum 5 words for the title\n\n# The document\n\n> {knowledgePieceContent}\n```\n\n`-> {title}`\n"}],sourceFile:"./books/prepare-knowledge-title.book"},{title:"Prepare Persona",pipelineUrl:"https://promptbook.studio/promptbook/prepare-persona.book",formfactorName:"GENERIC",parameters:[{name:"availableModels",description:"List of available model names together with their descriptions as JSON",isInput:true,isOutput:false},{name:"personaDescription",description:"Description of the persona",isInput:true,isOutput:false},{name:"modelsRequirements",description:"Specific requirements for the model",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"make-model-requirements",title:"Make modelRequirements",content:"You are an experienced AI engineer, you need to find the best models for virtual assistants:\n\n## Example\n\n```json\n[\n {\n \"modelName\": \"gpt-4o\",\n \"systemMessage\": \"You are experienced AI engineer and helpfull assistant.\",\n \"temperature\": 0.7\n },\n {\n \"modelName\": \"claude-3-5-sonnet\",\n \"systemMessage\": \"You are a friendly and knowledgeable chatbot.\",\n \"temperature\": 0.5\n }\n]\n```\n\n## Instructions\n\n- Your output format is JSON array\n- Sort best-fitting models first\n- Omit any models that are not suitable\n- Write just the JSON, no other text should be present\n- Array contain items with following keys:\n - `modelName`: The name of the model to use\n - `systemMessage`: The system message to provide context to the model\n - `temperature`: The sampling temperature to use\n\n### Key `modelName`\n\nHere are the available models:\n\n```json\n{availableModels}\n```\n\n### Key `systemMessage`\n\nThe system message is used to communicate instructions or provide context to the model at the beginning of a conversation. It is displayed in a different format compared to user messages, helping the model understand its role in the conversation. The system message typically guides the model's behavior, sets the tone, or specifies desired output from the model. By utilizing the system message effectively, users can steer the model towards generating more accurate and relevant responses.\n\nFor example:\n\n> You are an experienced AI engineer and helpful assistant.\n\n> You are a friendly and knowledgeable chatbot.\n\n### Key `temperature`\n\nThe sampling temperature, between 0 and 1. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. If set to 0, the model will use log probability to automatically increase the temperature until certain thresholds are hit.\n\nYou can pick a value between 0 and 2. For example:\n\n- `0.1`: Low temperature, extremely conservative and deterministic\n- `0.5`: Medium temperature, balanced between conservative and creative\n- `1.0`: High temperature, creative and bit random\n- `1.5`: Very high temperature, extremely creative and often chaotic and unpredictable\n- `2.0`: Maximum temperature, completely random and unpredictable, for some extreme creative use cases\n\n# The assistant\n\nTake this description of the persona:\n\n> {personaDescription}",resultingParameterName:"modelsRequirements",format:"JSON",dependentParameterNames:["availableModels","personaDescription"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Persona\n\n- PIPELINE URL `https://promptbook.studio/promptbook/prepare-persona.book`\n- INPUT PARAMETER `{availableModels}` List of available model names together with their descriptions as JSON\n- INPUT PARAMETER `{personaDescription}` Description of the persona\n- OUTPUT PARAMETER `{modelsRequirements}` Specific requirements for the model\n\n## Make modelRequirements\n\n- FORMAT JSON\n\n```markdown\nYou are an experienced AI engineer, you need to find the best models for virtual assistants:\n\n## Example\n\n\\`\\`\\`json\n[\n {\n \"modelName\": \"gpt-4o\",\n \"systemMessage\": \"You are experienced AI engineer and helpfull assistant.\",\n \"temperature\": 0.7\n },\n {\n \"modelName\": \"claude-3-5-sonnet\",\n \"systemMessage\": \"You are a friendly and knowledgeable chatbot.\",\n \"temperature\": 0.5\n }\n]\n\\`\\`\\`\n\n## Instructions\n\n- Your output format is JSON array\n- Sort best-fitting models first\n- Omit any models that are not suitable\n- Write just the JSON, no other text should be present\n- Array contain items with following keys:\n - `modelName`: The name of the model to use\n - `systemMessage`: The system message to provide context to the model\n - `temperature`: The sampling temperature to use\n\n### Key `modelName`\n\nHere are the available models:\n\n\\`\\`\\`json\n{availableModels}\n\\`\\`\\`\n\n### Key `systemMessage`\n\nThe system message is used to communicate instructions or provide context to the model at the beginning of a conversation. It is displayed in a different format compared to user messages, helping the model understand its role in the conversation. The system message typically guides the model's behavior, sets the tone, or specifies desired output from the model. By utilizing the system message effectively, users can steer the model towards generating more accurate and relevant responses.\n\nFor example:\n\n> You are an experienced AI engineer and helpful assistant.\n\n> You are a friendly and knowledgeable chatbot.\n\n### Key `temperature`\n\nThe sampling temperature, between 0 and 1. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. If set to 0, the model will use log probability to automatically increase the temperature until certain thresholds are hit.\n\nYou can pick a value between 0 and 2. For example:\n\n- `0.1`: Low temperature, extremely conservative and deterministic\n- `0.5`: Medium temperature, balanced between conservative and creative\n- `1.0`: High temperature, creative and bit random\n- `1.5`: Very high temperature, extremely creative and often chaotic and unpredictable\n- `2.0`: Maximum temperature, completely random and unpredictable, for some extreme creative use cases\n\n# The assistant\n\nTake this description of the persona:\n\n> {personaDescription}\n```\n\n`-> {modelsRequirements}`\n"}],sourceFile:"./books/prepare-persona.book"},{title:"Prepare Title",pipelineUrl:"https://promptbook.studio/promptbook/prepare-title.book",formfactorName:"GENERIC",parameters:[{name:"book",description:"The book to prepare the title for",isInput:true,isOutput:false},{name:"title",description:"Best title for the book",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"make-title",title:"Make title",content:"Make best title for given text which describes the workflow:\n\n## Rules\n\n- Write just title, nothing else\n- Title should be concise and clear - Write maximum ideally 2 words, maximum 5 words\n- Title starts with emoticon\n- Title should not mention the input and output of the workflow but the main purpose of the workflow\n _For example, not \"✍ Convert Knowledge-piece to title\" but \"✍ Title\"_\n\n## The workflow\n\n> {book}",resultingParameterName:"title",expectations:{words:{min:1,max:8},lines:{min:1,max:1}},dependentParameterNames:["book"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Title\n\n- PIPELINE URL `https://promptbook.studio/promptbook/prepare-title.book`\n- INPUT PARAMETER `{book}` The book to prepare the title for\n- OUTPUT PARAMETER `{title}` Best title for the book\n\n## Make title\n\n- EXPECT MIN 1 Word\n- EXPECT MAX 8 Words\n- EXPECT EXACTLY 1 Line\n\n```markdown\nMake best title for given text which describes the workflow:\n\n## Rules\n\n- Write just title, nothing else\n- Title should be concise and clear - Write maximum ideally 2 words, maximum 5 words\n- Title starts with emoticon\n- Title should not mention the input and output of the workflow but the main purpose of the workflow\n _For example, not \"✍ Convert Knowledge-piece to title\" but \"✍ Title\"_\n\n## The workflow\n\n> {book}\n```\n\n`-> {title}`\n"}],sourceFile:"./books/prepare-title.book"}];
|
|
1040
|
+
var PipelineCollection = [{title:"Prepare Knowledge from Markdown",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.book",formfactorName:"GENERIC",parameters:[{name:"knowledgeContent",description:"Markdown document content",isInput:true,isOutput:false},{name:"knowledgePieces",description:"The knowledge JSON object",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, extract the important knowledge from the document.\n\n# Rules\n\n- Make pieces of information concise, clear, and easy to understand\n- One piece of information should be approximately 1 paragraph\n- Divide the paragraphs by markdown horizontal lines ---\n- Omit irrelevant information\n- Group redundant information\n- Write just extracted information, nothing else\n\n# The document\n\nTake information from this document:\n\n> {knowledgeContent}",resultingParameterName:"knowledgePieces",dependentParameterNames:["knowledgeContent"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Knowledge from Markdown\n\n- PIPELINE URL `https://promptbook.studio/promptbook/prepare-knowledge-from-markdown.book`\n- INPUT PARAMETER `{knowledgeContent}` Markdown document content\n- OUTPUT PARAMETER `{knowledgePieces}` The knowledge JSON object\n\n## Knowledge\n\n<!-- TODO: [🍆] -FORMAT JSON -->\n\n```markdown\nYou are experienced data researcher, extract the important knowledge from the document.\n\n# Rules\n\n- Make pieces of information concise, clear, and easy to understand\n- One piece of information should be approximately 1 paragraph\n- Divide the paragraphs by markdown horizontal lines ---\n- Omit irrelevant information\n- Group redundant information\n- Write just extracted information, nothing else\n\n# The document\n\nTake information from this document:\n\n> {knowledgeContent}\n```\n\n`-> {knowledgePieces}`\n"}],sourceFile:"./books/prepare-knowledge-from-markdown.book"},{title:"Prepare Keywords",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-keywords.book",formfactorName:"GENERIC",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"keywords",description:"Keywords separated by comma",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"knowledge",title:"Knowledge",content:"You are experienced data researcher, detect the important keywords in the document.\n\n# Rules\n\n- Write just keywords separated by comma\n\n# The document\n\nTake information from this document:\n\n> {knowledgePieceContent}",resultingParameterName:"keywords",dependentParameterNames:["knowledgePieceContent"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Keywords\n\n- PIPELINE URL `https://promptbook.studio/promptbook/prepare-knowledge-keywords.book`\n- INPUT PARAMETER `{knowledgePieceContent}` The content\n- OUTPUT PARAMETER `{keywords}` Keywords separated by comma\n\n## Knowledge\n\n<!-- TODO: [🍆] -FORMAT JSON -->\n\n```markdown\nYou are experienced data researcher, detect the important keywords in the document.\n\n# Rules\n\n- Write just keywords separated by comma\n\n# The document\n\nTake information from this document:\n\n> {knowledgePieceContent}\n```\n\n`-> {keywords}`\n"}],sourceFile:"./books/prepare-knowledge-keywords.book"},{title:"Prepare Knowledge-piece Title",pipelineUrl:"https://promptbook.studio/promptbook/prepare-knowledge-title.book",formfactorName:"GENERIC",parameters:[{name:"knowledgePieceContent",description:"The content",isInput:true,isOutput:false},{name:"title",description:"The title of the document",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"knowledge",title:"Knowledge",content:"You are experienced content creator, write best title for the document.\n\n# Rules\n\n- Write just title, nothing else\n- Write maximum 5 words for the title\n\n# The document\n\n> {knowledgePieceContent}",resultingParameterName:"title",expectations:{words:{min:1,max:8}},dependentParameterNames:["knowledgePieceContent"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Knowledge-piece Title\n\n- PIPELINE URL `https://promptbook.studio/promptbook/prepare-knowledge-title.book`\n- INPUT PARAMETER `{knowledgePieceContent}` The content\n- OUTPUT PARAMETER `{title}` The title of the document\n\n## Knowledge\n\n- EXPECT MIN 1 WORD\n- EXPECT MAX 8 WORDS\n\n```markdown\nYou are experienced content creator, write best title for the document.\n\n# Rules\n\n- Write just title, nothing else\n- Write maximum 5 words for the title\n\n# The document\n\n> {knowledgePieceContent}\n```\n\n`-> {title}`\n"}],sourceFile:"./books/prepare-knowledge-title.book"},{title:"Prepare Persona",pipelineUrl:"https://promptbook.studio/promptbook/prepare-persona.book",formfactorName:"GENERIC",parameters:[{name:"availableModels",description:"List of available model names together with their descriptions as JSON",isInput:true,isOutput:false},{name:"personaDescription",description:"Description of the persona",isInput:true,isOutput:false},{name:"modelsRequirements",description:"Specific requirements for the model",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"make-model-requirements",title:"Make modelRequirements",content:"You are an experienced AI engineer, you need to find the best models for virtual assistants:\n\n## Example\n\n```json\n[\n {\n \"modelName\": \"gpt-4o\",\n \"systemMessage\": \"You are experienced AI engineer and helpful assistant.\",\n \"temperature\": 0.7\n },\n {\n \"modelName\": \"claude-3-5-sonnet\",\n \"systemMessage\": \"You are a friendly and knowledgeable chatbot.\",\n \"temperature\": 0.5\n }\n]\n```\n\n## Instructions\n\n- Your output format is JSON array\n- Sort best-fitting models first\n- Omit any models that are not suitable\n- Write just the JSON, no other text should be present\n- Array contain items with following keys:\n - `modelName`: The name of the model to use\n - `systemMessage`: The system message to provide context to the model\n - `temperature`: The sampling temperature to use\n\n### Key `modelName`\n\nHere are the available models:\n\n```json\n{availableModels}\n```\n\n### Key `systemMessage`\n\nThe system message is used to communicate instructions or provide context to the model at the beginning of a conversation. It is displayed in a different format compared to user messages, helping the model understand its role in the conversation. The system message typically guides the model's behavior, sets the tone, or specifies desired output from the model. By utilizing the system message effectively, users can steer the model towards generating more accurate and relevant responses.\n\nFor example:\n\n> You are an experienced AI engineer and helpful assistant.\n\n> You are a friendly and knowledgeable chatbot.\n\n### Key `temperature`\n\nThe sampling temperature, between 0 and 1. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. If set to 0, the model will use log probability to automatically increase the temperature until certain thresholds are hit.\n\nYou can pick a value between 0 and 2. For example:\n\n- `0.1`: Low temperature, extremely conservative and deterministic\n- `0.5`: Medium temperature, balanced between conservative and creative\n- `1.0`: High temperature, creative and bit random\n- `1.5`: Very high temperature, extremely creative and often chaotic and unpredictable\n- `2.0`: Maximum temperature, completely random and unpredictable, for some extreme creative use cases\n\n# The assistant\n\nTake this description of the persona:\n\n> {personaDescription}",resultingParameterName:"modelsRequirements",format:"JSON",dependentParameterNames:["availableModels","personaDescription"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Persona\n\n- PIPELINE URL `https://promptbook.studio/promptbook/prepare-persona.book`\n- INPUT PARAMETER `{availableModels}` List of available model names together with their descriptions as JSON\n- INPUT PARAMETER `{personaDescription}` Description of the persona\n- OUTPUT PARAMETER `{modelsRequirements}` Specific requirements for the model\n\n## Make modelRequirements\n\n- FORMAT JSON\n\n```markdown\nYou are an experienced AI engineer, you need to find the best models for virtual assistants:\n\n## Example\n\n\\`\\`\\`json\n[\n {\n \"modelName\": \"gpt-4o\",\n \"systemMessage\": \"You are experienced AI engineer and helpful assistant.\",\n \"temperature\": 0.7\n },\n {\n \"modelName\": \"claude-3-5-sonnet\",\n \"systemMessage\": \"You are a friendly and knowledgeable chatbot.\",\n \"temperature\": 0.5\n }\n]\n\\`\\`\\`\n\n## Instructions\n\n- Your output format is JSON array\n- Sort best-fitting models first\n- Omit any models that are not suitable\n- Write just the JSON, no other text should be present\n- Array contain items with following keys:\n - `modelName`: The name of the model to use\n - `systemMessage`: The system message to provide context to the model\n - `temperature`: The sampling temperature to use\n\n### Key `modelName`\n\nHere are the available models:\n\n\\`\\`\\`json\n{availableModels}\n\\`\\`\\`\n\n### Key `systemMessage`\n\nThe system message is used to communicate instructions or provide context to the model at the beginning of a conversation. It is displayed in a different format compared to user messages, helping the model understand its role in the conversation. The system message typically guides the model's behavior, sets the tone, or specifies desired output from the model. By utilizing the system message effectively, users can steer the model towards generating more accurate and relevant responses.\n\nFor example:\n\n> You are an experienced AI engineer and helpful assistant.\n\n> You are a friendly and knowledgeable chatbot.\n\n### Key `temperature`\n\nThe sampling temperature, between 0 and 1. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic. If set to 0, the model will use log probability to automatically increase the temperature until certain thresholds are hit.\n\nYou can pick a value between 0 and 2. For example:\n\n- `0.1`: Low temperature, extremely conservative and deterministic\n- `0.5`: Medium temperature, balanced between conservative and creative\n- `1.0`: High temperature, creative and bit random\n- `1.5`: Very high temperature, extremely creative and often chaotic and unpredictable\n- `2.0`: Maximum temperature, completely random and unpredictable, for some extreme creative use cases\n\n# The assistant\n\nTake this description of the persona:\n\n> {personaDescription}\n```\n\n`-> {modelsRequirements}`\n"}],sourceFile:"./books/prepare-persona.book"},{title:"Prepare Title",pipelineUrl:"https://promptbook.studio/promptbook/prepare-title.book",formfactorName:"GENERIC",parameters:[{name:"book",description:"The book to prepare the title for",isInput:true,isOutput:false},{name:"title",description:"Best title for the book",isInput:false,isOutput:true}],tasks:[{taskType:"PROMPT_TASK",name:"make-title",title:"Make title",content:"Make best title for given text which describes the workflow:\n\n## Rules\n\n- Write just title, nothing else\n- Title should be concise and clear - Write maximum ideally 2 words, maximum 5 words\n- Title starts with emoticon\n- Title should not mention the input and output of the workflow but the main purpose of the workflow\n _For example, not \"✍ Convert Knowledge-piece to title\" but \"✍ Title\"_\n\n## The workflow\n\n> {book}",resultingParameterName:"title",expectations:{words:{min:1,max:8},lines:{min:1,max:1}},dependentParameterNames:["book"]}],personas:[],preparations:[],knowledgeSources:[],knowledgePieces:[],sources:[{type:"BOOK",path:null,content:"# Prepare Title\n\n- PIPELINE URL `https://promptbook.studio/promptbook/prepare-title.book`\n- INPUT PARAMETER `{book}` The book to prepare the title for\n- OUTPUT PARAMETER `{title}` Best title for the book\n\n## Make title\n\n- EXPECT MIN 1 Word\n- EXPECT MAX 8 Words\n- EXPECT EXACTLY 1 Line\n\n```markdown\nMake best title for given text which describes the workflow:\n\n## Rules\n\n- Write just title, nothing else\n- Title should be concise and clear - Write maximum ideally 2 words, maximum 5 words\n- Title starts with emoticon\n- Title should not mention the input and output of the workflow but the main purpose of the workflow\n _For example, not \"✍ Convert Knowledge-piece to title\" but \"✍ Title\"_\n\n## The workflow\n\n> {book}\n```\n\n`-> {title}`\n"}],sourceFile:"./books/prepare-title.book"}];
|
|
1041
1041
|
|
|
1042
1042
|
/**
|
|
1043
1043
|
* Checks if value is valid email
|
|
@@ -1194,7 +1194,7 @@
|
|
|
1194
1194
|
});
|
|
1195
1195
|
}
|
|
1196
1196
|
catch (error) {
|
|
1197
|
-
// TODO: [🟥] Detect browser / node and make it
|
|
1197
|
+
// TODO: [🟥] Detect browser / node and make it colorful
|
|
1198
1198
|
console.error('There was an error with prettifying the markdown, using the original as the fallback', {
|
|
1199
1199
|
error,
|
|
1200
1200
|
html: content,
|
|
@@ -1449,7 +1449,7 @@
|
|
|
1449
1449
|
else {
|
|
1450
1450
|
for (const [subName, subValue] of Object.entries(value)) {
|
|
1451
1451
|
if (subValue === undefined) {
|
|
1452
|
-
// Note: undefined in object is serializable - it is just
|
|
1452
|
+
// Note: undefined in object is serializable - it is just omitted
|
|
1453
1453
|
continue;
|
|
1454
1454
|
}
|
|
1455
1455
|
checkSerializableAsJson({ name: `${name}.${subName}`, value: subValue, message });
|
|
@@ -2139,7 +2139,7 @@
|
|
|
2139
2139
|
|
|
2140
2140
|
Note: You have probably forgotten to run "ptbk make" to update the collection
|
|
2141
2141
|
Note: Pipelines with the same URL are not allowed
|
|
2142
|
-
Only
|
|
2142
|
+
Only exception is when the pipelines are identical
|
|
2143
2143
|
|
|
2144
2144
|
`));
|
|
2145
2145
|
}
|
|
@@ -2884,12 +2884,12 @@
|
|
|
2884
2884
|
get title() {
|
|
2885
2885
|
return `${llmTools.title} (+usage)`;
|
|
2886
2886
|
// <- TODO: [🧈] Maybe standartize the suffix when wrapping `LlmExecutionTools` up
|
|
2887
|
-
// <- TODO: [🧈][🧠] Does it make
|
|
2887
|
+
// <- TODO: [🧈][🧠] Does it make sense to suffix "(+usage)"?
|
|
2888
2888
|
},
|
|
2889
2889
|
get description() {
|
|
2890
2890
|
return `${llmTools.description} (+usage)`;
|
|
2891
2891
|
// <- TODO: [🧈] Maybe standartize the suffix when wrapping `LlmExecutionTools` up
|
|
2892
|
-
// <- TODO: [🧈][🧠] Does it make
|
|
2892
|
+
// <- TODO: [🧈][🧠] Does it make sense to suffix "(+usage)"?
|
|
2893
2893
|
},
|
|
2894
2894
|
checkConfiguration() {
|
|
2895
2895
|
return /* not await */ llmTools.checkConfiguration();
|
|
@@ -3116,13 +3116,13 @@
|
|
|
3116
3116
|
|
|
3117
3117
|
Technically, it's not an error, but it's probably not what you want because it does not make sense to use Promptbook without language models.
|
|
3118
3118
|
`);
|
|
3119
|
-
// TODO: [🟥] Detect browser / node and make it
|
|
3119
|
+
// TODO: [🟥] Detect browser / node and make it colorful
|
|
3120
3120
|
console.warn(warningMessage);
|
|
3121
3121
|
// <- TODO: [🏮] Some standard way how to transform errors into warnings and how to handle non-critical fails during the tasks
|
|
3122
3122
|
/*
|
|
3123
3123
|
return {
|
|
3124
3124
|
async listModels() {
|
|
3125
|
-
// TODO: [🟥] Detect browser / node and make it
|
|
3125
|
+
// TODO: [🟥] Detect browser / node and make it colorful
|
|
3126
3126
|
console.warn(
|
|
3127
3127
|
spaceTrim(
|
|
3128
3128
|
(block) => `
|
|
@@ -3258,17 +3258,17 @@
|
|
|
3258
3258
|
* Mixes registered scrapers from $scrapersMetadataRegister and $scrapersRegister
|
|
3259
3259
|
*/
|
|
3260
3260
|
const all = [];
|
|
3261
|
-
for (const { packageName, className, mimeTypes, documentationUrl,
|
|
3261
|
+
for (const { packageName, className, mimeTypes, documentationUrl, isAvailableInBrowser, } of $scrapersMetadataRegister.list()) {
|
|
3262
3262
|
if (all.some((item) => item.packageName === packageName && item.className === className)) {
|
|
3263
3263
|
continue;
|
|
3264
3264
|
}
|
|
3265
|
-
all.push({ packageName, className, mimeTypes, documentationUrl,
|
|
3265
|
+
all.push({ packageName, className, mimeTypes, documentationUrl, isAvailableInBrowser });
|
|
3266
3266
|
}
|
|
3267
|
-
for (const { packageName, className, mimeTypes, documentationUrl,
|
|
3267
|
+
for (const { packageName, className, mimeTypes, documentationUrl, isAvailableInBrowser, } of $scrapersRegister.list()) {
|
|
3268
3268
|
if (all.some((item) => item.packageName === packageName && item.className === className)) {
|
|
3269
3269
|
continue;
|
|
3270
3270
|
}
|
|
3271
|
-
all.push({ packageName, className, mimeTypes, documentationUrl,
|
|
3271
|
+
all.push({ packageName, className, mimeTypes, documentationUrl, isAvailableInBrowser });
|
|
3272
3272
|
}
|
|
3273
3273
|
for (const { metadata } of availableScrapers) {
|
|
3274
3274
|
all.push(metadata);
|
|
@@ -3280,8 +3280,8 @@
|
|
|
3280
3280
|
const isInstalled = $scrapersRegister
|
|
3281
3281
|
.list()
|
|
3282
3282
|
.find(({ packageName, className }) => metadata.packageName === packageName && metadata.className === className);
|
|
3283
|
-
const
|
|
3284
|
-
return { ...metadata, isMetadataAviailable, isInstalled,
|
|
3283
|
+
const isAvailableInTools = availableScrapers.some(({ metadata: { packageName, className } }) => metadata.packageName === packageName && metadata.className === className);
|
|
3284
|
+
return { ...metadata, isMetadataAviailable, isInstalled, isAvailableInTools };
|
|
3285
3285
|
});
|
|
3286
3286
|
if (metadata.length === 0) {
|
|
3287
3287
|
return spaceTrim__default["default"](`
|
|
@@ -3294,7 +3294,7 @@
|
|
|
3294
3294
|
return spaceTrim__default["default"]((block) => `
|
|
3295
3295
|
Available scrapers are:
|
|
3296
3296
|
${block(metadata
|
|
3297
|
-
.map(({ packageName, className, isMetadataAviailable, isInstalled, mimeTypes,
|
|
3297
|
+
.map(({ packageName, className, isMetadataAviailable, isInstalled, mimeTypes, isAvailableInBrowser, isAvailableInTools, }, i) => {
|
|
3298
3298
|
const more = [];
|
|
3299
3299
|
// TODO: [🧠] Maybe use `documentationUrl`
|
|
3300
3300
|
if (isMetadataAviailable) {
|
|
@@ -3303,16 +3303,16 @@
|
|
|
3303
3303
|
if (isInstalled) {
|
|
3304
3304
|
more.push(`🟩 Installed`);
|
|
3305
3305
|
} // not else
|
|
3306
|
-
if (
|
|
3306
|
+
if (isAvailableInTools) {
|
|
3307
3307
|
more.push(`🟦 Available in tools`);
|
|
3308
3308
|
} // not else
|
|
3309
3309
|
if (!isMetadataAviailable && isInstalled) {
|
|
3310
3310
|
more.push(`When no metadata registered but scraper is installed, it is an unexpected behavior`);
|
|
3311
3311
|
} // not else
|
|
3312
|
-
if (!isInstalled &&
|
|
3312
|
+
if (!isInstalled && isAvailableInTools) {
|
|
3313
3313
|
more.push(`When the scraper is not installed but available in tools, it is an unexpected compatibility behavior`);
|
|
3314
3314
|
} // not else
|
|
3315
|
-
if (!
|
|
3315
|
+
if (!isAvailableInBrowser) {
|
|
3316
3316
|
more.push(`Not usable in browser`);
|
|
3317
3317
|
}
|
|
3318
3318
|
const moreText = more.length === 0 ? '' : ` *(${more.join('; ')})*`;
|
|
@@ -3677,7 +3677,7 @@
|
|
|
3677
3677
|
/**
|
|
3678
3678
|
* TODO: [🧊] In future one preparation can take data from previous preparation and save tokens and time
|
|
3679
3679
|
* Put `knowledgePieces` into `PrepareKnowledgeOptions`
|
|
3680
|
-
* TODO: [🪂] More than max things can run in parallel by
|
|
3680
|
+
* TODO: [🪂] More than max things can run in parallel by accident [1,[2a,2b,_],[3a,3b,_]]
|
|
3681
3681
|
* TODO: [🧠][❎] Do here proper M:N mapping
|
|
3682
3682
|
* [x] One source can make multiple pieces
|
|
3683
3683
|
* [ ] One piece can have multiple sources
|
|
@@ -5349,10 +5349,10 @@
|
|
|
5349
5349
|
*/
|
|
5350
5350
|
async function getKnowledgeForTask(options) {
|
|
5351
5351
|
const { tools, preparedPipeline, task, parameters } = options;
|
|
5352
|
-
const
|
|
5353
|
-
const
|
|
5352
|
+
const firstKnowledgePiece = preparedPipeline.knowledgePieces[0];
|
|
5353
|
+
const firstKnowledgeIndex = firstKnowledgePiece === null || firstKnowledgePiece === void 0 ? void 0 : firstKnowledgePiece.index[0];
|
|
5354
5354
|
// <- TODO: Do not use just first knowledge piece and first index to determine embedding model, use also keyword search
|
|
5355
|
-
if (
|
|
5355
|
+
if (firstKnowledgePiece === undefined || firstKnowledgeIndex === undefined) {
|
|
5356
5356
|
return ''; // <- Note: Np knowledge present, return empty string
|
|
5357
5357
|
}
|
|
5358
5358
|
try {
|
|
@@ -5363,7 +5363,7 @@
|
|
|
5363
5363
|
title: 'Knowledge Search',
|
|
5364
5364
|
modelRequirements: {
|
|
5365
5365
|
modelVariant: 'EMBEDDING',
|
|
5366
|
-
modelName:
|
|
5366
|
+
modelName: firstKnowledgeIndex.modelName,
|
|
5367
5367
|
},
|
|
5368
5368
|
content: task.content,
|
|
5369
5369
|
parameters,
|
|
@@ -5371,7 +5371,7 @@
|
|
|
5371
5371
|
const taskEmbeddingResult = await llmTools.callEmbeddingModel(taskEmbeddingPrompt);
|
|
5372
5372
|
const knowledgePiecesWithRelevance = preparedPipeline.knowledgePieces.map((knowledgePiece) => {
|
|
5373
5373
|
const { index } = knowledgePiece;
|
|
5374
|
-
const knowledgePieceIndex = index.find((i) => i.modelName ===
|
|
5374
|
+
const knowledgePieceIndex = index.find((i) => i.modelName === firstKnowledgeIndex.modelName);
|
|
5375
5375
|
// <- TODO: Do not use just first knowledge piece and first index to determine embedding model
|
|
5376
5376
|
if (knowledgePieceIndex === undefined) {
|
|
5377
5377
|
return {
|
|
@@ -5392,8 +5392,8 @@
|
|
|
5392
5392
|
task,
|
|
5393
5393
|
taskEmbeddingPrompt,
|
|
5394
5394
|
taskEmbeddingResult,
|
|
5395
|
-
|
|
5396
|
-
|
|
5395
|
+
firstKnowledgePiece,
|
|
5396
|
+
firstKnowledgeIndex,
|
|
5397
5397
|
knowledgePiecesWithRelevance,
|
|
5398
5398
|
knowledgePiecesSorted,
|
|
5399
5399
|
knowledgePiecesLimited,
|
|
@@ -5462,7 +5462,7 @@
|
|
|
5462
5462
|
* @private internal utility of `createPipelineExecutor`
|
|
5463
5463
|
*/
|
|
5464
5464
|
async function executeTask(options) {
|
|
5465
|
-
const { currentTask, preparedPipeline, parametersToPass, tools, onProgress, $executionReport, pipelineIdentification, maxExecutionAttempts, maxParallelCount, csvSettings, isVerbose, rootDirname, cacheDirname, intermediateFilesStrategy, isAutoInstalled,
|
|
5465
|
+
const { currentTask, preparedPipeline, parametersToPass, tools, onProgress, $executionReport, pipelineIdentification, maxExecutionAttempts, maxParallelCount, csvSettings, isVerbose, rootDirname, cacheDirname, intermediateFilesStrategy, isAutoInstalled, isNotPreparedWarningSuppressed, } = options;
|
|
5466
5466
|
const priority = preparedPipeline.tasks.length - preparedPipeline.tasks.indexOf(currentTask);
|
|
5467
5467
|
// Note: Check consistency of used and dependent parameters which was also done in `validatePipeline`, but it’s good to doublecheck
|
|
5468
5468
|
const usedParameterNames = extractParameterNamesFromTask(currentTask);
|
|
@@ -5550,7 +5550,7 @@
|
|
|
5550
5550
|
cacheDirname,
|
|
5551
5551
|
intermediateFilesStrategy,
|
|
5552
5552
|
isAutoInstalled,
|
|
5553
|
-
|
|
5553
|
+
isNotPreparedWarningSuppressed,
|
|
5554
5554
|
});
|
|
5555
5555
|
await onProgress({
|
|
5556
5556
|
outputParameters: {
|
|
@@ -5645,7 +5645,7 @@
|
|
|
5645
5645
|
}
|
|
5646
5646
|
return exportJson({
|
|
5647
5647
|
name: `executionReport`,
|
|
5648
|
-
message: `
|
|
5648
|
+
message: `Unsuccessful PipelineExecutorResult (with missing parameter {${parameter.name}}) PipelineExecutorResult`,
|
|
5649
5649
|
order: [],
|
|
5650
5650
|
value: {
|
|
5651
5651
|
isSuccessful: false,
|
|
@@ -5682,7 +5682,7 @@
|
|
|
5682
5682
|
return exportJson({
|
|
5683
5683
|
name: 'pipelineExecutorResult',
|
|
5684
5684
|
message: spaceTrim.spaceTrim((block) => `
|
|
5685
|
-
|
|
5685
|
+
Unsuccessful PipelineExecutorResult (with extra parameter {${parameter.name}}) PipelineExecutorResult
|
|
5686
5686
|
|
|
5687
5687
|
${block(pipelineIdentification)}
|
|
5688
5688
|
`),
|
|
@@ -5823,7 +5823,7 @@
|
|
|
5823
5823
|
}
|
|
5824
5824
|
return exportJson({
|
|
5825
5825
|
name: 'pipelineExecutorResult',
|
|
5826
|
-
message: `
|
|
5826
|
+
message: `Unsuccessful PipelineExecutorResult (with misc errors) PipelineExecutorResult`,
|
|
5827
5827
|
order: [],
|
|
5828
5828
|
value: {
|
|
5829
5829
|
isSuccessful: false,
|
|
@@ -5874,7 +5874,7 @@
|
|
|
5874
5874
|
* @public exported from `@promptbook/core`
|
|
5875
5875
|
*/
|
|
5876
5876
|
function createPipelineExecutor(options) {
|
|
5877
|
-
const { pipeline, tools, maxExecutionAttempts = DEFAULT_MAX_EXECUTION_ATTEMPTS, maxParallelCount = DEFAULT_MAX_PARALLEL_COUNT, csvSettings = DEFAULT_CSV_SETTINGS, isVerbose = DEFAULT_IS_VERBOSE,
|
|
5877
|
+
const { pipeline, tools, maxExecutionAttempts = DEFAULT_MAX_EXECUTION_ATTEMPTS, maxParallelCount = DEFAULT_MAX_PARALLEL_COUNT, csvSettings = DEFAULT_CSV_SETTINGS, isVerbose = DEFAULT_IS_VERBOSE, isNotPreparedWarningSuppressed = false, cacheDirname = DEFAULT_SCRAPE_CACHE_DIRNAME, intermediateFilesStrategy = DEFAULT_INTERMEDIATE_FILES_STRATEGY, isAutoInstalled = DEFAULT_IS_AUTO_INSTALLED, rootDirname = null, } = options;
|
|
5878
5878
|
validatePipeline(pipeline);
|
|
5879
5879
|
const pipelineIdentification = (() => {
|
|
5880
5880
|
// Note: This is a 😐 implementation of [🚞]
|
|
@@ -5891,7 +5891,7 @@
|
|
|
5891
5891
|
if (isPipelinePrepared(pipeline)) {
|
|
5892
5892
|
preparedPipeline = pipeline;
|
|
5893
5893
|
}
|
|
5894
|
-
else if (
|
|
5894
|
+
else if (isNotPreparedWarningSuppressed !== true) {
|
|
5895
5895
|
console.warn(spaceTrim.spaceTrim((block) => `
|
|
5896
5896
|
Pipeline is not prepared
|
|
5897
5897
|
|
|
@@ -5924,7 +5924,7 @@
|
|
|
5924
5924
|
maxParallelCount,
|
|
5925
5925
|
csvSettings,
|
|
5926
5926
|
isVerbose,
|
|
5927
|
-
|
|
5927
|
+
isNotPreparedWarningSuppressed,
|
|
5928
5928
|
rootDirname,
|
|
5929
5929
|
cacheDirname,
|
|
5930
5930
|
intermediateFilesStrategy,
|
|
@@ -5933,7 +5933,7 @@
|
|
|
5933
5933
|
assertsError(error);
|
|
5934
5934
|
return exportJson({
|
|
5935
5935
|
name: 'pipelineExecutorResult',
|
|
5936
|
-
message: `
|
|
5936
|
+
message: `Unsuccessful PipelineExecutorResult, last catch`,
|
|
5937
5937
|
order: [],
|
|
5938
5938
|
value: {
|
|
5939
5939
|
isSuccessful: false,
|
|
@@ -5971,7 +5971,7 @@
|
|
|
5971
5971
|
className: 'MarkdownScraper',
|
|
5972
5972
|
mimeTypes: ['text/markdown', 'text/plain'],
|
|
5973
5973
|
documentationUrl: 'https://github.com/webgptorg/promptbook/discussions/@@',
|
|
5974
|
-
|
|
5974
|
+
isAvailableInBrowser: true,
|
|
5975
5975
|
// <- Note: [🌏] This is the only scraper which makes sense to be available in the browser, for scraping non-markdown sources in the browser use a remote server
|
|
5976
5976
|
requiredExecutables: [],
|
|
5977
5977
|
}); /* <- Note: [🤛] */
|
|
@@ -5981,7 +5981,7 @@
|
|
|
5981
5981
|
* Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available known scrapers
|
|
5982
5982
|
*
|
|
5983
5983
|
* @public exported from `@promptbook/core`
|
|
5984
|
-
* @public exported from `@promptbook/
|
|
5984
|
+
* @public exported from `@promptbook/wizard`
|
|
5985
5985
|
* @public exported from `@promptbook/cli`
|
|
5986
5986
|
*/
|
|
5987
5987
|
$scrapersMetadataRegister.register(markdownScraperMetadata);
|
|
@@ -6080,7 +6080,7 @@
|
|
|
6080
6080
|
}
|
|
6081
6081
|
// ---
|
|
6082
6082
|
if (!llmTools.callEmbeddingModel) {
|
|
6083
|
-
// TODO: [🟥] Detect browser / node and make it
|
|
6083
|
+
// TODO: [🟥] Detect browser / node and make it colorful
|
|
6084
6084
|
console.error('No callEmbeddingModel function provided');
|
|
6085
6085
|
}
|
|
6086
6086
|
else {
|
|
@@ -6106,7 +6106,7 @@
|
|
|
6106
6106
|
if (!(error instanceof PipelineExecutionError)) {
|
|
6107
6107
|
throw error;
|
|
6108
6108
|
}
|
|
6109
|
-
// TODO: [🟥] Detect browser / node and make it
|
|
6109
|
+
// TODO: [🟥] Detect browser / node and make it colorful
|
|
6110
6110
|
console.error(error, "<- Note: This error is not critical to prepare the pipeline, just knowledge pieces won't have embeddings");
|
|
6111
6111
|
}
|
|
6112
6112
|
return {
|
|
@@ -6265,7 +6265,7 @@
|
|
|
6265
6265
|
* Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available known scrapers
|
|
6266
6266
|
*
|
|
6267
6267
|
* @public exported from `@promptbook/website-crawler`
|
|
6268
|
-
* @public exported from `@promptbook/
|
|
6268
|
+
* @public exported from `@promptbook/wizard`
|
|
6269
6269
|
* @public exported from `@promptbook/cli`
|
|
6270
6270
|
*/
|
|
6271
6271
|
const _WebsiteScraperRegistration = $scrapersRegister.register(createWebsiteScraper);
|