@promptbook/website-crawler 0.100.0-2 → 0.100.0-21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/esm/index.es.js +39 -5
- package/esm/index.es.js.map +1 -1
- package/esm/typings/src/_packages/core.index.d.ts +24 -0
- package/esm/typings/src/_packages/types.index.d.ts +28 -0
- package/esm/typings/src/book-2.0/agent-source/parseAgentSource.d.ts +30 -0
- package/esm/typings/src/book-2.0/agent-source/parseAgentSource.test.d.ts +1 -0
- package/esm/typings/src/book-2.0/agent-source/string_book.d.ts +26 -0
- package/esm/typings/src/book-2.0/commitments/ACTION/ACTION.d.ts +30 -0
- package/esm/typings/src/book-2.0/commitments/FORMAT/FORMAT.d.ts +31 -0
- package/esm/typings/src/book-2.0/commitments/KNOWLEDGE/FrontendRAGService.d.ts +48 -0
- package/esm/typings/src/book-2.0/commitments/KNOWLEDGE/KNOWLEDGE.d.ts +43 -0
- package/esm/typings/src/book-2.0/commitments/KNOWLEDGE/RAGService.d.ts +54 -0
- package/esm/typings/src/book-2.0/commitments/KNOWLEDGE/processors/BaseKnowledgeProcessor.d.ts +45 -0
- package/esm/typings/src/book-2.0/commitments/KNOWLEDGE/processors/PdfProcessor.d.ts +31 -0
- package/esm/typings/src/book-2.0/commitments/KNOWLEDGE/processors/ProcessorFactory.d.ts +23 -0
- package/esm/typings/src/book-2.0/commitments/KNOWLEDGE/processors/TextProcessor.d.ts +18 -0
- package/esm/typings/src/book-2.0/commitments/KNOWLEDGE/types.d.ts +56 -0
- package/esm/typings/src/book-2.0/commitments/KNOWLEDGE/utils/ragHelper.d.ts +34 -0
- package/esm/typings/src/book-2.0/commitments/META_IMAGE/META_IMAGE.d.ts +36 -0
- package/esm/typings/src/book-2.0/commitments/META_LINK/META_LINK.d.ts +48 -0
- package/esm/typings/src/book-2.0/commitments/MODEL/MODEL.d.ts +31 -0
- package/esm/typings/src/book-2.0/commitments/NOTE/NOTE.d.ts +41 -0
- package/esm/typings/src/book-2.0/commitments/PERSONA/PERSONA.d.ts +38 -0
- package/esm/typings/src/book-2.0/commitments/RULE/RULE.d.ts +36 -0
- package/esm/typings/src/book-2.0/commitments/SAMPLE/SAMPLE.d.ts +36 -0
- package/esm/typings/src/book-2.0/commitments/STYLE/STYLE.d.ts +30 -0
- package/esm/typings/src/book-2.0/commitments/_base/BaseCommitmentDefinition.d.ts +42 -0
- package/esm/typings/src/book-2.0/commitments/_base/BookCommitment.d.ts +5 -0
- package/esm/typings/src/book-2.0/commitments/_base/CommitmentDefinition.d.ts +37 -0
- package/esm/typings/src/book-2.0/commitments/_base/NotYetImplementedCommitmentDefinition.d.ts +14 -0
- package/esm/typings/src/book-2.0/commitments/_base/createEmptyAgentModelRequirements.d.ts +19 -0
- package/esm/typings/src/book-2.0/commitments/_misc/AgentModelRequirements.d.ts +37 -0
- package/esm/typings/src/book-2.0/commitments/_misc/AgentSourceParseResult.d.ts +18 -0
- package/esm/typings/src/book-2.0/commitments/_misc/ParsedCommitment.d.ts +22 -0
- package/esm/typings/src/book-2.0/commitments/_misc/createAgentModelRequirements.d.ts +61 -0
- package/esm/typings/src/book-2.0/commitments/_misc/createAgentModelRequirementsWithCommitments.d.ts +35 -0
- package/esm/typings/src/book-2.0/commitments/_misc/createCommitmentRegex.d.ts +20 -0
- package/esm/typings/src/book-2.0/commitments/_misc/parseAgentSourceWithCommitments.d.ts +24 -0
- package/esm/typings/src/book-2.0/commitments/_misc/removeCommentsFromSystemMessage.d.ts +11 -0
- package/esm/typings/src/book-2.0/commitments/index.d.ts +56 -0
- package/esm/typings/src/book-2.0/utils/profileImageUtils.d.ts +39 -0
- package/esm/typings/src/pipeline/book-notation.d.ts +2 -1
- package/esm/typings/src/types/typeAliases.d.ts +6 -0
- package/esm/typings/src/version.d.ts +1 -1
- package/esm/typings/src/wizard/wizard.d.ts +14 -4
- package/package.json +2 -2
- package/umd/index.umd.js +39 -5
- package/umd/index.umd.js.map +1 -1
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import type { string_url_image } from '../../types/typeAliases';
|
|
2
|
+
/**
|
|
3
|
+
* Extracts profile image URL from agent definition text and returns cleaned system message
|
|
4
|
+
* @param systemMessage The original system message that may contain META IMAGE line
|
|
5
|
+
* @returns Object with profileImageUrl (if found) and cleanedSystemMessage (without META IMAGE line)
|
|
6
|
+
*
|
|
7
|
+
* @private - TODO: [🧠] Maybe should be public?
|
|
8
|
+
*/
|
|
9
|
+
export declare function extractProfileImageFromSystemMessage(systemMessage: string): {
|
|
10
|
+
profileImageUrl?: string_url_image;
|
|
11
|
+
cleanedSystemMessage: string;
|
|
12
|
+
};
|
|
13
|
+
/**
|
|
14
|
+
* Extracts persona, examples, and profile image from agent definition text
|
|
15
|
+
* @param systemMessage The original system message that may contain PERSONA, EXAMPLE, and META IMAGE lines
|
|
16
|
+
* @returns Object with extracted information and cleaned system message
|
|
17
|
+
*
|
|
18
|
+
* @private - TODO: [🧠] Maybe should be public?
|
|
19
|
+
*/
|
|
20
|
+
export declare function extractAgentMetadata(systemMessage: string): {
|
|
21
|
+
persona?: {
|
|
22
|
+
name: string;
|
|
23
|
+
description?: string;
|
|
24
|
+
};
|
|
25
|
+
examples: string[];
|
|
26
|
+
profileImageUrl?: string_url_image;
|
|
27
|
+
cleanedSystemMessage: string;
|
|
28
|
+
};
|
|
29
|
+
/**
|
|
30
|
+
* Generates a gravatar URL based on agent name for fallback avatar
|
|
31
|
+
* @param name The agent name to generate avatar for
|
|
32
|
+
* @returns Gravatar URL
|
|
33
|
+
*
|
|
34
|
+
* @private - TODO: [🧠] Maybe should be public?
|
|
35
|
+
*/
|
|
36
|
+
export declare function generateGravatarUrl(name?: string | null): string;
|
|
37
|
+
/**
|
|
38
|
+
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
39
|
+
*/
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { string_book } from '../book-2.0/agent-source/string_book';
|
|
1
2
|
import type { PipelineString } from './PipelineString';
|
|
2
3
|
/**
|
|
3
4
|
* Tag function for notating a pipeline with a book\`...\ notation as template literal
|
|
@@ -12,7 +13,7 @@ import type { PipelineString } from './PipelineString';
|
|
|
12
13
|
* @returns the pipeline string
|
|
13
14
|
* @public exported from `@promptbook/core`
|
|
14
15
|
*/
|
|
15
|
-
export declare function book(strings: TemplateStringsArray, ...values: Array<string>): PipelineString;
|
|
16
|
+
export declare function book(strings: TemplateStringsArray, ...values: Array<string>): string_book & PipelineString;
|
|
16
17
|
/**
|
|
17
18
|
* TODO: [🧠][🈴] Where is the best location for this file
|
|
18
19
|
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
@@ -136,6 +136,12 @@ export type ReservedParameters = Record<string_reserved_parameter_name, string_p
|
|
|
136
136
|
* For example `"Ai*nautes"`
|
|
137
137
|
*/
|
|
138
138
|
export type string_title = string;
|
|
139
|
+
/**
|
|
140
|
+
* Semantic helper
|
|
141
|
+
*
|
|
142
|
+
* For example `"My AI Assistant"`
|
|
143
|
+
*/
|
|
144
|
+
export type string_agent_name = string;
|
|
139
145
|
/**
|
|
140
146
|
* Unstructured description of the persona
|
|
141
147
|
*
|
|
@@ -15,7 +15,7 @@ export declare const BOOK_LANGUAGE_VERSION: string_semantic_version;
|
|
|
15
15
|
export declare const PROMPTBOOK_ENGINE_VERSION: string_promptbook_version;
|
|
16
16
|
/**
|
|
17
17
|
* Represents the version string of the Promptbook engine.
|
|
18
|
-
* It follows semantic versioning (e.g., `0.100.0-
|
|
18
|
+
* It follows semantic versioning (e.g., `0.100.0-20`).
|
|
19
19
|
*
|
|
20
20
|
* @generated
|
|
21
21
|
*/
|
|
@@ -6,6 +6,15 @@ import type { InputParameters } from '../types/typeAliases';
|
|
|
6
6
|
import type { string_filename } from '../types/typeAliases';
|
|
7
7
|
import type { string_parameter_value } from '../types/typeAliases';
|
|
8
8
|
import type { string_pipeline_url } from '../types/typeAliases';
|
|
9
|
+
/**
|
|
10
|
+
* Options for wizard methods
|
|
11
|
+
*/
|
|
12
|
+
interface WizardOptions {
|
|
13
|
+
/**
|
|
14
|
+
* Whether to enable verbose logging
|
|
15
|
+
*/
|
|
16
|
+
isVerbose?: boolean;
|
|
17
|
+
}
|
|
9
18
|
/**
|
|
10
19
|
* Wizard for simple usage of the Promptbook
|
|
11
20
|
* Look at `wizard` for more details
|
|
@@ -26,7 +35,7 @@ declare class Wizard {
|
|
|
26
35
|
*
|
|
27
36
|
* Note: This works similar to the `ptbk run` command
|
|
28
37
|
*/
|
|
29
|
-
execute(book: string_pipeline_url | string_filename | PipelineString, inputParameters: InputParameters): Promise<{
|
|
38
|
+
execute(book: string_pipeline_url | string_filename | PipelineString, inputParameters: InputParameters, options?: WizardOptions): Promise<{
|
|
30
39
|
/**
|
|
31
40
|
* Simple result of the execution
|
|
32
41
|
*/
|
|
@@ -36,9 +45,9 @@ declare class Wizard {
|
|
|
36
45
|
/**
|
|
37
46
|
* Provides the tools automatically for the Node.js environment
|
|
38
47
|
*
|
|
39
|
-
* @param
|
|
48
|
+
* @param options
|
|
40
49
|
*/
|
|
41
|
-
getExecutionTools(): Promise<Required<Pick<ExecutionTools, 'fs' | 'fetch'>>>;
|
|
50
|
+
getExecutionTools(options?: WizardOptions): Promise<Required<Pick<ExecutionTools, 'fs' | 'fetch'>>>;
|
|
42
51
|
/**
|
|
43
52
|
* Load book from the source
|
|
44
53
|
*
|
|
@@ -49,8 +58,9 @@ declare class Wizard {
|
|
|
49
58
|
* 3) As a string
|
|
50
59
|
*
|
|
51
60
|
* @param pipelineSource
|
|
61
|
+
* @param options
|
|
52
62
|
*/
|
|
53
|
-
getCompiledBook(pipelineSource: string_filename | string_pipeline_url | PipelineString): Promise<PipelineJson>;
|
|
63
|
+
getCompiledBook(pipelineSource: string_filename | string_pipeline_url | PipelineString, options?: WizardOptions): Promise<PipelineJson>;
|
|
54
64
|
}
|
|
55
65
|
/**
|
|
56
66
|
* Wizard for simple usage of the Promptbook
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@promptbook/website-crawler",
|
|
3
|
-
"version": "0.100.0-
|
|
3
|
+
"version": "0.100.0-21",
|
|
4
4
|
"description": "Promptbook: Run AI apps in plain human language across multiple models and platforms",
|
|
5
5
|
"private": false,
|
|
6
6
|
"sideEffects": false,
|
|
@@ -95,7 +95,7 @@
|
|
|
95
95
|
"module": "./esm/index.es.js",
|
|
96
96
|
"typings": "./esm/typings/src/_packages/website-crawler.index.d.ts",
|
|
97
97
|
"peerDependencies": {
|
|
98
|
-
"@promptbook/core": "0.100.0-
|
|
98
|
+
"@promptbook/core": "0.100.0-21"
|
|
99
99
|
},
|
|
100
100
|
"dependencies": {
|
|
101
101
|
"@mozilla/readability": "0.6.0",
|
package/umd/index.umd.js
CHANGED
|
@@ -25,7 +25,7 @@
|
|
|
25
25
|
* @generated
|
|
26
26
|
* @see https://github.com/webgptorg/promptbook
|
|
27
27
|
*/
|
|
28
|
-
const PROMPTBOOK_ENGINE_VERSION = '0.100.0-
|
|
28
|
+
const PROMPTBOOK_ENGINE_VERSION = '0.100.0-21';
|
|
29
29
|
/**
|
|
30
30
|
* TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
|
|
31
31
|
* Note: [💞] Ignore a discrepancy between file name and entity name
|
|
@@ -1009,7 +1009,23 @@
|
|
|
1009
1009
|
.join('/') +
|
|
1010
1010
|
'.' +
|
|
1011
1011
|
extension;
|
|
1012
|
-
|
|
1012
|
+
// Note: Try to create cache directory, but don't fail if filesystem has issues
|
|
1013
|
+
try {
|
|
1014
|
+
await promises.mkdir(path.dirname(cacheFilename), { recursive: true });
|
|
1015
|
+
}
|
|
1016
|
+
catch (error) {
|
|
1017
|
+
// Note: If we can't create cache directory, continue without it
|
|
1018
|
+
// This handles read-only filesystems, permission issues, and missing parent directories
|
|
1019
|
+
if (error instanceof Error && (error.message.includes('EROFS') ||
|
|
1020
|
+
error.message.includes('read-only') ||
|
|
1021
|
+
error.message.includes('EACCES') ||
|
|
1022
|
+
error.message.includes('EPERM') ||
|
|
1023
|
+
error.message.includes('ENOENT'))) ;
|
|
1024
|
+
else {
|
|
1025
|
+
// Re-throw other unexpected errors
|
|
1026
|
+
throw error;
|
|
1027
|
+
}
|
|
1028
|
+
}
|
|
1013
1029
|
let isDestroyed = true;
|
|
1014
1030
|
const fileHandler = {
|
|
1015
1031
|
filename: cacheFilename,
|
|
@@ -3494,7 +3510,23 @@
|
|
|
3494
3510
|
// <- TODO: [🥬] Encapsulate sha256 to some private utility function
|
|
3495
3511
|
const rootDirname = path.join(process.cwd(), DEFAULT_DOWNLOAD_CACHE_DIRNAME);
|
|
3496
3512
|
const filepath = path.join(...nameToSubfolderPath(hash /* <- TODO: [🎎] Maybe add some SHA256 prefix */), `${basename.substring(0, MAX_FILENAME_LENGTH)}.${mimeTypeToExtension(mimeType)}`);
|
|
3497
|
-
|
|
3513
|
+
// Note: Try to create cache directory, but don't fail if filesystem has issues
|
|
3514
|
+
try {
|
|
3515
|
+
await tools.fs.mkdir(path.dirname(path.join(rootDirname, filepath)), { recursive: true });
|
|
3516
|
+
}
|
|
3517
|
+
catch (error) {
|
|
3518
|
+
// Note: If we can't create cache directory, we'll handle it when trying to write the file
|
|
3519
|
+
// This handles read-only filesystems, permission issues, and missing parent directories
|
|
3520
|
+
if (error instanceof Error && (error.message.includes('EROFS') ||
|
|
3521
|
+
error.message.includes('read-only') ||
|
|
3522
|
+
error.message.includes('EACCES') ||
|
|
3523
|
+
error.message.includes('EPERM') ||
|
|
3524
|
+
error.message.includes('ENOENT'))) ;
|
|
3525
|
+
else {
|
|
3526
|
+
// Re-throw other unexpected errors
|
|
3527
|
+
throw error;
|
|
3528
|
+
}
|
|
3529
|
+
}
|
|
3498
3530
|
const fileContent = Buffer.from(await response.arrayBuffer());
|
|
3499
3531
|
if (fileContent.length > DEFAULT_MAX_FILE_SIZE /* <- TODO: Allow to pass different value to remote server */) {
|
|
3500
3532
|
throw new LimitReachedError(`File is too large (${Math.round(fileContent.length / 1024 / 1024)}MB). Maximum allowed size is ${Math.round(DEFAULT_MAX_FILE_SIZE / 1024 / 1024)}MB.`);
|
|
@@ -3509,7 +3541,8 @@
|
|
|
3509
3541
|
if (error instanceof Error && (error.message.includes('EROFS') ||
|
|
3510
3542
|
error.message.includes('read-only') ||
|
|
3511
3543
|
error.message.includes('EACCES') ||
|
|
3512
|
-
error.message.includes('EPERM')
|
|
3544
|
+
error.message.includes('EPERM') ||
|
|
3545
|
+
error.message.includes('ENOENT'))) {
|
|
3513
3546
|
// Return a handler that works directly with the downloaded content
|
|
3514
3547
|
return {
|
|
3515
3548
|
source: name,
|
|
@@ -6305,7 +6338,8 @@
|
|
|
6305
6338
|
if (error instanceof Error && (error.message.includes('EROFS') ||
|
|
6306
6339
|
error.message.includes('read-only') ||
|
|
6307
6340
|
error.message.includes('EACCES') ||
|
|
6308
|
-
error.message.includes('EPERM')
|
|
6341
|
+
error.message.includes('EPERM') ||
|
|
6342
|
+
error.message.includes('ENOENT'))) ;
|
|
6309
6343
|
else {
|
|
6310
6344
|
// Re-throw other unexpected errors
|
|
6311
6345
|
throw error;
|