@promptbook/node 0.92.0-23 → 0.92.0-25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/esm/index.es.js +117 -82
- package/esm/index.es.js.map +1 -1
- package/esm/typings/src/_packages/core.index.d.ts +0 -2
- package/esm/typings/src/collection/PipelineCollection.d.ts +0 -2
- package/esm/typings/src/collection/SimplePipelineCollection.d.ts +1 -1
- package/esm/typings/src/commands/FORMFACTOR/formfactorCommandParser.d.ts +1 -1
- package/esm/typings/src/commands/_common/types/CommandParser.d.ts +36 -28
- package/esm/typings/src/config.d.ts +8 -4
- package/esm/typings/src/constants.d.ts +2 -2
- package/esm/typings/src/errors/0-BoilerplateError.d.ts +2 -2
- package/esm/typings/src/execution/CommonToolsOptions.d.ts +3 -3
- package/esm/typings/src/formats/_common/FormatSubvalueParser.d.ts +10 -10
- package/esm/typings/src/formfactors/_boilerplate/BoilerplateFormfactorDefinition.d.ts +3 -2
- package/esm/typings/src/formfactors/_common/string_formfactor_name.d.ts +2 -1
- package/esm/typings/src/formfactors/index.d.ts +1 -1
- package/esm/typings/src/formfactors/sheets/SheetsFormfactorDefinition.d.ts +3 -2
- package/esm/typings/src/llm-providers/_common/register/$llmToolsMetadataRegister.d.ts +3 -3
- package/esm/typings/src/llm-providers/_common/register/$llmToolsRegister.d.ts +3 -3
- package/esm/typings/src/llm-providers/_common/register/$provideLlmToolsConfigurationFromEnv.d.ts +4 -4
- package/esm/typings/src/llm-providers/_common/utils/cache/CacheItem.d.ts +5 -5
- package/esm/typings/src/llm-providers/_common/utils/cache/CacheLlmToolsOptions.d.ts +5 -3
- package/esm/typings/src/llm-providers/_common/utils/cache/cacheLlmTools.d.ts +3 -3
- package/esm/typings/src/llm-providers/_common/utils/count-total-usage/limitTotalUsage.d.ts +5 -5
- package/esm/typings/src/other/templates/getBookTemplates.d.ts +2 -2
- package/esm/typings/src/pipeline/PipelineInterface/PipelineInterface.d.ts +3 -3
- package/esm/typings/src/pipeline/PipelineInterface/constants.d.ts +1 -1
- package/esm/typings/src/pipeline/PipelineInterface/getPipelineInterface.d.ts +1 -1
- package/esm/typings/src/pipeline/PipelineInterface/isPipelineImplementingInterface.d.ts +5 -4
- package/esm/typings/src/pipeline/PipelineInterface/isPipelineInterfacesEqual.d.ts +1 -1
- package/esm/typings/src/pipeline/PipelineJson/CommonTaskJson.d.ts +9 -6
- package/esm/typings/src/pipeline/PipelineJson/PipelineJson.d.ts +2 -2
- package/esm/typings/src/pipeline/PipelineString.d.ts +3 -1
- package/esm/typings/src/pipeline/book-notation.d.ts +2 -2
- package/esm/typings/src/prepare/prepareTasks.d.ts +7 -4
- package/esm/typings/src/remote-server/types/RemoteServerOptions.d.ts +2 -1
- package/esm/typings/src/scrapers/_boilerplate/BoilerplateScraper.d.ts +3 -3
- package/esm/typings/src/scrapers/_boilerplate/createBoilerplateScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/_boilerplate/register-metadata.d.ts +1 -1
- package/esm/typings/src/scrapers/_common/Converter.d.ts +3 -1
- package/esm/typings/src/scrapers/_common/Scraper.d.ts +4 -3
- package/esm/typings/src/scrapers/_common/ScraperIntermediateSource.d.ts +4 -2
- package/esm/typings/src/scrapers/_common/register/$provideFilesystemForNode.d.ts +2 -1
- package/esm/typings/src/scrapers/_common/register/$provideScrapersForBrowser.d.ts +6 -3
- package/esm/typings/src/scrapers/_common/register/$provideScrapersForNode.d.ts +3 -5
- package/esm/typings/src/scrapers/_common/register/$scrapersRegister.d.ts +3 -2
- package/esm/typings/src/scrapers/_common/register/ScraperAndConverterMetadata.d.ts +8 -5
- package/esm/typings/src/scrapers/_common/register/ScraperConstructor.d.ts +2 -1
- package/esm/typings/src/scrapers/_common/utils/getScraperIntermediateSource.d.ts +6 -5
- package/esm/typings/src/scrapers/_common/utils/makeKnowledgeSourceHandler.d.ts +3 -1
- package/esm/typings/src/scrapers/document/createDocumentScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/document-legacy/createLegacyDocumentScraper.d.ts +2 -1
- package/esm/typings/src/scrapers/markdown/createMarkdownScraper.d.ts +4 -1
- package/esm/typings/src/scrapers/markitdown/MarkitdownScraper.d.ts +1 -1
- package/esm/typings/src/scrapers/pdf/createPdfScraper.d.ts +2 -1
- package/esm/typings/src/scrapers/website/createWebsiteScraper.d.ts +3 -4
- package/esm/typings/src/scripting/javascript/postprocessing-functions.d.ts +5 -1
- package/esm/typings/src/storage/file-cache-storage/FileCacheStorage.d.ts +12 -5
- package/esm/typings/src/storage/file-cache-storage/FileCacheStorageOptions.d.ts +4 -2
- package/esm/typings/src/storage/file-cache-storage/utils/nameToSubfolderPath.d.ts +2 -1
- package/esm/typings/src/storage/local-storage/utils/makePromptbookStorageFromWebStorage.d.ts +2 -1
- package/esm/typings/src/types/IntermediateFilesStrategy.d.ts +2 -1
- package/esm/typings/src/types/ModelVariant.d.ts +5 -5
- package/esm/typings/src/types/typeAliases.d.ts +8 -6
- package/esm/typings/src/utils/editable/edit-pipeline-string/addPipelineCommand.d.ts +2 -2
- package/esm/typings/src/utils/editable/edit-pipeline-string/deflatePipeline.d.ts +4 -1
- package/esm/typings/src/utils/editable/utils/isFlatPipeline.d.ts +2 -1
- package/esm/typings/src/utils/environment/$getGlobalScope.d.ts +2 -1
- package/esm/typings/src/utils/markdown/extractAllListItemsFromMarkdown.d.ts +1 -1
- package/esm/typings/src/utils/normalization/nameToUriPart.d.ts +4 -4
- package/esm/typings/src/utils/normalization/nameToUriParts.d.ts +4 -4
- package/esm/typings/src/utils/normalization/normalize-to-kebab-case.d.ts +3 -3
- package/esm/typings/src/utils/normalization/normalizeTo_SCREAMING_CASE.d.ts +3 -3
- package/esm/typings/src/utils/normalization/normalizeTo_camelCase.d.ts +4 -4
- package/esm/typings/src/utils/normalization/normalizeTo_snake_case.d.ts +3 -3
- package/esm/typings/src/utils/normalization/removeDiacritics.d.ts +3 -3
- package/esm/typings/src/utils/normalization/searchKeywords.d.ts +4 -1
- package/esm/typings/src/utils/normalization/titleToName.d.ts +4 -4
- package/esm/typings/src/utils/organization/empty_object.d.ts +2 -2
- package/esm/typings/src/utils/organization/just_empty_object.d.ts +4 -4
- package/esm/typings/src/version.d.ts +2 -1
- package/package.json +2 -2
- package/umd/index.umd.js +117 -82
- package/umd/index.umd.js.map +1 -1
|
@@ -1,25 +1,26 @@
|
|
|
1
1
|
import type { PipelineJson } from '../PipelineJson/PipelineJson';
|
|
2
2
|
import type { PipelineInterface } from './PipelineInterface';
|
|
3
3
|
/**
|
|
4
|
-
*
|
|
4
|
+
* Options for the `isPipelineImplementingInterface` function.
|
|
5
5
|
*
|
|
6
6
|
* @see https://github.com/webgptorg/promptbook/discussions/171
|
|
7
7
|
*/
|
|
8
8
|
export type IsPipelineImplementingInterfaceOptions = {
|
|
9
9
|
/**
|
|
10
|
-
*
|
|
10
|
+
* @param pipeline The pipeline to check.
|
|
11
11
|
*/
|
|
12
12
|
pipeline: PipelineJson;
|
|
13
13
|
/**
|
|
14
|
-
*
|
|
14
|
+
* @param pipelineInterface The interface to check against.
|
|
15
15
|
*/
|
|
16
16
|
pipelineInterface: PipelineInterface;
|
|
17
17
|
};
|
|
18
18
|
/**
|
|
19
|
-
*
|
|
19
|
+
* Checks if a given pipeline satisfies the requirements of a specified pipeline interface.
|
|
20
20
|
*
|
|
21
21
|
* @deprecated https://github.com/webgptorg/promptbook/pull/186
|
|
22
22
|
* @see https://github.com/webgptorg/promptbook/discussions/171
|
|
23
|
+
* @returns `true` if the pipeline implements the interface, `false` otherwise.
|
|
23
24
|
*
|
|
24
25
|
* @public exported from `@promptbook/core`
|
|
25
26
|
*/
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { PipelineInterface } from './PipelineInterface';
|
|
2
2
|
/**
|
|
3
|
-
*
|
|
3
|
+
* Checks if two pipeline interfaces are structurally identical.
|
|
4
4
|
*
|
|
5
5
|
* @deprecated https://github.com/webgptorg/promptbook/pull/186
|
|
6
6
|
* @see https://github.com/webgptorg/promptbook/discussions/171
|
|
@@ -38,13 +38,15 @@ export type CommonTaskJson = {
|
|
|
38
38
|
*/
|
|
39
39
|
readonly dependentParameterNames: Array<string_parameter_name>;
|
|
40
40
|
/**
|
|
41
|
-
*
|
|
41
|
+
* List of parameter names that act as jokers.
|
|
42
|
+
* If a joker parameter meets the expectations, its value is used instead of executing the task.
|
|
42
43
|
*
|
|
43
44
|
* @see https://github.com/webgptorg/promptbook/discussions/66
|
|
44
45
|
*/
|
|
45
46
|
readonly jokerParameterNames?: Array<string_parameter_name>;
|
|
46
47
|
/**
|
|
47
|
-
*
|
|
48
|
+
* Configuration for the FOREACH command, if used.
|
|
49
|
+
* Allows iterating over a list parameter.
|
|
48
50
|
*/
|
|
49
51
|
readonly foreach?: ForeachJson;
|
|
50
52
|
/**
|
|
@@ -53,15 +55,16 @@ export type CommonTaskJson = {
|
|
|
53
55
|
*/
|
|
54
56
|
readonly taskType: SectionType;
|
|
55
57
|
/**
|
|
56
|
-
*
|
|
58
|
+
* Raw content of the task with {placeholders} for parameters before any preparation.
|
|
57
59
|
*
|
|
58
|
-
*
|
|
60
|
+
* @see preparedContent
|
|
59
61
|
*/
|
|
60
62
|
readonly content: (string_prompt | string_javascript | string_markdown) & string_template;
|
|
61
63
|
/**
|
|
62
|
-
*
|
|
64
|
+
* Content of the task after preparation, with {placeholders} for parameters.
|
|
65
|
+
* This is the content used during execution.
|
|
63
66
|
*
|
|
64
|
-
*
|
|
67
|
+
* @see content
|
|
65
68
|
*
|
|
66
69
|
* @default "{content}"
|
|
67
70
|
*/
|
|
@@ -15,12 +15,12 @@ import type { PersonaPreparedJson } from './PersonaJson';
|
|
|
15
15
|
import type { PreparationJson } from './PreparationJson';
|
|
16
16
|
import type { TaskJson } from './TaskJson';
|
|
17
17
|
/**
|
|
18
|
-
*
|
|
18
|
+
* Represents a Promptbook pipeline defined in JSON format.
|
|
19
19
|
* It represents a series of tasks chained together to form a pipeline / one big task with input and result parameters.
|
|
20
20
|
*
|
|
21
21
|
* Note: [🚉] This is fully serializable as JSON
|
|
22
22
|
*
|
|
23
|
-
* @see
|
|
23
|
+
* @see The main Promptbook documentation for more details: https://github.com/webgptorg/promptbook#promptbook
|
|
24
24
|
*/
|
|
25
25
|
export type PipelineJson = {
|
|
26
26
|
/**
|
|
@@ -2,7 +2,9 @@
|
|
|
2
2
|
* Promptbook is the **core concept of this package**.
|
|
3
3
|
* It represents a series of tasks chained together to form a pipeline / one big task with input and result parameters.
|
|
4
4
|
*
|
|
5
|
-
*
|
|
5
|
+
* Represents a Promptbook pipeline defined as a string.
|
|
6
|
+
*
|
|
7
|
+
* @see The main Promptbook documentation for more details: https://github.com/webgptorg/promptbook#promptbook
|
|
6
8
|
*/
|
|
7
9
|
export type PipelineString = string & {
|
|
8
10
|
readonly _type: 'Promptbook';
|
|
@@ -7,8 +7,8 @@ import type { PipelineString } from './PipelineString';
|
|
|
7
7
|
* 2) `promptTemplate` alias for `prompt`
|
|
8
8
|
* 3) `book` for notating and validating entire books exported from `@promptbook/utils`
|
|
9
9
|
*
|
|
10
|
-
* @param strings
|
|
11
|
-
* @param values
|
|
10
|
+
* @param strings The static string parts of the template literal
|
|
11
|
+
* @param values The dynamic values embedded within the template literal used as data
|
|
12
12
|
* @returns the pipeline string
|
|
13
13
|
* @public exported from `@promptbook/core`
|
|
14
14
|
*/
|
|
@@ -4,20 +4,23 @@ import type { TaskJson } from '../pipeline/PipelineJson/TaskJson';
|
|
|
4
4
|
import type { PrepareAndScrapeOptions } from './PrepareAndScrapeOptions';
|
|
5
5
|
type PrepareTaskInput = Pick<PipelineJson, 'tasks' | 'parameters'> & {
|
|
6
6
|
/**
|
|
7
|
-
*
|
|
7
|
+
* The number of knowledge pieces available for the pipeline.
|
|
8
8
|
*/
|
|
9
9
|
readonly knowledgePiecesCount: number;
|
|
10
10
|
};
|
|
11
11
|
type PreparedTasks = {
|
|
12
12
|
/**
|
|
13
|
-
*
|
|
13
|
+
* The sequence of tasks after preparation.
|
|
14
14
|
*/
|
|
15
15
|
readonly tasksPrepared: ReadonlyArray<TaskJson>;
|
|
16
16
|
};
|
|
17
17
|
/**
|
|
18
|
-
*
|
|
18
|
+
* Prepares tasks by adding knowledge to the prompt and ensuring all necessary parameters are included.
|
|
19
19
|
*
|
|
20
|
-
* @
|
|
20
|
+
* @param tasks Sequence of tasks that are chained together to form a pipeline
|
|
21
|
+
* @returns A promise that resolves to the prepared tasks.
|
|
22
|
+
*
|
|
23
|
+
* @private internal utility of `preparePipeline`
|
|
21
24
|
*/
|
|
22
25
|
export declare function prepareTasks(pipeline: PrepareTaskInput, tools: Pick<ExecutionTools, 'llm' | 'fs' | 'scrapers'>, options: PrepareAndScrapeOptions): Promise<PreparedTasks>;
|
|
23
26
|
export {};
|
|
@@ -13,7 +13,7 @@ import type { string_user_id } from '../../types/typeAliases';
|
|
|
13
13
|
import type { ApplicationModeIdentification } from '../socket-types/_subtypes/Identification';
|
|
14
14
|
import type { Identification } from '../socket-types/_subtypes/Identification';
|
|
15
15
|
/**
|
|
16
|
-
*
|
|
16
|
+
* Options for configuring the Promptbook remote server.
|
|
17
17
|
*
|
|
18
18
|
* There are two modes of remote server:
|
|
19
19
|
*
|
|
@@ -28,6 +28,7 @@ import type { Identification } from '../socket-types/_subtypes/Identification';
|
|
|
28
28
|
export type RemoteServerOptions<TCustomOptions> = CommonToolsOptions & {
|
|
29
29
|
/**
|
|
30
30
|
* Port on which the server will listen
|
|
31
|
+
* @property {number} port The port number the server will listen on.
|
|
31
32
|
*/
|
|
32
33
|
readonly port: number;
|
|
33
34
|
/**
|
|
@@ -7,7 +7,7 @@ import type { Scraper } from '../_common/Scraper';
|
|
|
7
7
|
import type { ScraperSourceHandler } from '../_common/Scraper';
|
|
8
8
|
import type { ScraperIntermediateSource } from '../_common/ScraperIntermediateSource';
|
|
9
9
|
/**
|
|
10
|
-
* Scraper of
|
|
10
|
+
* Scraper of @@ files
|
|
11
11
|
*
|
|
12
12
|
* @see `documentationUrl` for more details
|
|
13
13
|
* @public exported from `@promptbook/boilerplate`
|
|
@@ -25,7 +25,7 @@ export declare class BoilerplateScraper implements Converter, Scraper {
|
|
|
25
25
|
private readonly markdownScraper;
|
|
26
26
|
constructor(tools: Pick<ExecutionTools, 'fs' | 'llm' | 'executables'>, options: PrepareAndScrapeOptions);
|
|
27
27
|
/**
|
|
28
|
-
* Convert the
|
|
28
|
+
* Convert the `.@@` to `.md` file and returns intermediate source
|
|
29
29
|
*
|
|
30
30
|
* Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object
|
|
31
31
|
*/
|
|
@@ -39,5 +39,5 @@ export declare class BoilerplateScraper implements Converter, Scraper {
|
|
|
39
39
|
* TODO: [👣] Converted documents can act as cached items - there is no need to run conversion each time
|
|
40
40
|
* TODO: [🪂] Do it in parallel
|
|
41
41
|
* Note: No need to aggregate usage here, it is done by intercepting the llmTools
|
|
42
|
-
*
|
|
42
|
+
* @@ Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
|
|
43
43
|
*/
|
|
@@ -11,7 +11,7 @@ export declare const createBoilerplateScraper: ((tools: Pick<ExecutionTools, 'll
|
|
|
11
11
|
packageName: string;
|
|
12
12
|
className: string;
|
|
13
13
|
mimeTypes: string[];
|
|
14
|
-
documentationUrl: "https://github.com/webgptorg/promptbook/discussions
|
|
14
|
+
documentationUrl: "https://github.com/webgptorg/promptbook/discussions/@@";
|
|
15
15
|
isAvilableInBrowser: false;
|
|
16
16
|
requiredExecutables: never[];
|
|
17
17
|
}>;
|
|
@@ -9,7 +9,7 @@ export declare const boilerplateScraperMetadata: import("type-fest/source/readon
|
|
|
9
9
|
packageName: string;
|
|
10
10
|
className: string;
|
|
11
11
|
mimeTypes: string[];
|
|
12
|
-
documentationUrl: "https://github.com/webgptorg/promptbook/discussions
|
|
12
|
+
documentationUrl: "https://github.com/webgptorg/promptbook/discussions/@@";
|
|
13
13
|
isAvilableInBrowser: false;
|
|
14
14
|
requiredExecutables: never[];
|
|
15
15
|
}>;
|
|
@@ -2,7 +2,9 @@ import type { ScraperAndConverterMetadata } from './register/ScraperAndConverter
|
|
|
2
2
|
import type { ScraperSourceHandler } from './Scraper';
|
|
3
3
|
import type { ScraperIntermediateSource } from './ScraperIntermediateSource';
|
|
4
4
|
/**
|
|
5
|
-
*
|
|
5
|
+
* Interface defining the requirements for converter implementations.
|
|
6
|
+
* Converters transform scraped content from one format to another,
|
|
7
|
+
* optimizing it for knowledge extraction and processing.
|
|
6
8
|
*
|
|
7
9
|
* Note: [🌏] Converters are not usable in browser because they produce a files
|
|
8
10
|
*/
|
|
@@ -6,7 +6,8 @@ import type { string_mime_type } from '../../types/typeAliases';
|
|
|
6
6
|
import type { string_url } from '../../types/typeAliases';
|
|
7
7
|
import type { ScraperAndConverterMetadata } from './register/ScraperAndConverterMetadata';
|
|
8
8
|
/**
|
|
9
|
-
*
|
|
9
|
+
* Interface defining the requirements for scraper implementations.
|
|
10
|
+
* Scrapers are responsible for extracting structured content from various knowledge sources.
|
|
10
11
|
*/
|
|
11
12
|
export type Scraper = {
|
|
12
13
|
/**
|
|
@@ -19,7 +20,8 @@ export type Scraper = {
|
|
|
19
20
|
scrape(source: ScraperSourceHandler): Promisable<ReadonlyArray<Omit<KnowledgePiecePreparedJson, 'sources' | 'preparationIds'>> | null>;
|
|
20
21
|
};
|
|
21
22
|
/**
|
|
22
|
-
*
|
|
23
|
+
* Handler for accessing and processing content from diverse knowledge sources.
|
|
24
|
+
* Provides standardized methods to interact with files, URLs and raw text sources.
|
|
23
25
|
*/
|
|
24
26
|
export type ScraperSourceHandler = {
|
|
25
27
|
/**
|
|
@@ -54,6 +56,5 @@ export type ScraperSourceHandler = {
|
|
|
54
56
|
/**
|
|
55
57
|
* TODO: [🧠] Maybe split `ScraperSourceHandler` into `ScraperWebsiteSourceHandler` + `ScraperFileSourceHandler`
|
|
56
58
|
* TODO: [🥽] Add ` asBlob(): Promisable<Blob>;` or asFile
|
|
57
|
-
* TODO: [🐝] @@@ Annotate all
|
|
58
59
|
* TODO: [🔼] Export via types
|
|
59
60
|
*/
|
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
import type { IDestroyable } from 'destroyable';
|
|
2
2
|
import type { string_absolute_filename } from '../../types/typeAliases';
|
|
3
3
|
/**
|
|
4
|
-
*
|
|
4
|
+
* Interface representing an intermediate storage location for scraper results.
|
|
5
|
+
* Provides functionality for caching and managing temporary data during scraping operations.
|
|
5
6
|
*/
|
|
6
7
|
export type ScraperIntermediateSource = IDestroyable & {
|
|
7
8
|
/**
|
|
8
|
-
*
|
|
9
|
+
* The absolute path to the file where intermediate data is stored.
|
|
10
|
+
* This file serves as a cache for scraped content to avoid redundant processing.
|
|
9
11
|
*/
|
|
10
12
|
readonly filename: string_absolute_filename;
|
|
11
13
|
};
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import type { FilesystemTools } from '../../../execution/FilesystemTools';
|
|
2
2
|
import type { PrepareAndScrapeOptions } from '../../../prepare/PrepareAndScrapeOptions';
|
|
3
3
|
/**
|
|
4
|
-
*
|
|
4
|
+
* Provides filesystem access (for example for Node.js-based scrapers)
|
|
5
|
+
* Creates a standardized filesystem interface that scrapers can use for file operations.
|
|
5
6
|
*
|
|
6
7
|
* @public exported from `@promptbook/node`
|
|
7
8
|
*/
|
|
@@ -2,10 +2,13 @@ import type { ExecutionTools } from '../../../execution/ExecutionTools';
|
|
|
2
2
|
import type { PrepareAndScrapeOptions } from '../../../prepare/PrepareAndScrapeOptions';
|
|
3
3
|
import type { Scraper } from '../Scraper';
|
|
4
4
|
/**
|
|
5
|
-
*
|
|
5
|
+
* Provides a collection of scrapers optimized for browser environments.
|
|
6
|
+
* Only includes scrapers that can safely run in a browser context.
|
|
6
7
|
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
8
|
+
* Note: Browser scrapers have limitations compared to Node.js scrapers.
|
|
9
|
+
*
|
|
10
|
+
* 1) `provideScrapersForNode` use as default
|
|
11
|
+
* 2) `provideScrapersForBrowser` use in limited browser environment
|
|
9
12
|
*
|
|
10
13
|
* @public exported from `@promptbook/browser`
|
|
11
14
|
*/
|
|
@@ -2,11 +2,9 @@ import type { ExecutionTools } from '../../../execution/ExecutionTools';
|
|
|
2
2
|
import type { PrepareAndScrapeOptions } from '../../../prepare/PrepareAndScrapeOptions';
|
|
3
3
|
import type { Scraper } from '../Scraper';
|
|
4
4
|
/**
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
* 2) @@@
|
|
9
|
-
*
|
|
5
|
+
* Provides a collection of scrapers optimized for Node.js environment.
|
|
6
|
+
* 1) `provideScrapersForNode` use as default
|
|
7
|
+
* 2) `provideScrapersForBrowser` use in limited browser environment *
|
|
10
8
|
* @public exported from `@promptbook/node`
|
|
11
9
|
*/
|
|
12
10
|
export declare function $provideScrapersForNode(tools: Pick<ExecutionTools, 'fs' | 'llm' | 'executables'>, options?: PrepareAndScrapeOptions): Promise<ReadonlyArray<Scraper>>;
|
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
import { $Register } from '../../../utils/$Register';
|
|
2
2
|
import type { ScraperConstructor } from './ScraperConstructor';
|
|
3
3
|
/**
|
|
4
|
-
*
|
|
4
|
+
* Registry for all available scrapers in the system.
|
|
5
|
+
* Central point for registering and accessing different types of content scrapers.
|
|
5
6
|
*
|
|
6
7
|
* Note: `$` is used to indicate that this interacts with the global scope
|
|
7
|
-
* @singleton Only one instance of each register is created per build, but
|
|
8
|
+
* @singleton Only one instance of each register is created per build, but there can be more than one in different build modules
|
|
8
9
|
* @public exported from `@promptbook/core`
|
|
9
10
|
*/
|
|
10
11
|
export declare const $scrapersRegister: $Register<ScraperConstructor>;
|
|
@@ -4,9 +4,9 @@ import type { string_promptbook_documentation_url } from '../../../types/typeAli
|
|
|
4
4
|
import type { string_title } from '../../../types/typeAliases';
|
|
5
5
|
import type { Registered } from '../../../utils/$Register';
|
|
6
6
|
/**
|
|
7
|
-
*
|
|
7
|
+
* Metadata interface for scrapers and converters in the system.
|
|
8
|
+
* Contains information about the capabilities and requirements of a scraper or converter.
|
|
8
9
|
*
|
|
9
|
-
* @@@
|
|
10
10
|
* x) `Scraper`
|
|
11
11
|
* x) `Converter`
|
|
12
12
|
* x) `ScraperConstructor`
|
|
@@ -19,7 +19,8 @@ import type { Registered } from '../../../utils/$Register';
|
|
|
19
19
|
*/
|
|
20
20
|
export type ScraperAndConverterMetadata = Registered & {
|
|
21
21
|
/**
|
|
22
|
-
*
|
|
22
|
+
* Human-readable title of the scraper or converter.
|
|
23
|
+
* Used for display purposes in logs and interfaces.
|
|
23
24
|
*/
|
|
24
25
|
readonly title: string_title;
|
|
25
26
|
/**
|
|
@@ -27,13 +28,15 @@ export type ScraperAndConverterMetadata = Registered & {
|
|
|
27
28
|
*/
|
|
28
29
|
readonly mimeTypes: ReadonlyArray<string_mime_type>;
|
|
29
30
|
/**
|
|
30
|
-
*
|
|
31
|
+
* Flag indicating whether this scraper or converter can run in a browser environment.
|
|
32
|
+
* Some scrapers require Node.js capabilities and cannot run client-side.
|
|
31
33
|
*
|
|
32
34
|
* Note: [🌏] Only `MarkdownScraper` makes sense to be available in the browser, for scraping non-markdown sources in the browser use a remote server
|
|
33
35
|
*/
|
|
34
36
|
readonly isAvilableInBrowser: boolean;
|
|
35
37
|
/**
|
|
36
|
-
*
|
|
38
|
+
* List of executables required by this scraper or converter to function properly.
|
|
39
|
+
* For example, PDF scrapers may require 'pandoc' to be installed on the system.
|
|
37
40
|
*/
|
|
38
41
|
readonly requiredExecutables: ReadonlyArray<Capitalize<keyof Executables extends `${infer N}Path` ? N : never>>;
|
|
39
42
|
/**
|
|
@@ -4,7 +4,8 @@ import type { Registered } from '../../../utils/$Register';
|
|
|
4
4
|
import type { Scraper } from '../Scraper';
|
|
5
5
|
import type { ScraperAndConverterMetadata } from './ScraperAndConverterMetadata';
|
|
6
6
|
/**
|
|
7
|
-
*
|
|
7
|
+
* Type definition for a constructor function that creates a Scraper instance.
|
|
8
|
+
* Used for registering scrapers in the system to handle different content types.
|
|
8
9
|
*/
|
|
9
10
|
export type ScraperConstructor = Registered & ScraperAndConverterMetadata & ((tools: Pick<ExecutionTools, 'llm'>, options: PrepareAndScrapeOptions) => Scraper);
|
|
10
11
|
/**
|
|
@@ -3,13 +3,15 @@ import type { string_file_extension } from '../../../types/typeAliases';
|
|
|
3
3
|
import type { ScraperSourceHandler } from '../Scraper';
|
|
4
4
|
import type { ScraperIntermediateSource } from '../ScraperIntermediateSource';
|
|
5
5
|
/**
|
|
6
|
-
*
|
|
6
|
+
* Type representing source information for generating intermediate storage.
|
|
7
|
+
* Contains essential file and URL information needed to create a deterministic storage path.
|
|
7
8
|
*
|
|
8
9
|
* @private internal utility of `getScraperIntermediateSource`
|
|
9
10
|
*/
|
|
10
11
|
type GetScraperIntermediateSourceSource = Pick<ScraperSourceHandler, 'filename' | 'url'>;
|
|
11
12
|
/**
|
|
12
|
-
*
|
|
13
|
+
* Options for intermediate source generation and management.
|
|
14
|
+
* Configures how intermediate files are named, stored, and cleaned up.
|
|
13
15
|
*
|
|
14
16
|
* @private internal utility of `getScraperIntermediateSource`
|
|
15
17
|
*/
|
|
@@ -17,9 +19,8 @@ type GetScraperIntermediateSourceHandler = Required<Pick<PrepareAndScrapeOptions
|
|
|
17
19
|
readonly extension: string_file_extension;
|
|
18
20
|
};
|
|
19
21
|
/**
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
* Note: It also checks if directory exists and creates it if not
|
|
22
|
+
* Retrieves an intermediate source for a scraper based on the knowledge source.
|
|
23
|
+
* Manages the caching and retrieval of intermediate scraper results for optimized performance.
|
|
23
24
|
*
|
|
24
25
|
* @private as internal utility for scrapers
|
|
25
26
|
*/
|
|
@@ -4,7 +4,9 @@ import type { KnowledgeSourceJson } from '../../../pipeline/PipelineJson/Knowled
|
|
|
4
4
|
import type { PrepareAndScrapeOptions } from '../../../prepare/PrepareAndScrapeOptions';
|
|
5
5
|
import type { ScraperSourceHandler } from '../Scraper';
|
|
6
6
|
/**
|
|
7
|
-
*
|
|
7
|
+
* Factory function that creates a handler for processing knowledge sources.
|
|
8
|
+
* Provides standardized processing of different types of knowledge sources
|
|
9
|
+
* across various scraper implementations.
|
|
8
10
|
*
|
|
9
11
|
* @public exported from `@promptbook/core`
|
|
10
12
|
*/
|
|
@@ -2,7 +2,7 @@ import type { ExecutionTools } from '../../execution/ExecutionTools';
|
|
|
2
2
|
import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions';
|
|
3
3
|
import { DocumentScraper } from './DocumentScraper';
|
|
4
4
|
/**
|
|
5
|
-
*
|
|
5
|
+
* Creates a scraper for document content.
|
|
6
6
|
*
|
|
7
7
|
* @public exported from `@promptbook/documents`
|
|
8
8
|
*/
|
|
@@ -2,7 +2,8 @@ import type { ExecutionTools } from '../../execution/ExecutionTools';
|
|
|
2
2
|
import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions';
|
|
3
3
|
import { LegacyDocumentScraper } from './LegacyDocumentScraper';
|
|
4
4
|
/**
|
|
5
|
-
*
|
|
5
|
+
* Creates a scraper for legacy document formats (.doc, .rtf, etc).
|
|
6
|
+
* Uses LibreOffice for conversion to extract content from older document formats.
|
|
6
7
|
*
|
|
7
8
|
* @public exported from `@promptbook/legacy-documents`
|
|
8
9
|
*/
|
|
@@ -2,7 +2,7 @@ import type { ExecutionTools } from '../../execution/ExecutionTools';
|
|
|
2
2
|
import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions';
|
|
3
3
|
import { MarkdownScraper } from './MarkdownScraper';
|
|
4
4
|
/**
|
|
5
|
-
*
|
|
5
|
+
* Creates a scraper for markdown content.
|
|
6
6
|
*
|
|
7
7
|
* @public exported from `@promptbook/markdown-utils`
|
|
8
8
|
*/
|
|
@@ -13,6 +13,9 @@ export declare const createMarkdownScraper: ((tools: Pick<ExecutionTools, 'llm'>
|
|
|
13
13
|
mimeTypes: string[];
|
|
14
14
|
documentationUrl: "https://github.com/webgptorg/promptbook/discussions/@@";
|
|
15
15
|
isAvilableInBrowser: true;
|
|
16
|
+
/**
|
|
17
|
+
* TODO: [🎶] Naming "constructor" vs "creator" vs "factory"
|
|
18
|
+
*/
|
|
16
19
|
requiredExecutables: never[];
|
|
17
20
|
}>;
|
|
18
21
|
/**
|
|
@@ -37,7 +37,7 @@ export declare class MarkitdownScraper implements Converter, Scraper {
|
|
|
37
37
|
*/
|
|
38
38
|
$convert(source: ScraperSourceHandler): Promise<ScraperIntermediateSource>;
|
|
39
39
|
/**
|
|
40
|
-
* Scrapes the
|
|
40
|
+
* Scrapes the source document (PDF, DOCX, etc.) and returns the knowledge pieces or `null` if it can't scrape it.
|
|
41
41
|
*/
|
|
42
42
|
scrape(source: ScraperSourceHandler): Promise<ReadonlyArray<Omit<KnowledgePiecePreparedJson, 'sources' | 'preparationIds'>> | null>;
|
|
43
43
|
}
|
|
@@ -2,7 +2,8 @@ import type { ExecutionTools } from '../../execution/ExecutionTools';
|
|
|
2
2
|
import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions';
|
|
3
3
|
import { PdfScraper } from './PdfScraper';
|
|
4
4
|
/**
|
|
5
|
-
*
|
|
5
|
+
* Factory function to create an instance of PdfScraper.
|
|
6
|
+
* It bundles the scraper class with its metadata.
|
|
6
7
|
*
|
|
7
8
|
* @public exported from `@promptbook/pdf`
|
|
8
9
|
*/
|
|
@@ -2,7 +2,8 @@ import type { ExecutionTools } from '../../execution/ExecutionTools';
|
|
|
2
2
|
import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions';
|
|
3
3
|
import { WebsiteScraper } from './WebsiteScraper';
|
|
4
4
|
/**
|
|
5
|
-
*
|
|
5
|
+
* Factory function to create an instance of WebsiteScraper.
|
|
6
|
+
* It bundles the scraper class with its metadata.
|
|
6
7
|
*
|
|
7
8
|
* @public exported from `@promptbook/website-crawler`
|
|
8
9
|
*/
|
|
@@ -12,9 +13,7 @@ export declare const createWebsiteScraper: ((tools: Pick<ExecutionTools, 'llm'>,
|
|
|
12
13
|
className: string;
|
|
13
14
|
mimeTypes: string[];
|
|
14
15
|
documentationUrl: "https://github.com/webgptorg/promptbook/discussions/@@";
|
|
15
|
-
isAvilableInBrowser: false;
|
|
16
|
-
* TODO: [🎶] Naming "constructor" vs "creator" vs "factory"
|
|
17
|
-
*/
|
|
16
|
+
isAvilableInBrowser: false;
|
|
18
17
|
requiredExecutables: never[];
|
|
19
18
|
}>;
|
|
20
19
|
/**
|
|
@@ -18,7 +18,11 @@ import { trimCodeBlock } from '../../utils/trimCodeBlock';
|
|
|
18
18
|
import { trimEndOfCodeBlock } from '../../utils/trimEndOfCodeBlock';
|
|
19
19
|
import { unwrapResult } from '../../utils/unwrapResult';
|
|
20
20
|
/**
|
|
21
|
-
*
|
|
21
|
+
* Collection of utility functions that can be used for post-processing model outputs.
|
|
22
|
+
* These functions help transform, extract, or format the raw model responses.
|
|
23
|
+
*
|
|
24
|
+
* These utilities range from simple string manipulations to markdown processing
|
|
25
|
+
* and formatting functions for specific types of output.
|
|
22
26
|
*
|
|
23
27
|
* @public exported from `@promptbook/javascript`
|
|
24
28
|
*/
|
|
@@ -2,7 +2,10 @@ import type { ExecutionTools } from '../../execution/ExecutionTools';
|
|
|
2
2
|
import type { PromptbookStorage } from '../_common/PromptbookStorage';
|
|
3
3
|
import type { FileCacheStorageOptions } from './FileCacheStorageOptions';
|
|
4
4
|
/**
|
|
5
|
-
*
|
|
5
|
+
* A storage implementation that caches data in files organized in a directory structure.
|
|
6
|
+
* Provides methods for retrieving, storing, and managing cached data on the filesystem.
|
|
7
|
+
*
|
|
8
|
+
* This class implements the PromptbookStorage interface for filesystem-based caching.
|
|
6
9
|
*
|
|
7
10
|
* @public exported from `@promptbook/node`
|
|
8
11
|
*/
|
|
@@ -11,19 +14,23 @@ export declare class FileCacheStorage<TItem> implements PromptbookStorage<TItem>
|
|
|
11
14
|
private readonly options;
|
|
12
15
|
constructor(tools: Required<Pick<ExecutionTools, 'fs'>>, options: FileCacheStorageOptions);
|
|
13
16
|
/**
|
|
14
|
-
*
|
|
17
|
+
* Converts a storage key to a filesystem path where the data should be stored.
|
|
18
|
+
* Creates a consistent, deterministic file path based on the key string.
|
|
15
19
|
*/
|
|
16
20
|
private getFilenameForKey;
|
|
17
21
|
/**
|
|
18
|
-
*
|
|
22
|
+
* Returns the current value associated with the given key, or null if the given key does not exist.
|
|
23
|
+
* Retrieves the cached data from the file system storage.
|
|
19
24
|
*/
|
|
20
25
|
getItem(key: string): Promise<TItem | null>;
|
|
21
26
|
/**
|
|
22
|
-
*
|
|
27
|
+
* Sets the value of the pair identified by key to value, creating a new key/value pair if none existed for key previously.
|
|
28
|
+
* Persists data to the file system, creating necessary directory structure if it doesn't exist.
|
|
23
29
|
*/
|
|
24
30
|
setItem(key: string, value: TItem): Promise<void>;
|
|
25
31
|
/**
|
|
26
|
-
*
|
|
32
|
+
* Removes the key/value pair with the given key from the storage, if a key/value pair with the given key exists.
|
|
33
|
+
* Deletes the corresponding file from the filesystem.
|
|
27
34
|
*/
|
|
28
35
|
removeItem(key: string): Promise<void>;
|
|
29
36
|
}
|
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
import type { string_dirname } from '../../types/typeAliases';
|
|
2
2
|
/**
|
|
3
|
-
*
|
|
3
|
+
* Configuration options for the FileCacheStorage implementation.
|
|
4
|
+
* Defines how and where file cache data should be stored and managed.
|
|
4
5
|
*/
|
|
5
6
|
export type FileCacheStorageOptions = {
|
|
6
7
|
/**
|
|
7
|
-
*
|
|
8
|
+
* The absolute path to the root directory where cache files will be stored.
|
|
9
|
+
* This directory must exist and be writable by the application.
|
|
8
10
|
*/
|
|
9
11
|
rootFolderPath: string_dirname;
|
|
10
12
|
};
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { string_name } from '../../../types/typeAliases';
|
|
2
2
|
/**
|
|
3
|
-
*
|
|
3
|
+
* Converts a name to a properly formatted subfolder path for cache storage.
|
|
4
|
+
* Handles normalization and path formatting to create consistent cache directory structures.
|
|
4
5
|
*
|
|
5
6
|
* @private for `FileCacheStorage`
|
|
6
7
|
*/
|
package/esm/typings/src/storage/local-storage/utils/makePromptbookStorageFromWebStorage.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { PromptbookStorage } from '../../_common/PromptbookStorage';
|
|
2
2
|
/**
|
|
3
|
-
*
|
|
3
|
+
* Creates a Promptbook storage interface from a web storage object.
|
|
4
|
+
* Facilitates using Web Storage (localStorage/sessionStorage) as a storage backend.
|
|
4
5
|
*
|
|
5
6
|
* @private for `getLocalStorage` and `getSessionStorage`
|
|
6
7
|
*/
|
|
@@ -2,14 +2,14 @@ import type { TupleToUnion } from 'type-fest';
|
|
|
2
2
|
/**
|
|
3
3
|
* Model variant describes the very general type of the model
|
|
4
4
|
*
|
|
5
|
-
* There are
|
|
6
|
-
* -
|
|
7
|
-
* -
|
|
5
|
+
* There are theese supported model variants:
|
|
6
|
+
* - `COMPLETION` - Model that takes prompt and writes the rest of the text
|
|
7
|
+
* - `CHAT` - Model that takes prompt and previous messages and returns response
|
|
8
|
+
* - `EMBEDDING` - Model that convert text into vector representations
|
|
8
9
|
*/
|
|
9
10
|
export type ModelVariant = TupleToUnion<typeof MODEL_VARIANTS>;
|
|
10
11
|
/**
|
|
11
|
-
*
|
|
12
|
-
*
|
|
12
|
+
* @see {@link ModelVariant}
|
|
13
13
|
* @public exported from `@promptbook/core`
|
|
14
14
|
*/
|
|
15
15
|
export declare const MODEL_VARIANTS: readonly ["COMPLETION", "CHAT", "EMBEDDING"];
|