@promptbook/browser 0.72.0-7 → 0.72.0-9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -0
- package/esm/index.es.js +46 -2
- package/esm/index.es.js.map +1 -1
- package/esm/typings/src/_packages/browser.index.d.ts +2 -0
- package/esm/typings/src/_packages/cli.index.d.ts +20 -0
- package/esm/typings/src/_packages/core.index.d.ts +20 -16
- package/esm/typings/src/_packages/documents.index.d.ts +8 -0
- package/esm/typings/src/_packages/legacy-documents.index.d.ts +8 -0
- package/esm/typings/src/_packages/markdown-utils.index.d.ts +6 -0
- package/esm/typings/src/_packages/node.index.d.ts +10 -4
- package/esm/typings/src/_packages/pdf.index.d.ts +8 -0
- package/esm/typings/src/_packages/types.index.d.ts +15 -5
- package/esm/typings/src/_packages/website-crawler.index.d.ts +8 -0
- package/esm/typings/src/collection/constructors/createCollectionFromDirectory.d.ts +5 -3
- package/esm/typings/src/config.d.ts +6 -0
- package/esm/typings/src/conversion/pipelineStringToJson.d.ts +3 -1
- package/esm/typings/src/dialogs/callback/CallbackInterfaceToolsOptions.d.ts +2 -2
- package/esm/typings/src/dialogs/simple-prompt/SimplePromptInterfaceTools.d.ts +3 -3
- package/esm/typings/src/execution/{CommonExecutionToolsOptions.d.ts → CommonToolsOptions.d.ts} +1 -1
- package/esm/typings/src/execution/ExecutionTools.d.ts +26 -6
- package/esm/typings/src/execution/FilesystemTools.d.ts +9 -0
- package/esm/typings/src/execution/createPipelineExecutor/10-executePipeline.d.ts +1 -1
- package/esm/typings/src/execution/createPipelineExecutor/20-executeTemplate.d.ts +1 -6
- package/esm/typings/src/execution/createPipelineExecutor/40-executeAttempts.d.ts +1 -6
- package/esm/typings/src/execution/translation/automatic-translate/translateMessages.d.ts +1 -0
- package/esm/typings/src/execution/utils/$provideExecutionToolsForNode.d.ts +13 -0
- package/esm/typings/src/llm-providers/_common/{$llmToolsMetadataRegister.d.ts → register/$llmToolsMetadataRegister.d.ts} +4 -1
- package/esm/typings/src/llm-providers/_common/{$llmToolsRegister.d.ts → register/$llmToolsRegister.d.ts} +5 -2
- package/esm/typings/src/llm-providers/_common/{createLlmToolsFromConfigurationFromEnv.d.ts → register/$provideLlmToolsConfigurationFromEnv.d.ts} +3 -3
- package/esm/typings/src/llm-providers/_common/{getLlmToolsForCli.d.ts → register/$provideLlmToolsForCli.d.ts} +4 -11
- package/esm/typings/src/llm-providers/_common/{getLlmToolsForTestingAndScriptsAndPlayground.d.ts → register/$provideLlmToolsForTestingAndScriptsAndPlayground.d.ts} +4 -3
- package/esm/typings/src/llm-providers/_common/{createLlmToolsFromEnv.d.ts → register/$provideLlmToolsFromEnv.d.ts} +6 -5
- package/esm/typings/src/llm-providers/_common/{$registeredLlmToolsMessage.d.ts → register/$registeredLlmToolsMessage.d.ts} +5 -2
- package/esm/typings/src/llm-providers/_common/{LlmToolsConfiguration.d.ts → register/LlmToolsConfiguration.d.ts} +5 -4
- package/esm/typings/src/llm-providers/_common/{LlmToolsMetadata.d.ts → register/LlmToolsMetadata.d.ts} +5 -4
- package/esm/typings/src/llm-providers/_common/{LlmToolsOptions.d.ts → register/LlmToolsOptions.d.ts} +4 -1
- package/esm/typings/src/llm-providers/_common/{createLlmToolsFromConfiguration.d.ts → register/createLlmToolsFromConfiguration.d.ts} +5 -4
- package/esm/typings/src/llm-providers/anthropic-claude/AnthropicClaudeExecutionToolsOptions.d.ts +3 -3
- package/esm/typings/src/llm-providers/anthropic-claude/register-configuration.d.ts +4 -3
- package/esm/typings/src/llm-providers/anthropic-claude/register-constructor.d.ts +4 -3
- package/esm/typings/src/llm-providers/azure-openai/AzureOpenAiExecutionToolsOptions.d.ts +2 -2
- package/esm/typings/src/llm-providers/azure-openai/register-configuration.d.ts +4 -3
- package/esm/typings/src/llm-providers/azure-openai/register-constructor.d.ts +4 -3
- package/esm/typings/src/llm-providers/mocked/MockedEchoLlmExecutionTools.d.ts +3 -3
- package/esm/typings/src/llm-providers/mocked/MockedFackedLlmExecutionTools.d.ts +3 -3
- package/esm/typings/src/llm-providers/multiple/MultipleLlmExecutionTools.d.ts +1 -0
- package/esm/typings/src/llm-providers/openai/OpenAiExecutionToolsOptions.d.ts +2 -2
- package/esm/typings/src/llm-providers/openai/register-configuration.d.ts +5 -4
- package/esm/typings/src/llm-providers/openai/register-constructor.d.ts +5 -4
- package/esm/typings/src/llm-providers/remote/interfaces/PromptbookServer_ListModels_Request.d.ts +1 -1
- package/esm/typings/src/llm-providers/remote/interfaces/PromptbookServer_Prompt_Request.d.ts +1 -1
- package/esm/typings/src/llm-providers/remote/interfaces/RemoteLlmExecutionToolsOptions.d.ts +3 -3
- package/esm/typings/src/llm-providers/remote/interfaces/RemoteServerOptions.d.ts +2 -2
- package/esm/typings/src/personas/preparePersona.d.ts +2 -1
- package/esm/typings/src/prepare/PrepareAndScrapeOptions.d.ts +8 -7
- package/esm/typings/src/prepare/preparePipeline.d.ts +2 -1
- package/esm/typings/src/prepare/prepareTemplates.d.ts +2 -1
- package/esm/typings/src/scrapers/_common/Converter.d.ts +4 -10
- package/esm/typings/src/scrapers/_common/Scraper.d.ts +4 -9
- package/esm/typings/src/scrapers/_common/prepareKnowledgePieces.d.ts +2 -1
- package/esm/typings/src/scrapers/_common/register/$provideFilesystemForNode.d.ts +11 -0
- package/esm/typings/src/scrapers/_common/register/$provideScrapersForBrowser.d.ts +12 -0
- package/esm/typings/src/scrapers/_common/register/$provideScrapersForNode.d.ts +15 -0
- package/esm/typings/src/scrapers/_common/register/$registeredScrapersMessage.d.ts +12 -0
- package/esm/typings/src/scrapers/_common/register/$scrapersMetadataRegister.d.ts +13 -0
- package/esm/typings/src/scrapers/_common/register/$scrapersRegister.d.ts +13 -0
- package/esm/typings/src/scrapers/_common/register/ScraperAndConverterMetadata.d.ts +41 -0
- package/esm/typings/src/scrapers/_common/register/ScraperConstructor.d.ts +12 -0
- package/esm/typings/src/scrapers/_common/utils/getScraperIntermediateSource.d.ts +1 -0
- package/esm/typings/src/scrapers/_common/utils/makeKnowledgeSourceHandler.d.ts +2 -1
- package/esm/typings/src/scrapers/document/{documentScraper.d.ts → DocumentScraper.d.ts} +18 -12
- package/esm/typings/src/scrapers/document/createDocumentScraper.d.ts +20 -0
- package/esm/typings/src/scrapers/document/register-constructor.d.ts +13 -0
- package/esm/typings/src/scrapers/document/register-metadata.d.ts +24 -0
- package/esm/typings/src/scrapers/document-legacy/{legacyDocumentScraper.d.ts → LegacyDocumentScraper.d.ts} +18 -12
- package/esm/typings/src/scrapers/document-legacy/createLegacyDocumentScraper.d.ts +20 -0
- package/esm/typings/src/scrapers/document-legacy/register-constructor.d.ts +13 -0
- package/esm/typings/src/scrapers/document-legacy/register-metadata.d.ts +24 -0
- package/esm/typings/src/scrapers/markdown/MarkdownScraper.d.ts +29 -0
- package/esm/typings/src/scrapers/markdown/createMarkdownScraper.d.ts +20 -0
- package/esm/typings/src/scrapers/markdown/register-constructor.d.ts +13 -0
- package/esm/typings/src/scrapers/markdown/register-metadata.d.ts +24 -0
- package/esm/typings/src/scrapers/pdf/PdfScraper.d.ts +40 -0
- package/esm/typings/src/scrapers/pdf/createPdfScraper.d.ts +20 -0
- package/esm/typings/src/scrapers/pdf/register-constructor.d.ts +13 -0
- package/esm/typings/src/scrapers/pdf/register-metadata.d.ts +24 -0
- package/esm/typings/src/scrapers/website/{websiteScraper.d.ts → WebsiteScraper.d.ts} +18 -14
- package/esm/typings/src/scrapers/website/createWebsiteScraper.d.ts +20 -0
- package/esm/typings/src/scrapers/website/register-constructor.d.ts +13 -0
- package/esm/typings/src/scrapers/website/register-metadata.d.ts +24 -0
- package/esm/typings/src/scripting/javascript/JavascriptExecutionToolsOptions.d.ts +2 -2
- package/esm/typings/src/scripting/python/PythonExecutionTools.d.ts +3 -3
- package/esm/typings/src/scripting/typescript/TypescriptExecutionTools.d.ts +3 -3
- package/esm/typings/src/storage/file-cache-storage/FileCacheStorage.d.ts +5 -3
- package/esm/typings/src/storage/{utils → memory/utils}/PrefixStorage.d.ts +1 -1
- package/esm/typings/src/storage/{utils → memory/utils}/makePromptbookStorageFromWebStorage.d.ts +1 -1
- package/esm/typings/src/types/typeAliases.d.ts +7 -0
- package/esm/typings/src/utils/$Register.d.ts +19 -6
- package/esm/typings/src/utils/execCommand/$execCommand.d.ts +1 -1
- package/esm/typings/src/utils/execCommand/$execCommands.d.ts +1 -1
- package/esm/typings/src/utils/files/isDirectoryExisting.d.ts +14 -0
- package/esm/typings/src/utils/files/isFileExisting.d.ts +13 -0
- package/esm/typings/src/utils/files/{$listAllFiles.d.ts → listAllFiles.d.ts} +3 -4
- package/package.json +2 -2
- package/umd/index.umd.js +46 -1
- package/umd/index.umd.js.map +1 -1
- package/esm/typings/src/scrapers/index.d.ts +0 -7
- package/esm/typings/src/scrapers/markdown/markdownScraper.d.ts +0 -29
- package/esm/typings/src/scrapers/pdf/pdfScraper.d.ts +0 -35
- package/esm/typings/src/utils/files/$isDirectoryExisting.d.ts +0 -15
- package/esm/typings/src/utils/files/$isFileExisting.d.ts +0 -14
- /package/esm/typings/src/scrapers/document/{documentScraper.test.d.ts → DocumentScraper.test.d.ts} +0 -0
- /package/esm/typings/src/scrapers/document-legacy/{legacyDocumentScraper.test.d.ts → LegacyDocumentScraper.test.d.ts} +0 -0
- /package/esm/typings/src/scrapers/markdown/{markdownScraper.test.d.ts → MarkdownScraper.test.d.ts} +0 -0
- /package/esm/typings/src/scrapers/website/{websiteScraper.test.d.ts → WebsiteScraper.test.d.ts} +0 -0
- /package/esm/typings/src/utils/files/{$isDirectoryExisting.test.d.ts → isDirectoryExisting.test.d.ts} +0 -0
- /package/esm/typings/src/utils/files/{$isFileExisting.test.d.ts → isFileExisting.test.d.ts} +0 -0
- /package/esm/typings/src/utils/files/{$listAllFiles.test.d.ts → listAllFiles.test.d.ts} +0 -0
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { ExecutionTools } from '../../execution/ExecutionTools';
|
|
2
|
+
import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions';
|
|
3
|
+
import { MarkdownScraper } from './MarkdownScraper';
|
|
4
|
+
/**
|
|
5
|
+
* @@@
|
|
6
|
+
*
|
|
7
|
+
* @public exported from `@promptbook/markdown-utils`
|
|
8
|
+
*/
|
|
9
|
+
export declare const createMarkdownScraper: ((tools: Pick<ExecutionTools, 'llm'>, options: PrepareAndScrapeOptions) => MarkdownScraper) & import("type-fest/source/readonly-deep").ReadonlyObjectDeep<{
|
|
10
|
+
title: string;
|
|
11
|
+
packageName: string;
|
|
12
|
+
className: string;
|
|
13
|
+
mimeTypes: string[];
|
|
14
|
+
documentationUrl: "https://github.com/webgptorg/promptbook/discussions/@@";
|
|
15
|
+
isAvilableInBrowser: true;
|
|
16
|
+
requiredExecutables: string[];
|
|
17
|
+
}>;
|
|
18
|
+
/**
|
|
19
|
+
* TODO: [🎶] Naming "constructor" vs "creator" vs "factory"
|
|
20
|
+
*/
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { Registration } from '../../utils/$Register';
|
|
2
|
+
/**
|
|
3
|
+
* Registration of known scraper
|
|
4
|
+
*
|
|
5
|
+
* Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available known scrapers
|
|
6
|
+
*
|
|
7
|
+
* @public exported from `@promptbook/markdown-utils`
|
|
8
|
+
* @public exported from `@promptbook/cli`
|
|
9
|
+
*/
|
|
10
|
+
export declare const _MarkdownScraperRegistration: Registration;
|
|
11
|
+
/**
|
|
12
|
+
* TODO: [🎶] Naming "constructor" vs "creator" vs "factory"
|
|
13
|
+
*/
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type { Registration } from '../../utils/$Register';
|
|
2
|
+
/**
|
|
3
|
+
* Metadata of the scraper
|
|
4
|
+
*
|
|
5
|
+
* @private within the scraper directory
|
|
6
|
+
*/
|
|
7
|
+
export declare const markdownScraperMetadata: import("type-fest/source/readonly-deep").ReadonlyObjectDeep<{
|
|
8
|
+
title: string;
|
|
9
|
+
packageName: string;
|
|
10
|
+
className: string;
|
|
11
|
+
mimeTypes: string[];
|
|
12
|
+
documentationUrl: "https://github.com/webgptorg/promptbook/discussions/@@";
|
|
13
|
+
isAvilableInBrowser: true;
|
|
14
|
+
requiredExecutables: string[];
|
|
15
|
+
}>;
|
|
16
|
+
/**
|
|
17
|
+
* Registration of known scraper metadata
|
|
18
|
+
*
|
|
19
|
+
* Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available known scrapers
|
|
20
|
+
*
|
|
21
|
+
* @public exported from `@promptbook/core`
|
|
22
|
+
* @public exported from `@promptbook/cli`
|
|
23
|
+
*/
|
|
24
|
+
export declare const _MarkdownScraperMetadataRegistration: Registration;
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import type { KnowledgePiecePreparedJson } from '../../types/PipelineJson/KnowledgePieceJson';
|
|
2
|
+
import type { Scraper } from '../_common/Scraper';
|
|
3
|
+
import type { ScraperSourceHandler } from '../_common/Scraper';
|
|
4
|
+
import type { ExecutionTools } from '../../execution/ExecutionTools';
|
|
5
|
+
import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions';
|
|
6
|
+
import type { Converter } from '../_common/Converter';
|
|
7
|
+
import type { ScraperAndConverterMetadata } from '../_common/register/ScraperAndConverterMetadata';
|
|
8
|
+
import type { ScraperIntermediateSource } from '../_common/ScraperIntermediateSource';
|
|
9
|
+
/**
|
|
10
|
+
* Scraper for .docx files
|
|
11
|
+
*
|
|
12
|
+
* @see `documentationUrl` for more details
|
|
13
|
+
* @public exported from `@promptbook/pdf`
|
|
14
|
+
*/
|
|
15
|
+
export declare class PdfScraper implements Converter, Scraper {
|
|
16
|
+
private readonly tools;
|
|
17
|
+
private readonly options;
|
|
18
|
+
/**
|
|
19
|
+
* Metadata of the scraper which includes title, mime types, etc.
|
|
20
|
+
*/
|
|
21
|
+
get metadata(): ScraperAndConverterMetadata;
|
|
22
|
+
/**
|
|
23
|
+
* Markdown scraper is used internally
|
|
24
|
+
*/
|
|
25
|
+
private readonly markdownScraper;
|
|
26
|
+
constructor(tools: Pick<ExecutionTools, 'llm'>, options: PrepareAndScrapeOptions);
|
|
27
|
+
/**
|
|
28
|
+
* Converts the `.pdf` file to `.md` file and returns intermediate source
|
|
29
|
+
*/
|
|
30
|
+
$convert(source: ScraperSourceHandler): Promise<ScraperIntermediateSource>;
|
|
31
|
+
/**
|
|
32
|
+
* Scrapes the `.pdf` file and returns the knowledge pieces or `null` if it can't scrape it
|
|
33
|
+
*/
|
|
34
|
+
scrape(source: ScraperSourceHandler): Promise<Array<Omit<KnowledgePiecePreparedJson, 'sources' | 'preparationIds'>> | null>;
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* TODO: [👣] Converted pdf documents can act as cached items - there is no need to run conversion each time
|
|
38
|
+
* TODO: [🪂] Do it in parallel 11:11
|
|
39
|
+
* Note: No need to aggregate usage here, it is done by intercepting the llmTools
|
|
40
|
+
*/
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { ExecutionTools } from '../../execution/ExecutionTools';
|
|
2
|
+
import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions';
|
|
3
|
+
import { PdfScraper } from './PdfScraper';
|
|
4
|
+
/**
|
|
5
|
+
* @@@
|
|
6
|
+
*
|
|
7
|
+
* @public exported from `@promptbook/pdf`
|
|
8
|
+
*/
|
|
9
|
+
export declare const createPdfScraper: ((tools: Pick<ExecutionTools, 'llm'>, options: PrepareAndScrapeOptions) => PdfScraper) & import("type-fest/source/readonly-deep").ReadonlyObjectDeep<{
|
|
10
|
+
title: string;
|
|
11
|
+
packageName: string;
|
|
12
|
+
className: string;
|
|
13
|
+
mimeTypes: string[];
|
|
14
|
+
documentationUrl: "https://github.com/webgptorg/promptbook/discussions/@@";
|
|
15
|
+
isAvilableInBrowser: true;
|
|
16
|
+
requiredExecutables: string[];
|
|
17
|
+
}>;
|
|
18
|
+
/**
|
|
19
|
+
* TODO: [🎶] Naming "constructor" vs "creator" vs "factory"
|
|
20
|
+
*/
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { Registration } from '../../utils/$Register';
|
|
2
|
+
/**
|
|
3
|
+
* Registration of known scraper
|
|
4
|
+
*
|
|
5
|
+
* Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available known scrapers
|
|
6
|
+
*
|
|
7
|
+
* @public exported from `@promptbook/pdf`
|
|
8
|
+
* @public exported from `@promptbook/cli`
|
|
9
|
+
*/
|
|
10
|
+
export declare const _PdfScraperRegistration: Registration;
|
|
11
|
+
/**
|
|
12
|
+
* TODO: [🎶] Naming "constructor" vs "creator" vs "factory"
|
|
13
|
+
*/
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type { Registration } from '../../utils/$Register';
|
|
2
|
+
/**
|
|
3
|
+
* Metadata of the scraper
|
|
4
|
+
*
|
|
5
|
+
* @private within the scraper directory
|
|
6
|
+
*/
|
|
7
|
+
export declare const pdfScraperMetadata: import("type-fest/source/readonly-deep").ReadonlyObjectDeep<{
|
|
8
|
+
title: string;
|
|
9
|
+
packageName: string;
|
|
10
|
+
className: string;
|
|
11
|
+
mimeTypes: string[];
|
|
12
|
+
documentationUrl: "https://github.com/webgptorg/promptbook/discussions/@@";
|
|
13
|
+
isAvilableInBrowser: true;
|
|
14
|
+
requiredExecutables: string[];
|
|
15
|
+
}>;
|
|
16
|
+
/**
|
|
17
|
+
* Registration of known scraper metadata
|
|
18
|
+
*
|
|
19
|
+
* Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available known scrapers
|
|
20
|
+
*
|
|
21
|
+
* @public exported from `@promptbook/core`
|
|
22
|
+
* @public exported from `@promptbook/cli`
|
|
23
|
+
*/
|
|
24
|
+
export declare const _PdfScraperMetadataRegistration: Registration;
|
|
@@ -1,43 +1,47 @@
|
|
|
1
|
-
import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions';
|
|
2
1
|
import type { KnowledgePiecePreparedJson } from '../../types/PipelineJson/KnowledgePieceJson';
|
|
3
2
|
import type { string_markdown } from '../../types/typeAliases';
|
|
3
|
+
import type { Converter } from '../_common/Converter';
|
|
4
|
+
import type { Scraper } from '../_common/Scraper';
|
|
4
5
|
import type { ScraperSourceHandler } from '../_common/Scraper';
|
|
6
|
+
import type { ExecutionTools } from '../../execution/ExecutionTools';
|
|
7
|
+
import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions';
|
|
8
|
+
import type { ScraperAndConverterMetadata } from '../_common/register/ScraperAndConverterMetadata';
|
|
5
9
|
import type { ScraperIntermediateSource } from '../_common/ScraperIntermediateSource';
|
|
6
10
|
/**
|
|
7
11
|
* Scraper for .docx files
|
|
8
12
|
*
|
|
9
13
|
* @see `documentationUrl` for more details
|
|
10
|
-
* @public exported from `@promptbook/
|
|
14
|
+
* @public exported from `@promptbook/website-crawler`
|
|
11
15
|
*/
|
|
12
|
-
export declare
|
|
16
|
+
export declare class WebsiteScraper implements Converter, Scraper {
|
|
17
|
+
private readonly tools;
|
|
18
|
+
private readonly options;
|
|
13
19
|
/**
|
|
14
|
-
*
|
|
20
|
+
* Metadata of the scraper which includes title, mime types, etc.
|
|
15
21
|
*/
|
|
16
|
-
|
|
22
|
+
get metadata(): ScraperAndConverterMetadata;
|
|
17
23
|
/**
|
|
18
|
-
*
|
|
24
|
+
* Markdown scraper is used internally
|
|
19
25
|
*/
|
|
20
|
-
|
|
26
|
+
private readonly markdownScraper;
|
|
27
|
+
constructor(tools: Pick<ExecutionTools, 'fs' | 'llm'>, options: PrepareAndScrapeOptions);
|
|
21
28
|
/**
|
|
22
29
|
* Convert the website to `.md` file and returns intermediate source
|
|
23
30
|
*
|
|
24
31
|
* Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object
|
|
25
32
|
*/
|
|
26
|
-
$convert(source: ScraperSourceHandler
|
|
27
|
-
readonly filename: string;
|
|
28
|
-
} & {
|
|
33
|
+
$convert(source: ScraperSourceHandler): Promise<ScraperIntermediateSource & {
|
|
29
34
|
markdown: string_markdown;
|
|
30
35
|
}>;
|
|
31
36
|
/**
|
|
32
37
|
* Scrapes the website and returns the knowledge pieces or `null` if it can't scrape it
|
|
33
38
|
*/
|
|
34
|
-
scrape(source: ScraperSourceHandler
|
|
35
|
-
}
|
|
39
|
+
scrape(source: ScraperSourceHandler): Promise<Array<Omit<KnowledgePiecePreparedJson, 'sources' | 'preparationIds'>> | null>;
|
|
40
|
+
}
|
|
36
41
|
/**
|
|
37
42
|
* TODO: !!!!!! Put into separate package
|
|
38
43
|
* TODO: [👣] Scraped website in .md can act as cache item - there is no need to run conversion each time
|
|
39
|
-
* TODO: [🦖] Make some system for putting scrapers to separete packages
|
|
40
44
|
* TODO: [🪂] Do it in parallel 11:11
|
|
41
|
-
* TODO: [🦷] Ideally use `as const satisfies Converter & Scraper` BUT this combination throws errors
|
|
42
45
|
* Note: No need to aggregate usage here, it is done by intercepting the llmTools
|
|
46
|
+
* Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
|
|
43
47
|
*/
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { ExecutionTools } from '../../execution/ExecutionTools';
|
|
2
|
+
import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions';
|
|
3
|
+
import { WebsiteScraper } from './WebsiteScraper';
|
|
4
|
+
/**
|
|
5
|
+
* @@@
|
|
6
|
+
*
|
|
7
|
+
* @public exported from `@promptbook/website-crawler`
|
|
8
|
+
*/
|
|
9
|
+
export declare const createWebsiteScraper: ((tools: Pick<ExecutionTools, 'llm'>, options: PrepareAndScrapeOptions) => WebsiteScraper) & import("type-fest/source/readonly-deep").ReadonlyObjectDeep<{
|
|
10
|
+
title: string;
|
|
11
|
+
packageName: string;
|
|
12
|
+
className: string;
|
|
13
|
+
mimeTypes: string[];
|
|
14
|
+
documentationUrl: "https://github.com/webgptorg/promptbook/discussions/@@";
|
|
15
|
+
isAvilableInBrowser: false;
|
|
16
|
+
requiredExecutables: string[];
|
|
17
|
+
}>;
|
|
18
|
+
/**
|
|
19
|
+
* TODO: [🎶] Naming "constructor" vs "creator" vs "factory"
|
|
20
|
+
*/
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { Registration } from '../../utils/$Register';
|
|
2
|
+
/**
|
|
3
|
+
* Registration of known scraper
|
|
4
|
+
*
|
|
5
|
+
* Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available known scrapers
|
|
6
|
+
*
|
|
7
|
+
* @public exported from `@promptbook/website-crawler`
|
|
8
|
+
* @public exported from `@promptbook/cli`
|
|
9
|
+
*/
|
|
10
|
+
export declare const _WebsiteScraperRegistration: Registration;
|
|
11
|
+
/**
|
|
12
|
+
* TODO: [🎶] Naming "constructor" vs "creator" vs "factory"
|
|
13
|
+
*/
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type { Registration } from '../../utils/$Register';
|
|
2
|
+
/**
|
|
3
|
+
* Metadata of the scraper
|
|
4
|
+
*
|
|
5
|
+
* @private within the scraper directory
|
|
6
|
+
*/
|
|
7
|
+
export declare const websiteScraperMetadata: import("type-fest/source/readonly-deep").ReadonlyObjectDeep<{
|
|
8
|
+
title: string;
|
|
9
|
+
packageName: string;
|
|
10
|
+
className: string;
|
|
11
|
+
mimeTypes: string[];
|
|
12
|
+
documentationUrl: "https://github.com/webgptorg/promptbook/discussions/@@";
|
|
13
|
+
isAvilableInBrowser: false;
|
|
14
|
+
requiredExecutables: string[];
|
|
15
|
+
}>;
|
|
16
|
+
/**
|
|
17
|
+
* Registration of known scraper metadata
|
|
18
|
+
*
|
|
19
|
+
* Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available known scrapers
|
|
20
|
+
*
|
|
21
|
+
* @public exported from `@promptbook/core`
|
|
22
|
+
* @public exported from `@promptbook/cli`
|
|
23
|
+
*/
|
|
24
|
+
export declare const _WebsiteScraperMetadataRegistration: Registration;
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import type { Promisable } from 'type-fest';
|
|
2
|
-
import type {
|
|
2
|
+
import type { CommonToolsOptions } from '../../execution/CommonToolsOptions';
|
|
3
3
|
import type { string_postprocessing_function_name } from '../../types/typeAliases';
|
|
4
4
|
/**
|
|
5
5
|
* Options for `JavascriptExecutionTools`
|
|
6
6
|
*/
|
|
7
|
-
export type JavascriptExecutionToolsOptions =
|
|
7
|
+
export type JavascriptExecutionToolsOptions = CommonToolsOptions & {
|
|
8
8
|
/**
|
|
9
9
|
* Functions to be executed in the JavaScript evaluation.
|
|
10
10
|
*
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { CommonToolsOptions } from '../../execution/CommonToolsOptions';
|
|
2
2
|
import type { ScriptExecutionTools } from '../../execution/ScriptExecutionTools';
|
|
3
3
|
import type { ScriptExecutionToolsExecuteOptions } from '../../execution/ScriptExecutionTools';
|
|
4
4
|
/**
|
|
@@ -9,8 +9,8 @@ import type { ScriptExecutionToolsExecuteOptions } from '../../execution/ScriptE
|
|
|
9
9
|
* @private still in development
|
|
10
10
|
*/
|
|
11
11
|
export declare class PythonExecutionTools implements ScriptExecutionTools {
|
|
12
|
-
protected readonly options:
|
|
13
|
-
constructor(options?:
|
|
12
|
+
protected readonly options: CommonToolsOptions;
|
|
13
|
+
constructor(options?: CommonToolsOptions);
|
|
14
14
|
/**
|
|
15
15
|
* Executes a Python
|
|
16
16
|
*/
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { CommonToolsOptions } from '../../execution/CommonToolsOptions';
|
|
2
2
|
import type { ScriptExecutionTools } from '../../execution/ScriptExecutionTools';
|
|
3
3
|
import type { ScriptExecutionToolsExecuteOptions } from '../../execution/ScriptExecutionTools';
|
|
4
4
|
/**
|
|
@@ -9,8 +9,8 @@ import type { ScriptExecutionToolsExecuteOptions } from '../../execution/ScriptE
|
|
|
9
9
|
* @private still in development
|
|
10
10
|
*/
|
|
11
11
|
export declare class TypescriptExecutionTools implements ScriptExecutionTools {
|
|
12
|
-
protected readonly options:
|
|
13
|
-
constructor(options?:
|
|
12
|
+
protected readonly options: CommonToolsOptions;
|
|
13
|
+
constructor(options?: CommonToolsOptions);
|
|
14
14
|
/**
|
|
15
15
|
* Executes a TypeScript
|
|
16
16
|
*/
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { ExecutionTools } from '../../execution/ExecutionTools';
|
|
1
2
|
import type { PromptbookStorage } from '../_common/PromptbookStorage';
|
|
2
3
|
import type { FileCacheStorageOptions } from './FileCacheStorageOptions';
|
|
3
4
|
/**
|
|
@@ -6,8 +7,9 @@ import type { FileCacheStorageOptions } from './FileCacheStorageOptions';
|
|
|
6
7
|
* @public exported from `@promptbook/node`
|
|
7
8
|
*/
|
|
8
9
|
export declare class FileCacheStorage<TItem> implements PromptbookStorage<TItem> {
|
|
9
|
-
protected readonly
|
|
10
|
-
|
|
10
|
+
protected readonly tools: Required<Pick<ExecutionTools, 'fs'>>;
|
|
11
|
+
private readonly options;
|
|
12
|
+
constructor(tools: Required<Pick<ExecutionTools, 'fs'>>, options: FileCacheStorageOptions);
|
|
11
13
|
/**
|
|
12
14
|
* @@@
|
|
13
15
|
*/
|
|
@@ -27,5 +29,5 @@ export declare class FileCacheStorage<TItem> implements PromptbookStorage<TItem>
|
|
|
27
29
|
}
|
|
28
30
|
/**
|
|
29
31
|
* TODO: [🌗] Maybe some checkers, not all valid JSONs are desired and valid values
|
|
30
|
-
* Note: [🟢] Code in this file should never be
|
|
32
|
+
* Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
|
|
31
33
|
*/
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { Promisable } from 'type-fest';
|
|
2
|
-
import type { PromptbookStorage } from '
|
|
2
|
+
import type { PromptbookStorage } from '../../_common/PromptbookStorage';
|
|
3
3
|
/**
|
|
4
4
|
* This class behaves like LocalStorage but separates keys by prefix
|
|
5
5
|
*
|
|
@@ -331,6 +331,13 @@ export type string_href = string;
|
|
|
331
331
|
* For example `"https://collboard.com/9SeSQTupmQHwuSrLi.png?width=1200&height=630"`
|
|
332
332
|
*/
|
|
333
333
|
export type string_url_image = string;
|
|
334
|
+
/**
|
|
335
|
+
* Semantic helper
|
|
336
|
+
*
|
|
337
|
+
* For example `"C:/Users/me/AppData/Local/Pandoc/pandoc.exe"`
|
|
338
|
+
* For example `"C:/Program Files/LibreOffice/program/swriter.exe"`
|
|
339
|
+
*/
|
|
340
|
+
export type string_executable_path = string;
|
|
334
341
|
/**
|
|
335
342
|
* Semantic helper
|
|
336
343
|
*
|
|
@@ -1,14 +1,27 @@
|
|
|
1
|
-
import type
|
|
1
|
+
import { type IDestroyable } from 'destroyable';
|
|
2
|
+
import type { string_name } from '../types/typeAliases';
|
|
2
3
|
import type { TODO_string } from './organization/TODO_string';
|
|
4
|
+
/**
|
|
5
|
+
* @@@
|
|
6
|
+
*/
|
|
3
7
|
export type Registered = {
|
|
4
8
|
/**
|
|
5
9
|
* @@@
|
|
6
10
|
*/
|
|
7
|
-
packageName: TODO_string;
|
|
11
|
+
readonly packageName: TODO_string;
|
|
12
|
+
/**
|
|
13
|
+
* @@@
|
|
14
|
+
*/
|
|
15
|
+
readonly className: TODO_string;
|
|
16
|
+
};
|
|
17
|
+
/**
|
|
18
|
+
* @@@
|
|
19
|
+
*/
|
|
20
|
+
export type Registration = Registered & IDestroyable & {
|
|
8
21
|
/**
|
|
9
22
|
* @@@
|
|
10
23
|
*/
|
|
11
|
-
|
|
24
|
+
readonly registerName: string_name;
|
|
12
25
|
};
|
|
13
26
|
/**
|
|
14
27
|
* Register is @@@
|
|
@@ -18,9 +31,9 @@ export type Registered = {
|
|
|
18
31
|
* @private internal utility, exported are only signleton instances of this class
|
|
19
32
|
*/
|
|
20
33
|
export declare class $Register<TRegistered extends Registered> {
|
|
21
|
-
private readonly
|
|
34
|
+
private readonly registerName;
|
|
22
35
|
private readonly storage;
|
|
23
|
-
constructor(
|
|
36
|
+
constructor(registerName: string_name);
|
|
24
37
|
list(): Array<TRegistered>;
|
|
25
|
-
register(registered: TRegistered):
|
|
38
|
+
register(registered: TRegistered): Registration;
|
|
26
39
|
}
|
|
@@ -10,5 +10,5 @@ import type { IExecCommandOptions } from './IExecCommandOptions';
|
|
|
10
10
|
*/
|
|
11
11
|
export declare function $execCommand(options: IExecCommandOptions): Promise<string>;
|
|
12
12
|
/**
|
|
13
|
-
* Note: [🟢
|
|
13
|
+
* Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
|
|
14
14
|
*/
|
|
@@ -13,5 +13,5 @@ export declare function $execCommands({ commands, cwd, crashOnError, }: {
|
|
|
13
13
|
readonly crashOnError?: boolean;
|
|
14
14
|
}): Promise<void>;
|
|
15
15
|
/**
|
|
16
|
-
* Note: [🟢
|
|
16
|
+
* Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
|
|
17
17
|
*/
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { FilesystemTools } from '../../execution/FilesystemTools';
|
|
2
|
+
import type { string_dirname } from '../../types/typeAliases';
|
|
3
|
+
/**
|
|
4
|
+
* Checks if the directory exists
|
|
5
|
+
*
|
|
6
|
+
* @private within the repository
|
|
7
|
+
*/
|
|
8
|
+
export declare function isDirectoryExisting(directoryPath: string_dirname, fs: FilesystemTools): Promise<boolean>;
|
|
9
|
+
/**
|
|
10
|
+
* Note: Not [~🟢~] because it is not directly dependent on `fs
|
|
11
|
+
* TODO: [🐠] This can be a validator - with variants that return true/false and variants that throw errors with meaningless messages
|
|
12
|
+
* TODO: [🧠][📂] "directory" vs "folder"
|
|
13
|
+
* TODO: [🖇] What about symlinks?
|
|
14
|
+
*/
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { FilesystemTools } from '../../execution/FilesystemTools';
|
|
2
|
+
import type { string_filename } from '../../types/typeAliases';
|
|
3
|
+
/**
|
|
4
|
+
* Checks if the file exists
|
|
5
|
+
*
|
|
6
|
+
* @private within the repository
|
|
7
|
+
*/
|
|
8
|
+
export declare function isFileExisting(filename: string_filename, fs: FilesystemTools): Promise<boolean>;
|
|
9
|
+
/**
|
|
10
|
+
* Note: Not [~🟢~] because it is not directly dependent on `fs
|
|
11
|
+
* TODO: [🐠] This can be a validator - with variants that return true/false and variants that throw errors with meaningless messages
|
|
12
|
+
* TODO: [🖇] What about symlinks?
|
|
13
|
+
*/
|
|
@@ -1,18 +1,17 @@
|
|
|
1
|
+
import type { FilesystemTools } from '../../execution/FilesystemTools';
|
|
1
2
|
import type { string_dirname } from '../../types/typeAliases';
|
|
2
3
|
import type { string_filename } from '../../types/typeAliases';
|
|
3
4
|
/**
|
|
4
5
|
* Reads all files in the directory
|
|
5
6
|
*
|
|
6
|
-
* Note: `$` is used to indicate that this function is not a pure function - it looks at the filesystem
|
|
7
|
-
*
|
|
8
7
|
* @param path
|
|
9
8
|
* @param isRecursive
|
|
10
9
|
* @returns List of all files in the directory
|
|
11
10
|
* @private internal function of `createCollectionFromDirectory`
|
|
12
11
|
*/
|
|
13
|
-
export declare function
|
|
12
|
+
export declare function listAllFiles(path: string_dirname, isRecursive: boolean, fs: FilesystemTools): Promise<Array<string_filename>>;
|
|
14
13
|
/**
|
|
15
14
|
* TODO: [😶] Unite floder listing
|
|
16
|
-
* Note:
|
|
15
|
+
* Note: Not [~🟢~] because it is not directly dependent on `fs
|
|
17
16
|
* TODO: [🖇] What about symlinks?
|
|
18
17
|
*/
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@promptbook/browser",
|
|
3
|
-
"version": "0.72.0-
|
|
3
|
+
"version": "0.72.0-9",
|
|
4
4
|
"description": "Supercharge your use of large language models",
|
|
5
5
|
"private": false,
|
|
6
6
|
"sideEffects": false,
|
|
@@ -51,7 +51,7 @@
|
|
|
51
51
|
"module": "./esm/index.es.js",
|
|
52
52
|
"typings": "./esm/typings/src/_packages/browser.index.d.ts",
|
|
53
53
|
"peerDependencies": {
|
|
54
|
-
"@promptbook/core": "0.72.0-
|
|
54
|
+
"@promptbook/core": "0.72.0-9"
|
|
55
55
|
},
|
|
56
56
|
"dependencies": {
|
|
57
57
|
"spacetrim": "0.11.39"
|
package/umd/index.umd.js
CHANGED
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
/**
|
|
13
13
|
* The version of the Promptbook library
|
|
14
14
|
*/
|
|
15
|
-
var PROMPTBOOK_VERSION = '0.72.0-
|
|
15
|
+
var PROMPTBOOK_VERSION = '0.72.0-8';
|
|
16
16
|
// TODO: [main] !!!! List here all the versions and annotate + put into script
|
|
17
17
|
|
|
18
18
|
/*! *****************************************************************************
|
|
@@ -182,6 +182,50 @@
|
|
|
182
182
|
return EnvironmentMismatchError;
|
|
183
183
|
}(Error));
|
|
184
184
|
|
|
185
|
+
/**
|
|
186
|
+
* Detects if the code is running in a browser environment in main thread (Not in a web worker)
|
|
187
|
+
*
|
|
188
|
+
* Note: `$` is used to indicate that this function is not a pure function - it looks at the global object to determine the environment
|
|
189
|
+
*
|
|
190
|
+
* @public exported from `@promptbook/utils`
|
|
191
|
+
*/
|
|
192
|
+
var $isRunningInBrowser = new Function("\n try {\n return this === window;\n } catch (e) {\n return false;\n }\n");
|
|
193
|
+
|
|
194
|
+
/**
|
|
195
|
+
* Detects if the code is running in a web worker
|
|
196
|
+
*
|
|
197
|
+
* Note: `$` is used to indicate that this function is not a pure function - it looks at the global object to determine the environment
|
|
198
|
+
*
|
|
199
|
+
* @public exported from `@promptbook/utils`
|
|
200
|
+
*/
|
|
201
|
+
var $isRunningInWebWorker = new Function("\n try {\n if (typeof WorkerGlobalScope !== 'undefined' && self instanceof WorkerGlobalScope) {\n return true;\n } else {\n return false;\n }\n } catch (e) {\n return false;\n }\n");
|
|
202
|
+
|
|
203
|
+
/**
|
|
204
|
+
* @@@!!!!!!
|
|
205
|
+
*
|
|
206
|
+
* 1) @@@
|
|
207
|
+
* 2) @@@
|
|
208
|
+
*
|
|
209
|
+
* @public exported from `@promptbook/browser`
|
|
210
|
+
*/
|
|
211
|
+
function $provideScrapersForBrowser(tools, options) {
|
|
212
|
+
return __awaiter(this, void 0, void 0, function () {
|
|
213
|
+
var isAutoInstalled /* Note: [0] Intentionally not assigning a default value = IS_AUTO_INSTALLED */;
|
|
214
|
+
return __generator(this, function (_a) {
|
|
215
|
+
if (!$isRunningInBrowser() || $isRunningInWebWorker()) {
|
|
216
|
+
throw new EnvironmentMismatchError('Function `$provideScrapersForBrowser` works only in browser environment');
|
|
217
|
+
}
|
|
218
|
+
isAutoInstalled = (options || {}).isAutoInstalled;
|
|
219
|
+
if (isAutoInstalled === true /* <- Note: [0] Ignoring undefined, just checking EXPLICIT requirement for install */) {
|
|
220
|
+
throw new EnvironmentMismatchError('Auto-installing is not supported in browser environment');
|
|
221
|
+
}
|
|
222
|
+
return [2 /*return*/, [
|
|
223
|
+
// TODO: !!!!!!! Implement
|
|
224
|
+
]];
|
|
225
|
+
});
|
|
226
|
+
});
|
|
227
|
+
}
|
|
228
|
+
|
|
185
229
|
/**
|
|
186
230
|
* @@@
|
|
187
231
|
*
|
|
@@ -543,6 +587,7 @@
|
|
|
543
587
|
* Note: [🔵] Code in this file should never be published outside of `@promptbook/browser`
|
|
544
588
|
*/
|
|
545
589
|
|
|
590
|
+
exports.$provideScrapersForBrowser = $provideScrapersForBrowser;
|
|
546
591
|
exports.PROMPTBOOK_VERSION = PROMPTBOOK_VERSION;
|
|
547
592
|
exports.SimplePromptInterfaceTools = SimplePromptInterfaceTools;
|
|
548
593
|
exports.getLocalStorage = getLocalStorage;
|