@promptbook/remote-server 0.72.0-6 → 0.72.0-8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -0
- package/esm/index.es.js +1 -1
- package/esm/index.es.js.map +1 -1
- package/esm/typings/src/_packages/browser.index.d.ts +1 -1
- package/esm/typings/src/_packages/core.index.d.ts +22 -10
- package/esm/typings/src/_packages/node.index.d.ts +6 -2
- package/esm/typings/src/_packages/types.index.d.ts +28 -20
- package/esm/typings/src/cli/cli-commands/about.d.ts +1 -1
- package/esm/typings/src/cli/cli-commands/hello.d.ts +2 -1
- package/esm/typings/src/cli/cli-commands/make.d.ts +1 -1
- package/esm/typings/src/cli/cli-commands/prettify.d.ts +2 -1
- package/esm/typings/src/cli/cli-commands/test-command.d.ts +13 -0
- package/esm/typings/src/cli/main.d.ts +1 -1
- package/esm/typings/src/cli/promptbookCli.d.ts +1 -1
- package/esm/typings/src/collection/constructors/createCollectionFromDirectory.d.ts +8 -5
- package/esm/typings/src/commands/EXPECT/expectCommandParser.d.ts +1 -1
- package/esm/typings/src/commands/FOREACH/foreachCommandParser.d.ts +1 -1
- package/esm/typings/src/commands/FORMAT/formatCommandParser.d.ts +1 -1
- package/esm/typings/src/commands/JOKER/jokerCommandParser.d.ts +1 -1
- package/esm/typings/src/commands/KNOWLEDGE/knowledgeCommandParser.d.ts +1 -1
- package/esm/typings/src/commands/KNOWLEDGE/utils/sourceContentToName.d.ts +11 -0
- package/esm/typings/src/commands/KNOWLEDGE/utils/sourceContentToName.test.d.ts +4 -0
- package/esm/typings/src/commands/MODEL/modelCommandParser.d.ts +1 -1
- package/esm/typings/src/commands/PARAMETER/parameterCommandParser.d.ts +1 -1
- package/esm/typings/src/commands/PERSONA/personaCommandParser.d.ts +1 -1
- package/esm/typings/src/commands/POSTPROCESS/postprocessCommandParser.d.ts +1 -1
- package/esm/typings/src/commands/PROMPTBOOK_VERSION/promptbookVersionCommandParser.d.ts +1 -1
- package/esm/typings/src/commands/TEMPLATE/templateCommandParser.d.ts +1 -1
- package/esm/typings/src/commands/URL/urlCommandParser.d.ts +1 -1
- package/esm/typings/src/commands/X_ACTION/actionCommandParser.d.ts +1 -1
- package/esm/typings/src/commands/X_INSTRUMENT/instrumentCommandParser.d.ts +1 -1
- package/esm/typings/src/commands/_BOILERPLATE/boilerplateCommandParser.d.ts +1 -1
- package/esm/typings/src/config.d.ts +10 -0
- package/esm/typings/src/conversion/pipelineStringToJson.d.ts +2 -15
- package/esm/typings/src/conversion/validation/_importPipeline.d.ts +1 -1
- package/esm/typings/src/conversion/validation/validatePipeline.d.ts +5 -5
- package/esm/typings/src/{knowledge/dialogs → dialogs}/callback/CallbackInterfaceTools.d.ts +2 -2
- package/esm/typings/src/{knowledge/dialogs → dialogs}/callback/CallbackInterfaceToolsOptions.d.ts +2 -2
- package/esm/typings/src/{knowledge/dialogs → dialogs}/simple-prompt/SimplePromptInterfaceTools.d.ts +4 -4
- package/esm/typings/src/errors/KnowledgeScrapeError.d.ts +9 -0
- package/esm/typings/src/errors/MissingToolsError.d.ts +9 -0
- package/esm/typings/src/execution/ExecutionTools.d.ts +3 -3
- package/esm/typings/src/execution/createPipelineExecutor/00-CreatePipelineExecutorOptions.d.ts +5 -2
- package/esm/typings/src/execution/createPipelineExecutor/00-CreatePipelineExecutorSettings.d.ts +2 -13
- package/esm/typings/src/execution/createPipelineExecutor/00-createPipelineExecutor.d.ts +1 -1
- package/esm/typings/src/execution/createPipelineExecutor/10-executePipeline.d.ts +1 -1
- package/esm/typings/src/execution/translation/automatic-translate/translateMessages.d.ts +3 -0
- package/esm/typings/src/execution/utils/forEachAsync.d.ts +1 -0
- package/esm/typings/src/llm-providers/_common/createLlmToolsFromConfigurationFromEnv.d.ts +1 -1
- package/esm/typings/src/llm-providers/_common/createLlmToolsFromEnv.d.ts +1 -1
- package/esm/typings/src/llm-providers/_common/getLlmToolsForCli.d.ts +1 -1
- package/esm/typings/src/llm-providers/anthropic-claude/playground/playground.d.ts +1 -0
- package/esm/typings/src/llm-providers/azure-openai/AzureOpenAiExecutionTools.d.ts +6 -0
- package/esm/typings/src/llm-providers/azure-openai/playground/playground.d.ts +1 -0
- package/esm/typings/src/llm-providers/langtail/playground/playground.d.ts +3 -0
- package/esm/typings/src/llm-providers/multiple/playground/playground.d.ts +3 -0
- package/esm/typings/src/llm-providers/openai/playground/playground.d.ts +1 -0
- package/esm/typings/src/llm-providers/remote/playground/playground.d.ts +3 -0
- package/esm/typings/src/personas/preparePersona.d.ts +2 -2
- package/esm/typings/src/prepare/PrepareAndScrapeOptions.d.ts +67 -0
- package/esm/typings/src/prepare/preparePipeline.d.ts +2 -2
- package/esm/typings/src/prepare/prepareTemplates.d.ts +2 -2
- package/esm/typings/src/scrapers/_common/Converter.d.ts +28 -0
- package/esm/typings/src/scrapers/_common/Scraper.d.ts +71 -0
- package/esm/typings/src/scrapers/_common/ScraperIntermediateSource.d.ts +11 -0
- package/esm/typings/src/{knowledge/prepare-knowledge → scrapers}/_common/prepareKnowledgePieces.d.ts +4 -4
- package/esm/typings/src/scrapers/_common/utils/getScraperIntermediateSource.d.ts +33 -0
- package/esm/typings/src/scrapers/_common/utils/getScraperIntermediateSource.test.d.ts +4 -0
- package/esm/typings/src/scrapers/_common/utils/makeKnowledgeSourceHandler.d.ts +10 -0
- package/esm/typings/src/scrapers/document/documentScraper.d.ts +37 -0
- package/esm/typings/src/scrapers/document/documentScraper.test.d.ts +4 -0
- package/esm/typings/src/scrapers/document/playground/document-scraper-playground.d.ts +5 -0
- package/esm/typings/src/scrapers/document-legacy/legacyDocumentScraper.d.ts +37 -0
- package/esm/typings/src/scrapers/document-legacy/legacyDocumentScraper.test.d.ts +4 -0
- package/esm/typings/src/scrapers/document-legacy/playground/legacy-document-scraper-playground.d.ts +5 -0
- package/esm/typings/src/scrapers/index.d.ts +8 -0
- package/esm/typings/src/scrapers/markdown/markdownScraper.d.ts +29 -0
- package/esm/typings/src/scrapers/markdown/playground/markdown-scraper-playground.d.ts +5 -0
- package/esm/typings/src/scrapers/pdf/pdfScraper.d.ts +35 -0
- package/esm/typings/src/scrapers/pdf/playground/pdf-scraper-playground.d.ts +5 -0
- package/esm/typings/src/scrapers/website/playground/website-scraper-playground.d.ts +5 -0
- package/esm/typings/src/scrapers/website/utils/markdownConverter.d.ts +12 -0
- package/esm/typings/src/scrapers/website/websiteScraper.d.ts +43 -0
- package/esm/typings/src/storage/{files-storage/FilesStorage.d.ts → file-cache-storage/FileCacheStorage.d.ts} +5 -5
- package/esm/typings/src/storage/file-cache-storage/FileCacheStorageOptions.d.ts +10 -0
- package/esm/typings/src/storage/{files-storage → file-cache-storage}/utils/nameToSubfolderPath.d.ts +1 -1
- package/esm/typings/src/storage/file-cache-storage/utils/nameToSubfolderPath.test.d.ts +1 -0
- package/esm/typings/src/storage/local-storage/getLocalStorage.d.ts +1 -1
- package/esm/typings/src/storage/local-storage/getSessionStorage.d.ts +1 -1
- package/esm/typings/src/types/PipelineJson/KnowledgeSourceJson.d.ts +9 -2
- package/esm/typings/src/types/PipelineJson/PipelineJson.d.ts +2 -2
- package/esm/typings/src/types/typeAliases.d.ts +8 -11
- package/esm/typings/src/utils/execCommand/$execCommand.d.ts +14 -0
- package/esm/typings/src/utils/execCommand/$execCommands.d.ts +17 -0
- package/esm/typings/src/utils/execCommand/IExecCommandOptions.d.ts +23 -0
- package/esm/typings/src/utils/execCommand/execCommand.test.d.ts +1 -0
- package/esm/typings/src/utils/execCommand/execCommandNormalizeOptions.d.ts +10 -0
- package/esm/typings/src/utils/execCommand/execCommandNormalizeOptions.test.d.ts +1 -0
- package/esm/typings/src/utils/files/$isDirectoryExisting.d.ts +3 -3
- package/esm/typings/src/utils/files/$isFileExisting.d.ts +3 -3
- package/esm/typings/src/utils/files/$listAllFiles.d.ts +5 -4
- package/esm/typings/src/utils/files/extensionToMimeType.d.ts +8 -0
- package/esm/typings/src/utils/files/extensionToMimeType.test.d.ts +1 -0
- package/esm/typings/src/utils/files/getFileExtension.d.ts +8 -0
- package/esm/typings/src/utils/files/getFileExtension.test.d.ts +1 -0
- package/esm/typings/src/utils/validators/filePath/isValidFilePath.d.ts +2 -2
- package/package.json +2 -2
- package/umd/index.umd.js +1 -1
- package/umd/index.umd.js.map +1 -1
- package/esm/typings/src/knowledge/prepare-knowledge/_common/Scraper.d.ts +0 -37
- package/esm/typings/src/knowledge/prepare-knowledge/markdown/playground/markdown-knowledge-playground.d.ts +0 -2
- package/esm/typings/src/knowledge/prepare-knowledge/markdown/prepareKnowledgeFromMarkdown.d.ts +0 -14
- package/esm/typings/src/knowledge/prepare-knowledge/pdf/prepareKnowledgeFromPdf.d.ts +0 -15
- package/esm/typings/src/prepare/PrepareOptions.d.ts +0 -22
- package/esm/typings/src/storage/files-storage/FilesStorageOptions.d.ts +0 -10
- /package/esm/typings/src/{knowledge/dialogs → dialogs}/user-interface-execution-tools.test.d.ts +0 -0
- /package/esm/typings/src/{knowledge/prepare-knowledge → scrapers}/_common/prepareKnowledgePieces.test.d.ts +0 -0
- /package/esm/typings/src/{knowledge/prepare-knowledge/markdown/prepareKnowledgeFromMarkdown.test.d.ts → scrapers/markdown/markdownScraper.test.d.ts} +0 -0
- /package/esm/typings/src/{knowledge/prepare-knowledge/pdf/prepareKnowledgeFromPdf.test.d.ts → scrapers/website/utils/markdownConverter.test.d.ts} +0 -0
- /package/esm/typings/src/{storage/files-storage/utils/nameToSubfolderPath.test.d.ts → scrapers/website/websiteScraper.test.d.ts} +0 -0
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import type { Promisable } from 'type-fest';
|
|
2
|
+
import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions';
|
|
3
|
+
import type { KnowledgePiecePreparedJson } from '../../types/PipelineJson/KnowledgePieceJson';
|
|
4
|
+
import type { string_filename } from '../../types/typeAliases';
|
|
5
|
+
import type { string_knowledge_source_link } from '../../types/typeAliases';
|
|
6
|
+
import type { string_mime_type } from '../../types/typeAliases';
|
|
7
|
+
import type { string_promptbook_documentation_url } from '../../types/typeAliases';
|
|
8
|
+
import type { string_url } from '../../types/typeAliases';
|
|
9
|
+
/**
|
|
10
|
+
* @@@
|
|
11
|
+
*
|
|
12
|
+
*/
|
|
13
|
+
export type Scraper = {
|
|
14
|
+
/**
|
|
15
|
+
* Mime types that this scraper can handle
|
|
16
|
+
*/
|
|
17
|
+
readonly mimeTypes: Array<string_mime_type>;
|
|
18
|
+
/**
|
|
19
|
+
* Link to documentation
|
|
20
|
+
*/
|
|
21
|
+
readonly documentationUrl: string_promptbook_documentation_url;
|
|
22
|
+
/**
|
|
23
|
+
* Scrapes the markdown file and returns the knowledge pieces or `null` if it can't scrape it
|
|
24
|
+
*/
|
|
25
|
+
scrape(source: ScraperSourceHandler, options: PrepareAndScrapeOptions): Promisable<Array<Omit<KnowledgePiecePreparedJson, 'sources' | 'preparationIds'>> | null>;
|
|
26
|
+
};
|
|
27
|
+
/**
|
|
28
|
+
* @@@
|
|
29
|
+
*/
|
|
30
|
+
export type ScraperSourceHandler = {
|
|
31
|
+
/**
|
|
32
|
+
* The source of the knowledge
|
|
33
|
+
*/
|
|
34
|
+
readonly source: string_knowledge_source_link;
|
|
35
|
+
/**
|
|
36
|
+
* The path to the file, if it is a file
|
|
37
|
+
*
|
|
38
|
+
* Note: Typically one of the `filename` or `url` is set and the other is `null`
|
|
39
|
+
*/
|
|
40
|
+
readonly filename: string_filename | null;
|
|
41
|
+
/**
|
|
42
|
+
* The URL, if it is online
|
|
43
|
+
*
|
|
44
|
+
* Note: Typically one of the `filename` or `url` is set and the other is `null`
|
|
45
|
+
*/
|
|
46
|
+
readonly url: string_url | null;
|
|
47
|
+
/**
|
|
48
|
+
* Mime type of the source
|
|
49
|
+
*/
|
|
50
|
+
readonly mimeType: string_mime_type;
|
|
51
|
+
/**
|
|
52
|
+
* Get the content as parsed JSON
|
|
53
|
+
*/
|
|
54
|
+
asJson(): Promisable<unknown>;
|
|
55
|
+
/**
|
|
56
|
+
* Get the content as a utf-8 string
|
|
57
|
+
*/
|
|
58
|
+
asText(): Promisable<string>;
|
|
59
|
+
/**
|
|
60
|
+
* Get the content as a blob
|
|
61
|
+
*/
|
|
62
|
+
asBlob(): Promisable<Blob>;
|
|
63
|
+
};
|
|
64
|
+
/**
|
|
65
|
+
* TODO: [🐝] @@@ Annotate all
|
|
66
|
+
* TODO: [🔼] Export via types
|
|
67
|
+
*/
|
|
68
|
+
/**
|
|
69
|
+
* TODO: !!!!!! Test that this is catched
|
|
70
|
+
* Note: [⚫] Code in this file should never be published in any package
|
|
71
|
+
*/
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { IDestroyable } from 'destroyable';
|
|
2
|
+
import type { string_absolute_filename } from '../../types/typeAliases';
|
|
3
|
+
/**
|
|
4
|
+
* @@@
|
|
5
|
+
*/
|
|
6
|
+
export type ScraperIntermediateSource = IDestroyable & {
|
|
7
|
+
/**
|
|
8
|
+
* @@@
|
|
9
|
+
*/
|
|
10
|
+
readonly filename: string_absolute_filename;
|
|
11
|
+
};
|
package/esm/typings/src/{knowledge/prepare-knowledge → scrapers}/_common/prepareKnowledgePieces.d.ts
RENAMED
|
@@ -1,13 +1,13 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
import type { KnowledgePiecePreparedJson } from '
|
|
3
|
-
import type { KnowledgeSourceJson } from '
|
|
1
|
+
import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions';
|
|
2
|
+
import type { KnowledgePiecePreparedJson } from '../../types/PipelineJson/KnowledgePieceJson';
|
|
3
|
+
import type { KnowledgeSourceJson } from '../../types/PipelineJson/KnowledgeSourceJson';
|
|
4
4
|
/**
|
|
5
5
|
* Prepares the knowle
|
|
6
6
|
*
|
|
7
7
|
* @see https://github.com/webgptorg/promptbook/discussions/41
|
|
8
8
|
* @public exported from `@promptbook/core`
|
|
9
9
|
*/
|
|
10
|
-
export declare function prepareKnowledgePieces(knowledgeSources: Array<KnowledgeSourceJson>, options:
|
|
10
|
+
export declare function prepareKnowledgePieces(knowledgeSources: Array<KnowledgeSourceJson>, options: PrepareAndScrapeOptions): Promise<Array<Omit<KnowledgePiecePreparedJson, 'preparationIds'>>>;
|
|
11
11
|
/**
|
|
12
12
|
* TODO: [🧊] In future one preparation can take data from previous preparation and save tokens and time
|
|
13
13
|
* Put `knowledgePieces` into `PrepareKnowledgeOptions`
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import type { PrepareAndScrapeOptions } from '../../../prepare/PrepareAndScrapeOptions';
|
|
2
|
+
import type { string_file_extension } from '../../../types/typeAliases';
|
|
3
|
+
import type { ScraperSourceHandler } from '../Scraper';
|
|
4
|
+
import type { ScraperIntermediateSource } from '../ScraperIntermediateSource';
|
|
5
|
+
/**
|
|
6
|
+
* @@@
|
|
7
|
+
*
|
|
8
|
+
* @private internal utility of `getScraperIntermediateSource`
|
|
9
|
+
*/
|
|
10
|
+
type GetScraperIntermediateSourceSource = Pick<ScraperSourceHandler, 'filename' | 'url'>;
|
|
11
|
+
/**
|
|
12
|
+
* @@@
|
|
13
|
+
*
|
|
14
|
+
* @private internal utility of `getScraperIntermediateSource`
|
|
15
|
+
*/
|
|
16
|
+
type GetScraperIntermediateSourceHandler = Required<Pick<PrepareAndScrapeOptions, 'rootDirname' | 'cacheDirname' | 'isCacheCleaned' | 'isVerbose'>> & {
|
|
17
|
+
readonly extension: string_file_extension;
|
|
18
|
+
};
|
|
19
|
+
/**
|
|
20
|
+
* Create a filename for intermediate cache for scrapers
|
|
21
|
+
*
|
|
22
|
+
* Note: It also checks if directory exists and creates it if not
|
|
23
|
+
*
|
|
24
|
+
* @private as internal utility for scrapers
|
|
25
|
+
*/
|
|
26
|
+
export declare function getScraperIntermediateSource(source: GetScraperIntermediateSourceSource, options: GetScraperIntermediateSourceHandler): Promise<ScraperIntermediateSource>;
|
|
27
|
+
export {};
|
|
28
|
+
/**
|
|
29
|
+
* Note: Not using `FileCacheStorage` for two reasons:
|
|
30
|
+
* 1) Need to store more than serialized JSONs
|
|
31
|
+
* 2) Need to switch between a `rootDirname` and `cacheDirname` <- TODO: !!!!
|
|
32
|
+
* TODO: [🐱🐉][🧠] Make some smart crop
|
|
33
|
+
*/
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { SetOptional } from 'type-fest';
|
|
2
|
+
import type { PrepareAndScrapeOptions } from '../../../prepare/PrepareAndScrapeOptions';
|
|
3
|
+
import type { KnowledgeSourceJson } from '../../../types/PipelineJson/KnowledgeSourceJson';
|
|
4
|
+
import type { ScraperSourceHandler } from '../Scraper';
|
|
5
|
+
/**
|
|
6
|
+
* @@@
|
|
7
|
+
*
|
|
8
|
+
* @private for scraper utilities
|
|
9
|
+
*/
|
|
10
|
+
export declare function makeKnowledgeSourceHandler(knowledgeSource: SetOptional<KnowledgeSourceJson, 'name'>, options?: Pick<PrepareAndScrapeOptions, 'rootDirname' | 'isVerbose'>): Promise<ScraperSourceHandler>;
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions';
|
|
2
|
+
import type { KnowledgePiecePreparedJson } from '../../types/PipelineJson/KnowledgePieceJson';
|
|
3
|
+
import type { ScraperSourceHandler } from '../_common/Scraper';
|
|
4
|
+
import type { ScraperIntermediateSource } from '../_common/ScraperIntermediateSource';
|
|
5
|
+
/**
|
|
6
|
+
* Scraper of .docx and .odt files
|
|
7
|
+
*
|
|
8
|
+
* @see `documentationUrl` for more details
|
|
9
|
+
* @public exported from `@promptbook/core`
|
|
10
|
+
*/
|
|
11
|
+
export declare const documentScraper: {
|
|
12
|
+
/**
|
|
13
|
+
* Mime types that this scraper can handle
|
|
14
|
+
*/
|
|
15
|
+
mimeTypes: string[];
|
|
16
|
+
/**
|
|
17
|
+
* Link to documentation
|
|
18
|
+
*/
|
|
19
|
+
documentationUrl: "https://github.com/webgptorg/promptbook/discussions/@@";
|
|
20
|
+
/**
|
|
21
|
+
* Convert the `.docx` or `.odt` to `.md` file and returns intermediate source
|
|
22
|
+
*
|
|
23
|
+
* Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object
|
|
24
|
+
*/
|
|
25
|
+
$convert(source: ScraperSourceHandler, options: PrepareAndScrapeOptions): Promise<ScraperIntermediateSource>;
|
|
26
|
+
/**
|
|
27
|
+
* Scrapes the docx file and returns the knowledge pieces or `null` if it can't scrape it
|
|
28
|
+
*/
|
|
29
|
+
scrape(source: ScraperSourceHandler, options: PrepareAndScrapeOptions): Promise<Array<Omit<KnowledgePiecePreparedJson, 'sources' | 'preparationIds'>> | null>;
|
|
30
|
+
};
|
|
31
|
+
/**
|
|
32
|
+
* TODO: [👣] Converted documents can act as cached items - there is no need to run conversion each time
|
|
33
|
+
* TODO: [🦖] Make some system for putting scrapers to separete packages
|
|
34
|
+
* TODO: [🪂] Do it in parallel 11:11
|
|
35
|
+
* TODO: [🦷] Ideally use `as const satisfies Converter & Scraper` BUT this combination throws errors
|
|
36
|
+
* Note: No need to aggregate usage here, it is done by intercepting the llmTools
|
|
37
|
+
*/
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions';
|
|
2
|
+
import type { KnowledgePiecePreparedJson } from '../../types/PipelineJson/KnowledgePieceJson';
|
|
3
|
+
import type { ScraperSourceHandler } from '../_common/Scraper';
|
|
4
|
+
import type { ScraperIntermediateSource } from '../_common/ScraperIntermediateSource';
|
|
5
|
+
/**
|
|
6
|
+
* Scraper for .docx files
|
|
7
|
+
*
|
|
8
|
+
* @see `documentationUrl` for more details
|
|
9
|
+
* @public exported from `@promptbook/core`
|
|
10
|
+
*/
|
|
11
|
+
export declare const legacyDocumentScraper: {
|
|
12
|
+
/**
|
|
13
|
+
* Mime types that this scraper can handle
|
|
14
|
+
*/
|
|
15
|
+
mimeTypes: string[];
|
|
16
|
+
/**
|
|
17
|
+
* Link to documentation
|
|
18
|
+
*/
|
|
19
|
+
documentationUrl: "https://github.com/webgptorg/promptbook/discussions/@@";
|
|
20
|
+
/**
|
|
21
|
+
* Convert the `.doc` or `.rtf` to `.doc` file and returns intermediate source
|
|
22
|
+
*
|
|
23
|
+
* Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object
|
|
24
|
+
*/
|
|
25
|
+
$convert(source: ScraperSourceHandler, options: PrepareAndScrapeOptions): Promise<ScraperIntermediateSource>;
|
|
26
|
+
/**
|
|
27
|
+
* Scrapes the `.doc` or `.rtf` file and returns the knowledge pieces or `null` if it can't scrape it
|
|
28
|
+
*/
|
|
29
|
+
scrape(source: ScraperSourceHandler, options: PrepareAndScrapeOptions): Promise<Array<Omit<KnowledgePiecePreparedJson, 'sources' | 'preparationIds'>> | null>;
|
|
30
|
+
};
|
|
31
|
+
/**
|
|
32
|
+
* TODO: [👣] Converted documents can act as cached items - there is no need to run conversion each time
|
|
33
|
+
* TODO: [🦖] Make some system for putting scrapers to separete packages
|
|
34
|
+
* TODO: [🪂] Do it in parallel 11:11
|
|
35
|
+
* TODO: [🦷] Ideally use `as const satisfies Converter & Scraper` BUT this combination throws errors
|
|
36
|
+
* Note: No need to aggregate usage here, it is done by intercepting the llmTools
|
|
37
|
+
*/
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { Scraper } from './_common/Scraper';
|
|
2
|
+
/**
|
|
3
|
+
* @@@
|
|
4
|
+
*
|
|
5
|
+
* @private because this will be replaced by a system of one scraper per package [🦖]
|
|
6
|
+
* TODO: [🦖] System for scrapers NOT public exported from `@promptbook/core`
|
|
7
|
+
*/
|
|
8
|
+
export declare const SCRAPERS: Array<Scraper>;
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions';
|
|
2
|
+
import type { KnowledgePiecePreparedJson } from '../../types/PipelineJson/KnowledgePieceJson';
|
|
3
|
+
import type { ScraperSourceHandler } from '../_common/Scraper';
|
|
4
|
+
/**
|
|
5
|
+
* Scraper for markdown files
|
|
6
|
+
*
|
|
7
|
+
* @see `documentationUrl` for more details
|
|
8
|
+
* @public exported from `@promptbook/core`
|
|
9
|
+
*/
|
|
10
|
+
export declare const markdownScraper: {
|
|
11
|
+
/**
|
|
12
|
+
* Mime types that this scraper can handle
|
|
13
|
+
*/
|
|
14
|
+
mimeTypes: string[];
|
|
15
|
+
/**
|
|
16
|
+
* Link to documentation
|
|
17
|
+
*/
|
|
18
|
+
documentationUrl: "https://github.com/webgptorg/promptbook/discussions/@@";
|
|
19
|
+
/**
|
|
20
|
+
* Scrapes the markdown file and returns the knowledge pieces or `null` if it can't scrape it
|
|
21
|
+
*/
|
|
22
|
+
scrape(source: ScraperSourceHandler, options: PrepareAndScrapeOptions): Promise<Array<Omit<KnowledgePiecePreparedJson, 'sources' | 'preparationIds'>> | null>;
|
|
23
|
+
};
|
|
24
|
+
/**
|
|
25
|
+
* TODO: [🦖] Make some system for putting scrapers to separete packages
|
|
26
|
+
* TODO: [🪂] Do it in parallel 11:11
|
|
27
|
+
* TODO: [🦷] Ideally use `as const satisfies Scraper` BUT this combination throws errors
|
|
28
|
+
* Note: No need to aggregate usage here, it is done by intercepting the llmTools
|
|
29
|
+
*/
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions';
|
|
2
|
+
import type { KnowledgePiecePreparedJson } from '../../types/PipelineJson/KnowledgePieceJson';
|
|
3
|
+
import type { ScraperSourceHandler } from '../_common/Scraper';
|
|
4
|
+
import type { ScraperIntermediateSource } from '../_common/ScraperIntermediateSource';
|
|
5
|
+
/**
|
|
6
|
+
* Scraper for .docx files
|
|
7
|
+
*
|
|
8
|
+
* @see `documentationUrl` for more details
|
|
9
|
+
* @public exported from `@promptbook/core`
|
|
10
|
+
*/
|
|
11
|
+
export declare const pdfScraper: {
|
|
12
|
+
/**
|
|
13
|
+
* Mime types that this scraper can handle
|
|
14
|
+
*/
|
|
15
|
+
mimeTypes: string[];
|
|
16
|
+
/**
|
|
17
|
+
* Link to documentation
|
|
18
|
+
*/
|
|
19
|
+
documentationUrl: "https://github.com/webgptorg/promptbook/discussions/@@";
|
|
20
|
+
/**
|
|
21
|
+
* Converts the `.pdf` file to `.md` file and returns intermediate source
|
|
22
|
+
*/
|
|
23
|
+
$convert(source: ScraperSourceHandler, options: PrepareAndScrapeOptions): Promise<ScraperIntermediateSource>;
|
|
24
|
+
/**
|
|
25
|
+
* Scrapes the `.pdf` file and returns the knowledge pieces or `null` if it can't scrape it
|
|
26
|
+
*/
|
|
27
|
+
scrape(source: ScraperSourceHandler, options: PrepareAndScrapeOptions): Promise<Array<Omit<KnowledgePiecePreparedJson, 'sources' | 'preparationIds'>> | null>;
|
|
28
|
+
};
|
|
29
|
+
/**
|
|
30
|
+
* TODO: [👣] Converted documents can act as cached items - there is no need to run conversion each time
|
|
31
|
+
* TODO: [🦖] Make some system for putting scrapers to separete packages
|
|
32
|
+
* TODO: [🪂] Do it in parallel 11:11
|
|
33
|
+
* TODO: [🦷] Ideally use `as const satisfies Converter & Scraper` BUT this combination throws errors
|
|
34
|
+
* Note: No need to aggregate usage here, it is done by intercepting the llmTools
|
|
35
|
+
*/
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { Converter } from 'showdown';
|
|
2
|
+
/**
|
|
3
|
+
* A converter instance that uses showdown and highlight extensions
|
|
4
|
+
*
|
|
5
|
+
* @type {Converter}
|
|
6
|
+
* @private for markdown and html knowledge scrapers
|
|
7
|
+
*/
|
|
8
|
+
export declare const markdownConverter: Converter;
|
|
9
|
+
/**
|
|
10
|
+
* TODO: !!!!!! Figure out better name not to confuse with `Converter`
|
|
11
|
+
* TODO: !!!!!! Lazy-make converter
|
|
12
|
+
*/
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions';
|
|
2
|
+
import type { KnowledgePiecePreparedJson } from '../../types/PipelineJson/KnowledgePieceJson';
|
|
3
|
+
import type { string_markdown } from '../../types/typeAliases';
|
|
4
|
+
import type { ScraperSourceHandler } from '../_common/Scraper';
|
|
5
|
+
import type { ScraperIntermediateSource } from '../_common/ScraperIntermediateSource';
|
|
6
|
+
/**
|
|
7
|
+
* Scraper for .docx files
|
|
8
|
+
*
|
|
9
|
+
* @see `documentationUrl` for more details
|
|
10
|
+
* @public exported from `@promptbook/core`
|
|
11
|
+
*/
|
|
12
|
+
export declare const websiteScraper: {
|
|
13
|
+
/**
|
|
14
|
+
* Mime types that this scraper can handle
|
|
15
|
+
*/
|
|
16
|
+
mimeTypes: string[];
|
|
17
|
+
/**
|
|
18
|
+
* Link to documentation
|
|
19
|
+
*/
|
|
20
|
+
documentationUrl: "https://github.com/webgptorg/promptbook/discussions/@@";
|
|
21
|
+
/**
|
|
22
|
+
* Convert the website to `.md` file and returns intermediate source
|
|
23
|
+
*
|
|
24
|
+
* Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object
|
|
25
|
+
*/
|
|
26
|
+
$convert(source: ScraperSourceHandler, options: PrepareAndScrapeOptions): Promise<import("destroyable").IDestroyable & {
|
|
27
|
+
readonly filename: string;
|
|
28
|
+
} & {
|
|
29
|
+
markdown: string_markdown;
|
|
30
|
+
}>;
|
|
31
|
+
/**
|
|
32
|
+
* Scrapes the website and returns the knowledge pieces or `null` if it can't scrape it
|
|
33
|
+
*/
|
|
34
|
+
scrape(source: ScraperSourceHandler, options: PrepareAndScrapeOptions): Promise<Array<Omit<KnowledgePiecePreparedJson, 'sources' | 'preparationIds'>> | null>;
|
|
35
|
+
};
|
|
36
|
+
/**
|
|
37
|
+
* TODO: !!!!!! Put into separate package
|
|
38
|
+
* TODO: [👣] Scraped website in .md can act as cache item - there is no need to run conversion each time
|
|
39
|
+
* TODO: [🦖] Make some system for putting scrapers to separete packages
|
|
40
|
+
* TODO: [🪂] Do it in parallel 11:11
|
|
41
|
+
* TODO: [🦷] Ideally use `as const satisfies Converter & Scraper` BUT this combination throws errors
|
|
42
|
+
* Note: No need to aggregate usage here, it is done by intercepting the llmTools
|
|
43
|
+
*/
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
import type { PromptbookStorage } from '../_common/PromptbookStorage';
|
|
2
|
-
import type {
|
|
2
|
+
import type { FileCacheStorageOptions } from './FileCacheStorageOptions';
|
|
3
3
|
/**
|
|
4
4
|
* @@@
|
|
5
5
|
*
|
|
6
6
|
* @public exported from `@promptbook/node`
|
|
7
7
|
*/
|
|
8
|
-
export declare class
|
|
9
|
-
protected readonly options:
|
|
10
|
-
constructor(options:
|
|
8
|
+
export declare class FileCacheStorage<TItem> implements PromptbookStorage<TItem> {
|
|
9
|
+
protected readonly options: FileCacheStorageOptions;
|
|
10
|
+
constructor(options: FileCacheStorageOptions);
|
|
11
11
|
/**
|
|
12
12
|
* @@@
|
|
13
13
|
*/
|
|
@@ -27,5 +27,5 @@ export declare class FilesStorage<TItem> implements PromptbookStorage<TItem> {
|
|
|
27
27
|
}
|
|
28
28
|
/**
|
|
29
29
|
* TODO: [🌗] Maybe some checkers, not all valid JSONs are desired and valid values
|
|
30
|
-
* Note: [🟢]
|
|
30
|
+
* Note: [🟢] Code in this file should never be published outside of `@promptbook/node` and `@promptbook/cli`
|
|
31
31
|
*/
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -6,5 +6,5 @@ import type { PromptbookStorage } from '../_common/PromptbookStorage';
|
|
|
6
6
|
*/
|
|
7
7
|
export declare function getLocalStorage<TItem>(): PromptbookStorage<TItem>;
|
|
8
8
|
/**
|
|
9
|
-
* Note: [🔵]
|
|
9
|
+
* Note: [🔵] Code in this file should never be published outside of `@promptbook/browser`
|
|
10
10
|
*/
|
|
@@ -6,5 +6,5 @@ import type { PromptbookStorage } from '../_common/PromptbookStorage';
|
|
|
6
6
|
*/
|
|
7
7
|
export declare function getSessionStorage<TItem>(): PromptbookStorage<TItem>;
|
|
8
8
|
/**
|
|
9
|
-
* Note: [🔵]
|
|
9
|
+
* Note: [🔵] Code in this file should never be published outside of `@promptbook/browser`
|
|
10
10
|
*/
|
|
@@ -11,11 +11,18 @@ import type { string_name } from '../typeAliases';
|
|
|
11
11
|
*/
|
|
12
12
|
export type KnowledgeSourceJson = {
|
|
13
13
|
/**
|
|
14
|
-
*
|
|
14
|
+
* Unique identifier of the knowledge source
|
|
15
15
|
*/
|
|
16
16
|
readonly name: string_name;
|
|
17
17
|
/**
|
|
18
|
-
*
|
|
18
|
+
* Source of one knowledge
|
|
19
|
+
*
|
|
20
|
+
* It can be a link, a relative path to file or direct text or combination of those
|
|
21
|
+
*
|
|
22
|
+
* For example `"https://pavolhejny.com/"`
|
|
23
|
+
* For example `"./pavol-hejny-cv.pdf"`
|
|
24
|
+
* For example `"Pavol Hejný has web https://pavolhejny.com/"`
|
|
25
|
+
* For example `"Pavol Hejný is web developer and creator of Promptbook and Collboard"`
|
|
19
26
|
*/
|
|
20
27
|
readonly sourceContent: string_knowledge_source_content;
|
|
21
28
|
};
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { ModelRequirements } from '../ModelRequirements';
|
|
2
|
-
import type {
|
|
2
|
+
import type { string_filename } from '../typeAliases';
|
|
3
3
|
import type { string_markdown_text } from '../typeAliases';
|
|
4
4
|
import type { string_pipeline_url } from '../typeAliases';
|
|
5
5
|
import type { string_semantic_version } from '../typeAliases';
|
|
@@ -33,7 +33,7 @@ export type PipelineJson = {
|
|
|
33
33
|
/**
|
|
34
34
|
* Internal helper for tracking the source `.ptbk.md` file of the pipeline
|
|
35
35
|
*/
|
|
36
|
-
readonly sourceFile?:
|
|
36
|
+
readonly sourceFile?: string_filename;
|
|
37
37
|
/**
|
|
38
38
|
* Title of the promptbook
|
|
39
39
|
* -It can use simple markdown formatting like **bold**, *italic*, [link](https://example.com), ... BUT not code blocks and structure
|
|
@@ -156,7 +156,7 @@ export type string_knowledge_source_content = string_knowledge_source_link | str
|
|
|
156
156
|
*
|
|
157
157
|
* @@@ string_knowledge_source vs string_knowledge_source_link
|
|
158
158
|
*/
|
|
159
|
-
export type string_knowledge_source_link = string_url |
|
|
159
|
+
export type string_knowledge_source_link = string_url | string_filename;
|
|
160
160
|
/**
|
|
161
161
|
* Semantic helper
|
|
162
162
|
*
|
|
@@ -386,6 +386,7 @@ export type string_emails = string;
|
|
|
386
386
|
* - `isValidUuid to check validity
|
|
387
387
|
*
|
|
388
388
|
* For example `"5a0a153d-7be9-4018-9eda-e0e2e2b89bd9"`
|
|
389
|
+
* TODO: [🥬] Make some system for hashes and ids of promptbook
|
|
389
390
|
*/
|
|
390
391
|
export type string_uuid = string & {
|
|
391
392
|
readonly _type: 'uuid';
|
|
@@ -423,37 +424,33 @@ export type string_file_extension = string;
|
|
|
423
424
|
*
|
|
424
425
|
* For example `"C:/Users/me/work/collboard/modules-sdk/src/colldev/commands/develop/ColldevDevelop.tsx"`
|
|
425
426
|
*/
|
|
426
|
-
export type
|
|
427
|
+
export type string_absolute_filename = string;
|
|
427
428
|
/**
|
|
428
429
|
* Semantic helper
|
|
429
430
|
*
|
|
430
431
|
* For example `"./src/colldev/commands/develop/ColldevDevelop.tsx"`
|
|
431
432
|
*/
|
|
432
|
-
export type
|
|
433
|
+
export type string_relative_filename = string;
|
|
433
434
|
/**
|
|
434
435
|
* Semantic helper
|
|
435
436
|
*/
|
|
436
|
-
export type
|
|
437
|
+
export type string_filename = string_absolute_filename | string_relative_filename;
|
|
437
438
|
/**
|
|
438
439
|
* Semantic helper
|
|
439
440
|
*
|
|
440
441
|
* For example `"C:/Users/me/work/collboard/modules-sdk/src/colldev/commands/develop/ColldevDevelop.tsx"`
|
|
441
442
|
*/
|
|
442
|
-
export type
|
|
443
|
+
export type string_absolute_dirname = string;
|
|
443
444
|
/**
|
|
444
445
|
* Semantic helper
|
|
445
446
|
*
|
|
446
447
|
* For example `"./src/colldev/commands/develop/ColldevDevelop.tsx"`
|
|
447
448
|
*/
|
|
448
|
-
export type
|
|
449
|
+
export type string_relative_dirname = string;
|
|
449
450
|
/**
|
|
450
451
|
* Semantic helper
|
|
451
452
|
*/
|
|
452
|
-
export type
|
|
453
|
-
/**
|
|
454
|
-
* Semantic helper
|
|
455
|
-
*/
|
|
456
|
-
export type string_filename = string;
|
|
453
|
+
export type string_dirname = string_absolute_dirname | string_relative_dirname;
|
|
457
454
|
/**
|
|
458
455
|
* Semantic helper
|
|
459
456
|
*
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { IExecCommandOptions } from './IExecCommandOptions';
|
|
2
|
+
/**
|
|
3
|
+
* Run one command in a shell
|
|
4
|
+
*
|
|
5
|
+
* Note: There are 2 similar functions in the codebase:
|
|
6
|
+
* - `$execCommand` which runs a single command
|
|
7
|
+
* - `$execCommands` which runs multiple commands
|
|
8
|
+
*
|
|
9
|
+
* @public exported from `@promptbook/node`
|
|
10
|
+
*/
|
|
11
|
+
export declare function $execCommand(options: IExecCommandOptions): Promise<string>;
|
|
12
|
+
/**
|
|
13
|
+
* Note: [🟢 <- TODO: [🦖] !!!!!! Split scrapers into packages and enable] Code in this file should never be published outside of `@promptbook/node` and `@promptbook/cli`
|
|
14
|
+
*/
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Run multiple commands in a shell in sequence
|
|
3
|
+
*
|
|
4
|
+
* Note: There are 2 similar functions in the codebase:
|
|
5
|
+
* - `$execCommand` which runs a single command
|
|
6
|
+
* - `$execCommands` which runs multiple commands
|
|
7
|
+
*
|
|
8
|
+
* @public exported from `@promptbook/node`
|
|
9
|
+
*/
|
|
10
|
+
export declare function $execCommands({ commands, cwd, crashOnError, }: {
|
|
11
|
+
readonly commands: string[];
|
|
12
|
+
readonly cwd: string;
|
|
13
|
+
readonly crashOnError?: boolean;
|
|
14
|
+
}): Promise<void>;
|
|
15
|
+
/**
|
|
16
|
+
* Note: [🟢 <- TODO: [🦖] !!!!!! Split scrapers into packages and enable] Code in this file should never be published outside of `@promptbook/node` and `@promptbook/cli`
|
|
17
|
+
*/
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @deprecated Use sth from type-fest or move this to type helpers
|
|
3
|
+
*/
|
|
4
|
+
type RequiredAndOptional<TBase, TRequired extends keyof TBase, TOptional extends keyof TBase> = Pick<TBase, TRequired> & Partial<Pick<TBase, TOptional>>;
|
|
5
|
+
/**
|
|
6
|
+
* Simple options for `execCommand`
|
|
7
|
+
*
|
|
8
|
+
* TODO: !!!!!! Rename - remove I prefix
|
|
9
|
+
*/
|
|
10
|
+
export type IExecCommandOptions = string | RequiredAndOptional<IExecCommandOptionsAdvanced, 'command', 'args' | 'cwd' | 'crashOnError' | 'timeout'>;
|
|
11
|
+
/**
|
|
12
|
+
* Advanced options for `execCommand`
|
|
13
|
+
*
|
|
14
|
+
* TODO: !!!!!! Rename - remove I prefix
|
|
15
|
+
*/
|
|
16
|
+
export type IExecCommandOptionsAdvanced = {
|
|
17
|
+
readonly command: string;
|
|
18
|
+
readonly args: string[];
|
|
19
|
+
readonly cwd: string;
|
|
20
|
+
readonly crashOnError: boolean;
|
|
21
|
+
readonly timeout: number;
|
|
22
|
+
};
|
|
23
|
+
export {};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|