@promptbook/utils 0.71.0-0 → 0.71.0-7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -0
- package/esm/index.es.js +68 -73
- package/esm/index.es.js.map +1 -1
- package/esm/typings/src/_packages/browser.index.d.ts +1 -1
- package/esm/typings/src/_packages/core.index.d.ts +28 -10
- package/esm/typings/src/_packages/node.index.d.ts +6 -2
- package/esm/typings/src/_packages/types.index.d.ts +28 -20
- package/esm/typings/src/cli/cli-commands/about.d.ts +1 -1
- package/esm/typings/src/cli/cli-commands/hello.d.ts +1 -1
- package/esm/typings/src/cli/cli-commands/make.d.ts +1 -1
- package/esm/typings/src/cli/cli-commands/prettify.d.ts +1 -1
- package/esm/typings/src/cli/main.d.ts +1 -1
- package/esm/typings/src/cli/promptbookCli.d.ts +1 -1
- package/esm/typings/src/collection/constructors/createCollectionFromDirectory.d.ts +8 -5
- package/esm/typings/src/collection/constructors/createCollectionFromUrl.d.ts +1 -1
- package/esm/typings/src/commands/EXPECT/expectCommandParser.d.ts +1 -1
- package/esm/typings/src/commands/FOREACH/ForeachJson.d.ts +5 -1
- package/esm/typings/src/commands/FOREACH/foreachCommandParser.d.ts +1 -1
- package/esm/typings/src/commands/FORMAT/formatCommandParser.d.ts +1 -1
- package/esm/typings/src/commands/JOKER/jokerCommandParser.d.ts +1 -1
- package/esm/typings/src/commands/KNOWLEDGE/knowledgeCommandParser.d.ts +1 -1
- package/esm/typings/src/commands/KNOWLEDGE/utils/sourceContentToName.d.ts +11 -0
- package/esm/typings/src/commands/KNOWLEDGE/utils/sourceContentToName.test.d.ts +4 -0
- package/esm/typings/src/commands/MODEL/modelCommandParser.d.ts +1 -1
- package/esm/typings/src/commands/PARAMETER/parameterCommandParser.d.ts +1 -1
- package/esm/typings/src/commands/PERSONA/personaCommandParser.d.ts +1 -1
- package/esm/typings/src/commands/POSTPROCESS/postprocessCommandParser.d.ts +1 -1
- package/esm/typings/src/commands/PROMPTBOOK_VERSION/promptbookVersionCommandParser.d.ts +1 -1
- package/esm/typings/src/commands/TEMPLATE/templateCommandParser.d.ts +1 -1
- package/esm/typings/src/commands/URL/urlCommandParser.d.ts +1 -1
- package/esm/typings/src/commands/X_ACTION/actionCommandParser.d.ts +1 -1
- package/esm/typings/src/commands/X_INSTRUMENT/instrumentCommandParser.d.ts +1 -1
- package/esm/typings/src/commands/_BOILERPLATE/boilerplateCommandParser.d.ts +1 -1
- package/esm/typings/src/config.d.ts +10 -0
- package/esm/typings/src/conversion/pipelineStringToJson.d.ts +2 -15
- package/esm/typings/src/conversion/pipelineStringToJsonSync.d.ts +1 -1
- package/esm/typings/src/conversion/validation/_importPipeline.d.ts +1 -1
- package/esm/typings/src/{knowledge/dialogs → dialogs}/callback/CallbackInterfaceTools.d.ts +2 -2
- package/esm/typings/src/{knowledge/dialogs → dialogs}/callback/CallbackInterfaceToolsOptions.d.ts +2 -2
- package/esm/typings/src/{knowledge/dialogs → dialogs}/simple-prompt/SimplePromptInterfaceTools.d.ts +4 -4
- package/esm/typings/src/errors/AbstractFormatError.d.ts +11 -0
- package/esm/typings/src/errors/KnowledgeScrapeError.d.ts +9 -0
- package/esm/typings/src/errors/MissingToolsError.d.ts +9 -0
- package/esm/typings/src/execution/ExecutionTools.d.ts +3 -3
- package/esm/typings/src/execution/PipelineExecutor.d.ts +1 -0
- package/esm/typings/src/execution/createPipelineExecutor/00-CreatePipelineExecutorOptions.d.ts +5 -2
- package/esm/typings/src/execution/createPipelineExecutor/00-CreatePipelineExecutorSettings.d.ts +2 -13
- package/esm/typings/src/execution/createPipelineExecutor/00-createPipelineExecutor.d.ts +3 -0
- package/esm/typings/src/execution/createPipelineExecutor/10-executePipeline.d.ts +4 -1
- package/esm/typings/src/execution/createPipelineExecutor/20-executeTemplate.d.ts +3 -0
- package/esm/typings/src/execution/createPipelineExecutor/{30-executeFormatCells.d.ts → 30-executeFormatSubvalues.d.ts} +2 -6
- package/esm/typings/src/execution/embeddingVectorToString.d.ts +1 -1
- package/esm/typings/src/execution/utils/forEachAsync.d.ts +1 -0
- package/esm/typings/src/formats/_common/FormatSubvalueDefinition.d.ts +2 -1
- package/esm/typings/src/formats/csv/CsvFormatError.d.ts +10 -0
- package/esm/typings/src/llm-providers/_common/createLlmToolsFromConfigurationFromEnv.d.ts +1 -1
- package/esm/typings/src/llm-providers/_common/createLlmToolsFromEnv.d.ts +1 -1
- package/esm/typings/src/llm-providers/_common/getLlmToolsForCli.d.ts +1 -1
- package/esm/typings/src/llm-providers/anthropic-claude/AnthropicClaudeExecutionTools.d.ts +2 -1
- package/esm/typings/src/llm-providers/anthropic-claude/playground/playground.d.ts +3 -2
- package/esm/typings/src/llm-providers/azure-openai/AzureOpenAiExecutionTools.d.ts +8 -1
- package/esm/typings/src/llm-providers/azure-openai/playground/playground.d.ts +1 -0
- package/esm/typings/src/llm-providers/langtail/playground/playground.d.ts +3 -0
- package/esm/typings/src/llm-providers/multiple/playground/playground.d.ts +3 -0
- package/esm/typings/src/llm-providers/openai/OpenAiExecutionTools.d.ts +2 -1
- package/esm/typings/src/llm-providers/openai/playground/playground.d.ts +2 -1
- package/esm/typings/src/llm-providers/remote/playground/playground.d.ts +3 -0
- package/esm/typings/src/personas/preparePersona.d.ts +2 -2
- package/esm/typings/src/prepare/PrepareAndScrapeOptions.d.ts +67 -0
- package/esm/typings/src/prepare/preparePipeline.d.ts +2 -2
- package/esm/typings/src/prepare/prepareTemplates.d.ts +2 -2
- package/esm/typings/src/scrapers/_common/Converter.d.ts +28 -0
- package/esm/typings/src/scrapers/_common/Scraper.d.ts +71 -0
- package/esm/typings/src/scrapers/_common/ScraperIntermediateSource.d.ts +11 -0
- package/esm/typings/src/{knowledge/prepare-knowledge → scrapers}/_common/prepareKnowledgePieces.d.ts +4 -4
- package/esm/typings/src/scrapers/_common/utils/getScraperIntermediateSource.d.ts +33 -0
- package/esm/typings/src/scrapers/_common/utils/getScraperIntermediateSource.test.d.ts +4 -0
- package/esm/typings/src/scrapers/_common/utils/makeKnowledgeSourceHandler.d.ts +10 -0
- package/esm/typings/src/scrapers/document/documentScraper.d.ts +37 -0
- package/esm/typings/src/scrapers/document/documentScraper.test.d.ts +4 -0
- package/esm/typings/src/scrapers/document/playground/document-scraper-playground.d.ts +5 -0
- package/esm/typings/src/scrapers/document-legacy/legacyDocumentScraper.d.ts +37 -0
- package/esm/typings/src/scrapers/document-legacy/legacyDocumentScraper.test.d.ts +4 -0
- package/esm/typings/src/scrapers/document-legacy/playground/legacy-document-scraper-playground.d.ts +5 -0
- package/esm/typings/src/scrapers/index.d.ts +7 -0
- package/esm/typings/src/scrapers/markdown/markdownScraper.d.ts +29 -0
- package/esm/typings/src/scrapers/markdown/playground/markdown-scraper-playground.d.ts +5 -0
- package/esm/typings/src/scrapers/pdf/pdfScraper.d.ts +35 -0
- package/esm/typings/src/scrapers/pdf/playground/pdf-scraper-playground.d.ts +5 -0
- package/esm/typings/src/scrapers/website/playground/website-scraper-playground.d.ts +5 -0
- package/esm/typings/src/scrapers/website/utils/markdownConverter.d.ts +12 -0
- package/esm/typings/src/scrapers/website/websiteScraper.d.ts +43 -0
- package/esm/typings/src/storage/{files-storage/FilesStorage.d.ts → file-cache-storage/FileCacheStorage.d.ts} +4 -4
- package/esm/typings/src/storage/file-cache-storage/FileCacheStorageOptions.d.ts +10 -0
- package/esm/typings/src/storage/{files-storage → file-cache-storage}/utils/nameToSubfolderPath.d.ts +1 -1
- package/esm/typings/src/storage/file-cache-storage/utils/nameToSubfolderPath.test.d.ts +1 -0
- package/esm/typings/src/storage/local-storage/getLocalStorage.d.ts +1 -1
- package/esm/typings/src/storage/local-storage/getSessionStorage.d.ts +1 -1
- package/esm/typings/src/types/PipelineJson/ParameterJson.d.ts +1 -1
- package/esm/typings/src/types/PipelineJson/PipelineJson.d.ts +2 -2
- package/esm/typings/src/types/execution-report/ExecutionReportJson.d.ts +0 -3
- package/esm/typings/src/types/execution-report/executionReportJsonToString.d.ts +2 -1
- package/esm/typings/src/types/typeAliases.d.ts +9 -12
- package/esm/typings/src/utils/execCommand/$execCommand.d.ts +14 -0
- package/esm/typings/src/utils/execCommand/$execCommands.d.ts +17 -0
- package/esm/typings/src/utils/execCommand/IExecCommandOptions.d.ts +23 -0
- package/esm/typings/src/utils/execCommand/execCommand.test.d.ts +1 -0
- package/esm/typings/src/utils/execCommand/execCommandNormalizeOptions.d.ts +10 -0
- package/esm/typings/src/utils/execCommand/execCommandNormalizeOptions.test.d.ts +1 -0
- package/esm/typings/src/utils/files/$isDirectoryExisting.d.ts +3 -3
- package/esm/typings/src/utils/files/$isFileExisting.d.ts +3 -3
- package/esm/typings/src/utils/files/$listAllFiles.d.ts +4 -4
- package/esm/typings/src/utils/files/extensionToMimeType.d.ts +8 -0
- package/esm/typings/src/utils/files/extensionToMimeType.test.d.ts +1 -0
- package/esm/typings/src/utils/files/getFileExtension.d.ts +8 -0
- package/esm/typings/src/utils/files/getFileExtension.test.d.ts +1 -0
- package/esm/typings/src/utils/validators/filePath/isValidFilePath.d.ts +2 -2
- package/package.json +1 -1
- package/umd/index.umd.js +71 -77
- package/umd/index.umd.js.map +1 -1
- package/esm/typings/src/knowledge/prepare-knowledge/_common/Scraper.d.ts +0 -37
- package/esm/typings/src/knowledge/prepare-knowledge/markdown/playground/markdown-knowledge-playground.d.ts +0 -2
- package/esm/typings/src/knowledge/prepare-knowledge/markdown/prepareKnowledgeFromMarkdown.d.ts +0 -14
- package/esm/typings/src/knowledge/prepare-knowledge/pdf/prepareKnowledgeFromPdf.d.ts +0 -15
- package/esm/typings/src/prepare/PrepareOptions.d.ts +0 -22
- package/esm/typings/src/storage/files-storage/FilesStorageOptions.d.ts +0 -10
- /package/esm/typings/src/{knowledge/dialogs → dialogs}/user-interface-execution-tools.test.d.ts +0 -0
- /package/esm/typings/src/{knowledge/prepare-knowledge → scrapers}/_common/prepareKnowledgePieces.test.d.ts +0 -0
- /package/esm/typings/src/{knowledge/prepare-knowledge/markdown/prepareKnowledgeFromMarkdown.test.d.ts → scrapers/markdown/markdownScraper.test.d.ts} +0 -0
- /package/esm/typings/src/{knowledge/prepare-knowledge/pdf/prepareKnowledgeFromPdf.test.d.ts → scrapers/website/utils/markdownConverter.test.d.ts} +0 -0
- /package/esm/typings/src/{storage/files-storage/utils/nameToSubfolderPath.test.d.ts → scrapers/website/websiteScraper.test.d.ts} +0 -0
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import type { PrepareAndScrapeOptions } from '../../../prepare/PrepareAndScrapeOptions';
|
|
2
|
+
import type { string_file_extension } from '../../../types/typeAliases';
|
|
3
|
+
import type { ScraperSourceHandler } from '../Scraper';
|
|
4
|
+
import type { ScraperIntermediateSource } from '../ScraperIntermediateSource';
|
|
5
|
+
/**
|
|
6
|
+
* @@@
|
|
7
|
+
*
|
|
8
|
+
* @private internal utility of `getScraperIntermediateSource`
|
|
9
|
+
*/
|
|
10
|
+
type GetScraperIntermediateSourceSource = Pick<ScraperSourceHandler, 'filename' | 'url'>;
|
|
11
|
+
/**
|
|
12
|
+
* @@@
|
|
13
|
+
*
|
|
14
|
+
* @private internal utility of `getScraperIntermediateSource`
|
|
15
|
+
*/
|
|
16
|
+
type GetScraperIntermediateSourceHandler = Required<Pick<PrepareAndScrapeOptions, 'rootDirname' | 'cacheDirname' | 'isCacheCleaned' | 'isVerbose'>> & {
|
|
17
|
+
readonly extension: string_file_extension;
|
|
18
|
+
};
|
|
19
|
+
/**
|
|
20
|
+
* Create a filename for intermediate cache for scrapers
|
|
21
|
+
*
|
|
22
|
+
* Note: It also checks if directory exists and creates it if not
|
|
23
|
+
*
|
|
24
|
+
* @private as internal utility for scrapers
|
|
25
|
+
*/
|
|
26
|
+
export declare function getScraperIntermediateSource(source: GetScraperIntermediateSourceSource, options: GetScraperIntermediateSourceHandler): Promise<ScraperIntermediateSource>;
|
|
27
|
+
export {};
|
|
28
|
+
/**
|
|
29
|
+
* Note: Not using `FileCacheStorage` for two reasons:
|
|
30
|
+
* 1) Need to store more than serialized JSONs
|
|
31
|
+
* 2) Need to switch between a `rootDirname` and `cacheDirname` <- TODO: !!!!
|
|
32
|
+
* TODO: [🐱🐉][🧠] Make some smart crop
|
|
33
|
+
*/
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { SetOptional } from 'type-fest';
|
|
2
|
+
import type { PrepareAndScrapeOptions } from '../../../prepare/PrepareAndScrapeOptions';
|
|
3
|
+
import type { KnowledgeSourceJson } from '../../../types/PipelineJson/KnowledgeSourceJson';
|
|
4
|
+
import type { ScraperSourceHandler } from '../Scraper';
|
|
5
|
+
/**
|
|
6
|
+
* @@@
|
|
7
|
+
*
|
|
8
|
+
* @private for scraper utilities
|
|
9
|
+
*/
|
|
10
|
+
export declare function makeKnowledgeSourceHandler(knowledgeSource: SetOptional<KnowledgeSourceJson, 'name'>, options?: Pick<PrepareAndScrapeOptions, 'rootDirname' | 'isVerbose'>): Promise<ScraperSourceHandler>;
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions';
|
|
2
|
+
import type { KnowledgePiecePreparedJson } from '../../types/PipelineJson/KnowledgePieceJson';
|
|
3
|
+
import type { ScraperSourceHandler } from '../_common/Scraper';
|
|
4
|
+
import type { ScraperIntermediateSource } from '../_common/ScraperIntermediateSource';
|
|
5
|
+
/**
|
|
6
|
+
* Scraper of .docx and .odt files
|
|
7
|
+
*
|
|
8
|
+
* @see `documentationUrl` for more details
|
|
9
|
+
* @public exported from `@promptbook/core`
|
|
10
|
+
*/
|
|
11
|
+
export declare const documentScraper: {
|
|
12
|
+
/**
|
|
13
|
+
* Mime types that this scraper can handle
|
|
14
|
+
*/
|
|
15
|
+
mimeTypes: string[];
|
|
16
|
+
/**
|
|
17
|
+
* Link to documentation
|
|
18
|
+
*/
|
|
19
|
+
documentationUrl: "https://github.com/webgptorg/promptbook/discussions/@@";
|
|
20
|
+
/**
|
|
21
|
+
* Convert the `.docx` or `.odt` to `.md` file and returns intermediate source
|
|
22
|
+
*
|
|
23
|
+
* Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object
|
|
24
|
+
*/
|
|
25
|
+
$convert(source: ScraperSourceHandler, options: PrepareAndScrapeOptions): Promise<ScraperIntermediateSource>;
|
|
26
|
+
/**
|
|
27
|
+
* Scrapes the docx file and returns the knowledge pieces or `null` if it can't scrape it
|
|
28
|
+
*/
|
|
29
|
+
scrape(source: ScraperSourceHandler, options: PrepareAndScrapeOptions): Promise<Array<Omit<KnowledgePiecePreparedJson, 'sources' | 'preparationIds'>> | null>;
|
|
30
|
+
};
|
|
31
|
+
/**
|
|
32
|
+
* TODO: [👣] Converted documents can act as cached items - there is no need to run conversion each time
|
|
33
|
+
* TODO: [🦖] Make some system for putting scrapers to separete packages
|
|
34
|
+
* TODO: [🪂] Do it in parallel 11:11
|
|
35
|
+
* TODO: [🦷] Ideally use `as const satisfies Converter & Scraper` BUT this combination throws errors
|
|
36
|
+
* Note: No need to aggregate usage here, it is done by intercepting the llmTools
|
|
37
|
+
*/
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions';
|
|
2
|
+
import type { KnowledgePiecePreparedJson } from '../../types/PipelineJson/KnowledgePieceJson';
|
|
3
|
+
import type { ScraperSourceHandler } from '../_common/Scraper';
|
|
4
|
+
import type { ScraperIntermediateSource } from '../_common/ScraperIntermediateSource';
|
|
5
|
+
/**
|
|
6
|
+
* Scraper for .docx files
|
|
7
|
+
*
|
|
8
|
+
* @see `documentationUrl` for more details
|
|
9
|
+
* @public exported from `@promptbook/core`
|
|
10
|
+
*/
|
|
11
|
+
export declare const legacyDocumentScraper: {
|
|
12
|
+
/**
|
|
13
|
+
* Mime types that this scraper can handle
|
|
14
|
+
*/
|
|
15
|
+
mimeTypes: string[];
|
|
16
|
+
/**
|
|
17
|
+
* Link to documentation
|
|
18
|
+
*/
|
|
19
|
+
documentationUrl: "https://github.com/webgptorg/promptbook/discussions/@@";
|
|
20
|
+
/**
|
|
21
|
+
* Convert the `.doc` or `.rtf` to `.doc` file and returns intermediate source
|
|
22
|
+
*
|
|
23
|
+
* Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object
|
|
24
|
+
*/
|
|
25
|
+
$convert(source: ScraperSourceHandler, options: PrepareAndScrapeOptions): Promise<ScraperIntermediateSource>;
|
|
26
|
+
/**
|
|
27
|
+
* Scrapes the `.doc` or `.rtf` file and returns the knowledge pieces or `null` if it can't scrape it
|
|
28
|
+
*/
|
|
29
|
+
scrape(source: ScraperSourceHandler, options: PrepareAndScrapeOptions): Promise<Array<Omit<KnowledgePiecePreparedJson, 'sources' | 'preparationIds'>> | null>;
|
|
30
|
+
};
|
|
31
|
+
/**
|
|
32
|
+
* TODO: [👣] Converted documents can act as cached items - there is no need to run conversion each time
|
|
33
|
+
* TODO: [🦖] Make some system for putting scrapers to separete packages
|
|
34
|
+
* TODO: [🪂] Do it in parallel 11:11
|
|
35
|
+
* TODO: [🦷] Ideally use `as const satisfies Converter & Scraper` BUT this combination throws errors
|
|
36
|
+
* Note: No need to aggregate usage here, it is done by intercepting the llmTools
|
|
37
|
+
*/
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions';
|
|
2
|
+
import type { KnowledgePiecePreparedJson } from '../../types/PipelineJson/KnowledgePieceJson';
|
|
3
|
+
import type { ScraperSourceHandler } from '../_common/Scraper';
|
|
4
|
+
/**
|
|
5
|
+
* Scraper for markdown files
|
|
6
|
+
*
|
|
7
|
+
* @see `documentationUrl` for more details
|
|
8
|
+
* @public exported from `@promptbook/core`
|
|
9
|
+
*/
|
|
10
|
+
export declare const markdownScraper: {
|
|
11
|
+
/**
|
|
12
|
+
* Mime types that this scraper can handle
|
|
13
|
+
*/
|
|
14
|
+
mimeTypes: string[];
|
|
15
|
+
/**
|
|
16
|
+
* Link to documentation
|
|
17
|
+
*/
|
|
18
|
+
documentationUrl: "https://github.com/webgptorg/promptbook/discussions/@@";
|
|
19
|
+
/**
|
|
20
|
+
* Scrapes the markdown file and returns the knowledge pieces or `null` if it can't scrape it
|
|
21
|
+
*/
|
|
22
|
+
scrape(source: ScraperSourceHandler, options: PrepareAndScrapeOptions): Promise<Array<Omit<KnowledgePiecePreparedJson, 'sources' | 'preparationIds'>> | null>;
|
|
23
|
+
};
|
|
24
|
+
/**
|
|
25
|
+
* TODO: [🦖] Make some system for putting scrapers to separete packages
|
|
26
|
+
* TODO: [🪂] Do it in parallel 11:11
|
|
27
|
+
* TODO: [🦷] Ideally use `as const satisfies Scraper` BUT this combination throws errors
|
|
28
|
+
* Note: No need to aggregate usage here, it is done by intercepting the llmTools
|
|
29
|
+
*/
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions';
|
|
2
|
+
import type { KnowledgePiecePreparedJson } from '../../types/PipelineJson/KnowledgePieceJson';
|
|
3
|
+
import type { ScraperSourceHandler } from '../_common/Scraper';
|
|
4
|
+
import type { ScraperIntermediateSource } from '../_common/ScraperIntermediateSource';
|
|
5
|
+
/**
|
|
6
|
+
* Scraper for .docx files
|
|
7
|
+
*
|
|
8
|
+
* @see `documentationUrl` for more details
|
|
9
|
+
* @public exported from `@promptbook/core`
|
|
10
|
+
*/
|
|
11
|
+
export declare const pdfScraper: {
|
|
12
|
+
/**
|
|
13
|
+
* Mime types that this scraper can handle
|
|
14
|
+
*/
|
|
15
|
+
mimeTypes: string[];
|
|
16
|
+
/**
|
|
17
|
+
* Link to documentation
|
|
18
|
+
*/
|
|
19
|
+
documentationUrl: "https://github.com/webgptorg/promptbook/discussions/@@";
|
|
20
|
+
/**
|
|
21
|
+
* Converts the `.pdf` file to `.md` file and returns intermediate source
|
|
22
|
+
*/
|
|
23
|
+
$convert(source: ScraperSourceHandler, options: PrepareAndScrapeOptions): Promise<ScraperIntermediateSource>;
|
|
24
|
+
/**
|
|
25
|
+
* Scrapes the `.pdf` file and returns the knowledge pieces or `null` if it can't scrape it
|
|
26
|
+
*/
|
|
27
|
+
scrape(source: ScraperSourceHandler, options: PrepareAndScrapeOptions): Promise<Array<Omit<KnowledgePiecePreparedJson, 'sources' | 'preparationIds'>> | null>;
|
|
28
|
+
};
|
|
29
|
+
/**
|
|
30
|
+
* TODO: [👣] Converted documents can act as cached items - there is no need to run conversion each time
|
|
31
|
+
* TODO: [🦖] Make some system for putting scrapers to separete packages
|
|
32
|
+
* TODO: [🪂] Do it in parallel 11:11
|
|
33
|
+
* TODO: [🦷] Ideally use `as const satisfies Converter & Scraper` BUT this combination throws errors
|
|
34
|
+
* Note: No need to aggregate usage here, it is done by intercepting the llmTools
|
|
35
|
+
*/
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { Converter } from 'showdown';
|
|
2
|
+
/**
|
|
3
|
+
* A converter instance that uses showdown and highlight extensions
|
|
4
|
+
*
|
|
5
|
+
* @type {Converter}
|
|
6
|
+
* @private for markdown and html knowledge scrapers
|
|
7
|
+
*/
|
|
8
|
+
export declare const markdownConverter: Converter;
|
|
9
|
+
/**
|
|
10
|
+
* TODO: !!!!!! Figure out better name not to confuse with `Converter`
|
|
11
|
+
* TODO: !!!!!! Lazy-make converter
|
|
12
|
+
*/
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions';
|
|
2
|
+
import type { KnowledgePiecePreparedJson } from '../../types/PipelineJson/KnowledgePieceJson';
|
|
3
|
+
import type { string_markdown } from '../../types/typeAliases';
|
|
4
|
+
import type { ScraperSourceHandler } from '../_common/Scraper';
|
|
5
|
+
import type { ScraperIntermediateSource } from '../_common/ScraperIntermediateSource';
|
|
6
|
+
/**
|
|
7
|
+
* Scraper for .docx files
|
|
8
|
+
*
|
|
9
|
+
* @see `documentationUrl` for more details
|
|
10
|
+
* @public exported from `@promptbook/core`
|
|
11
|
+
*/
|
|
12
|
+
export declare const websiteScraper: {
|
|
13
|
+
/**
|
|
14
|
+
* Mime types that this scraper can handle
|
|
15
|
+
*/
|
|
16
|
+
mimeTypes: string[];
|
|
17
|
+
/**
|
|
18
|
+
* Link to documentation
|
|
19
|
+
*/
|
|
20
|
+
documentationUrl: "https://github.com/webgptorg/promptbook/discussions/@@";
|
|
21
|
+
/**
|
|
22
|
+
* Convert the website to `.md` file and returns intermediate source
|
|
23
|
+
*
|
|
24
|
+
* Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object
|
|
25
|
+
*/
|
|
26
|
+
$convert(source: ScraperSourceHandler, options: PrepareAndScrapeOptions): Promise<import("destroyable").IDestroyable & {
|
|
27
|
+
readonly filename: string;
|
|
28
|
+
} & {
|
|
29
|
+
markdown: string_markdown;
|
|
30
|
+
}>;
|
|
31
|
+
/**
|
|
32
|
+
* Scrapes the website and returns the knowledge pieces or `null` if it can't scrape it
|
|
33
|
+
*/
|
|
34
|
+
scrape(source: ScraperSourceHandler, options: PrepareAndScrapeOptions): Promise<Array<Omit<KnowledgePiecePreparedJson, 'sources' | 'preparationIds'>> | null>;
|
|
35
|
+
};
|
|
36
|
+
/**
|
|
37
|
+
* TODO: !!!!!! Put into separate package
|
|
38
|
+
* TODO: [👣] Scraped website in .md can act as cache item - there is no need to run conversion each time
|
|
39
|
+
* TODO: [🦖] Make some system for putting scrapers to separete packages
|
|
40
|
+
* TODO: [🪂] Do it in parallel 11:11
|
|
41
|
+
* TODO: [🦷] Ideally use `as const satisfies Converter & Scraper` BUT this combination throws errors
|
|
42
|
+
* Note: No need to aggregate usage here, it is done by intercepting the llmTools
|
|
43
|
+
*/
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
import type { PromptbookStorage } from '../_common/PromptbookStorage';
|
|
2
|
-
import type {
|
|
2
|
+
import type { FileCacheStorageOptions } from './FileCacheStorageOptions';
|
|
3
3
|
/**
|
|
4
4
|
* @@@
|
|
5
5
|
*
|
|
6
6
|
* @public exported from `@promptbook/node`
|
|
7
7
|
*/
|
|
8
|
-
export declare class
|
|
8
|
+
export declare class FileCacheStorage<TItem> implements PromptbookStorage<TItem> {
|
|
9
9
|
private readonly options;
|
|
10
|
-
constructor(options:
|
|
10
|
+
constructor(options: FileCacheStorageOptions);
|
|
11
11
|
/**
|
|
12
12
|
* @@@
|
|
13
13
|
*/
|
|
@@ -27,5 +27,5 @@ export declare class FilesStorage<TItem> implements PromptbookStorage<TItem> {
|
|
|
27
27
|
}
|
|
28
28
|
/**
|
|
29
29
|
* TODO: [🌗] Maybe some checkers, not all valid JSONs are desired and valid values
|
|
30
|
-
* Note: [🟢]
|
|
30
|
+
* Note: [🟢] Code in this file should never be published outside of `@promptbook/node` and `@promptbook/cli`
|
|
31
31
|
*/
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -6,5 +6,5 @@ import type { PromptbookStorage } from '../_common/PromptbookStorage';
|
|
|
6
6
|
*/
|
|
7
7
|
export declare function getLocalStorage<TItem>(): PromptbookStorage<TItem>;
|
|
8
8
|
/**
|
|
9
|
-
* Note: [🔵]
|
|
9
|
+
* Note: [🔵] Code in this file should never be published outside of `@promptbook/browser`
|
|
10
10
|
*/
|
|
@@ -6,5 +6,5 @@ import type { PromptbookStorage } from '../_common/PromptbookStorage';
|
|
|
6
6
|
*/
|
|
7
7
|
export declare function getSessionStorage<TItem>(): PromptbookStorage<TItem>;
|
|
8
8
|
/**
|
|
9
|
-
* Note: [🔵]
|
|
9
|
+
* Note: [🔵] Code in this file should never be published outside of `@promptbook/browser`
|
|
10
10
|
*/
|
|
@@ -33,7 +33,7 @@ export type ParameterJson = {
|
|
|
33
33
|
readonly sampleValues?: Array<string_parameter_value>;
|
|
34
34
|
};
|
|
35
35
|
/**
|
|
36
|
-
* TODO: [🧠]
|
|
36
|
+
* TODO: [🧠] Should be here registered subparameters from foreach or not?
|
|
37
37
|
* TODO: [♈] Probbably move expectations from templates to parameters
|
|
38
38
|
* TODO: [🍙] Make some standard order of json properties
|
|
39
39
|
*/
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { ModelRequirements } from '../ModelRequirements';
|
|
2
|
-
import type {
|
|
2
|
+
import type { string_filename } from '../typeAliases';
|
|
3
3
|
import type { string_markdown_text } from '../typeAliases';
|
|
4
4
|
import type { string_pipeline_url } from '../typeAliases';
|
|
5
5
|
import type { string_semantic_version } from '../typeAliases';
|
|
@@ -33,7 +33,7 @@ export type PipelineJson = {
|
|
|
33
33
|
/**
|
|
34
34
|
* Internal helper for tracking the source `.ptbk.md` file of the pipeline
|
|
35
35
|
*/
|
|
36
|
-
readonly sourceFile?:
|
|
36
|
+
readonly sourceFile?: string_filename;
|
|
37
37
|
/**
|
|
38
38
|
* Title of the promptbook
|
|
39
39
|
* -It can use simple markdown formatting like **bold**, *italic*, [link](https://example.com), ... BUT not code blocks and structure
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { ReadonlyDeep } from 'type-fest';
|
|
1
2
|
import type { ExecutionReportJson } from './ExecutionReportJson';
|
|
2
3
|
import type { ExecutionReportString } from './ExecutionReportString';
|
|
3
4
|
import type { ExecutionReportStringOptions } from './ExecutionReportStringOptions';
|
|
@@ -6,7 +7,7 @@ import type { ExecutionReportStringOptions } from './ExecutionReportStringOption
|
|
|
6
7
|
*
|
|
7
8
|
* @public exported from `@promptbook/core`
|
|
8
9
|
*/
|
|
9
|
-
export declare function executionReportJsonToString(executionReportJson: ExecutionReportJson
|
|
10
|
+
export declare function executionReportJsonToString(executionReportJson: ReadonlyDeep<ExecutionReportJson>, options?: Partial<ExecutionReportStringOptions>): ExecutionReportString;
|
|
10
11
|
/**
|
|
11
12
|
* TODO: Add mermaid chart for every report
|
|
12
13
|
* TODO: [🧠] Allow to filter out some parts of the report by options
|
|
@@ -156,7 +156,7 @@ export type string_knowledge_source_content = string_knowledge_source_link | str
|
|
|
156
156
|
*
|
|
157
157
|
* @@@ string_knowledge_source vs string_knowledge_source_link
|
|
158
158
|
*/
|
|
159
|
-
export type string_knowledge_source_link = string_url |
|
|
159
|
+
export type string_knowledge_source_link = string_url | string_filename;
|
|
160
160
|
/**
|
|
161
161
|
* Semantic helper
|
|
162
162
|
*
|
|
@@ -386,6 +386,7 @@ export type string_emails = string;
|
|
|
386
386
|
* - `isValidUuid to check validity
|
|
387
387
|
*
|
|
388
388
|
* For example `"5a0a153d-7be9-4018-9eda-e0e2e2b89bd9"`
|
|
389
|
+
* TODO: [🥬] Make some system for hashes and ids of promptbook
|
|
389
390
|
*/
|
|
390
391
|
export type string_uuid = string & {
|
|
391
392
|
readonly _type: 'uuid';
|
|
@@ -423,37 +424,33 @@ export type string_file_extension = string;
|
|
|
423
424
|
*
|
|
424
425
|
* For example `"C:/Users/me/work/collboard/modules-sdk/src/colldev/commands/develop/ColldevDevelop.tsx"`
|
|
425
426
|
*/
|
|
426
|
-
export type
|
|
427
|
+
export type string_absolute_filename = string;
|
|
427
428
|
/**
|
|
428
429
|
* Semantic helper
|
|
429
430
|
*
|
|
430
431
|
* For example `"./src/colldev/commands/develop/ColldevDevelop.tsx"`
|
|
431
432
|
*/
|
|
432
|
-
export type
|
|
433
|
+
export type string_relative_filename = string;
|
|
433
434
|
/**
|
|
434
435
|
* Semantic helper
|
|
435
436
|
*/
|
|
436
|
-
export type
|
|
437
|
+
export type string_filename = string_absolute_filename | string_relative_filename;
|
|
437
438
|
/**
|
|
438
439
|
* Semantic helper
|
|
439
440
|
*
|
|
440
441
|
* For example `"C:/Users/me/work/collboard/modules-sdk/src/colldev/commands/develop/ColldevDevelop.tsx"`
|
|
441
442
|
*/
|
|
442
|
-
export type
|
|
443
|
+
export type string_absolute_dirname = string;
|
|
443
444
|
/**
|
|
444
445
|
* Semantic helper
|
|
445
446
|
*
|
|
446
447
|
* For example `"./src/colldev/commands/develop/ColldevDevelop.tsx"`
|
|
447
448
|
*/
|
|
448
|
-
export type
|
|
449
|
+
export type string_relative_dirname = string;
|
|
449
450
|
/**
|
|
450
451
|
* Semantic helper
|
|
451
452
|
*/
|
|
452
|
-
export type
|
|
453
|
-
/**
|
|
454
|
-
* Semantic helper
|
|
455
|
-
*/
|
|
456
|
-
export type string_filename = string;
|
|
453
|
+
export type string_dirname = string_absolute_dirname | string_relative_dirname;
|
|
457
454
|
/**
|
|
458
455
|
* Semantic helper
|
|
459
456
|
*
|
|
@@ -592,7 +589,7 @@ export type number_megabytes = number_positive;
|
|
|
592
589
|
export type number_gigabytes = number_positive;
|
|
593
590
|
export type number_terabytes = number_positive;
|
|
594
591
|
/**.
|
|
595
|
-
* TODO:[main] !!! Change "For example" to @example
|
|
592
|
+
* TODO: [main] !!! Change "For example" to @example
|
|
596
593
|
* TODO: !! Change to branded types
|
|
597
594
|
* TODO: Delete type aliases that are not exported or used internally
|
|
598
595
|
*/
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { IExecCommandOptions } from './IExecCommandOptions';
|
|
2
|
+
/**
|
|
3
|
+
* Run one command in a shell
|
|
4
|
+
*
|
|
5
|
+
* Note: There are 2 similar functions in the codebase:
|
|
6
|
+
* - `$execCommand` which runs a single command
|
|
7
|
+
* - `$execCommands` which runs multiple commands
|
|
8
|
+
*
|
|
9
|
+
* @public exported from `@promptbook/node`
|
|
10
|
+
*/
|
|
11
|
+
export declare function $execCommand(options: IExecCommandOptions): Promise<string>;
|
|
12
|
+
/**
|
|
13
|
+
* Note: [🟢 <- TODO: [🦖] !!!!!! Split scrapers into packages and enable] Code in this file should never be published outside of `@promptbook/node` and `@promptbook/cli`
|
|
14
|
+
*/
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Run multiple commands in a shell in sequence
|
|
3
|
+
*
|
|
4
|
+
* Note: There are 2 similar functions in the codebase:
|
|
5
|
+
* - `$execCommand` which runs a single command
|
|
6
|
+
* - `$execCommands` which runs multiple commands
|
|
7
|
+
*
|
|
8
|
+
* @public exported from `@promptbook/node`
|
|
9
|
+
*/
|
|
10
|
+
export declare function $execCommands({ commands, cwd, crashOnError, }: {
|
|
11
|
+
readonly commands: string[];
|
|
12
|
+
readonly cwd: string;
|
|
13
|
+
readonly crashOnError?: boolean;
|
|
14
|
+
}): Promise<void>;
|
|
15
|
+
/**
|
|
16
|
+
* Note: [🟢 <- TODO: [🦖] !!!!!! Split scrapers into packages and enable] Code in this file should never be published outside of `@promptbook/node` and `@promptbook/cli`
|
|
17
|
+
*/
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @deprecated Use sth from type-fest or move this to type helpers
|
|
3
|
+
*/
|
|
4
|
+
type RequiredAndOptional<TBase, TRequired extends keyof TBase, TOptional extends keyof TBase> = Pick<TBase, TRequired> & Partial<Pick<TBase, TOptional>>;
|
|
5
|
+
/**
|
|
6
|
+
* Simple options for `execCommand`
|
|
7
|
+
*
|
|
8
|
+
* TODO: !!!!!! Rename - remove I prefix
|
|
9
|
+
*/
|
|
10
|
+
export type IExecCommandOptions = string | RequiredAndOptional<IExecCommandOptionsAdvanced, 'command', 'args' | 'cwd' | 'crashOnError' | 'timeout'>;
|
|
11
|
+
/**
|
|
12
|
+
* Advanced options for `execCommand`
|
|
13
|
+
*
|
|
14
|
+
* TODO: !!!!!! Rename - remove I prefix
|
|
15
|
+
*/
|
|
16
|
+
export type IExecCommandOptionsAdvanced = {
|
|
17
|
+
readonly command: string;
|
|
18
|
+
readonly args: string[];
|
|
19
|
+
readonly cwd: string;
|
|
20
|
+
readonly crashOnError: boolean;
|
|
21
|
+
readonly timeout: number;
|
|
22
|
+
};
|
|
23
|
+
export {};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { IExecCommandOptions } from './IExecCommandOptions';
|
|
2
|
+
import type { IExecCommandOptionsAdvanced } from './IExecCommandOptions';
|
|
3
|
+
/**
|
|
4
|
+
* Normalize options for `execCommand` and `execCommands`
|
|
5
|
+
*
|
|
6
|
+
* @private internal utility of `execCommand` and `execCommands`
|
|
7
|
+
*/
|
|
8
|
+
export declare function execCommandNormalizeOptions(options: IExecCommandOptions): Pick<IExecCommandOptionsAdvanced, 'command' | 'args' | 'cwd' | 'crashOnError' | 'timeout'> & {
|
|
9
|
+
humanReadableCommand: string;
|
|
10
|
+
};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { string_dirname } from '../../types/typeAliases';
|
|
2
2
|
/**
|
|
3
3
|
* Checks if the directory exists
|
|
4
4
|
*
|
|
@@ -6,9 +6,9 @@ import type { string_folder_path } from '../../types/typeAliases';
|
|
|
6
6
|
*
|
|
7
7
|
* @private within the repository
|
|
8
8
|
*/
|
|
9
|
-
export declare function $isDirectoryExisting(directoryPath:
|
|
9
|
+
export declare function $isDirectoryExisting(directoryPath: string_dirname): Promise<boolean>;
|
|
10
10
|
/**
|
|
11
|
-
* Note: [🟢]
|
|
11
|
+
* Note: [🟢 <- TODO: [🦖] !!!!!! Split scrapers into packages and enable] Code in this file should never be published outside of `@promptbook/node` and `@promptbook/cli`
|
|
12
12
|
* TODO: [🐠] This can be a validator - with variants that return true/false and variants that throw errors with meaningless messages
|
|
13
13
|
* TODO: [🧠][📂] "directory" vs "folder"
|
|
14
14
|
* TODO: [🖇] What about symlinks?
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { string_filename } from '../../types/typeAliases';
|
|
2
2
|
/**
|
|
3
3
|
* Checks if the file exists
|
|
4
4
|
*
|
|
@@ -6,9 +6,9 @@ import type { string_file_path } from '../../types/typeAliases';
|
|
|
6
6
|
*
|
|
7
7
|
* @private within the repository
|
|
8
8
|
*/
|
|
9
|
-
export declare function $isFileExisting(
|
|
9
|
+
export declare function $isFileExisting(filename: string_filename): Promise<boolean>;
|
|
10
10
|
/**
|
|
11
|
-
* Note: [🟢]
|
|
11
|
+
* Note: [🟢 <- TODO: [🦖] !!!!!! Split scrapers into packages and enable] Code in this file should never be published outside of `@promptbook/node` and `@promptbook/cli`
|
|
12
12
|
* TODO: [🐠] This can be a validator - with variants that return true/false and variants that throw errors with meaningless messages
|
|
13
13
|
* TODO: [🖇] What about symlinks?
|
|
14
14
|
*/
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
import type {
|
|
1
|
+
import type { string_dirname } from '../../types/typeAliases';
|
|
2
|
+
import type { string_filename } from '../../types/typeAliases';
|
|
3
3
|
/**
|
|
4
4
|
* Reads all files in the directory
|
|
5
5
|
*
|
|
@@ -10,8 +10,8 @@ import type { string_folder_path } from '../../types/typeAliases';
|
|
|
10
10
|
* @returns List of all files in the directory
|
|
11
11
|
* @private internal function of `createCollectionFromDirectory`
|
|
12
12
|
*/
|
|
13
|
-
export declare function $listAllFiles(path:
|
|
13
|
+
export declare function $listAllFiles(path: string_dirname, isRecursive: boolean): Promise<Array<string_filename>>;
|
|
14
14
|
/**
|
|
15
|
-
* Note: [🟢]
|
|
15
|
+
* Note: [🟢 <- TODO: [🦖] !!!!!! Split scrapers into packages and enable] Code in this file should never be published outside of `@promptbook/node` and `@promptbook/cli`
|
|
16
16
|
* TODO: [🖇] What about symlinks?
|
|
17
17
|
*/
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { string_file_extension } from '../../types/typeAliases';
|
|
2
|
+
import type { string_mime_type } from '../../types/typeAliases';
|
|
3
|
+
/**
|
|
4
|
+
* Convert file extension to mime type
|
|
5
|
+
*
|
|
6
|
+
* @private within the repository
|
|
7
|
+
*/
|
|
8
|
+
export declare function extensionToMimeType(value: string_file_extension): string_mime_type;
|