@promptbook/remote-server 0.72.0-8 → 0.72.0-9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (118) hide show
  1. package/README.md +4 -0
  2. package/esm/index.es.js +111 -13
  3. package/esm/index.es.js.map +1 -1
  4. package/esm/typings/src/_packages/browser.index.d.ts +2 -0
  5. package/esm/typings/src/_packages/cli.index.d.ts +20 -0
  6. package/esm/typings/src/_packages/core.index.d.ts +20 -14
  7. package/esm/typings/src/_packages/documents.index.d.ts +8 -0
  8. package/esm/typings/src/_packages/legacy-documents.index.d.ts +8 -0
  9. package/esm/typings/src/_packages/markdown-utils.index.d.ts +6 -0
  10. package/esm/typings/src/_packages/node.index.d.ts +10 -4
  11. package/esm/typings/src/_packages/pdf.index.d.ts +8 -0
  12. package/esm/typings/src/_packages/types.index.d.ts +15 -5
  13. package/esm/typings/src/_packages/website-crawler.index.d.ts +8 -0
  14. package/esm/typings/src/collection/constructors/createCollectionFromDirectory.d.ts +5 -3
  15. package/esm/typings/src/config.d.ts +6 -0
  16. package/esm/typings/src/conversion/pipelineStringToJson.d.ts +3 -1
  17. package/esm/typings/src/dialogs/callback/CallbackInterfaceToolsOptions.d.ts +2 -2
  18. package/esm/typings/src/dialogs/simple-prompt/SimplePromptInterfaceTools.d.ts +3 -3
  19. package/esm/typings/src/execution/{CommonExecutionToolsOptions.d.ts → CommonToolsOptions.d.ts} +1 -1
  20. package/esm/typings/src/execution/ExecutionTools.d.ts +26 -6
  21. package/esm/typings/src/execution/FilesystemTools.d.ts +9 -0
  22. package/esm/typings/src/execution/createPipelineExecutor/10-executePipeline.d.ts +1 -1
  23. package/esm/typings/src/execution/createPipelineExecutor/20-executeTemplate.d.ts +1 -6
  24. package/esm/typings/src/execution/createPipelineExecutor/40-executeAttempts.d.ts +1 -6
  25. package/esm/typings/src/execution/translation/automatic-translate/translateMessages.d.ts +1 -0
  26. package/esm/typings/src/execution/utils/$provideExecutionToolsForNode.d.ts +13 -0
  27. package/esm/typings/src/llm-providers/_common/{$llmToolsMetadataRegister.d.ts → register/$llmToolsMetadataRegister.d.ts} +4 -1
  28. package/esm/typings/src/llm-providers/_common/{$llmToolsRegister.d.ts → register/$llmToolsRegister.d.ts} +5 -2
  29. package/esm/typings/src/llm-providers/_common/{createLlmToolsFromConfigurationFromEnv.d.ts → register/$provideLlmToolsConfigurationFromEnv.d.ts} +3 -3
  30. package/esm/typings/src/llm-providers/_common/{getLlmToolsForCli.d.ts → register/$provideLlmToolsForCli.d.ts} +4 -11
  31. package/esm/typings/src/llm-providers/_common/{getLlmToolsForTestingAndScriptsAndPlayground.d.ts → register/$provideLlmToolsForTestingAndScriptsAndPlayground.d.ts} +4 -3
  32. package/esm/typings/src/llm-providers/_common/{createLlmToolsFromEnv.d.ts → register/$provideLlmToolsFromEnv.d.ts} +6 -5
  33. package/esm/typings/src/llm-providers/_common/{$registeredLlmToolsMessage.d.ts → register/$registeredLlmToolsMessage.d.ts} +5 -2
  34. package/esm/typings/src/llm-providers/_common/{LlmToolsConfiguration.d.ts → register/LlmToolsConfiguration.d.ts} +5 -4
  35. package/esm/typings/src/llm-providers/_common/{LlmToolsMetadata.d.ts → register/LlmToolsMetadata.d.ts} +5 -4
  36. package/esm/typings/src/llm-providers/_common/{LlmToolsOptions.d.ts → register/LlmToolsOptions.d.ts} +4 -1
  37. package/esm/typings/src/llm-providers/_common/{createLlmToolsFromConfiguration.d.ts → register/createLlmToolsFromConfiguration.d.ts} +5 -4
  38. package/esm/typings/src/llm-providers/anthropic-claude/AnthropicClaudeExecutionToolsOptions.d.ts +3 -3
  39. package/esm/typings/src/llm-providers/anthropic-claude/register-configuration.d.ts +4 -3
  40. package/esm/typings/src/llm-providers/anthropic-claude/register-constructor.d.ts +4 -3
  41. package/esm/typings/src/llm-providers/azure-openai/AzureOpenAiExecutionToolsOptions.d.ts +2 -2
  42. package/esm/typings/src/llm-providers/azure-openai/register-configuration.d.ts +4 -3
  43. package/esm/typings/src/llm-providers/azure-openai/register-constructor.d.ts +4 -3
  44. package/esm/typings/src/llm-providers/mocked/MockedEchoLlmExecutionTools.d.ts +3 -3
  45. package/esm/typings/src/llm-providers/mocked/MockedFackedLlmExecutionTools.d.ts +3 -3
  46. package/esm/typings/src/llm-providers/multiple/MultipleLlmExecutionTools.d.ts +1 -0
  47. package/esm/typings/src/llm-providers/openai/OpenAiExecutionToolsOptions.d.ts +2 -2
  48. package/esm/typings/src/llm-providers/openai/register-configuration.d.ts +5 -4
  49. package/esm/typings/src/llm-providers/openai/register-constructor.d.ts +5 -4
  50. package/esm/typings/src/llm-providers/remote/interfaces/PromptbookServer_ListModels_Request.d.ts +1 -1
  51. package/esm/typings/src/llm-providers/remote/interfaces/PromptbookServer_Prompt_Request.d.ts +1 -1
  52. package/esm/typings/src/llm-providers/remote/interfaces/RemoteLlmExecutionToolsOptions.d.ts +3 -3
  53. package/esm/typings/src/llm-providers/remote/interfaces/RemoteServerOptions.d.ts +2 -2
  54. package/esm/typings/src/personas/preparePersona.d.ts +2 -1
  55. package/esm/typings/src/prepare/PrepareAndScrapeOptions.d.ts +8 -7
  56. package/esm/typings/src/prepare/preparePipeline.d.ts +2 -1
  57. package/esm/typings/src/prepare/prepareTemplates.d.ts +2 -1
  58. package/esm/typings/src/scrapers/_common/Converter.d.ts +4 -10
  59. package/esm/typings/src/scrapers/_common/Scraper.d.ts +4 -9
  60. package/esm/typings/src/scrapers/_common/prepareKnowledgePieces.d.ts +2 -1
  61. package/esm/typings/src/scrapers/_common/register/$provideFilesystemForNode.d.ts +11 -0
  62. package/esm/typings/src/scrapers/_common/register/$provideScrapersForBrowser.d.ts +12 -0
  63. package/esm/typings/src/scrapers/_common/register/$provideScrapersForNode.d.ts +15 -0
  64. package/esm/typings/src/scrapers/_common/register/$registeredScrapersMessage.d.ts +12 -0
  65. package/esm/typings/src/scrapers/_common/register/$scrapersMetadataRegister.d.ts +13 -0
  66. package/esm/typings/src/scrapers/_common/register/$scrapersRegister.d.ts +13 -0
  67. package/esm/typings/src/scrapers/_common/register/ScraperAndConverterMetadata.d.ts +41 -0
  68. package/esm/typings/src/scrapers/_common/register/ScraperConstructor.d.ts +12 -0
  69. package/esm/typings/src/scrapers/_common/utils/getScraperIntermediateSource.d.ts +1 -0
  70. package/esm/typings/src/scrapers/_common/utils/makeKnowledgeSourceHandler.d.ts +2 -1
  71. package/esm/typings/src/scrapers/document/{documentScraper.d.ts → DocumentScraper.d.ts} +18 -12
  72. package/esm/typings/src/scrapers/document/createDocumentScraper.d.ts +20 -0
  73. package/esm/typings/src/scrapers/document/register-constructor.d.ts +13 -0
  74. package/esm/typings/src/scrapers/document/register-metadata.d.ts +24 -0
  75. package/esm/typings/src/scrapers/document-legacy/{legacyDocumentScraper.d.ts → LegacyDocumentScraper.d.ts} +18 -12
  76. package/esm/typings/src/scrapers/document-legacy/createLegacyDocumentScraper.d.ts +20 -0
  77. package/esm/typings/src/scrapers/document-legacy/register-constructor.d.ts +13 -0
  78. package/esm/typings/src/scrapers/document-legacy/register-metadata.d.ts +24 -0
  79. package/esm/typings/src/scrapers/markdown/MarkdownScraper.d.ts +29 -0
  80. package/esm/typings/src/scrapers/markdown/createMarkdownScraper.d.ts +20 -0
  81. package/esm/typings/src/scrapers/markdown/register-constructor.d.ts +13 -0
  82. package/esm/typings/src/scrapers/markdown/register-metadata.d.ts +24 -0
  83. package/esm/typings/src/scrapers/pdf/PdfScraper.d.ts +40 -0
  84. package/esm/typings/src/scrapers/pdf/createPdfScraper.d.ts +20 -0
  85. package/esm/typings/src/scrapers/pdf/register-constructor.d.ts +13 -0
  86. package/esm/typings/src/scrapers/pdf/register-metadata.d.ts +24 -0
  87. package/esm/typings/src/scrapers/website/{websiteScraper.d.ts → WebsiteScraper.d.ts} +18 -14
  88. package/esm/typings/src/scrapers/website/createWebsiteScraper.d.ts +20 -0
  89. package/esm/typings/src/scrapers/website/register-constructor.d.ts +13 -0
  90. package/esm/typings/src/scrapers/website/register-metadata.d.ts +24 -0
  91. package/esm/typings/src/scripting/javascript/JavascriptExecutionToolsOptions.d.ts +2 -2
  92. package/esm/typings/src/scripting/python/PythonExecutionTools.d.ts +3 -3
  93. package/esm/typings/src/scripting/typescript/TypescriptExecutionTools.d.ts +3 -3
  94. package/esm/typings/src/storage/file-cache-storage/FileCacheStorage.d.ts +5 -3
  95. package/esm/typings/src/storage/{utils → memory/utils}/PrefixStorage.d.ts +1 -1
  96. package/esm/typings/src/storage/{utils → memory/utils}/makePromptbookStorageFromWebStorage.d.ts +1 -1
  97. package/esm/typings/src/types/typeAliases.d.ts +7 -0
  98. package/esm/typings/src/utils/$Register.d.ts +19 -6
  99. package/esm/typings/src/utils/execCommand/$execCommand.d.ts +1 -1
  100. package/esm/typings/src/utils/execCommand/$execCommands.d.ts +1 -1
  101. package/esm/typings/src/utils/files/isDirectoryExisting.d.ts +14 -0
  102. package/esm/typings/src/utils/files/isFileExisting.d.ts +13 -0
  103. package/esm/typings/src/utils/files/{$listAllFiles.d.ts → listAllFiles.d.ts} +3 -4
  104. package/package.json +2 -2
  105. package/umd/index.umd.js +111 -13
  106. package/umd/index.umd.js.map +1 -1
  107. package/esm/typings/src/scrapers/index.d.ts +0 -8
  108. package/esm/typings/src/scrapers/markdown/markdownScraper.d.ts +0 -29
  109. package/esm/typings/src/scrapers/pdf/pdfScraper.d.ts +0 -35
  110. package/esm/typings/src/utils/files/$isDirectoryExisting.d.ts +0 -15
  111. package/esm/typings/src/utils/files/$isFileExisting.d.ts +0 -14
  112. /package/esm/typings/src/scrapers/document/{documentScraper.test.d.ts → DocumentScraper.test.d.ts} +0 -0
  113. /package/esm/typings/src/scrapers/document-legacy/{legacyDocumentScraper.test.d.ts → LegacyDocumentScraper.test.d.ts} +0 -0
  114. /package/esm/typings/src/scrapers/markdown/{markdownScraper.test.d.ts → MarkdownScraper.test.d.ts} +0 -0
  115. /package/esm/typings/src/scrapers/website/{websiteScraper.test.d.ts → WebsiteScraper.test.d.ts} +0 -0
  116. /package/esm/typings/src/utils/files/{$isDirectoryExisting.test.d.ts → isDirectoryExisting.test.d.ts} +0 -0
  117. /package/esm/typings/src/utils/files/{$isFileExisting.test.d.ts → isFileExisting.test.d.ts} +0 -0
  118. /package/esm/typings/src/utils/files/{$listAllFiles.test.d.ts → listAllFiles.test.d.ts} +0 -0
@@ -0,0 +1,20 @@
1
+ import type { ExecutionTools } from '../../execution/ExecutionTools';
2
+ import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions';
3
+ import { DocumentScraper } from './DocumentScraper';
4
+ /**
5
+ * @@@
6
+ *
7
+ * @public exported from `@promptbook/documents`
8
+ */
9
+ export declare const createDocumentScraper: ((tools: Pick<ExecutionTools, 'llm'>, options: PrepareAndScrapeOptions) => DocumentScraper) & import("type-fest/source/readonly-deep").ReadonlyObjectDeep<{
10
+ title: string;
11
+ packageName: string;
12
+ className: string;
13
+ mimeTypes: string[];
14
+ documentationUrl: "https://github.com/webgptorg/promptbook/discussions/@@";
15
+ isAvilableInBrowser: false;
16
+ requiredExecutables: string[];
17
+ }>;
18
+ /**
19
+ * TODO: [🎶] Naming "constructor" vs "creator" vs "factory"
20
+ */
@@ -0,0 +1,13 @@
1
+ import type { Registration } from '../../utils/$Register';
2
+ /**
3
+ * Registration of known scraper
4
+ *
5
+ * Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available known scrapers
6
+ *
7
+ * @public exported from `@promptbook/documents`
8
+ * @public exported from `@promptbook/cli`
9
+ */
10
+ export declare const _DocumentScraperRegistration: Registration;
11
+ /**
12
+ * TODO: [🎶] Naming "constructor" vs "creator" vs "factory"
13
+ */
@@ -0,0 +1,24 @@
1
+ import type { Registration } from '../../utils/$Register';
2
+ /**
3
+ * Metadata of the scraper
4
+ *
5
+ * @private within the scraper directory
6
+ */
7
+ export declare const documentScraperMetadata: import("type-fest/source/readonly-deep").ReadonlyObjectDeep<{
8
+ title: string;
9
+ packageName: string;
10
+ className: string;
11
+ mimeTypes: string[];
12
+ documentationUrl: "https://github.com/webgptorg/promptbook/discussions/@@";
13
+ isAvilableInBrowser: false;
14
+ requiredExecutables: string[];
15
+ }>;
16
+ /**
17
+ * Registration of known scraper metadata
18
+ *
19
+ * Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available known scrapers
20
+ *
21
+ * @public exported from `@promptbook/core`
22
+ * @public exported from `@promptbook/cli`
23
+ */
24
+ export declare const _DocumentScraperMetadataRegistration: Registration;
@@ -1,37 +1,43 @@
1
- import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions';
2
1
  import type { KnowledgePiecePreparedJson } from '../../types/PipelineJson/KnowledgePieceJson';
2
+ import type { ExecutionTools } from '../../execution/ExecutionTools';
3
+ import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions';
4
+ import type { Converter } from '../_common/Converter';
5
+ import type { ScraperAndConverterMetadata } from '../_common/register/ScraperAndConverterMetadata';
6
+ import type { Scraper } from '../_common/Scraper';
3
7
  import type { ScraperSourceHandler } from '../_common/Scraper';
4
8
  import type { ScraperIntermediateSource } from '../_common/ScraperIntermediateSource';
5
9
  /**
6
10
  * Scraper for .docx files
7
11
  *
8
12
  * @see `documentationUrl` for more details
9
- * @public exported from `@promptbook/core`
13
+ * @public exported from `@promptbook/legacy-documents`
10
14
  */
11
- export declare const legacyDocumentScraper: {
15
+ export declare class LegacyDocumentScraper implements Converter, Scraper {
16
+ private readonly tools;
17
+ private readonly options;
12
18
  /**
13
- * Mime types that this scraper can handle
19
+ * Metadata of the scraper which includes title, mime types, etc.
14
20
  */
15
- mimeTypes: string[];
21
+ get metadata(): ScraperAndConverterMetadata;
16
22
  /**
17
- * Link to documentation
23
+ * Document scraper is used internally
18
24
  */
19
- documentationUrl: "https://github.com/webgptorg/promptbook/discussions/@@";
25
+ private readonly documentScraper;
26
+ constructor(tools: Pick<ExecutionTools, 'fs' | 'llm'>, options: PrepareAndScrapeOptions);
20
27
  /**
21
28
  * Convert the `.doc` or `.rtf` to `.doc` file and returns intermediate source
22
29
  *
23
30
  * Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object
24
31
  */
25
- $convert(source: ScraperSourceHandler, options: PrepareAndScrapeOptions): Promise<ScraperIntermediateSource>;
32
+ $convert(source: ScraperSourceHandler): Promise<ScraperIntermediateSource>;
26
33
  /**
27
34
  * Scrapes the `.doc` or `.rtf` file and returns the knowledge pieces or `null` if it can't scrape it
28
35
  */
29
- scrape(source: ScraperSourceHandler, options: PrepareAndScrapeOptions): Promise<Array<Omit<KnowledgePiecePreparedJson, 'sources' | 'preparationIds'>> | null>;
30
- };
36
+ scrape(source: ScraperSourceHandler): Promise<Array<Omit<KnowledgePiecePreparedJson, 'sources' | 'preparationIds'>> | null>;
37
+ }
31
38
  /**
32
39
  * TODO: [👣] Converted documents can act as cached items - there is no need to run conversion each time
33
- * TODO: [🦖] Make some system for putting scrapers to separete packages
34
40
  * TODO: [🪂] Do it in parallel 11:11
35
- * TODO: [🦷] Ideally use `as const satisfies Converter & Scraper` BUT this combination throws errors
36
41
  * Note: No need to aggregate usage here, it is done by intercepting the llmTools
42
+ * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
37
43
  */
@@ -0,0 +1,20 @@
1
+ import type { ExecutionTools } from '../../execution/ExecutionTools';
2
+ import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions';
3
+ import { LegacyDocumentScraper } from './LegacyDocumentScraper';
4
+ /**
5
+ * @@@
6
+ *
7
+ * @public exported from `@promptbook/legacy-documents`
8
+ */
9
+ export declare const createLegacyDocumentScraper: ((tools: Pick<ExecutionTools, 'llm'>, options: PrepareAndScrapeOptions) => LegacyDocumentScraper) & import("type-fest/source/readonly-deep").ReadonlyObjectDeep<{
10
+ title: string;
11
+ packageName: string;
12
+ className: string;
13
+ mimeTypes: string[];
14
+ documentationUrl: "https://github.com/webgptorg/promptbook/discussions/@@";
15
+ isAvilableInBrowser: false;
16
+ requiredExecutables: string[];
17
+ }>;
18
+ /**
19
+ * TODO: [🎶] Naming "constructor" vs "creator" vs "factory"
20
+ */
@@ -0,0 +1,13 @@
1
+ import type { Registration } from '../../utils/$Register';
2
+ /**
3
+ * Registration of known scraper
4
+ *
5
+ * Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available known scrapers
6
+ *
7
+ * @public exported from `@promptbook/legacy-documents`
8
+ * @public exported from `@promptbook/cli`
9
+ */
10
+ export declare const _LegacyDocumentScraperRegistration: Registration;
11
+ /**
12
+ * TODO: [🎶] Naming "constructor" vs "creator" vs "factory"
13
+ */
@@ -0,0 +1,24 @@
1
+ import type { Registration } from '../../utils/$Register';
2
+ /**
3
+ * Metadata of the scraper
4
+ *
5
+ * @private within the scraper directory
6
+ */
7
+ export declare const legacyDocumentScraperMetadata: import("type-fest/source/readonly-deep").ReadonlyObjectDeep<{
8
+ title: string;
9
+ packageName: string;
10
+ className: string;
11
+ mimeTypes: string[];
12
+ documentationUrl: "https://github.com/webgptorg/promptbook/discussions/@@";
13
+ isAvilableInBrowser: false;
14
+ requiredExecutables: string[];
15
+ }>;
16
+ /**
17
+ * Registration of known scraper metadata
18
+ *
19
+ * Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available known scrapers
20
+ *
21
+ * @public exported from `@promptbook/core`
22
+ * @public exported from `@promptbook/cli`
23
+ */
24
+ export declare const _LegacyDocumentScraperMetadataRegistration: Registration;
@@ -0,0 +1,29 @@
1
+ import type { KnowledgePiecePreparedJson } from '../../types/PipelineJson/KnowledgePieceJson';
2
+ import type { Scraper } from '../_common/Scraper';
3
+ import type { ScraperSourceHandler } from '../_common/Scraper';
4
+ import type { ExecutionTools } from '../../execution/ExecutionTools';
5
+ import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions';
6
+ import type { ScraperAndConverterMetadata } from '../_common/register/ScraperAndConverterMetadata';
7
+ /**
8
+ * Scraper for markdown files
9
+ *
10
+ * @see `documentationUrl` for more details
11
+ * @public exported from `@promptbook/markdown-utils`
12
+ */
13
+ export declare class MarkdownScraper implements Scraper {
14
+ private readonly tools;
15
+ private readonly options;
16
+ /**
17
+ * Metadata of the scraper which includes title, mime types, etc.
18
+ */
19
+ get metadata(): ScraperAndConverterMetadata;
20
+ constructor(tools: Pick<ExecutionTools, 'llm'>, options: PrepareAndScrapeOptions);
21
+ /**
22
+ * Scrapes the markdown file and returns the knowledge pieces or `null` if it can't scrape it
23
+ */
24
+ scrape(source: ScraperSourceHandler): Promise<Array<Omit<KnowledgePiecePreparedJson, 'sources' | 'preparationIds'>> | null>;
25
+ }
26
+ /**
27
+ * TODO: [🪂] Do it in parallel 11:11
28
+ * Note: No need to aggregate usage here, it is done by intercepting the llmTools
29
+ */
@@ -0,0 +1,20 @@
1
+ import type { ExecutionTools } from '../../execution/ExecutionTools';
2
+ import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions';
3
+ import { MarkdownScraper } from './MarkdownScraper';
4
+ /**
5
+ * @@@
6
+ *
7
+ * @public exported from `@promptbook/markdown-utils`
8
+ */
9
+ export declare const createMarkdownScraper: ((tools: Pick<ExecutionTools, 'llm'>, options: PrepareAndScrapeOptions) => MarkdownScraper) & import("type-fest/source/readonly-deep").ReadonlyObjectDeep<{
10
+ title: string;
11
+ packageName: string;
12
+ className: string;
13
+ mimeTypes: string[];
14
+ documentationUrl: "https://github.com/webgptorg/promptbook/discussions/@@";
15
+ isAvilableInBrowser: true;
16
+ requiredExecutables: string[];
17
+ }>;
18
+ /**
19
+ * TODO: [🎶] Naming "constructor" vs "creator" vs "factory"
20
+ */
@@ -0,0 +1,13 @@
1
+ import type { Registration } from '../../utils/$Register';
2
+ /**
3
+ * Registration of known scraper
4
+ *
5
+ * Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available known scrapers
6
+ *
7
+ * @public exported from `@promptbook/markdown-utils`
8
+ * @public exported from `@promptbook/cli`
9
+ */
10
+ export declare const _MarkdownScraperRegistration: Registration;
11
+ /**
12
+ * TODO: [🎶] Naming "constructor" vs "creator" vs "factory"
13
+ */
@@ -0,0 +1,24 @@
1
+ import type { Registration } from '../../utils/$Register';
2
+ /**
3
+ * Metadata of the scraper
4
+ *
5
+ * @private within the scraper directory
6
+ */
7
+ export declare const markdownScraperMetadata: import("type-fest/source/readonly-deep").ReadonlyObjectDeep<{
8
+ title: string;
9
+ packageName: string;
10
+ className: string;
11
+ mimeTypes: string[];
12
+ documentationUrl: "https://github.com/webgptorg/promptbook/discussions/@@";
13
+ isAvilableInBrowser: true;
14
+ requiredExecutables: string[];
15
+ }>;
16
+ /**
17
+ * Registration of known scraper metadata
18
+ *
19
+ * Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available known scrapers
20
+ *
21
+ * @public exported from `@promptbook/core`
22
+ * @public exported from `@promptbook/cli`
23
+ */
24
+ export declare const _MarkdownScraperMetadataRegistration: Registration;
@@ -0,0 +1,40 @@
1
+ import type { KnowledgePiecePreparedJson } from '../../types/PipelineJson/KnowledgePieceJson';
2
+ import type { Scraper } from '../_common/Scraper';
3
+ import type { ScraperSourceHandler } from '../_common/Scraper';
4
+ import type { ExecutionTools } from '../../execution/ExecutionTools';
5
+ import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions';
6
+ import type { Converter } from '../_common/Converter';
7
+ import type { ScraperAndConverterMetadata } from '../_common/register/ScraperAndConverterMetadata';
8
+ import type { ScraperIntermediateSource } from '../_common/ScraperIntermediateSource';
9
+ /**
10
+ * Scraper for .docx files
11
+ *
12
+ * @see `documentationUrl` for more details
13
+ * @public exported from `@promptbook/pdf`
14
+ */
15
+ export declare class PdfScraper implements Converter, Scraper {
16
+ private readonly tools;
17
+ private readonly options;
18
+ /**
19
+ * Metadata of the scraper which includes title, mime types, etc.
20
+ */
21
+ get metadata(): ScraperAndConverterMetadata;
22
+ /**
23
+ * Markdown scraper is used internally
24
+ */
25
+ private readonly markdownScraper;
26
+ constructor(tools: Pick<ExecutionTools, 'llm'>, options: PrepareAndScrapeOptions);
27
+ /**
28
+ * Converts the `.pdf` file to `.md` file and returns intermediate source
29
+ */
30
+ $convert(source: ScraperSourceHandler): Promise<ScraperIntermediateSource>;
31
+ /**
32
+ * Scrapes the `.pdf` file and returns the knowledge pieces or `null` if it can't scrape it
33
+ */
34
+ scrape(source: ScraperSourceHandler): Promise<Array<Omit<KnowledgePiecePreparedJson, 'sources' | 'preparationIds'>> | null>;
35
+ }
36
+ /**
37
+ * TODO: [👣] Converted pdf documents can act as cached items - there is no need to run conversion each time
38
+ * TODO: [🪂] Do it in parallel 11:11
39
+ * Note: No need to aggregate usage here, it is done by intercepting the llmTools
40
+ */
@@ -0,0 +1,20 @@
1
+ import type { ExecutionTools } from '../../execution/ExecutionTools';
2
+ import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions';
3
+ import { PdfScraper } from './PdfScraper';
4
+ /**
5
+ * @@@
6
+ *
7
+ * @public exported from `@promptbook/pdf`
8
+ */
9
+ export declare const createPdfScraper: ((tools: Pick<ExecutionTools, 'llm'>, options: PrepareAndScrapeOptions) => PdfScraper) & import("type-fest/source/readonly-deep").ReadonlyObjectDeep<{
10
+ title: string;
11
+ packageName: string;
12
+ className: string;
13
+ mimeTypes: string[];
14
+ documentationUrl: "https://github.com/webgptorg/promptbook/discussions/@@";
15
+ isAvilableInBrowser: true;
16
+ requiredExecutables: string[];
17
+ }>;
18
+ /**
19
+ * TODO: [🎶] Naming "constructor" vs "creator" vs "factory"
20
+ */
@@ -0,0 +1,13 @@
1
+ import type { Registration } from '../../utils/$Register';
2
+ /**
3
+ * Registration of known scraper
4
+ *
5
+ * Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available known scrapers
6
+ *
7
+ * @public exported from `@promptbook/pdf`
8
+ * @public exported from `@promptbook/cli`
9
+ */
10
+ export declare const _PdfScraperRegistration: Registration;
11
+ /**
12
+ * TODO: [🎶] Naming "constructor" vs "creator" vs "factory"
13
+ */
@@ -0,0 +1,24 @@
1
+ import type { Registration } from '../../utils/$Register';
2
+ /**
3
+ * Metadata of the scraper
4
+ *
5
+ * @private within the scraper directory
6
+ */
7
+ export declare const pdfScraperMetadata: import("type-fest/source/readonly-deep").ReadonlyObjectDeep<{
8
+ title: string;
9
+ packageName: string;
10
+ className: string;
11
+ mimeTypes: string[];
12
+ documentationUrl: "https://github.com/webgptorg/promptbook/discussions/@@";
13
+ isAvilableInBrowser: true;
14
+ requiredExecutables: string[];
15
+ }>;
16
+ /**
17
+ * Registration of known scraper metadata
18
+ *
19
+ * Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available known scrapers
20
+ *
21
+ * @public exported from `@promptbook/core`
22
+ * @public exported from `@promptbook/cli`
23
+ */
24
+ export declare const _PdfScraperMetadataRegistration: Registration;
@@ -1,43 +1,47 @@
1
- import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions';
2
1
  import type { KnowledgePiecePreparedJson } from '../../types/PipelineJson/KnowledgePieceJson';
3
2
  import type { string_markdown } from '../../types/typeAliases';
3
+ import type { Converter } from '../_common/Converter';
4
+ import type { Scraper } from '../_common/Scraper';
4
5
  import type { ScraperSourceHandler } from '../_common/Scraper';
6
+ import type { ExecutionTools } from '../../execution/ExecutionTools';
7
+ import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions';
8
+ import type { ScraperAndConverterMetadata } from '../_common/register/ScraperAndConverterMetadata';
5
9
  import type { ScraperIntermediateSource } from '../_common/ScraperIntermediateSource';
6
10
  /**
7
11
  * Scraper for .docx files
8
12
  *
9
13
  * @see `documentationUrl` for more details
10
- * @public exported from `@promptbook/core`
14
+ * @public exported from `@promptbook/website-crawler`
11
15
  */
12
- export declare const websiteScraper: {
16
+ export declare class WebsiteScraper implements Converter, Scraper {
17
+ private readonly tools;
18
+ private readonly options;
13
19
  /**
14
- * Mime types that this scraper can handle
20
+ * Metadata of the scraper which includes title, mime types, etc.
15
21
  */
16
- mimeTypes: string[];
22
+ get metadata(): ScraperAndConverterMetadata;
17
23
  /**
18
- * Link to documentation
24
+ * Markdown scraper is used internally
19
25
  */
20
- documentationUrl: "https://github.com/webgptorg/promptbook/discussions/@@";
26
+ private readonly markdownScraper;
27
+ constructor(tools: Pick<ExecutionTools, 'fs' | 'llm'>, options: PrepareAndScrapeOptions);
21
28
  /**
22
29
  * Convert the website to `.md` file and returns intermediate source
23
30
  *
24
31
  * Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object
25
32
  */
26
- $convert(source: ScraperSourceHandler, options: PrepareAndScrapeOptions): Promise<import("destroyable").IDestroyable & {
27
- readonly filename: string;
28
- } & {
33
+ $convert(source: ScraperSourceHandler): Promise<ScraperIntermediateSource & {
29
34
  markdown: string_markdown;
30
35
  }>;
31
36
  /**
32
37
  * Scrapes the website and returns the knowledge pieces or `null` if it can't scrape it
33
38
  */
34
- scrape(source: ScraperSourceHandler, options: PrepareAndScrapeOptions): Promise<Array<Omit<KnowledgePiecePreparedJson, 'sources' | 'preparationIds'>> | null>;
35
- };
39
+ scrape(source: ScraperSourceHandler): Promise<Array<Omit<KnowledgePiecePreparedJson, 'sources' | 'preparationIds'>> | null>;
40
+ }
36
41
  /**
37
42
  * TODO: !!!!!! Put into separate package
38
43
  * TODO: [👣] Scraped website in .md can act as cache item - there is no need to run conversion each time
39
- * TODO: [🦖] Make some system for putting scrapers to separete packages
40
44
  * TODO: [🪂] Do it in parallel 11:11
41
- * TODO: [🦷] Ideally use `as const satisfies Converter & Scraper` BUT this combination throws errors
42
45
  * Note: No need to aggregate usage here, it is done by intercepting the llmTools
46
+ * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
43
47
  */
@@ -0,0 +1,20 @@
1
+ import type { ExecutionTools } from '../../execution/ExecutionTools';
2
+ import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions';
3
+ import { WebsiteScraper } from './WebsiteScraper';
4
+ /**
5
+ * @@@
6
+ *
7
+ * @public exported from `@promptbook/website-crawler`
8
+ */
9
+ export declare const createWebsiteScraper: ((tools: Pick<ExecutionTools, 'llm'>, options: PrepareAndScrapeOptions) => WebsiteScraper) & import("type-fest/source/readonly-deep").ReadonlyObjectDeep<{
10
+ title: string;
11
+ packageName: string;
12
+ className: string;
13
+ mimeTypes: string[];
14
+ documentationUrl: "https://github.com/webgptorg/promptbook/discussions/@@";
15
+ isAvilableInBrowser: false;
16
+ requiredExecutables: string[];
17
+ }>;
18
+ /**
19
+ * TODO: [🎶] Naming "constructor" vs "creator" vs "factory"
20
+ */
@@ -0,0 +1,13 @@
1
+ import type { Registration } from '../../utils/$Register';
2
+ /**
3
+ * Registration of known scraper
4
+ *
5
+ * Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available known scrapers
6
+ *
7
+ * @public exported from `@promptbook/website-crawler`
8
+ * @public exported from `@promptbook/cli`
9
+ */
10
+ export declare const _WebsiteScraperRegistration: Registration;
11
+ /**
12
+ * TODO: [🎶] Naming "constructor" vs "creator" vs "factory"
13
+ */
@@ -0,0 +1,24 @@
1
+ import type { Registration } from '../../utils/$Register';
2
+ /**
3
+ * Metadata of the scraper
4
+ *
5
+ * @private within the scraper directory
6
+ */
7
+ export declare const websiteScraperMetadata: import("type-fest/source/readonly-deep").ReadonlyObjectDeep<{
8
+ title: string;
9
+ packageName: string;
10
+ className: string;
11
+ mimeTypes: string[];
12
+ documentationUrl: "https://github.com/webgptorg/promptbook/discussions/@@";
13
+ isAvilableInBrowser: false;
14
+ requiredExecutables: string[];
15
+ }>;
16
+ /**
17
+ * Registration of known scraper metadata
18
+ *
19
+ * Warning: This is not useful for the end user, it is just a side effect of the mechanism that handles all available known scrapers
20
+ *
21
+ * @public exported from `@promptbook/core`
22
+ * @public exported from `@promptbook/cli`
23
+ */
24
+ export declare const _WebsiteScraperMetadataRegistration: Registration;
@@ -1,10 +1,10 @@
1
1
  import type { Promisable } from 'type-fest';
2
- import type { CommonExecutionToolsOptions } from '../../execution/CommonExecutionToolsOptions';
2
+ import type { CommonToolsOptions } from '../../execution/CommonToolsOptions';
3
3
  import type { string_postprocessing_function_name } from '../../types/typeAliases';
4
4
  /**
5
5
  * Options for `JavascriptExecutionTools`
6
6
  */
7
- export type JavascriptExecutionToolsOptions = CommonExecutionToolsOptions & {
7
+ export type JavascriptExecutionToolsOptions = CommonToolsOptions & {
8
8
  /**
9
9
  * Functions to be executed in the JavaScript evaluation.
10
10
  *
@@ -1,4 +1,4 @@
1
- import type { CommonExecutionToolsOptions } from '../../execution/CommonExecutionToolsOptions';
1
+ import type { CommonToolsOptions } from '../../execution/CommonToolsOptions';
2
2
  import type { ScriptExecutionTools } from '../../execution/ScriptExecutionTools';
3
3
  import type { ScriptExecutionToolsExecuteOptions } from '../../execution/ScriptExecutionTools';
4
4
  /**
@@ -9,8 +9,8 @@ import type { ScriptExecutionToolsExecuteOptions } from '../../execution/ScriptE
9
9
  * @private still in development
10
10
  */
11
11
  export declare class PythonExecutionTools implements ScriptExecutionTools {
12
- protected readonly options: CommonExecutionToolsOptions;
13
- constructor(options?: CommonExecutionToolsOptions);
12
+ protected readonly options: CommonToolsOptions;
13
+ constructor(options?: CommonToolsOptions);
14
14
  /**
15
15
  * Executes a Python
16
16
  */
@@ -1,4 +1,4 @@
1
- import type { CommonExecutionToolsOptions } from '../../execution/CommonExecutionToolsOptions';
1
+ import type { CommonToolsOptions } from '../../execution/CommonToolsOptions';
2
2
  import type { ScriptExecutionTools } from '../../execution/ScriptExecutionTools';
3
3
  import type { ScriptExecutionToolsExecuteOptions } from '../../execution/ScriptExecutionTools';
4
4
  /**
@@ -9,8 +9,8 @@ import type { ScriptExecutionToolsExecuteOptions } from '../../execution/ScriptE
9
9
  * @private still in development
10
10
  */
11
11
  export declare class TypescriptExecutionTools implements ScriptExecutionTools {
12
- protected readonly options: CommonExecutionToolsOptions;
13
- constructor(options?: CommonExecutionToolsOptions);
12
+ protected readonly options: CommonToolsOptions;
13
+ constructor(options?: CommonToolsOptions);
14
14
  /**
15
15
  * Executes a TypeScript
16
16
  */
@@ -1,3 +1,4 @@
1
+ import type { ExecutionTools } from '../../execution/ExecutionTools';
1
2
  import type { PromptbookStorage } from '../_common/PromptbookStorage';
2
3
  import type { FileCacheStorageOptions } from './FileCacheStorageOptions';
3
4
  /**
@@ -6,8 +7,9 @@ import type { FileCacheStorageOptions } from './FileCacheStorageOptions';
6
7
  * @public exported from `@promptbook/node`
7
8
  */
8
9
  export declare class FileCacheStorage<TItem> implements PromptbookStorage<TItem> {
9
- protected readonly options: FileCacheStorageOptions;
10
- constructor(options: FileCacheStorageOptions);
10
+ protected readonly tools: Required<Pick<ExecutionTools, 'fs'>>;
11
+ private readonly options;
12
+ constructor(tools: Required<Pick<ExecutionTools, 'fs'>>, options: FileCacheStorageOptions);
11
13
  /**
12
14
  * @@@
13
15
  */
@@ -27,5 +29,5 @@ export declare class FileCacheStorage<TItem> implements PromptbookStorage<TItem>
27
29
  }
28
30
  /**
29
31
  * TODO: [🌗] Maybe some checkers, not all valid JSONs are desired and valid values
30
- * Note: [🟢] Code in this file should never be published outside of `@promptbook/node` and `@promptbook/cli`
32
+ * Note: [🟢] Code in this file should never be never released in packages that could be imported into browser environment
31
33
  */
@@ -1,5 +1,5 @@
1
1
  import type { Promisable } from 'type-fest';
2
- import type { PromptbookStorage } from '../_common/PromptbookStorage';
2
+ import type { PromptbookStorage } from '../../_common/PromptbookStorage';
3
3
  /**
4
4
  * This class behaves like LocalStorage but separates keys by prefix
5
5
  *
@@ -1,4 +1,4 @@
1
- import type { PromptbookStorage } from '../_common/PromptbookStorage';
1
+ import type { PromptbookStorage } from '../../_common/PromptbookStorage';
2
2
  /**
3
3
  * @@@
4
4
  *
@@ -331,6 +331,13 @@ export type string_href = string;
331
331
  * For example `"https://collboard.com/9SeSQTupmQHwuSrLi.png?width=1200&height=630"`
332
332
  */
333
333
  export type string_url_image = string;
334
+ /**
335
+ * Semantic helper
336
+ *
337
+ * For example `"C:/Users/me/AppData/Local/Pandoc/pandoc.exe"`
338
+ * For example `"C:/Program Files/LibreOffice/program/swriter.exe"`
339
+ */
340
+ export type string_executable_path = string;
334
341
  /**
335
342
  * Semantic helper
336
343
  *
@@ -1,14 +1,27 @@
1
- import type { string_javascript_name } from '../types/typeAliases';
1
+ import { type IDestroyable } from 'destroyable';
2
+ import type { string_name } from '../types/typeAliases';
2
3
  import type { TODO_string } from './organization/TODO_string';
4
+ /**
5
+ * @@@
6
+ */
3
7
  export type Registered = {
4
8
  /**
5
9
  * @@@
6
10
  */
7
- packageName: TODO_string;
11
+ readonly packageName: TODO_string;
12
+ /**
13
+ * @@@
14
+ */
15
+ readonly className: TODO_string;
16
+ };
17
+ /**
18
+ * @@@
19
+ */
20
+ export type Registration = Registered & IDestroyable & {
8
21
  /**
9
22
  * @@@
10
23
  */
11
- className: TODO_string;
24
+ readonly registerName: string_name;
12
25
  };
13
26
  /**
14
27
  * Register is @@@
@@ -18,9 +31,9 @@ export type Registered = {
18
31
  * @private internal utility, exported are only signleton instances of this class
19
32
  */
20
33
  export declare class $Register<TRegistered extends Registered> {
21
- private readonly storageName;
34
+ private readonly registerName;
22
35
  private readonly storage;
23
- constructor(storageName: string_javascript_name);
36
+ constructor(registerName: string_name);
24
37
  list(): Array<TRegistered>;
25
- register(registered: TRegistered): void;
38
+ register(registered: TRegistered): Registration;
26
39
  }