@promptbook/remote-server 0.72.0-6 → 0.72.0-8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. package/README.md +6 -0
  2. package/esm/index.es.js +1 -1
  3. package/esm/index.es.js.map +1 -1
  4. package/esm/typings/src/_packages/browser.index.d.ts +1 -1
  5. package/esm/typings/src/_packages/core.index.d.ts +22 -10
  6. package/esm/typings/src/_packages/node.index.d.ts +6 -2
  7. package/esm/typings/src/_packages/types.index.d.ts +28 -20
  8. package/esm/typings/src/cli/cli-commands/about.d.ts +1 -1
  9. package/esm/typings/src/cli/cli-commands/hello.d.ts +2 -1
  10. package/esm/typings/src/cli/cli-commands/make.d.ts +1 -1
  11. package/esm/typings/src/cli/cli-commands/prettify.d.ts +2 -1
  12. package/esm/typings/src/cli/cli-commands/test-command.d.ts +13 -0
  13. package/esm/typings/src/cli/main.d.ts +1 -1
  14. package/esm/typings/src/cli/promptbookCli.d.ts +1 -1
  15. package/esm/typings/src/collection/constructors/createCollectionFromDirectory.d.ts +8 -5
  16. package/esm/typings/src/commands/EXPECT/expectCommandParser.d.ts +1 -1
  17. package/esm/typings/src/commands/FOREACH/foreachCommandParser.d.ts +1 -1
  18. package/esm/typings/src/commands/FORMAT/formatCommandParser.d.ts +1 -1
  19. package/esm/typings/src/commands/JOKER/jokerCommandParser.d.ts +1 -1
  20. package/esm/typings/src/commands/KNOWLEDGE/knowledgeCommandParser.d.ts +1 -1
  21. package/esm/typings/src/commands/KNOWLEDGE/utils/sourceContentToName.d.ts +11 -0
  22. package/esm/typings/src/commands/KNOWLEDGE/utils/sourceContentToName.test.d.ts +4 -0
  23. package/esm/typings/src/commands/MODEL/modelCommandParser.d.ts +1 -1
  24. package/esm/typings/src/commands/PARAMETER/parameterCommandParser.d.ts +1 -1
  25. package/esm/typings/src/commands/PERSONA/personaCommandParser.d.ts +1 -1
  26. package/esm/typings/src/commands/POSTPROCESS/postprocessCommandParser.d.ts +1 -1
  27. package/esm/typings/src/commands/PROMPTBOOK_VERSION/promptbookVersionCommandParser.d.ts +1 -1
  28. package/esm/typings/src/commands/TEMPLATE/templateCommandParser.d.ts +1 -1
  29. package/esm/typings/src/commands/URL/urlCommandParser.d.ts +1 -1
  30. package/esm/typings/src/commands/X_ACTION/actionCommandParser.d.ts +1 -1
  31. package/esm/typings/src/commands/X_INSTRUMENT/instrumentCommandParser.d.ts +1 -1
  32. package/esm/typings/src/commands/_BOILERPLATE/boilerplateCommandParser.d.ts +1 -1
  33. package/esm/typings/src/config.d.ts +10 -0
  34. package/esm/typings/src/conversion/pipelineStringToJson.d.ts +2 -15
  35. package/esm/typings/src/conversion/validation/_importPipeline.d.ts +1 -1
  36. package/esm/typings/src/conversion/validation/validatePipeline.d.ts +5 -5
  37. package/esm/typings/src/{knowledge/dialogs → dialogs}/callback/CallbackInterfaceTools.d.ts +2 -2
  38. package/esm/typings/src/{knowledge/dialogs → dialogs}/callback/CallbackInterfaceToolsOptions.d.ts +2 -2
  39. package/esm/typings/src/{knowledge/dialogs → dialogs}/simple-prompt/SimplePromptInterfaceTools.d.ts +4 -4
  40. package/esm/typings/src/errors/KnowledgeScrapeError.d.ts +9 -0
  41. package/esm/typings/src/errors/MissingToolsError.d.ts +9 -0
  42. package/esm/typings/src/execution/ExecutionTools.d.ts +3 -3
  43. package/esm/typings/src/execution/createPipelineExecutor/00-CreatePipelineExecutorOptions.d.ts +5 -2
  44. package/esm/typings/src/execution/createPipelineExecutor/00-CreatePipelineExecutorSettings.d.ts +2 -13
  45. package/esm/typings/src/execution/createPipelineExecutor/00-createPipelineExecutor.d.ts +1 -1
  46. package/esm/typings/src/execution/createPipelineExecutor/10-executePipeline.d.ts +1 -1
  47. package/esm/typings/src/execution/translation/automatic-translate/translateMessages.d.ts +3 -0
  48. package/esm/typings/src/execution/utils/forEachAsync.d.ts +1 -0
  49. package/esm/typings/src/llm-providers/_common/createLlmToolsFromConfigurationFromEnv.d.ts +1 -1
  50. package/esm/typings/src/llm-providers/_common/createLlmToolsFromEnv.d.ts +1 -1
  51. package/esm/typings/src/llm-providers/_common/getLlmToolsForCli.d.ts +1 -1
  52. package/esm/typings/src/llm-providers/anthropic-claude/playground/playground.d.ts +1 -0
  53. package/esm/typings/src/llm-providers/azure-openai/AzureOpenAiExecutionTools.d.ts +6 -0
  54. package/esm/typings/src/llm-providers/azure-openai/playground/playground.d.ts +1 -0
  55. package/esm/typings/src/llm-providers/langtail/playground/playground.d.ts +3 -0
  56. package/esm/typings/src/llm-providers/multiple/playground/playground.d.ts +3 -0
  57. package/esm/typings/src/llm-providers/openai/playground/playground.d.ts +1 -0
  58. package/esm/typings/src/llm-providers/remote/playground/playground.d.ts +3 -0
  59. package/esm/typings/src/personas/preparePersona.d.ts +2 -2
  60. package/esm/typings/src/prepare/PrepareAndScrapeOptions.d.ts +67 -0
  61. package/esm/typings/src/prepare/preparePipeline.d.ts +2 -2
  62. package/esm/typings/src/prepare/prepareTemplates.d.ts +2 -2
  63. package/esm/typings/src/scrapers/_common/Converter.d.ts +28 -0
  64. package/esm/typings/src/scrapers/_common/Scraper.d.ts +71 -0
  65. package/esm/typings/src/scrapers/_common/ScraperIntermediateSource.d.ts +11 -0
  66. package/esm/typings/src/{knowledge/prepare-knowledge → scrapers}/_common/prepareKnowledgePieces.d.ts +4 -4
  67. package/esm/typings/src/scrapers/_common/utils/getScraperIntermediateSource.d.ts +33 -0
  68. package/esm/typings/src/scrapers/_common/utils/getScraperIntermediateSource.test.d.ts +4 -0
  69. package/esm/typings/src/scrapers/_common/utils/makeKnowledgeSourceHandler.d.ts +10 -0
  70. package/esm/typings/src/scrapers/document/documentScraper.d.ts +37 -0
  71. package/esm/typings/src/scrapers/document/documentScraper.test.d.ts +4 -0
  72. package/esm/typings/src/scrapers/document/playground/document-scraper-playground.d.ts +5 -0
  73. package/esm/typings/src/scrapers/document-legacy/legacyDocumentScraper.d.ts +37 -0
  74. package/esm/typings/src/scrapers/document-legacy/legacyDocumentScraper.test.d.ts +4 -0
  75. package/esm/typings/src/scrapers/document-legacy/playground/legacy-document-scraper-playground.d.ts +5 -0
  76. package/esm/typings/src/scrapers/index.d.ts +8 -0
  77. package/esm/typings/src/scrapers/markdown/markdownScraper.d.ts +29 -0
  78. package/esm/typings/src/scrapers/markdown/playground/markdown-scraper-playground.d.ts +5 -0
  79. package/esm/typings/src/scrapers/pdf/pdfScraper.d.ts +35 -0
  80. package/esm/typings/src/scrapers/pdf/playground/pdf-scraper-playground.d.ts +5 -0
  81. package/esm/typings/src/scrapers/website/playground/website-scraper-playground.d.ts +5 -0
  82. package/esm/typings/src/scrapers/website/utils/markdownConverter.d.ts +12 -0
  83. package/esm/typings/src/scrapers/website/websiteScraper.d.ts +43 -0
  84. package/esm/typings/src/storage/{files-storage/FilesStorage.d.ts → file-cache-storage/FileCacheStorage.d.ts} +5 -5
  85. package/esm/typings/src/storage/file-cache-storage/FileCacheStorageOptions.d.ts +10 -0
  86. package/esm/typings/src/storage/{files-storage → file-cache-storage}/utils/nameToSubfolderPath.d.ts +1 -1
  87. package/esm/typings/src/storage/file-cache-storage/utils/nameToSubfolderPath.test.d.ts +1 -0
  88. package/esm/typings/src/storage/local-storage/getLocalStorage.d.ts +1 -1
  89. package/esm/typings/src/storage/local-storage/getSessionStorage.d.ts +1 -1
  90. package/esm/typings/src/types/PipelineJson/KnowledgeSourceJson.d.ts +9 -2
  91. package/esm/typings/src/types/PipelineJson/PipelineJson.d.ts +2 -2
  92. package/esm/typings/src/types/typeAliases.d.ts +8 -11
  93. package/esm/typings/src/utils/execCommand/$execCommand.d.ts +14 -0
  94. package/esm/typings/src/utils/execCommand/$execCommands.d.ts +17 -0
  95. package/esm/typings/src/utils/execCommand/IExecCommandOptions.d.ts +23 -0
  96. package/esm/typings/src/utils/execCommand/execCommand.test.d.ts +1 -0
  97. package/esm/typings/src/utils/execCommand/execCommandNormalizeOptions.d.ts +10 -0
  98. package/esm/typings/src/utils/execCommand/execCommandNormalizeOptions.test.d.ts +1 -0
  99. package/esm/typings/src/utils/files/$isDirectoryExisting.d.ts +3 -3
  100. package/esm/typings/src/utils/files/$isFileExisting.d.ts +3 -3
  101. package/esm/typings/src/utils/files/$listAllFiles.d.ts +5 -4
  102. package/esm/typings/src/utils/files/extensionToMimeType.d.ts +8 -0
  103. package/esm/typings/src/utils/files/extensionToMimeType.test.d.ts +1 -0
  104. package/esm/typings/src/utils/files/getFileExtension.d.ts +8 -0
  105. package/esm/typings/src/utils/files/getFileExtension.test.d.ts +1 -0
  106. package/esm/typings/src/utils/validators/filePath/isValidFilePath.d.ts +2 -2
  107. package/package.json +2 -2
  108. package/umd/index.umd.js +1 -1
  109. package/umd/index.umd.js.map +1 -1
  110. package/esm/typings/src/knowledge/prepare-knowledge/_common/Scraper.d.ts +0 -37
  111. package/esm/typings/src/knowledge/prepare-knowledge/markdown/playground/markdown-knowledge-playground.d.ts +0 -2
  112. package/esm/typings/src/knowledge/prepare-knowledge/markdown/prepareKnowledgeFromMarkdown.d.ts +0 -14
  113. package/esm/typings/src/knowledge/prepare-knowledge/pdf/prepareKnowledgeFromPdf.d.ts +0 -15
  114. package/esm/typings/src/prepare/PrepareOptions.d.ts +0 -22
  115. package/esm/typings/src/storage/files-storage/FilesStorageOptions.d.ts +0 -10
  116. /package/esm/typings/src/{knowledge/dialogs → dialogs}/user-interface-execution-tools.test.d.ts +0 -0
  117. /package/esm/typings/src/{knowledge/prepare-knowledge → scrapers}/_common/prepareKnowledgePieces.test.d.ts +0 -0
  118. /package/esm/typings/src/{knowledge/prepare-knowledge/markdown/prepareKnowledgeFromMarkdown.test.d.ts → scrapers/markdown/markdownScraper.test.d.ts} +0 -0
  119. /package/esm/typings/src/{knowledge/prepare-knowledge/pdf/prepareKnowledgeFromPdf.test.d.ts → scrapers/website/utils/markdownConverter.test.d.ts} +0 -0
  120. /package/esm/typings/src/{storage/files-storage/utils/nameToSubfolderPath.test.d.ts → scrapers/website/websiteScraper.test.d.ts} +0 -0
@@ -0,0 +1,71 @@
1
+ import type { Promisable } from 'type-fest';
2
+ import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions';
3
+ import type { KnowledgePiecePreparedJson } from '../../types/PipelineJson/KnowledgePieceJson';
4
+ import type { string_filename } from '../../types/typeAliases';
5
+ import type { string_knowledge_source_link } from '../../types/typeAliases';
6
+ import type { string_mime_type } from '../../types/typeAliases';
7
+ import type { string_promptbook_documentation_url } from '../../types/typeAliases';
8
+ import type { string_url } from '../../types/typeAliases';
9
+ /**
10
+ * @@@
11
+ *
12
+ */
13
+ export type Scraper = {
14
+ /**
15
+ * Mime types that this scraper can handle
16
+ */
17
+ readonly mimeTypes: Array<string_mime_type>;
18
+ /**
19
+ * Link to documentation
20
+ */
21
+ readonly documentationUrl: string_promptbook_documentation_url;
22
+ /**
23
+ * Scrapes the markdown file and returns the knowledge pieces or `null` if it can't scrape it
24
+ */
25
+ scrape(source: ScraperSourceHandler, options: PrepareAndScrapeOptions): Promisable<Array<Omit<KnowledgePiecePreparedJson, 'sources' | 'preparationIds'>> | null>;
26
+ };
27
+ /**
28
+ * @@@
29
+ */
30
+ export type ScraperSourceHandler = {
31
+ /**
32
+ * The source of the knowledge
33
+ */
34
+ readonly source: string_knowledge_source_link;
35
+ /**
36
+ * The path to the file, if it is a file
37
+ *
38
+ * Note: Typically one of the `filename` or `url` is set and the other is `null`
39
+ */
40
+ readonly filename: string_filename | null;
41
+ /**
42
+ * The URL, if it is online
43
+ *
44
+ * Note: Typically one of the `filename` or `url` is set and the other is `null`
45
+ */
46
+ readonly url: string_url | null;
47
+ /**
48
+ * Mime type of the source
49
+ */
50
+ readonly mimeType: string_mime_type;
51
+ /**
52
+ * Get the content as parsed JSON
53
+ */
54
+ asJson(): Promisable<unknown>;
55
+ /**
56
+ * Get the content as a utf-8 string
57
+ */
58
+ asText(): Promisable<string>;
59
+ /**
60
+ * Get the content as a blob
61
+ */
62
+ asBlob(): Promisable<Blob>;
63
+ };
64
+ /**
65
+ * TODO: [🐝] @@@ Annotate all
66
+ * TODO: [🔼] Export via types
67
+ */
68
+ /**
69
+ * TODO: !!!!!! Test that this is catched
70
+ * Note: [⚫] Code in this file should never be published in any package
71
+ */
@@ -0,0 +1,11 @@
1
+ import type { IDestroyable } from 'destroyable';
2
+ import type { string_absolute_filename } from '../../types/typeAliases';
3
+ /**
4
+ * @@@
5
+ */
6
+ export type ScraperIntermediateSource = IDestroyable & {
7
+ /**
8
+ * @@@
9
+ */
10
+ readonly filename: string_absolute_filename;
11
+ };
@@ -1,13 +1,13 @@
1
- import type { PrepareOptions } from '../../../prepare/PrepareOptions';
2
- import type { KnowledgePiecePreparedJson } from '../../../types/PipelineJson/KnowledgePieceJson';
3
- import type { KnowledgeSourceJson } from '../../../types/PipelineJson/KnowledgeSourceJson';
1
+ import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions';
2
+ import type { KnowledgePiecePreparedJson } from '../../types/PipelineJson/KnowledgePieceJson';
3
+ import type { KnowledgeSourceJson } from '../../types/PipelineJson/KnowledgeSourceJson';
4
4
  /**
5
5
  * Prepares the knowle
6
6
  *
7
7
  * @see https://github.com/webgptorg/promptbook/discussions/41
8
8
  * @public exported from `@promptbook/core`
9
9
  */
10
- export declare function prepareKnowledgePieces(knowledgeSources: Array<KnowledgeSourceJson>, options: PrepareOptions): Promise<Array<Omit<KnowledgePiecePreparedJson, 'preparationIds'>>>;
10
+ export declare function prepareKnowledgePieces(knowledgeSources: Array<KnowledgeSourceJson>, options: PrepareAndScrapeOptions): Promise<Array<Omit<KnowledgePiecePreparedJson, 'preparationIds'>>>;
11
11
  /**
12
12
  * TODO: [🧊] In future one preparation can take data from previous preparation and save tokens and time
13
13
  * Put `knowledgePieces` into `PrepareKnowledgeOptions`
@@ -0,0 +1,33 @@
1
+ import type { PrepareAndScrapeOptions } from '../../../prepare/PrepareAndScrapeOptions';
2
+ import type { string_file_extension } from '../../../types/typeAliases';
3
+ import type { ScraperSourceHandler } from '../Scraper';
4
+ import type { ScraperIntermediateSource } from '../ScraperIntermediateSource';
5
+ /**
6
+ * @@@
7
+ *
8
+ * @private internal utility of `getScraperIntermediateSource`
9
+ */
10
+ type GetScraperIntermediateSourceSource = Pick<ScraperSourceHandler, 'filename' | 'url'>;
11
+ /**
12
+ * @@@
13
+ *
14
+ * @private internal utility of `getScraperIntermediateSource`
15
+ */
16
+ type GetScraperIntermediateSourceHandler = Required<Pick<PrepareAndScrapeOptions, 'rootDirname' | 'cacheDirname' | 'isCacheCleaned' | 'isVerbose'>> & {
17
+ readonly extension: string_file_extension;
18
+ };
19
+ /**
20
+ * Create a filename for intermediate cache for scrapers
21
+ *
22
+ * Note: It also checks if directory exists and creates it if not
23
+ *
24
+ * @private as internal utility for scrapers
25
+ */
26
+ export declare function getScraperIntermediateSource(source: GetScraperIntermediateSourceSource, options: GetScraperIntermediateSourceHandler): Promise<ScraperIntermediateSource>;
27
+ export {};
28
+ /**
29
+ * Note: Not using `FileCacheStorage` for two reasons:
30
+ * 1) Need to store more than serialized JSONs
31
+ * 2) Need to switch between a `rootDirname` and `cacheDirname` <- TODO: !!!!
32
+ * TODO: [🐱‍🐉][🧠] Make some smart crop
33
+ */
@@ -0,0 +1,4 @@
1
+ export {};
2
+ /**
3
+ * TODO: [🐱‍🐉][🧠] Make some smart crop
4
+ */
@@ -0,0 +1,10 @@
1
+ import type { SetOptional } from 'type-fest';
2
+ import type { PrepareAndScrapeOptions } from '../../../prepare/PrepareAndScrapeOptions';
3
+ import type { KnowledgeSourceJson } from '../../../types/PipelineJson/KnowledgeSourceJson';
4
+ import type { ScraperSourceHandler } from '../Scraper';
5
+ /**
6
+ * @@@
7
+ *
8
+ * @private for scraper utilities
9
+ */
10
+ export declare function makeKnowledgeSourceHandler(knowledgeSource: SetOptional<KnowledgeSourceJson, 'name'>, options?: Pick<PrepareAndScrapeOptions, 'rootDirname' | 'isVerbose'>): Promise<ScraperSourceHandler>;
@@ -0,0 +1,37 @@
1
+ import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions';
2
+ import type { KnowledgePiecePreparedJson } from '../../types/PipelineJson/KnowledgePieceJson';
3
+ import type { ScraperSourceHandler } from '../_common/Scraper';
4
+ import type { ScraperIntermediateSource } from '../_common/ScraperIntermediateSource';
5
+ /**
6
+ * Scraper of .docx and .odt files
7
+ *
8
+ * @see `documentationUrl` for more details
9
+ * @public exported from `@promptbook/core`
10
+ */
11
+ export declare const documentScraper: {
12
+ /**
13
+ * Mime types that this scraper can handle
14
+ */
15
+ mimeTypes: string[];
16
+ /**
17
+ * Link to documentation
18
+ */
19
+ documentationUrl: "https://github.com/webgptorg/promptbook/discussions/@@";
20
+ /**
21
+ * Convert the `.docx` or `.odt` to `.md` file and returns intermediate source
22
+ *
23
+ * Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object
24
+ */
25
+ $convert(source: ScraperSourceHandler, options: PrepareAndScrapeOptions): Promise<ScraperIntermediateSource>;
26
+ /**
27
+ * Scrapes the docx file and returns the knowledge pieces or `null` if it can't scrape it
28
+ */
29
+ scrape(source: ScraperSourceHandler, options: PrepareAndScrapeOptions): Promise<Array<Omit<KnowledgePiecePreparedJson, 'sources' | 'preparationIds'>> | null>;
30
+ };
31
+ /**
32
+ * TODO: [👣] Converted documents can act as cached items - there is no need to run conversion each time
33
+ * TODO: [🦖] Make some system for putting scrapers to separete packages
34
+ * TODO: [🪂] Do it in parallel 11:11
35
+ * TODO: [🦷] Ideally use `as const satisfies Converter & Scraper` BUT this combination throws errors
36
+ * Note: No need to aggregate usage here, it is done by intercepting the llmTools
37
+ */
@@ -0,0 +1,4 @@
1
+ export {};
2
+ /**
3
+ * TODO: [📓] Maybe test all file in samples (not just 10-simple.docx)
4
+ */
@@ -0,0 +1,5 @@
1
+ #!/usr/bin/env ts-node
2
+ export {};
3
+ /**
4
+ * Note: [⚫] Code in this file should never be published in any package
5
+ */
@@ -0,0 +1,37 @@
1
+ import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions';
2
+ import type { KnowledgePiecePreparedJson } from '../../types/PipelineJson/KnowledgePieceJson';
3
+ import type { ScraperSourceHandler } from '../_common/Scraper';
4
+ import type { ScraperIntermediateSource } from '../_common/ScraperIntermediateSource';
5
+ /**
6
+ * Scraper for .docx files
7
+ *
8
+ * @see `documentationUrl` for more details
9
+ * @public exported from `@promptbook/core`
10
+ */
11
+ export declare const legacyDocumentScraper: {
12
+ /**
13
+ * Mime types that this scraper can handle
14
+ */
15
+ mimeTypes: string[];
16
+ /**
17
+ * Link to documentation
18
+ */
19
+ documentationUrl: "https://github.com/webgptorg/promptbook/discussions/@@";
20
+ /**
21
+ * Convert the `.doc` or `.rtf` to `.doc` file and returns intermediate source
22
+ *
23
+ * Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object
24
+ */
25
+ $convert(source: ScraperSourceHandler, options: PrepareAndScrapeOptions): Promise<ScraperIntermediateSource>;
26
+ /**
27
+ * Scrapes the `.doc` or `.rtf` file and returns the knowledge pieces or `null` if it can't scrape it
28
+ */
29
+ scrape(source: ScraperSourceHandler, options: PrepareAndScrapeOptions): Promise<Array<Omit<KnowledgePiecePreparedJson, 'sources' | 'preparationIds'>> | null>;
30
+ };
31
+ /**
32
+ * TODO: [👣] Converted documents can act as cached items - there is no need to run conversion each time
33
+ * TODO: [🦖] Make some system for putting scrapers to separete packages
34
+ * TODO: [🪂] Do it in parallel 11:11
35
+ * TODO: [🦷] Ideally use `as const satisfies Converter & Scraper` BUT this combination throws errors
36
+ * Note: No need to aggregate usage here, it is done by intercepting the llmTools
37
+ */
@@ -0,0 +1,4 @@
1
+ export {};
2
+ /**
3
+ * TODO: [📓] Maybe test all file in samples (not just 10-simple.doc)
4
+ */
@@ -0,0 +1,5 @@
1
+ #!/usr/bin/env ts-node
2
+ export {};
3
+ /**
4
+ * Note: [⚫] Code in this file should never be published in any package
5
+ */
@@ -0,0 +1,8 @@
1
+ import type { Scraper } from './_common/Scraper';
2
+ /**
3
+ * @@@
4
+ *
5
+ * @private because this will be replaced by a system of one scraper per package [🦖]
6
+ * TODO: [🦖] System for scrapers NOT public exported from `@promptbook/core`
7
+ */
8
+ export declare const SCRAPERS: Array<Scraper>;
@@ -0,0 +1,29 @@
1
+ import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions';
2
+ import type { KnowledgePiecePreparedJson } from '../../types/PipelineJson/KnowledgePieceJson';
3
+ import type { ScraperSourceHandler } from '../_common/Scraper';
4
+ /**
5
+ * Scraper for markdown files
6
+ *
7
+ * @see `documentationUrl` for more details
8
+ * @public exported from `@promptbook/core`
9
+ */
10
+ export declare const markdownScraper: {
11
+ /**
12
+ * Mime types that this scraper can handle
13
+ */
14
+ mimeTypes: string[];
15
+ /**
16
+ * Link to documentation
17
+ */
18
+ documentationUrl: "https://github.com/webgptorg/promptbook/discussions/@@";
19
+ /**
20
+ * Scrapes the markdown file and returns the knowledge pieces or `null` if it can't scrape it
21
+ */
22
+ scrape(source: ScraperSourceHandler, options: PrepareAndScrapeOptions): Promise<Array<Omit<KnowledgePiecePreparedJson, 'sources' | 'preparationIds'>> | null>;
23
+ };
24
+ /**
25
+ * TODO: [🦖] Make some system for putting scrapers to separete packages
26
+ * TODO: [🪂] Do it in parallel 11:11
27
+ * TODO: [🦷] Ideally use `as const satisfies Scraper` BUT this combination throws errors
28
+ * Note: No need to aggregate usage here, it is done by intercepting the llmTools
29
+ */
@@ -0,0 +1,5 @@
1
+ #!/usr/bin/env ts-node
2
+ export {};
3
+ /**
4
+ * Note: [⚫] Code in this file should never be published in any package
5
+ */
@@ -0,0 +1,35 @@
1
+ import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions';
2
+ import type { KnowledgePiecePreparedJson } from '../../types/PipelineJson/KnowledgePieceJson';
3
+ import type { ScraperSourceHandler } from '../_common/Scraper';
4
+ import type { ScraperIntermediateSource } from '../_common/ScraperIntermediateSource';
5
+ /**
6
+ * Scraper for .docx files
7
+ *
8
+ * @see `documentationUrl` for more details
9
+ * @public exported from `@promptbook/core`
10
+ */
11
+ export declare const pdfScraper: {
12
+ /**
13
+ * Mime types that this scraper can handle
14
+ */
15
+ mimeTypes: string[];
16
+ /**
17
+ * Link to documentation
18
+ */
19
+ documentationUrl: "https://github.com/webgptorg/promptbook/discussions/@@";
20
+ /**
21
+ * Converts the `.pdf` file to `.md` file and returns intermediate source
22
+ */
23
+ $convert(source: ScraperSourceHandler, options: PrepareAndScrapeOptions): Promise<ScraperIntermediateSource>;
24
+ /**
25
+ * Scrapes the `.pdf` file and returns the knowledge pieces or `null` if it can't scrape it
26
+ */
27
+ scrape(source: ScraperSourceHandler, options: PrepareAndScrapeOptions): Promise<Array<Omit<KnowledgePiecePreparedJson, 'sources' | 'preparationIds'>> | null>;
28
+ };
29
+ /**
30
+ * TODO: [👣] Converted documents can act as cached items - there is no need to run conversion each time
31
+ * TODO: [🦖] Make some system for putting scrapers to separete packages
32
+ * TODO: [🪂] Do it in parallel 11:11
33
+ * TODO: [🦷] Ideally use `as const satisfies Converter & Scraper` BUT this combination throws errors
34
+ * Note: No need to aggregate usage here, it is done by intercepting the llmTools
35
+ */
@@ -0,0 +1,5 @@
1
+ #!/usr/bin/env ts-node
2
+ export {};
3
+ /**
4
+ * Note: [⚫] Code in this file should never be published in any package
5
+ */
@@ -0,0 +1,5 @@
1
+ #!/usr/bin/env ts-node
2
+ export {};
3
+ /**
4
+ * Note: [⚫] Code in this file should never be published in any package
5
+ */
@@ -0,0 +1,12 @@
1
+ import { Converter } from 'showdown';
2
+ /**
3
+ * A converter instance that uses showdown and highlight extensions
4
+ *
5
+ * @type {Converter}
6
+ * @private for markdown and html knowledge scrapers
7
+ */
8
+ export declare const markdownConverter: Converter;
9
+ /**
10
+ * TODO: !!!!!! Figure out better name not to confuse with `Converter`
11
+ * TODO: !!!!!! Lazy-make converter
12
+ */
@@ -0,0 +1,43 @@
1
+ import type { PrepareAndScrapeOptions } from '../../prepare/PrepareAndScrapeOptions';
2
+ import type { KnowledgePiecePreparedJson } from '../../types/PipelineJson/KnowledgePieceJson';
3
+ import type { string_markdown } from '../../types/typeAliases';
4
+ import type { ScraperSourceHandler } from '../_common/Scraper';
5
+ import type { ScraperIntermediateSource } from '../_common/ScraperIntermediateSource';
6
+ /**
7
+ * Scraper for .docx files
8
+ *
9
+ * @see `documentationUrl` for more details
10
+ * @public exported from `@promptbook/core`
11
+ */
12
+ export declare const websiteScraper: {
13
+ /**
14
+ * Mime types that this scraper can handle
15
+ */
16
+ mimeTypes: string[];
17
+ /**
18
+ * Link to documentation
19
+ */
20
+ documentationUrl: "https://github.com/webgptorg/promptbook/discussions/@@";
21
+ /**
22
+ * Convert the website to `.md` file and returns intermediate source
23
+ *
24
+ * Note: `$` is used to indicate that this function is not a pure function - it leaves files on the disk and you are responsible for cleaning them by calling `destroy` method of returned object
25
+ */
26
+ $convert(source: ScraperSourceHandler, options: PrepareAndScrapeOptions): Promise<import("destroyable").IDestroyable & {
27
+ readonly filename: string;
28
+ } & {
29
+ markdown: string_markdown;
30
+ }>;
31
+ /**
32
+ * Scrapes the website and returns the knowledge pieces or `null` if it can't scrape it
33
+ */
34
+ scrape(source: ScraperSourceHandler, options: PrepareAndScrapeOptions): Promise<Array<Omit<KnowledgePiecePreparedJson, 'sources' | 'preparationIds'>> | null>;
35
+ };
36
+ /**
37
+ * TODO: !!!!!! Put into separate package
38
+ * TODO: [👣] Scraped website in .md can act as cache item - there is no need to run conversion each time
39
+ * TODO: [🦖] Make some system for putting scrapers to separete packages
40
+ * TODO: [🪂] Do it in parallel 11:11
41
+ * TODO: [🦷] Ideally use `as const satisfies Converter & Scraper` BUT this combination throws errors
42
+ * Note: No need to aggregate usage here, it is done by intercepting the llmTools
43
+ */
@@ -1,13 +1,13 @@
1
1
  import type { PromptbookStorage } from '../_common/PromptbookStorage';
2
- import type { FilesStorageOptions } from './FilesStorageOptions';
2
+ import type { FileCacheStorageOptions } from './FileCacheStorageOptions';
3
3
  /**
4
4
  * @@@
5
5
  *
6
6
  * @public exported from `@promptbook/node`
7
7
  */
8
- export declare class FilesStorage<TItem> implements PromptbookStorage<TItem> {
9
- protected readonly options: FilesStorageOptions;
10
- constructor(options: FilesStorageOptions);
8
+ export declare class FileCacheStorage<TItem> implements PromptbookStorage<TItem> {
9
+ protected readonly options: FileCacheStorageOptions;
10
+ constructor(options: FileCacheStorageOptions);
11
11
  /**
12
12
  * @@@
13
13
  */
@@ -27,5 +27,5 @@ export declare class FilesStorage<TItem> implements PromptbookStorage<TItem> {
27
27
  }
28
28
  /**
29
29
  * TODO: [🌗] Maybe some checkers, not all valid JSONs are desired and valid values
30
- * Note: [🟢] This code should never be published outside of `@promptbook/node` and `@promptbook/cli` and `@promptbook/cli`
30
+ * Note: [🟢] Code in this file should never be published outside of `@promptbook/node` and `@promptbook/cli`
31
31
  */
@@ -0,0 +1,10 @@
1
+ import type { string_dirname } from '../../types/typeAliases';
2
+ /**
3
+ * @@@
4
+ */
5
+ export type FileCacheStorageOptions = {
6
+ /**
7
+ * @@@
8
+ */
9
+ rootFolderPath: string_dirname;
10
+ };
@@ -2,6 +2,6 @@ import type { string_name } from '../../../types/typeAliases';
2
2
  /**
3
3
  * @@@
4
4
  *
5
- * @private for `FilesStorage`
5
+ * @private for `FileCacheStorage`
6
6
  */
7
7
  export declare function nameToSubfolderPath(name: string_name): Array<string>;
@@ -6,5 +6,5 @@ import type { PromptbookStorage } from '../_common/PromptbookStorage';
6
6
  */
7
7
  export declare function getLocalStorage<TItem>(): PromptbookStorage<TItem>;
8
8
  /**
9
- * Note: [🔵] This code should never be published outside of `@promptbook/browser`
9
+ * Note: [🔵] Code in this file should never be published outside of `@promptbook/browser`
10
10
  */
@@ -6,5 +6,5 @@ import type { PromptbookStorage } from '../_common/PromptbookStorage';
6
6
  */
7
7
  export declare function getSessionStorage<TItem>(): PromptbookStorage<TItem>;
8
8
  /**
9
- * Note: [🔵] This code should never be published outside of `@promptbook/browser`
9
+ * Note: [🔵] Code in this file should never be published outside of `@promptbook/browser`
10
10
  */
@@ -11,11 +11,18 @@ import type { string_name } from '../typeAliases';
11
11
  */
12
12
  export type KnowledgeSourceJson = {
13
13
  /**
14
- * @@@
14
+ * Unique identifier of the knowledge source
15
15
  */
16
16
  readonly name: string_name;
17
17
  /**
18
- * @@@
18
+ * Source of one knowledge
19
+ *
20
+ * It can be a link, a relative path to file or direct text or combination of those
21
+ *
22
+ * For example `"https://pavolhejny.com/"`
23
+ * For example `"./pavol-hejny-cv.pdf"`
24
+ * For example `"Pavol Hejný has web https://pavolhejny.com/"`
25
+ * For example `"Pavol Hejný is web developer and creator of Promptbook and Collboard"`
19
26
  */
20
27
  readonly sourceContent: string_knowledge_source_content;
21
28
  };
@@ -1,5 +1,5 @@
1
1
  import type { ModelRequirements } from '../ModelRequirements';
2
- import type { string_file_path } from '../typeAliases';
2
+ import type { string_filename } from '../typeAliases';
3
3
  import type { string_markdown_text } from '../typeAliases';
4
4
  import type { string_pipeline_url } from '../typeAliases';
5
5
  import type { string_semantic_version } from '../typeAliases';
@@ -33,7 +33,7 @@ export type PipelineJson = {
33
33
  /**
34
34
  * Internal helper for tracking the source `.ptbk.md` file of the pipeline
35
35
  */
36
- readonly sourceFile?: string_file_path;
36
+ readonly sourceFile?: string_filename;
37
37
  /**
38
38
  * Title of the promptbook
39
39
  * -It can use simple markdown formatting like **bold**, *italic*, [link](https://example.com), ... BUT not code blocks and structure
@@ -156,7 +156,7 @@ export type string_knowledge_source_content = string_knowledge_source_link | str
156
156
  *
157
157
  * @@@ string_knowledge_source vs string_knowledge_source_link
158
158
  */
159
- export type string_knowledge_source_link = string_url | string_file_path;
159
+ export type string_knowledge_source_link = string_url | string_filename;
160
160
  /**
161
161
  * Semantic helper
162
162
  *
@@ -386,6 +386,7 @@ export type string_emails = string;
386
386
  * - `isValidUuid to check validity
387
387
  *
388
388
  * For example `"5a0a153d-7be9-4018-9eda-e0e2e2b89bd9"`
389
+ * TODO: [🥬] Make some system for hashes and ids of promptbook
389
390
  */
390
391
  export type string_uuid = string & {
391
392
  readonly _type: 'uuid';
@@ -423,37 +424,33 @@ export type string_file_extension = string;
423
424
  *
424
425
  * For example `"C:/Users/me/work/collboard/modules-sdk/src/colldev/commands/develop/ColldevDevelop.tsx"`
425
426
  */
426
- export type string_file_absolute_path = string;
427
+ export type string_absolute_filename = string;
427
428
  /**
428
429
  * Semantic helper
429
430
  *
430
431
  * For example `"./src/colldev/commands/develop/ColldevDevelop.tsx"`
431
432
  */
432
- export type string_file_relative_path = string;
433
+ export type string_relative_filename = string;
433
434
  /**
434
435
  * Semantic helper
435
436
  */
436
- export type string_file_path = string_file_absolute_path | string_file_relative_path;
437
+ export type string_filename = string_absolute_filename | string_relative_filename;
437
438
  /**
438
439
  * Semantic helper
439
440
  *
440
441
  * For example `"C:/Users/me/work/collboard/modules-sdk/src/colldev/commands/develop/ColldevDevelop.tsx"`
441
442
  */
442
- export type string_folder_absolute_path = string;
443
+ export type string_absolute_dirname = string;
443
444
  /**
444
445
  * Semantic helper
445
446
  *
446
447
  * For example `"./src/colldev/commands/develop/ColldevDevelop.tsx"`
447
448
  */
448
- export type string_folder_relative_path = string;
449
+ export type string_relative_dirname = string;
449
450
  /**
450
451
  * Semantic helper
451
452
  */
452
- export type string_folder_path = string_file_absolute_path | string_file_relative_path;
453
- /**
454
- * Semantic helper
455
- */
456
- export type string_filename = string;
453
+ export type string_dirname = string_absolute_dirname | string_relative_dirname;
457
454
  /**
458
455
  * Semantic helper
459
456
  *
@@ -0,0 +1,14 @@
1
+ import type { IExecCommandOptions } from './IExecCommandOptions';
2
+ /**
3
+ * Run one command in a shell
4
+ *
5
+ * Note: There are 2 similar functions in the codebase:
6
+ * - `$execCommand` which runs a single command
7
+ * - `$execCommands` which runs multiple commands
8
+ *
9
+ * @public exported from `@promptbook/node`
10
+ */
11
+ export declare function $execCommand(options: IExecCommandOptions): Promise<string>;
12
+ /**
13
+ * Note: [🟢 <- TODO: [🦖] !!!!!! Split scrapers into packages and enable] Code in this file should never be published outside of `@promptbook/node` and `@promptbook/cli`
14
+ */
@@ -0,0 +1,17 @@
1
+ /**
2
+ * Run multiple commands in a shell in sequence
3
+ *
4
+ * Note: There are 2 similar functions in the codebase:
5
+ * - `$execCommand` which runs a single command
6
+ * - `$execCommands` which runs multiple commands
7
+ *
8
+ * @public exported from `@promptbook/node`
9
+ */
10
+ export declare function $execCommands({ commands, cwd, crashOnError, }: {
11
+ readonly commands: string[];
12
+ readonly cwd: string;
13
+ readonly crashOnError?: boolean;
14
+ }): Promise<void>;
15
+ /**
16
+ * Note: [🟢 <- TODO: [🦖] !!!!!! Split scrapers into packages and enable] Code in this file should never be published outside of `@promptbook/node` and `@promptbook/cli`
17
+ */
@@ -0,0 +1,23 @@
1
+ /**
2
+ * @deprecated Use sth from type-fest or move this to type helpers
3
+ */
4
+ type RequiredAndOptional<TBase, TRequired extends keyof TBase, TOptional extends keyof TBase> = Pick<TBase, TRequired> & Partial<Pick<TBase, TOptional>>;
5
+ /**
6
+ * Simple options for `execCommand`
7
+ *
8
+ * TODO: !!!!!! Rename - remove I prefix
9
+ */
10
+ export type IExecCommandOptions = string | RequiredAndOptional<IExecCommandOptionsAdvanced, 'command', 'args' | 'cwd' | 'crashOnError' | 'timeout'>;
11
+ /**
12
+ * Advanced options for `execCommand`
13
+ *
14
+ * TODO: !!!!!! Rename - remove I prefix
15
+ */
16
+ export type IExecCommandOptionsAdvanced = {
17
+ readonly command: string;
18
+ readonly args: string[];
19
+ readonly cwd: string;
20
+ readonly crashOnError: boolean;
21
+ readonly timeout: number;
22
+ };
23
+ export {};