@promptbook/cli 0.112.0-104 → 0.112.0-106

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (174) hide show
  1. package/apps/agents-server/src/app/admin/image-generator-test/ImageAttachmentsEditor.tsx +19 -3
  2. package/apps/agents-server/src/app/admin/limits/LimitsClient.tsx +11 -12
  3. package/apps/agents-server/src/app/admin/metadata/MetadataClient.tsx +34 -2
  4. package/apps/agents-server/src/app/admin/servers/CreateServerDialog.tsx +6 -1
  5. package/apps/agents-server/src/app/admin/servers/useCreateServerWizard.ts +13 -1
  6. package/apps/agents-server/src/app/agents/[agentName]/AgentChatWrapper.tsx +11 -2
  7. package/apps/agents-server/src/app/agents/[agentName]/AgentProfileChat.tsx +11 -1
  8. package/apps/agents-server/src/app/agents/[agentName]/book/BookEditorWrapper.tsx +7 -1
  9. package/apps/agents-server/src/app/agents/[agentName]/chat/CanonicalAgentChatSurface.tsx +11 -1
  10. package/apps/agents-server/src/app/agents/[agentName]/images/default-avatar.png/route.ts +6 -2
  11. package/apps/agents-server/src/app/api/images/[filename]/route.ts +6 -2
  12. package/apps/agents-server/src/app/api/internal/agent-runner-limits/route.ts +51 -0
  13. package/apps/agents-server/src/app/api/upload/route.ts +48 -12
  14. package/apps/agents-server/src/app/layout.tsx +13 -0
  15. package/apps/agents-server/src/components/FileUploadAvailability/FileUploadAvailabilityContext.tsx +50 -0
  16. package/apps/agents-server/src/components/FileUploadAvailability/FileUploadUnavailableNotice.tsx +45 -0
  17. package/apps/agents-server/src/components/LayoutWrapper/LayoutWrapper.tsx +85 -76
  18. package/apps/agents-server/src/components/NewAgentDialog/NewAgentDialog.tsx +7 -3
  19. package/apps/agents-server/src/components/NewAgentDialog/NewAgentWizardKnowledgeStep.tsx +6 -0
  20. package/apps/agents-server/src/components/NewAgentDialog/useNewAgentWizardKnowledgeState.ts +8 -1
  21. package/apps/agents-server/src/constants/serverLimits.ts +22 -2
  22. package/apps/agents-server/src/database/seedDefaultAgents.ts +218 -0
  23. package/apps/agents-server/src/tools/$provideCdnForServer.ts +72 -8
  24. package/apps/agents-server/src/utils/defaultAgents/loadDefaultAgentBooks.ts +103 -0
  25. package/apps/agents-server/src/utils/knowledge/createInlineKnowledgeSourceUploader.ts +24 -5
  26. package/apps/agents-server/src/utils/serverLimits.ts +26 -1
  27. package/apps/agents-server/src/utils/serverManagement/createManagedServer/seedServerDefaultAgents.ts +1 -85
  28. package/apps/agents-server/src/utils/shareTargetPayloads.ts +20 -2
  29. package/apps/agents-server/src/utils/upload/fileUploadAvailability.ts +91 -0
  30. package/apps/agents-server/src/utils/upload/uploadFileToServer.ts +46 -2
  31. package/esm/apps/agents-server/src/constants/federatedAgentImport.d.ts +42 -0
  32. package/esm/apps/agents-server/src/constants/serverLimits.d.ts +207 -0
  33. package/esm/apps/agents-server/src/constants/toolUsageLimits.d.ts +55 -0
  34. package/esm/index.es.js +602 -39
  35. package/esm/index.es.js.map +1 -1
  36. package/esm/scripts/run-agent-messages/main/AgentMessageFailureTracker.d.ts +27 -0
  37. package/esm/scripts/run-agent-messages/main/handleAgentWatchError.d.ts +4 -0
  38. package/esm/scripts/run-agent-messages/main/runAgentMessages.d.ts +1 -0
  39. package/esm/scripts/run-agent-messages/messages/moveAgentMessageToFailed.d.ts +17 -0
  40. package/esm/src/book-components/BookEditor/BookEditor.d.ts +5 -4
  41. package/esm/src/book-components/BookEditor/BookEditorTheme.d.ts +24 -0
  42. package/esm/src/book-components/BookEditor/useBookEditorMonacoLanguage.d.ts +1 -6
  43. package/esm/src/book-components/BookEditor/useBookEditorMonacoLifecycle.d.ts +1 -4
  44. package/esm/src/book-components/BookEditor/useBookEditorMonacoStyles.d.ts +2 -1
  45. package/esm/src/cli/cli-commands/agent-folder/agentProjectPaths.d.ts +6 -0
  46. package/esm/src/version.d.ts +1 -1
  47. package/package.json +1 -1
  48. package/src/avatars/visuals/octopus3d3AvatarVisual.ts +28 -29
  49. package/src/book-components/BookEditor/BookEditor.tsx +10 -7
  50. package/src/book-components/BookEditor/BookEditorMonaco.tsx +3 -1
  51. package/src/book-components/BookEditor/BookEditorTheme.ts +32 -0
  52. package/src/book-components/BookEditor/useBookEditorMonacoLanguage.ts +15 -15
  53. package/src/book-components/BookEditor/useBookEditorMonacoLifecycle.ts +1 -5
  54. package/src/book-components/BookEditor/useBookEditorMonacoStyles.ts +2 -1
  55. package/src/cli/cli-commands/agent-folder/agentProjectPaths.ts +7 -0
  56. package/src/cli/cli-commands/agents-server/buildAgentsServer.ts +109 -9
  57. package/src/cli/cli-commands/agents-server/startAgentsServer.ts +132 -4
  58. package/src/other/templates/getTemplatesPipelineCollection.ts +874 -760
  59. package/src/version.ts +2 -2
  60. package/src/versions.txt +2 -0
  61. package/umd/apps/agents-server/src/constants/federatedAgentImport.d.ts +42 -0
  62. package/umd/apps/agents-server/src/constants/serverLimits.d.ts +207 -0
  63. package/umd/apps/agents-server/src/constants/toolUsageLimits.d.ts +55 -0
  64. package/umd/index.umd.js +602 -39
  65. package/umd/index.umd.js.map +1 -1
  66. package/umd/scripts/run-agent-messages/main/AgentMessageFailureTracker.d.ts +27 -0
  67. package/umd/scripts/run-agent-messages/main/handleAgentWatchError.d.ts +4 -0
  68. package/umd/scripts/run-agent-messages/main/runAgentMessages.d.ts +1 -0
  69. package/umd/scripts/run-agent-messages/messages/moveAgentMessageToFailed.d.ts +17 -0
  70. package/umd/src/book-components/BookEditor/BookEditor.d.ts +5 -4
  71. package/umd/src/book-components/BookEditor/BookEditorTheme.d.ts +24 -0
  72. package/umd/src/book-components/BookEditor/useBookEditorMonacoLanguage.d.ts +1 -6
  73. package/umd/src/book-components/BookEditor/useBookEditorMonacoLifecycle.d.ts +1 -4
  74. package/umd/src/book-components/BookEditor/useBookEditorMonacoStyles.d.ts +2 -1
  75. package/umd/src/cli/cli-commands/agent-folder/agentProjectPaths.d.ts +6 -0
  76. package/umd/src/version.d.ts +1 -1
  77. package/apps/agents-server/src/message-providers/email/_common/utils/parseEmailAddress.test.ts.todo +0 -108
  78. package/apps/agents-server/src/message-providers/email/_common/utils/parseEmailAddresses.test.ts.todo +0 -117
  79. package/apps/agents-server/src/message-providers/email/_common/utils/stringifyEmailAddress.test.ts.todo +0 -119
  80. package/apps/agents-server/src/message-providers/email/_common/utils/stringifyEmailAddresses.test.ts.todo +0 -74
  81. package/apps/agents-server/src/utils/defaultAgents/defaultAgents.ts +0 -168
  82. package/apps/agents-server/src/utils/defaultAgents/installDefaultAgents.ts +0 -139
  83. package/apps/agents-server/tests/e2e/authentication-and-navigation.spec.ts.todo +0 -178
  84. package/src/_packages/browser.index.ts +0 -31
  85. package/src/_packages/browser.readme.md +0 -43
  86. package/src/book-2.0/agent-source/parseAgentSourceWithCommitments.test.ts.todo +0 -265
  87. package/src/book-components/BookEditor/BookEditorMonaco.test.tsx.todo +0 -115
  88. package/src/book-components/Chat/utils/renderMarkdown.test.ts.tmp +0 -199
  89. package/src/collection/agent-collection/constructors/agent-collection-in-directory/AgentCollectionInDirectory.test.ts.todo +0 -131
  90. package/src/commands/_common/parseCommand.test.ts.todo +0 -48
  91. package/src/commitments/META_LINK/META_LINK.test.ts.todo +0 -75
  92. package/src/conversion/validation/pipelineStringToJson-errors.test.ts.todo +0 -33
  93. package/src/dialogs/simple-prompt/SimplePromptInterfaceTools.ts +0 -51
  94. package/src/executables/browsers/locateSafari.test.ts.tmp +0 -15
  95. package/src/execution/PromptbookFetch.test-type.ts +0 -14
  96. package/src/execution/createPipelineExecutor/00-createPipelineExecutor.test.ts.todo +0 -0
  97. package/src/execution/execution-report/executionReportJsonToString.test.ts.todo +0 -83
  98. package/src/execution/utils/usageToHuman.test.ts.todo +0 -80
  99. package/src/llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground.ts +0 -76
  100. package/src/llm-providers/_common/utils/assertUniqueModels.ts +0 -27
  101. package/src/llm-providers/_multiple/playground/playground.ts +0 -141
  102. package/src/llm-providers/_multiple/playground/tsconfig.json +0 -19
  103. package/src/llm-providers/agent/playground/playground.ts +0 -190
  104. package/src/llm-providers/agent/playground/tsconfig.json +0 -19
  105. package/src/llm-providers/anthropic-claude/playground/playground.ts +0 -99
  106. package/src/llm-providers/anthropic-claude/playground/tsconfig.json +0 -19
  107. package/src/llm-providers/azure-openai/playground/playground.ts +0 -101
  108. package/src/llm-providers/azure-openai/playground/tsconfig.json +0 -19
  109. package/src/llm-providers/ollama/playground/playground.ts +0 -120
  110. package/src/llm-providers/ollama/playground/tsconfig.json +0 -19
  111. package/src/llm-providers/openai/playground/playground.ts +0 -406
  112. package/src/llm-providers/openai/playground/tsconfig.json +0 -19
  113. package/src/llm-providers/remote/playground/playground.ts +0 -144
  114. package/src/llm-providers/remote/playground/tsconfig.json +0 -19
  115. package/src/llm-providers/vercel/playground/playground.ts +0 -133
  116. package/src/llm-providers/vercel/playground/tsconfig.json +0 -19
  117. package/src/personas/preparePersona.test.ts.todo +0 -126
  118. package/src/playground/backup/_playground-boilerplate.ts.txt +0 -37
  119. package/src/playground/backup/playground-agent-os.txt +0 -62
  120. package/src/playground/backup/playground-brj-app.ts.txt +0 -302
  121. package/src/playground/backup/playground-browser-playwright.txt +0 -110
  122. package/src/playground/backup/playground-claude-mcp.txt +0 -43
  123. package/src/playground/backup/playground-document-conversion.txt +0 -84
  124. package/src/playground/backup/playground-glob.ts.txt +0 -42
  125. package/src/playground/backup/playground-mcp-server.txt +0 -1
  126. package/src/playground/backup/playground-openai-agent-kit.txt +0 -73
  127. package/src/playground/backup/playground-openai-function-calling.txt +0 -131
  128. package/src/playground/backup/playground-openai-streaming.ts.txt +0 -68
  129. package/src/playground/backup/playground-scrape-knowledge.txt +0 -65
  130. package/src/playground/backup/playground-scraperFetch.ts.txt +0 -44
  131. package/src/playground/backup/playground-using-openai-compatible-route-on-agents-server.ts.txt +0 -49
  132. package/src/playground/backup/playground-write-pavolhejny-bio.txt +0 -120
  133. package/src/playground/permanent/_boilerplate.ts +0 -54
  134. package/src/playground/permanent/agent-with-browser-playground.ts +0 -92
  135. package/src/playground/permanent/error-handling-playground.ts +0 -103
  136. package/src/playground/playground.ts +0 -36
  137. package/src/playground/tsconfig.json +0 -19
  138. package/src/scrapers/_boilerplate/BoilerplateScraper.test.ts.todo +0 -73
  139. package/src/scrapers/_boilerplate/playground/boilerplate-scraper-playground.ts +0 -79
  140. package/src/scrapers/_boilerplate/playground/tsconfig.json +0 -19
  141. package/src/scrapers/_common/utils/files/blobToDataurl.test.ts.todo +0 -17
  142. package/src/scrapers/_common/utils/files/dataurlToBlob.test.ts.todo +0 -52
  143. package/src/scrapers/_common/utils/files/isValidDataurl.test.ts.todo +0 -42
  144. package/src/scrapers/_common/utils/files/shorten.test.ts.todo +0 -13
  145. package/src/scrapers/document/playground/document-scraper-playground.ts +0 -80
  146. package/src/scrapers/document/playground/tsconfig.json +0 -19
  147. package/src/scrapers/document-legacy/playground/legacy-document-scraper-playground.ts +0 -80
  148. package/src/scrapers/document-legacy/playground/tsconfig.json +0 -19
  149. package/src/scrapers/markdown/playground/markdown-scraper-playground.ts +0 -74
  150. package/src/scrapers/markdown/playground/tsconfig.json +0 -19
  151. package/src/scrapers/markitdown/MarkitdownScraper.test.ts.todo +0 -132
  152. package/src/scrapers/markitdown/playground/markitdown-scraper-playground.ts +0 -91
  153. package/src/scrapers/markitdown/playground/tsconfig.json +0 -19
  154. package/src/scrapers/pdf/PdfScraper.test.ts.todo +0 -52
  155. package/src/scrapers/pdf/playground/pdf-scraper-playground.ts +0 -75
  156. package/src/scrapers/pdf/playground/tsconfig.json +0 -19
  157. package/src/scrapers/website/playground/tsconfig.json +0 -19
  158. package/src/scrapers/website/playground/website-scraper-playground.ts +0 -82
  159. package/src/storage/_common/PromptbookStorage.test-type.ts +0 -14
  160. package/src/storage/local-storage/getIndexedDbStorage.ts +0 -36
  161. package/src/storage/local-storage/getLocalStorage.ts +0 -33
  162. package/src/storage/local-storage/getSessionStorage.ts +0 -33
  163. package/src/storage/local-storage/utils/IndexedDbStorageOptions.ts +0 -16
  164. package/src/storage/local-storage/utils/makePromptbookStorageFromIndexedDb.ts +0 -58
  165. package/src/storage/local-storage/utils/makePromptbookStorageFromWebStorage.ts +0 -45
  166. package/src/transpilers/formatted-book-in-markdown/FormattedBookInMarkdownTranspiler.test.ts.todo +0 -35
  167. package/src/transpilers/openai-sdk/playground/playground.ts +0 -85
  168. package/src/transpilers/openai-sdk/playground/tmp/chatbot-openaisdk-1.js +0 -194
  169. package/src/transpilers/openai-sdk/playground/tmp/package.json +0 -3
  170. package/src/transpilers/openai-sdk/playground/tsconfig.json +0 -18
  171. package/src/utils/editable/utils/findUsableParameters.test.ts.todo +0 -43
  172. package/src/utils/editable/utils/stringifyPipelineJson.test.ts.todo +0 -38
  173. package/src/utils/markdown/prettifyMarkdown.test.ts.tmp +0 -42
  174. package/src/utils/serialization/serializeToPromptbookJavascript.test.ts.todo +0 -116
@@ -1,74 +0,0 @@
1
- #!/usr/bin/env ts-node
2
-
3
- import * as dotenv from 'dotenv';
4
-
5
- dotenv.config({ path: '.env' });
6
-
7
- import colors from 'colors'; // <- TODO: [🔶] Make system to put color and style to both node and browser
8
- import { writeFile } from 'fs/promises';
9
- import { join } from 'path';
10
- import { usageToHuman } from '../../../execution/utils/usageToHuman';
11
- import { $provideLlmToolsForTestingAndScriptsAndPlayground } from '../../../llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground';
12
- import { stringifyPipelineJson } from '../../../utils/editable/utils/stringifyPipelineJson';
13
- import { $provideFilesystemForNode } from '../../_common/register/$provideFilesystemForNode';
14
- import { makeKnowledgeSourceHandler } from '../../_common/utils/makeKnowledgeSourceHandler';
15
- import { MarkdownScraper } from '../MarkdownScraper';
16
-
17
- playground()
18
- .catch((error) => {
19
- console.error(colors.bgRed(error.name || 'NamelessError'));
20
- console.error(error);
21
- process.exit(1);
22
- })
23
- .then(() => {
24
- process.exit(0);
25
- });
26
-
27
- /**
28
- * Handles playground.
29
- */
30
- async function playground() {
31
- console.info(`🧸 Scrape knowledge from Markdown (playground)`);
32
-
33
- // Do here stuff you want to test
34
- //========================================>
35
-
36
- const example = '10-simple.md';
37
- // <- TODO: [👩🏿‍🤝‍👩🏼] Read here the examples directory and itterate through all of them
38
-
39
- const llmTools = await $provideLlmToolsForTestingAndScriptsAndPlayground({ isCacheReloaded: true });
40
- const rootDirname = join(__dirname, '..', 'examples');
41
-
42
- const markdownScraper = new MarkdownScraper(
43
- { llm: await $provideLlmToolsForTestingAndScriptsAndPlayground() },
44
- {
45
- rootDirname,
46
- },
47
- );
48
-
49
- const knowledge = await markdownScraper.scrape(
50
- await makeKnowledgeSourceHandler(
51
- { knowledgeSourceContent: example },
52
- { fs: $provideFilesystemForNode() },
53
- { rootDirname },
54
- ),
55
- );
56
-
57
- console.info(colors.cyan(usageToHuman(llmTools.getTotalUsage())));
58
- console.info(colors.bgGreen(' Knowledge: '));
59
- console.info(knowledge);
60
-
61
- await writeFile(
62
- join(
63
- __dirname,
64
- `../examples/${example}.knowledge.json` /* <- TODO: [👩🏿‍🤝‍👩🏼] Read here the examples directory and itterate through all of them */,
65
- ),
66
- stringifyPipelineJson(knowledge),
67
- 'utf-8',
68
- );
69
- /**/
70
-
71
- //========================================/
72
- }
73
-
74
- // Note: [⚫] Code for playground [markdown-scraper-playground](src/scrapers/markdown/playground/markdown-scraper-playground.ts) should never be published in any package
@@ -1,19 +0,0 @@
1
- {
2
- "compilerOptions": {
3
- "target": "esnext",
4
- "module": "commonjs",
5
- "downlevelIteration": true,
6
- "allowJs": true,
7
- "moduleResolution": "node",
8
- "forceConsistentCasingInFileNames": true,
9
- "noImplicitReturns": true,
10
- "noImplicitThis": true,
11
- "noImplicitAny": true,
12
- "strictNullChecks": true,
13
- "experimentalDecorators": true,
14
- "noUnusedLocals": false,
15
- "resolveJsonModule": true,
16
- "esModuleInterop": true
17
- },
18
- "exclude": ["node_modules"]
19
- }
@@ -1,132 +0,0 @@
1
- /*
2
- TODO: !! This test is not wotking in Jest:
3
-
4
-
5
- Jest (promptbook)
6
- npm warn cli npm v10.9.1 does not support Node.js v18.4.0. This version of npm supports the following node versions: `^18.17.0 || >=20.5.0`. You can find the latest version at https://nodejs.org/.
7
-
8
- > promptbook@0.84.0-0 test
9
- > jest --testLocationInResults --json --useStderr --outputFile C:\Users\me\AppData\Local\Temp\jest_runner_promptbook_me_2.json --testNamePattern how creating knowledge by MarkitdownScraper works should scrape simple information from a pdf file$ --no-coverage --reporters default --reporters c:\Users\me\.vscode\extensions\orta.vscode-jest-6.4.0\out\reporter.js --colors --watchAll=false --testPathPattern c:\\Users\\me\\work\\ai\\promptbook\\src\\scrapers\\markitdown\\MarkitdownScraper\.test\.ts
10
-
11
-
12
- > Test run "by-file-test-pattern:46" started at 1/22/2025, 4:30:15 PM <
13
-
14
- FAIL src/scrapers/markitdown/MarkitdownScraper.test.ts
15
- ● Test suite failed to run
16
-
17
- Jest encountered an unexpected token
18
-
19
- Jest failed to parse a file. This happens e.g. when your code or its dependencies use non-standard JavaScript syntax, or when Jest is not configured to support such syntax.
20
-
21
- Out of the box Jest supports Babel, which will be used to transform your files into valid JS based on your Babel configuration.
22
-
23
- By default "node_modules" folder is ignored by transformers.
24
-
25
- Here's what you can do:
26
- • If you are trying to use ECMAScript Modules, see https://jestjs.io/docs/ecmascript-modules for how to enable it.
27
- • If you are trying to use TypeScript, see https://jestjs.io/docs/getting-started#using-typescript
28
- • To have some of your "node_modules" files transformed, you can specify a custom "transformIgnorePatterns" in your config.
29
- • If you need a custom transformation specify a "transform" option in your config.
30
- • If you simply want to mock your non-JS modules (e.g. binary assets) you can stub them out with the "moduleNameMapper" config option.
31
-
32
- You'll find more details and examples of these config options in the docs:
33
- https://jestjs.io/docs/configuration
34
- For information about custom transformations, see:
35
- https://jestjs.io/docs/code-transformation
36
-
37
- Details:
38
-
39
- C:\Users\me\work\ai\promptbook\node_modules\eventsource-parser\stream.js:2
40
- export * from './dist/stream.esm'
41
- ^^^^^^
42
-
43
- SyntaxError: Unexpected token 'export'
44
-
45
- at Runtime.createScriptFromCode (node_modules/jest-runtime/build/index.js:1728:14)
46
- at Object.<anonymous> (node_modules/ai/node_modules/@ai-sdk/provider-utils/src/response-handler.ts:5:8)
47
-
48
- Test Suites: 1 failed, 1 total
49
- Tests: 0 total
50
- Snapshots: 0 total
51
- Time: 36.491 s
52
- Ran all test suites matching /c:\\Users\\me\\work\\ai\\promptbook\\src\\scrapers\\markitdown\\MarkitdownScraper\.test\.ts/i with tests matching "how creating knowledge by MarkitdownScraper works should scrape simple information from a pdf file$".
53
-
54
- > Test run "by-file-test-pattern:46" finished at 1/22/2025, 4:30:51 PM <
55
-
56
-
57
- > Test run "by-file-test-pattern:46" exited at 1/22/2025, 4:30:51 PM <
58
-
59
-
60
- */
61
-
62
- import { describe, expect, it } from '@jest/globals';
63
- import { join } from 'path';
64
- import { $provideExecutablesForNode } from '../../executables/$provideExecutablesForNode';
65
- import { $provideLlmToolsForTestingAndScriptsAndPlayground } from '../../llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground';
66
- import { $provideFilesystemForNode } from '../_common/register/$provideFilesystemForNode';
67
- import { makeKnowledgeSourceHandler } from '../_common/utils/makeKnowledgeSourceHandler';
68
- import { MarkitdownScraper } from './MarkitdownScraper';
69
-
70
- describe('how creating knowledge by MarkitdownScraper works', () => {
71
- const rootDirname = join(__dirname, 'examples');
72
-
73
- const markitdownScraperPromise = (async () =>
74
- new MarkitdownScraper(
75
- {
76
- fs: $provideFilesystemForNode(),
77
- llm: await $provideLlmToolsForTestingAndScriptsAndPlayground(),
78
- executables: await $provideExecutablesForNode(),
79
- },
80
- {
81
- rootDirname,
82
- },
83
- ))();
84
-
85
- it('should scrape simple information from a pdf file', () =>
86
- expect(
87
- Promise.all([
88
- markitdownScraperPromise,
89
- makeKnowledgeSourceHandler(
90
- {
91
- knowledgeSourceContent: '10-simple.pdf',
92
- },
93
- { fs: $provideFilesystemForNode() },
94
- { rootDirname },
95
- ),
96
- ])
97
- .then(([markitdownScraper, sourceHandler]) => markitdownScraper.scrape(sourceHandler))
98
- .then((knowledge) => knowledge?.map(({ content }) => ({ content })))
99
- .then((knowledge) => knowledge?.slice(0, 1)),
100
- ).resolves.toMatchObject([
101
- {
102
- content: expect.stringMatching(/Springfield (is )?.*/i),
103
- },
104
- ]));
105
-
106
- // TODO: !! Add all samples
107
-
108
- it('should NOT scrape irrelevant information', () =>
109
- expect(
110
- Promise.all([
111
- markitdownScraperPromise,
112
- makeKnowledgeSourceHandler(
113
- {
114
- knowledgeSourceContent: '10-simple.pdf',
115
- },
116
- { fs: $provideFilesystemForNode() },
117
- { rootDirname },
118
- ),
119
- ])
120
- .then(([markitdownScraper, sourceHandler]) => markitdownScraper.scrape(sourceHandler))
121
- .then((knowledge) => knowledge?.map(({ content }) => ({ content })))
122
- .then((knowledge) => knowledge?.slice(0, 1)),
123
- ).resolves.toMatchObject([
124
- {
125
- content: expect.not.stringMatching(/London (is )?.*/i),
126
- },
127
- ]));
128
- });
129
-
130
- /**
131
- * TODO: [📓] Maybe test all file in examples (not just one by one)
132
- */
@@ -1,91 +0,0 @@
1
- #!/usr/bin/env ts-node
2
-
3
- import * as dotenv from 'dotenv';
4
-
5
- dotenv.config({ path: '.env' });
6
-
7
- import colors from 'colors'; // <- TODO: [🔶] Make system to put color and style to both node and browser
8
- import { writeFile } from 'fs/promises';
9
- import { join } from 'path';
10
- import { $provideExecutablesForNode } from '../../../executables/$provideExecutablesForNode';
11
- import { usageToHuman } from '../../../execution/utils/usageToHuman';
12
- import { $provideLlmToolsForTestingAndScriptsAndPlayground } from '../../../llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground';
13
- import { stringifyPipelineJson } from '../../../utils/editable/utils/stringifyPipelineJson';
14
- import { titleToName } from '../../../utils/normalization/titleToName';
15
- import { isValidUrl } from '../../../utils/validators/url/isValidUrl';
16
- import { $provideFilesystemForNode } from '../../_common/register/$provideFilesystemForNode';
17
- import { makeKnowledgeSourceHandler } from '../../_common/utils/makeKnowledgeSourceHandler';
18
- import { MarkitdownScraper } from '../MarkitdownScraper';
19
-
20
- playground()
21
- .catch((error) => {
22
- console.error(colors.bgRed(error.name || 'NamelessError'));
23
- console.error(error);
24
- process.exit(1);
25
- })
26
- .then(() => {
27
- process.exit(0);
28
- });
29
-
30
- /**
31
- * Handles playground.
32
- */
33
- async function playground() {
34
- console.info(`🧸 Scrape knowledge by Markitdown (playground)`);
35
-
36
- // Do here stuff you want to test
37
- //========================================>
38
-
39
- // const example = '10-simple.pdf';
40
- // const example = 'https://collboard.fra1.cdn.digitaloceanspaces.com/ptbk/user/knowledge-source/4/c/pravidla-pro-zadatele-a-prijemce-3-0.pdf';
41
- const example = 'https://collboard.fra1.cdn.digitaloceanspaces.com/ptbk/user/knowledge-source/b/d/10-simple.pdf';
42
- // const example = 'https://collboard.fra1.cdn.digitaloceanspaces.com/ptbk/user/knowledge-source/4/3/10-simple.odt';
43
- // const example = 'https://collboard.fra1.cdn.digitaloceanspaces.com/ptbk/user/knowledge-source/4/3/10-simple.odt';
44
- // const example = 'https://collboard.fra1.cdn.digitaloceanspaces.com/ptbk/user/knowledge-source/7/f/10-simple.docx';
45
- // const example = 'https://collboard.fra1.cdn.digitaloceanspaces.com/ptbk/user/knowledge-source/2/9/10-simple.rtf';
46
- // const example = 'https://collboard.fra1.cdn.digitaloceanspaces.com/ptbk/user/knowledge-source/f/3/10-simple.doc';
47
-
48
- // <- TODO: [👩🏿‍🤝‍👩🏼] Read here the examples directory and itterate through all of them
49
-
50
- const llmTools = await $provideLlmToolsForTestingAndScriptsAndPlayground({ isCacheReloaded: true });
51
- const rootDirname = isValidUrl(example) ? null : join(__dirname, '..', 'examples');
52
-
53
- const markitdownScraper = new MarkitdownScraper(
54
- {
55
- fs: $provideFilesystemForNode(),
56
- llm: await $provideLlmToolsForTestingAndScriptsAndPlayground(),
57
- executables: await $provideExecutablesForNode(),
58
- },
59
- {
60
- rootDirname,
61
- },
62
- );
63
-
64
- const knowledge = await markitdownScraper.scrape(
65
- await makeKnowledgeSourceHandler(
66
- { knowledgeSourceContent: example },
67
- { fs: $provideFilesystemForNode() },
68
- { rootDirname },
69
- ),
70
- );
71
-
72
- console.info(colors.cyan(usageToHuman(llmTools.getTotalUsage())));
73
- console.info(colors.bgGreen(' Knowledge: '));
74
- console.info(knowledge);
75
-
76
- await writeFile(
77
- join(
78
- __dirname,
79
- `../examples/${
80
- isValidUrl(example) ? titleToName(example) : example
81
- }.knowledge.json` /* <- TODO: [👩🏿‍🤝‍👩🏼] Read here the examples directory and itterate through all of them */,
82
- ),
83
- stringifyPipelineJson(knowledge),
84
- 'utf-8',
85
- );
86
- /**/
87
-
88
- //========================================/
89
- }
90
-
91
- // Note: [⚫] Code for playground [markitdown-scraper-playground](src/scrapers/markitdown/playground/markitdown-scraper-playground.ts) should never be published in any package
@@ -1,19 +0,0 @@
1
- {
2
- "compilerOptions": {
3
- "target": "esnext",
4
- "module": "commonjs",
5
- "downlevelIteration": true,
6
- "allowJs": true,
7
- "moduleResolution": "node",
8
- "forceConsistentCasingInFileNames": true,
9
- "noImplicitReturns": true,
10
- "noImplicitThis": true,
11
- "noImplicitAny": true,
12
- "strictNullChecks": true,
13
- "experimentalDecorators": true,
14
- "noUnusedLocals": false,
15
- "resolveJsonModule": true,
16
- "esModuleInterop": true
17
- },
18
- "exclude": ["node_modules"]
19
- }
@@ -1,52 +0,0 @@
1
- import { describe, expect, it } from '@jest/globals';
2
- import { join } from 'path';
3
- import { $provideLlmToolsForTestingAndScriptsAndPlayground } from '../../llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground';
4
- import { makeKnowledgeSourceHandler } from '../_common/utils/makeKnowledgeSourceHandler';
5
- import { pdfScraper } from './PdfScraper';
6
-
7
- // TODO: [📇] Uncomment this test
8
-
9
- describe('how creating knowledge from pdf works', () => {
10
-
11
- const rootDirname = join(__dirname, 'examples');
12
- const pdfScraper = new PdfScraper(
13
- { llm: await $provideLlmToolsForTestingAndScriptsAndPlayground() },
14
- {
15
- rootDirname,
16
- },
17
- );
18
-
19
- it('should scrape simple information from a .pdf file', () =>
20
- expect(
21
- Promise.resolve()
22
- .then(() => makeKnowledgeSourceHandler(join(__dirname, 'examples/10-simple.pdf'),{rootDirname}))
23
- .then((sourceHandler) =>
24
- pdfScraper.scrape(sourceHandler),
25
- )
26
- .then((knowledge) => knowledge?.map(({ content }) => ({ content })))
27
- .then((knowledge) => knowledge?.slice(0, 1)),
28
- ).resolves.toMatchObject([
29
- {
30
- content: expect.stringMatching(/Springfield (is )?.*/i),
31
- },
32
- ]));
33
-
34
- it('should NOT scrape irrelevant information', () =>
35
- expect(
36
- Promise.resolve()
37
- .then(() => makeKnowledgeSourceHandler(join(__dirname, 'examples/10-simple.pdf'),{rootDirname}))
38
- .then((sourceHandler) =>
39
- pdfScraper.scrape(sourceHandler),
40
- )
41
- .then((knowledge) => knowledge?.map(({ content }) => ({ content })))
42
- .then((knowledge) => knowledge?.slice(0, 1)),
43
- ).resolves.toMatchObject([
44
- {
45
- content: expect.not.stringMatching(/London (is )?.*/i),
46
- },
47
- ]));
48
- });
49
-
50
- /**
51
- * TODO: [📓] Maybe test all file in examples (not just 10-simple.pdf)
52
- */
@@ -1,75 +0,0 @@
1
- #!/usr/bin/env ts-node
2
-
3
- import * as dotenv from 'dotenv';
4
-
5
- dotenv.config({ path: '.env' });
6
-
7
- import colors from 'colors'; // <- TODO: [🔶] Make system to put color and style to both node and browser
8
- import { writeFile } from 'fs/promises';
9
- import { join } from 'path';
10
- import { usageToHuman } from '../../../execution/utils/usageToHuman';
11
- import { $provideLlmToolsForTestingAndScriptsAndPlayground } from '../../../llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground';
12
- import { stringifyPipelineJson } from '../../../utils/editable/utils/stringifyPipelineJson';
13
- import { $provideFilesystemForNode } from '../../_common/register/$provideFilesystemForNode';
14
- import { makeKnowledgeSourceHandler } from '../../_common/utils/makeKnowledgeSourceHandler';
15
- import { PdfScraper } from '../PdfScraper';
16
-
17
- playground()
18
- .catch((error) => {
19
- console.error(colors.bgRed(error.name || 'NamelessError'));
20
- console.error(error);
21
- process.exit(1);
22
- })
23
- .then(() => {
24
- process.exit(0);
25
- });
26
-
27
- /**
28
- * Handles playground.
29
- */
30
- async function playground() {
31
- console.info(`🧸 Scrape knowledge from PDFs (playground)`);
32
-
33
- // Do here stuff you want to test
34
- //========================================>
35
-
36
- //const example = '10-simple.doc';
37
- const example = '10-simple.pdf';
38
- // <- TODO: [👩🏿‍🤝‍👩🏼] Read here the examples directory and itterate through all of them
39
-
40
- const llmTools = await $provideLlmToolsForTestingAndScriptsAndPlayground({ isCacheReloaded: true });
41
- const rootDirname = join(__dirname, '..', 'examples');
42
-
43
- const pdfScraper = new PdfScraper(
44
- { llm: await $provideLlmToolsForTestingAndScriptsAndPlayground() },
45
- {
46
- rootDirname,
47
- },
48
- );
49
-
50
- const knowledge = await pdfScraper.scrape(
51
- await makeKnowledgeSourceHandler(
52
- { knowledgeSourceContent: example },
53
- { fs: $provideFilesystemForNode() },
54
- { rootDirname },
55
- ),
56
- );
57
-
58
- console.info(colors.cyan(usageToHuman(llmTools.getTotalUsage())));
59
- console.info(colors.bgGreen(' Knowledge: '));
60
- console.info(knowledge);
61
-
62
- await writeFile(
63
- join(
64
- __dirname,
65
- `../examples/${example}.knowledge.json` /* <- TODO: [👩🏿‍🤝‍👩🏼] Read here the examples directory and itterate through all of them */,
66
- ),
67
- stringifyPipelineJson(knowledge),
68
- 'utf-8',
69
- );
70
- /**/
71
-
72
- //========================================/
73
- }
74
-
75
- // Note: [⚫] Code for playground [pdf-scraper-playground](src/scrapers/pdf/playground/pdf-scraper-playground.ts) should never be published in any package
@@ -1,19 +0,0 @@
1
- {
2
- "compilerOptions": {
3
- "target": "esnext",
4
- "module": "commonjs",
5
- "downlevelIteration": true,
6
- "allowJs": true,
7
- "moduleResolution": "node",
8
- "forceConsistentCasingInFileNames": true,
9
- "noImplicitReturns": true,
10
- "noImplicitThis": true,
11
- "noImplicitAny": true,
12
- "strictNullChecks": true,
13
- "experimentalDecorators": true,
14
- "noUnusedLocals": false,
15
- "resolveJsonModule": true,
16
- "esModuleInterop": true
17
- },
18
- "exclude": ["node_modules"]
19
- }
@@ -1,19 +0,0 @@
1
- {
2
- "compilerOptions": {
3
- "target": "esnext",
4
- "module": "commonjs",
5
- "downlevelIteration": true,
6
- "allowJs": true,
7
- "moduleResolution": "node",
8
- "forceConsistentCasingInFileNames": true,
9
- "noImplicitReturns": true,
10
- "noImplicitThis": true,
11
- "noImplicitAny": true,
12
- "strictNullChecks": true,
13
- "experimentalDecorators": true,
14
- "noUnusedLocals": false,
15
- "resolveJsonModule": true,
16
- "esModuleInterop": true
17
- },
18
- "exclude": ["node_modules"]
19
- }
@@ -1,82 +0,0 @@
1
- #!/usr/bin/env ts-node
2
-
3
- import * as dotenv from 'dotenv';
4
-
5
- dotenv.config({ path: '.env' });
6
-
7
- import colors from 'colors'; // <- TODO: [🔶] Make system to put color and style to both node and browser
8
- import { writeFile } from 'fs/promises';
9
- import { join } from 'path';
10
- import { usageToHuman } from '../../../execution/utils/usageToHuman';
11
- import { $provideLlmToolsForTestingAndScriptsAndPlayground } from '../../../llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground';
12
- import { stringifyPipelineJson } from '../../../utils/editable/utils/stringifyPipelineJson';
13
- import { titleToName } from '../../../utils/normalization/titleToName';
14
- import { $provideFilesystemForNode } from '../../_common/register/$provideFilesystemForNode';
15
- import { makeKnowledgeSourceHandler } from '../../_common/utils/makeKnowledgeSourceHandler';
16
- import { WebsiteScraper } from '../WebsiteScraper';
17
-
18
- playground()
19
- .catch((error) => {
20
- console.error(colors.bgRed(error.name || 'NamelessError'));
21
- console.error(error);
22
- process.exit(1);
23
- })
24
- .then(() => {
25
- process.exit(0);
26
- });
27
-
28
- /**
29
- * Handles playground.
30
- */
31
- async function playground() {
32
- console.info(`🧸 Scrape knowledge from websites (playground)`);
33
-
34
- // Do here stuff you want to test
35
- //========================================>
36
-
37
- //const example = 'https://www.pavolhejny.com/'; // <- TODO: Not scraping really important information, just one-two paragraph
38
- const example = 'https://koralkykatlas.cz/cs/blog/prispevek/-rijna-zhorseni-kvality-kovove-bizuterie.html';
39
- // <- TODO: [👩🏿‍🤝‍👩🏼] Read here website-scraper-playground.ts and itterate
40
-
41
- const llmTools = await $provideLlmToolsForTestingAndScriptsAndPlayground({ isCacheReloaded: true });
42
- const rootDirname = join(__dirname, 'examples');
43
-
44
- const websiteScraper = new WebsiteScraper(
45
- { llm: await $provideLlmToolsForTestingAndScriptsAndPlayground(), fs: $provideFilesystemForNode() },
46
- {
47
- rootDirname,
48
- },
49
- );
50
-
51
- const source = await makeKnowledgeSourceHandler(
52
- { knowledgeSourceContent: example },
53
- { fs: $provideFilesystemForNode() },
54
- { rootDirname },
55
- );
56
-
57
- const converted = await websiteScraper.$convert(source);
58
-
59
- console.info(colors.bgGreen(' Converted: '), converted);
60
-
61
- const knowledge = await websiteScraper.scrape(source);
62
-
63
- console.info(colors.cyan(usageToHuman(llmTools.getTotalUsage())));
64
- console.info(colors.bgGreen(' Knowledge: '));
65
- console.info(knowledge);
66
-
67
- await writeFile(
68
- join(
69
- __dirname,
70
- `../examples/${titleToName(
71
- example,
72
- )}.knowledge.json` /* <- TODO: [👩🏿‍🤝‍👩🏼] Read here the examples directory and itterate through all of them */,
73
- ),
74
- stringifyPipelineJson(knowledge),
75
- 'utf-8',
76
- );
77
- /**/
78
-
79
- //========================================/
80
- }
81
-
82
- // Note: [⚫] Code for playground [website-scraper-playground](src/scrapers/website/playground/website-scraper-playground.ts) should never be published in any package
@@ -1,14 +0,0 @@
1
- import { keepUnused } from '../../utils/organization/keepUnused';
2
- import type { PromptbookStorage } from './PromptbookStorage';
3
-
4
- // Note: Checking that `localStorage` and `sessionStorage` are implementing `PromptbookStorage<string>`
5
- /**
6
- * Constant for storage.
7
- */
8
- let storage: PromptbookStorage<string>;
9
- storage = localStorage;
10
- storage = sessionStorage;
11
- keepUnused(storage);
12
-
13
- // Note: [⚪] This should never be in any released package
14
- // TODO: Is this a good pattern to do type testing?
@@ -1,36 +0,0 @@
1
- import { EnvironmentMismatchError } from '../../errors/EnvironmentMismatchError';
2
- import { $isRunningInBrowser } from '../../utils/environment/$isRunningInBrowser';
3
- import type { chococake } from '../../utils/organization/really_any';
4
- import type { PromptbookStorage } from '../_common/PromptbookStorage';
5
- import type { IndexedDbStorageOptions } from './utils/IndexedDbStorageOptions';
6
- import { makePromptbookStorageFromIndexedDb } from './utils/makePromptbookStorageFromIndexedDb';
7
-
8
- /**
9
- * Cache storage
10
- *
11
- * @private internal cache for `getIndexedDbStorage`
12
- */
13
- const indexedDbStorageCache = new Map<string, PromptbookStorage<chococake>>();
14
-
15
- /**
16
- * Gets wrapper around IndexedDB which can be used as PromptbookStorage
17
- *
18
- * @public exported from `@promptbook/browser`
19
- */
20
- export function getIndexedDbStorage<TItem>(options: IndexedDbStorageOptions): PromptbookStorage<TItem> {
21
- if (!$isRunningInBrowser()) {
22
- throw new EnvironmentMismatchError(`You can get IndexedDB storage only in browser environment`);
23
- }
24
-
25
- const { databaseName, storeName } = options;
26
- const cacheKey = `${databaseName}/${storeName}`;
27
- if (indexedDbStorageCache.has(cacheKey)) {
28
- return indexedDbStorageCache.get(cacheKey) as PromptbookStorage<TItem>;
29
- }
30
-
31
- const storage = makePromptbookStorageFromIndexedDb<TItem>({ databaseName, storeName });
32
- indexedDbStorageCache.set(cacheKey, storage as PromptbookStorage<TItem>);
33
- return storage;
34
- }
35
-
36
- // Note: [🔵] Code for browser storage helper [getIndexedDbStorage](src/storage/local-storage/getIndexedDbStorage.ts) should never be published outside of `@promptbook/browser`