@promptbook/cli 0.112.0-103 → 0.112.0-105

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (190) hide show
  1. package/apps/agents-server/src/app/AddAgentButton.tsx +0 -5
  2. package/apps/agents-server/src/app/actions.ts +50 -0
  3. package/apps/agents-server/src/app/admin/image-generator-test/ImageAttachmentsEditor.tsx +19 -3
  4. package/apps/agents-server/src/app/admin/limits/LimitsClient.tsx +11 -12
  5. package/apps/agents-server/src/app/admin/metadata/MetadataClient.tsx +34 -2
  6. package/apps/agents-server/src/app/admin/servers/CreateServerDialog.tsx +6 -1
  7. package/apps/agents-server/src/app/admin/servers/useCreateServerWizard.ts +13 -1
  8. package/apps/agents-server/src/app/agents/[agentName]/AgentChatWrapper.tsx +11 -2
  9. package/apps/agents-server/src/app/agents/[agentName]/AgentProfileChat.tsx +14 -5
  10. package/apps/agents-server/src/app/agents/[agentName]/book/BookEditorWrapper.tsx +7 -1
  11. package/apps/agents-server/src/app/agents/[agentName]/chat/CanonicalAgentChatSurface.tsx +11 -1
  12. package/apps/agents-server/src/app/agents/[agentName]/images/default-avatar.png/route.ts +6 -2
  13. package/apps/agents-server/src/app/api/health/route.ts +18 -0
  14. package/apps/agents-server/src/app/api/images/[filename]/route.ts +6 -2
  15. package/apps/agents-server/src/app/api/internal/agent-runner-limits/route.ts +51 -0
  16. package/apps/agents-server/src/app/api/upload/route.ts +48 -12
  17. package/apps/agents-server/src/app/layout.tsx +13 -0
  18. package/apps/agents-server/src/components/AgentProfile/AgentProfile.tsx +1 -4
  19. package/apps/agents-server/src/components/FileUploadAvailability/FileUploadAvailabilityContext.tsx +50 -0
  20. package/apps/agents-server/src/components/FileUploadAvailability/FileUploadUnavailableNotice.tsx +45 -0
  21. package/apps/agents-server/src/components/Header/Header.tsx +0 -11
  22. package/apps/agents-server/src/components/Header/useHeaderAgentMenus.tsx +0 -5
  23. package/apps/agents-server/src/components/LayoutWrapper/LayoutWrapper.tsx +85 -76
  24. package/apps/agents-server/src/components/NewAgentDialog/NewAgentDialog.tsx +7 -3
  25. package/apps/agents-server/src/components/NewAgentDialog/NewAgentWizardKnowledgeStep.tsx +6 -0
  26. package/apps/agents-server/src/components/NewAgentDialog/useNewAgentDialog.tsx +39 -16
  27. package/apps/agents-server/src/components/NewAgentDialog/useNewAgentWizardKnowledgeState.ts +8 -1
  28. package/apps/agents-server/src/constants/defaultAgentAvatarVisual.ts +1 -1
  29. package/apps/agents-server/src/constants/serverLimits.ts +22 -2
  30. package/apps/agents-server/src/database/migrations/2026-06-0200-default-agent-avatar-visual-octopus3d3.sql +16 -0
  31. package/apps/agents-server/src/database/seedDefaultAgents.ts +218 -0
  32. package/apps/agents-server/src/middleware.ts +2 -1
  33. package/apps/agents-server/src/tools/$provideCdnForServer.ts +114 -9
  34. package/apps/agents-server/src/utils/agentRouting/resolveAgentRouteTarget.ts +27 -4
  35. package/apps/agents-server/src/utils/defaultAgents/loadDefaultAgentBooks.ts +103 -0
  36. package/apps/agents-server/src/utils/knowledge/createInlineKnowledgeSourceUploader.ts +24 -5
  37. package/apps/agents-server/src/utils/serverLimits.ts +26 -1
  38. package/apps/agents-server/src/utils/serverManagement/createManagedServer/seedServerDefaultAgents.ts +1 -85
  39. package/apps/agents-server/src/utils/shareTargetPayloads.ts +20 -2
  40. package/apps/agents-server/src/utils/upload/fileUploadAvailability.ts +91 -0
  41. package/apps/agents-server/src/utils/upload/uploadFileToServer.ts +46 -2
  42. package/esm/apps/agents-server/src/constants/federatedAgentImport.d.ts +42 -0
  43. package/esm/apps/agents-server/src/constants/serverLimits.d.ts +207 -0
  44. package/esm/apps/agents-server/src/constants/toolUsageLimits.d.ts +55 -0
  45. package/esm/index.es.js +1109 -35
  46. package/esm/index.es.js.map +1 -1
  47. package/esm/scripts/run-agent-messages/main/AgentMessageFailureTracker.d.ts +27 -0
  48. package/esm/scripts/run-agent-messages/main/handleAgentWatchError.d.ts +4 -0
  49. package/esm/scripts/run-agent-messages/main/runAgentMessages.d.ts +1 -0
  50. package/esm/scripts/run-agent-messages/messages/moveAgentMessageToFailed.d.ts +17 -0
  51. package/esm/src/avatars/types/AvatarVisualDefinition.d.ts +1 -1
  52. package/esm/src/avatars/visuals/octopus3d3AvatarVisual.d.ts +7 -0
  53. package/esm/src/book-components/BookEditor/BookEditor.d.ts +5 -4
  54. package/esm/src/book-components/BookEditor/BookEditorTheme.d.ts +24 -0
  55. package/esm/src/book-components/BookEditor/useBookEditorMonacoLanguage.d.ts +1 -6
  56. package/esm/src/book-components/BookEditor/useBookEditorMonacoLifecycle.d.ts +1 -4
  57. package/esm/src/book-components/BookEditor/useBookEditorMonacoStyles.d.ts +2 -1
  58. package/esm/src/cli/cli-commands/agent-folder/agentProjectPaths.d.ts +6 -0
  59. package/esm/src/version.d.ts +1 -1
  60. package/package.json +1 -1
  61. package/src/avatars/types/AvatarVisualDefinition.ts +1 -0
  62. package/src/avatars/visuals/avatarVisualRegistry.ts +2 -0
  63. package/src/avatars/visuals/octopus3d3AvatarVisual.ts +902 -0
  64. package/src/book-components/BookEditor/BookEditor.tsx +10 -7
  65. package/src/book-components/BookEditor/BookEditorMonaco.tsx +3 -1
  66. package/src/book-components/BookEditor/BookEditorTheme.ts +32 -0
  67. package/src/book-components/BookEditor/useBookEditorMonacoLanguage.ts +12 -15
  68. package/src/book-components/BookEditor/useBookEditorMonacoLifecycle.ts +1 -5
  69. package/src/book-components/BookEditor/useBookEditorMonacoStyles.ts +2 -1
  70. package/src/cli/cli-commands/agent-folder/agentProjectPaths.ts +7 -0
  71. package/src/cli/cli-commands/agents-server/buildAgentsServer.ts +109 -9
  72. package/src/cli/cli-commands/agents-server/startAgentsServer.ts +132 -4
  73. package/src/other/templates/getTemplatesPipelineCollection.ts +690 -747
  74. package/src/utils/agents/resolveAgentAvatarImageUrl.ts +1 -1
  75. package/src/version.ts +2 -2
  76. package/src/versions.txt +2 -1
  77. package/umd/apps/agents-server/src/constants/federatedAgentImport.d.ts +42 -0
  78. package/umd/apps/agents-server/src/constants/serverLimits.d.ts +207 -0
  79. package/umd/apps/agents-server/src/constants/toolUsageLimits.d.ts +55 -0
  80. package/umd/index.umd.js +1109 -35
  81. package/umd/index.umd.js.map +1 -1
  82. package/umd/scripts/run-agent-messages/main/AgentMessageFailureTracker.d.ts +27 -0
  83. package/umd/scripts/run-agent-messages/main/handleAgentWatchError.d.ts +4 -0
  84. package/umd/scripts/run-agent-messages/main/runAgentMessages.d.ts +1 -0
  85. package/umd/scripts/run-agent-messages/messages/moveAgentMessageToFailed.d.ts +17 -0
  86. package/umd/src/avatars/types/AvatarVisualDefinition.d.ts +1 -1
  87. package/umd/src/avatars/visuals/octopus3d3AvatarVisual.d.ts +7 -0
  88. package/umd/src/book-components/BookEditor/BookEditor.d.ts +5 -4
  89. package/umd/src/book-components/BookEditor/BookEditorTheme.d.ts +24 -0
  90. package/umd/src/book-components/BookEditor/useBookEditorMonacoLanguage.d.ts +1 -6
  91. package/umd/src/book-components/BookEditor/useBookEditorMonacoLifecycle.d.ts +1 -4
  92. package/umd/src/book-components/BookEditor/useBookEditorMonacoStyles.d.ts +2 -1
  93. package/umd/src/cli/cli-commands/agent-folder/agentProjectPaths.d.ts +6 -0
  94. package/umd/src/version.d.ts +1 -1
  95. package/apps/agents-server/src/message-providers/email/_common/utils/parseEmailAddress.test.ts.todo +0 -108
  96. package/apps/agents-server/src/message-providers/email/_common/utils/parseEmailAddresses.test.ts.todo +0 -117
  97. package/apps/agents-server/src/message-providers/email/_common/utils/stringifyEmailAddress.test.ts.todo +0 -119
  98. package/apps/agents-server/src/message-providers/email/_common/utils/stringifyEmailAddresses.test.ts.todo +0 -74
  99. package/apps/agents-server/tests/e2e/authentication-and-navigation.spec.ts.todo +0 -178
  100. package/src/_packages/browser.index.ts +0 -31
  101. package/src/_packages/browser.readme.md +0 -43
  102. package/src/book-2.0/agent-source/parseAgentSourceWithCommitments.test.ts.todo +0 -265
  103. package/src/book-components/BookEditor/BookEditorMonaco.test.tsx.todo +0 -115
  104. package/src/book-components/Chat/utils/renderMarkdown.test.ts.tmp +0 -199
  105. package/src/collection/agent-collection/constructors/agent-collection-in-directory/AgentCollectionInDirectory.test.ts.todo +0 -131
  106. package/src/commands/_common/parseCommand.test.ts.todo +0 -48
  107. package/src/commitments/META_LINK/META_LINK.test.ts.todo +0 -75
  108. package/src/conversion/validation/pipelineStringToJson-errors.test.ts.todo +0 -33
  109. package/src/dialogs/simple-prompt/SimplePromptInterfaceTools.ts +0 -51
  110. package/src/executables/browsers/locateSafari.test.ts.tmp +0 -15
  111. package/src/execution/PromptbookFetch.test-type.ts +0 -14
  112. package/src/execution/createPipelineExecutor/00-createPipelineExecutor.test.ts.todo +0 -0
  113. package/src/execution/execution-report/executionReportJsonToString.test.ts.todo +0 -83
  114. package/src/execution/utils/usageToHuman.test.ts.todo +0 -80
  115. package/src/llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground.ts +0 -76
  116. package/src/llm-providers/_common/utils/assertUniqueModels.ts +0 -27
  117. package/src/llm-providers/_multiple/playground/playground.ts +0 -141
  118. package/src/llm-providers/_multiple/playground/tsconfig.json +0 -19
  119. package/src/llm-providers/agent/playground/playground.ts +0 -190
  120. package/src/llm-providers/agent/playground/tsconfig.json +0 -19
  121. package/src/llm-providers/anthropic-claude/playground/playground.ts +0 -99
  122. package/src/llm-providers/anthropic-claude/playground/tsconfig.json +0 -19
  123. package/src/llm-providers/azure-openai/playground/playground.ts +0 -101
  124. package/src/llm-providers/azure-openai/playground/tsconfig.json +0 -19
  125. package/src/llm-providers/ollama/playground/playground.ts +0 -120
  126. package/src/llm-providers/ollama/playground/tsconfig.json +0 -19
  127. package/src/llm-providers/openai/playground/playground.ts +0 -406
  128. package/src/llm-providers/openai/playground/tsconfig.json +0 -19
  129. package/src/llm-providers/remote/playground/playground.ts +0 -144
  130. package/src/llm-providers/remote/playground/tsconfig.json +0 -19
  131. package/src/llm-providers/vercel/playground/playground.ts +0 -133
  132. package/src/llm-providers/vercel/playground/tsconfig.json +0 -19
  133. package/src/personas/preparePersona.test.ts.todo +0 -126
  134. package/src/playground/backup/_playground-boilerplate.ts.txt +0 -37
  135. package/src/playground/backup/playground-agent-os.txt +0 -62
  136. package/src/playground/backup/playground-brj-app.ts.txt +0 -302
  137. package/src/playground/backup/playground-browser-playwright.txt +0 -110
  138. package/src/playground/backup/playground-claude-mcp.txt +0 -43
  139. package/src/playground/backup/playground-document-conversion.txt +0 -84
  140. package/src/playground/backup/playground-glob.ts.txt +0 -42
  141. package/src/playground/backup/playground-mcp-server.txt +0 -1
  142. package/src/playground/backup/playground-openai-agent-kit.txt +0 -73
  143. package/src/playground/backup/playground-openai-function-calling.txt +0 -131
  144. package/src/playground/backup/playground-openai-streaming.ts.txt +0 -68
  145. package/src/playground/backup/playground-scrape-knowledge.txt +0 -65
  146. package/src/playground/backup/playground-scraperFetch.ts.txt +0 -44
  147. package/src/playground/backup/playground-using-openai-compatible-route-on-agents-server.ts.txt +0 -49
  148. package/src/playground/backup/playground-write-pavolhejny-bio.txt +0 -120
  149. package/src/playground/permanent/_boilerplate.ts +0 -54
  150. package/src/playground/permanent/agent-with-browser-playground.ts +0 -92
  151. package/src/playground/permanent/error-handling-playground.ts +0 -103
  152. package/src/playground/playground.ts +0 -36
  153. package/src/playground/tsconfig.json +0 -19
  154. package/src/scrapers/_boilerplate/BoilerplateScraper.test.ts.todo +0 -73
  155. package/src/scrapers/_boilerplate/playground/boilerplate-scraper-playground.ts +0 -79
  156. package/src/scrapers/_boilerplate/playground/tsconfig.json +0 -19
  157. package/src/scrapers/_common/utils/files/blobToDataurl.test.ts.todo +0 -17
  158. package/src/scrapers/_common/utils/files/dataurlToBlob.test.ts.todo +0 -52
  159. package/src/scrapers/_common/utils/files/isValidDataurl.test.ts.todo +0 -42
  160. package/src/scrapers/_common/utils/files/shorten.test.ts.todo +0 -13
  161. package/src/scrapers/document/playground/document-scraper-playground.ts +0 -80
  162. package/src/scrapers/document/playground/tsconfig.json +0 -19
  163. package/src/scrapers/document-legacy/playground/legacy-document-scraper-playground.ts +0 -80
  164. package/src/scrapers/document-legacy/playground/tsconfig.json +0 -19
  165. package/src/scrapers/markdown/playground/markdown-scraper-playground.ts +0 -74
  166. package/src/scrapers/markdown/playground/tsconfig.json +0 -19
  167. package/src/scrapers/markitdown/MarkitdownScraper.test.ts.todo +0 -132
  168. package/src/scrapers/markitdown/playground/markitdown-scraper-playground.ts +0 -91
  169. package/src/scrapers/markitdown/playground/tsconfig.json +0 -19
  170. package/src/scrapers/pdf/PdfScraper.test.ts.todo +0 -52
  171. package/src/scrapers/pdf/playground/pdf-scraper-playground.ts +0 -75
  172. package/src/scrapers/pdf/playground/tsconfig.json +0 -19
  173. package/src/scrapers/website/playground/tsconfig.json +0 -19
  174. package/src/scrapers/website/playground/website-scraper-playground.ts +0 -82
  175. package/src/storage/_common/PromptbookStorage.test-type.ts +0 -14
  176. package/src/storage/local-storage/getIndexedDbStorage.ts +0 -36
  177. package/src/storage/local-storage/getLocalStorage.ts +0 -33
  178. package/src/storage/local-storage/getSessionStorage.ts +0 -33
  179. package/src/storage/local-storage/utils/IndexedDbStorageOptions.ts +0 -16
  180. package/src/storage/local-storage/utils/makePromptbookStorageFromIndexedDb.ts +0 -58
  181. package/src/storage/local-storage/utils/makePromptbookStorageFromWebStorage.ts +0 -45
  182. package/src/transpilers/formatted-book-in-markdown/FormattedBookInMarkdownTranspiler.test.ts.todo +0 -35
  183. package/src/transpilers/openai-sdk/playground/playground.ts +0 -85
  184. package/src/transpilers/openai-sdk/playground/tmp/chatbot-openaisdk-1.js +0 -194
  185. package/src/transpilers/openai-sdk/playground/tmp/package.json +0 -3
  186. package/src/transpilers/openai-sdk/playground/tsconfig.json +0 -18
  187. package/src/utils/editable/utils/findUsableParameters.test.ts.todo +0 -43
  188. package/src/utils/editable/utils/stringifyPipelineJson.test.ts.todo +0 -38
  189. package/src/utils/markdown/prettifyMarkdown.test.ts.tmp +0 -42
  190. package/src/utils/serialization/serializeToPromptbookJavascript.test.ts.todo +0 -116
@@ -1,103 +0,0 @@
1
- #!/usr/bin/env ts-node
2
-
3
- // Note: [❌] Turning off some global checks for playground file:
4
- // spell-checker: disable
5
- /* eslint-disable */
6
-
7
- import * as dotenv from 'dotenv';
8
-
9
- dotenv.config({ path: '.env' });
10
-
11
- import colors from 'colors';
12
- import { join } from 'path';
13
- import { spaceTrim } from 'spacetrim';
14
- import { compilePipeline } from '../../conversion/compilePipeline';
15
- import { CallbackInterfaceTools } from '../../dialogs/callback/CallbackInterfaceTools';
16
- import { createPipelineExecutor } from '../../execution/createPipelineExecutor/00-createPipelineExecutor';
17
- import { MockedEchoLlmExecutionTools } from '../../llm-providers/mocked/MockedEchoLlmExecutionTools';
18
- import type { PipelineString } from '../../pipeline/PipelineString';
19
-
20
- if (process.cwd() !== join(__dirname, '../../..')) {
21
- console.error(
22
- colors.red(
23
- spaceTrim(`
24
- CWD must be root of the project
25
-
26
- Script: error-handling-playground.ts
27
- Current CWD: ${process.cwd()}
28
- Expected CWD: ${join(__dirname, '../../..')}
29
- `),
30
- ),
31
- );
32
- process.exit(1);
33
- }
34
-
35
- playground()
36
- .catch((error) => {
37
- console.error(colors.bgRed(error.name || 'NamelessError'));
38
- console.error(error);
39
- process.exit(1);
40
- })
41
- .then(() => {
42
- process.exit(0);
43
- });
44
-
45
- /**
46
- * Handles playground.
47
- */
48
- async function playground() {
49
- console.info(`🧸 Error handling Playground`);
50
-
51
- // Do here stuff you want to test
52
- //========================================>
53
-
54
- const pipelineExecutor = await getPipelineExecutor();
55
- const result = await pipelineExecutor({}).asPromise({ isCrashedOnError: false });
56
-
57
- console.info('result', result);
58
-
59
- async function getPipelineExecutor() {
60
- const pipeline = await compilePipeline(
61
- spaceTrim(`
62
- # Example prompt
63
-
64
- Show how to use a simple chat prompt
65
-
66
- - PROMPTBOOK VERSION 1.0.0
67
- - PIPELINE URL https://promptbook.studio/examples/pipeline.book
68
- - MODEL VARIANT Chat
69
- - MODEL NAME gpt-3.5-turbo
70
- - INPUT PARAMETER {thing} Any thing to buy
71
- - OUTPUT PARAMETER {response}
72
-
73
- ## Prompt
74
-
75
- \`\`\`
76
- One day I went to the shop and bought {thing}.
77
- Now I have {thing}.
78
- \`\`\`
79
-
80
- -> {response}
81
- `) as PipelineString,
82
- // <- TODO: [📼] Use`book\`` string literal notation
83
- );
84
- const pipelineExecutor = createPipelineExecutor({
85
- pipeline,
86
- tools: {
87
- llm: new MockedEchoLlmExecutionTools({ isVerbose: true }),
88
- script: [],
89
- userInterface: new CallbackInterfaceTools({
90
- isVerbose: true,
91
- async callback() {
92
- return 'Hello';
93
- },
94
- }),
95
- },
96
- });
97
- return pipelineExecutor;
98
- }
99
-
100
- //========================================/
101
- }
102
-
103
- // Note: [⚫] Code for permanent playground [error-handling-playground](src/playground/permanent/error-handling-playground.ts) should never be published in any package
@@ -1,36 +0,0 @@
1
- #!/usr/bin/env ts-node
2
-
3
- import * as dotenv from 'dotenv';
4
-
5
- dotenv.config({ path: '.env' });
6
-
7
- import colors from 'colors';
8
- import { join } from 'path';
9
-
10
- if (process.cwd() !== join(__dirname, '../..')) {
11
- console.error(colors.red(`CWD must be root of the project`));
12
- process.exit(1);
13
- }
14
-
15
- playground()
16
- .catch((error) => {
17
- console.error(colors.bgRed(error.name || 'NamelessError'));
18
- console.error(error);
19
- process.exit(1);
20
- })
21
- .then(() => {
22
- process.exit(0);
23
- });
24
-
25
- async function playground() {
26
- console.info(`🧸 Playground`);
27
-
28
- // Do here stuff you want to test
29
- //========================================>
30
-
31
- //========================================/
32
-
33
- console.info(`[ Done 🧸 Playground ]`);
34
- }
35
-
36
- /** Note: [⚫] Code for archived playground [_playground-boilerplate.ts](src/playground/backup/_playground-boilerplate.ts.txt) should never be published in any package */
@@ -1,19 +0,0 @@
1
- {
2
- "compilerOptions": {
3
- "target": "esnext",
4
- "module": "commonjs",
5
- "downlevelIteration": true,
6
- "allowJs": true,
7
- "moduleResolution": "node",
8
- "forceConsistentCasingInFileNames": true,
9
- "noImplicitReturns": true,
10
- "noImplicitThis": true,
11
- "noImplicitAny": true,
12
- "strictNullChecks": true,
13
- "experimentalDecorators": true,
14
- "noUnusedLocals": false,
15
- "resolveJsonModule": true,
16
- "esModuleInterop": true
17
- },
18
- "exclude": ["node_modules"]
19
- }
@@ -1,73 +0,0 @@
1
- // TODO: @@ Uncomment this test when making new scraper from boilerplate
2
-
3
- import { describe, expect, it } from '@jest/globals';
4
- import { join } from 'path';
5
- import { $provideExecutablesForNode } from '../../executables/$provideExecutablesForNode';
6
- import { $provideLlmToolsForTestingAndScriptsAndPlayground } from '../../llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground';
7
- import { $provideFilesystemForNode } from '../_common/register/$provideFilesystemForNode';
8
- import { makeKnowledgeSourceHandler } from '../_common/utils/makeKnowledgeSourceHandler';
9
- import { BoilerplateScraper } from './BoilerplateScraper';
10
-
11
- describe('how creating knowledge from @@ works', () => {
12
- const rootDirname = join(__dirname, 'examples');
13
-
14
- const boilerplateScraperPromise = (async () =>
15
- new BoilerplateScraper(
16
- {
17
- fs: $provideFilesystemForNode(),
18
- llm: await $provideLlmToolsForTestingAndScriptsAndPlayground(),
19
- executables: await $provideExecutablesForNode(),
20
- },
21
- {
22
- rootDirname,
23
- },
24
- ))();
25
-
26
- it('should scrape simple information from a @@ file', () =>
27
- expect(
28
- Promise.all([
29
- boilerplateScraperPromise,
30
- makeKnowledgeSourceHandler(
31
- {
32
- knowledgeSourceContent: '10-simple.@@',
33
- },
34
- { fs: $provideFilesystemForNode() },
35
- { rootDirname },
36
- ),
37
- ])
38
- .then(([boilerplateScraper, sourceHandler]) => boilerplateScraper.scrape(sourceHandler))
39
- .then((knowledge) => knowledge?.map(({ content }) => ({ content })))
40
- .then((knowledge) => knowledge?.slice(0, 1)),
41
- ).resolves.toMatchObject([
42
- {
43
- content: expect.stringMatching(/Springfield (is )?.*/i),
44
- },
45
- ]));
46
-
47
- // TODO: @@ Add all samples
48
-
49
- it('should NOT scrape irrelevant information', () =>
50
- expect(
51
- Promise.all([
52
- boilerplateScraperPromise,
53
- makeKnowledgeSourceHandler(
54
- {
55
- knowledgeSourceContent: '10-simple.@@',
56
- },
57
- { fs: $provideFilesystemForNode() },
58
- { rootDirname },
59
- ),
60
- ])
61
- .then(([boilerplateScraper, sourceHandler]) => boilerplateScraper.scrape(sourceHandler))
62
- .then((knowledge) => knowledge?.map(({ content }) => ({ content })))
63
- .then((knowledge) => knowledge?.slice(0, 1)),
64
- ).resolves.toMatchObject([
65
- {
66
- content: expect.not.stringMatching(/London (is )?.*/i),
67
- },
68
- ]));
69
- });
70
-
71
- /**
72
- * TODO: [📓] Maybe test all file in examples (not just one by one)
73
- */
@@ -1,79 +0,0 @@
1
- #!/usr/bin/env ts-node
2
-
3
- import * as dotenv from 'dotenv';
4
-
5
- dotenv.config({ path: '.env' });
6
-
7
- import colors from 'colors'; // <- TODO: [🔶] Make system to put color and style to both node and browser
8
- import { writeFile } from 'fs/promises';
9
- import { join } from 'path';
10
- import { $provideExecutablesForNode } from '../../../executables/$provideExecutablesForNode';
11
- import { usageToHuman } from '../../../execution/utils/usageToHuman';
12
- import { $provideLlmToolsForTestingAndScriptsAndPlayground } from '../../../llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground';
13
- import { stringifyPipelineJson } from '../../../utils/editable/utils/stringifyPipelineJson';
14
- import { $provideFilesystemForNode } from '../../_common/register/$provideFilesystemForNode';
15
- import { makeKnowledgeSourceHandler } from '../../_common/utils/makeKnowledgeSourceHandler';
16
- import { BoilerplateScraper } from '../BoilerplateScraper';
17
-
18
- playground()
19
- .catch((error) => {
20
- console.error(colors.bgRed(error.name || 'NamelessError'));
21
- console.error(error);
22
- process.exit(1);
23
- })
24
- .then(() => {
25
- process.exit(0);
26
- });
27
-
28
- /**
29
- * Handles playground.
30
- */
31
- async function playground() {
32
- console.info(`🧸 Scrape knowledge from @@ (playground)`);
33
-
34
- // Do here stuff you want to test
35
- //========================================>
36
-
37
- const example = '10-simple.@@';
38
- // <- TODO: [👩🏿‍🤝‍👩🏼] Read here the examples directory and itterate through all of them
39
-
40
- const llmTools = await $provideLlmToolsForTestingAndScriptsAndPlayground({ isCacheReloaded: true });
41
- const rootDirname = join(__dirname, '..', 'examples');
42
-
43
- const boilerplateScraper = new BoilerplateScraper(
44
- {
45
- fs: $provideFilesystemForNode(),
46
- llm: await $provideLlmToolsForTestingAndScriptsAndPlayground(),
47
- executables: await $provideExecutablesForNode(),
48
- },
49
- {
50
- rootDirname,
51
- },
52
- );
53
-
54
- const knowledge = await boilerplateScraper.scrape(
55
- await makeKnowledgeSourceHandler(
56
- { knowledgeSourceContent: example },
57
- { fs: $provideFilesystemForNode() },
58
- { rootDirname },
59
- ),
60
- );
61
-
62
- console.info(colors.cyan(usageToHuman(llmTools.getTotalUsage())));
63
- console.info(colors.bgGreen(' Knowledge: '));
64
- console.info(knowledge);
65
-
66
- await writeFile(
67
- join(
68
- __dirname,
69
- `../examples/${example}.knowledge.json` /* <- TODO: [👩🏿‍🤝‍👩🏼] Read here the examples directory and itterate through all of them */,
70
- ),
71
- stringifyPipelineJson(knowledge),
72
- 'utf-8',
73
- );
74
- /**/
75
-
76
- //========================================/
77
- }
78
-
79
- // Note: [⚫] Code for playground [boilerplate-scraper-playground](src/scrapers/_boilerplate/playground/boilerplate-scraper-playground.ts) should never be published in any package
@@ -1,19 +0,0 @@
1
- {
2
- "compilerOptions": {
3
- "target": "esnext",
4
- "module": "commonjs",
5
- "downlevelIteration": true,
6
- "allowJs": true,
7
- "moduleResolution": "node",
8
- "forceConsistentCasingInFileNames": true,
9
- "noImplicitReturns": true,
10
- "noImplicitThis": true,
11
- "noImplicitAny": true,
12
- "strictNullChecks": true,
13
- "experimentalDecorators": true,
14
- "noUnusedLocals": false,
15
- "resolveJsonModule": true,
16
- "esModuleInterop": true
17
- },
18
- "exclude": ["node_modules"]
19
- }
@@ -1,17 +0,0 @@
1
- import { describe, expect, it } from '@jest/globals';
2
- import { blobToDataurl } from './blobToDataurl';
3
-
4
- describe(`conversion of blob to dataurl`, () => {
5
- it(`converts blob to base64 encoded dataurl`, () =>
6
- expect(blobToDataurl(new Blob(
7
- [`<h1>Hello, World!</h1>`], { type: 'text/html' }))).resolves.toBe(
8
- `data:text/html;base64,PGgxPkhlbGxvLCBXb3JsZCE8L2gxPg==`,
9
- ));
10
- });
11
-
12
-
13
-
14
- /**
15
- * TODO: [🧺] In future uncomment
16
- * TODO: [🧺] Export via `@promptbook/pdf` and `@promptbook/docx`
17
- */
@@ -1,52 +0,0 @@
1
- import { blobToBinaryString } from './blobToBinaryString';
2
- import { dataurlToBlob } from './dataurlToBlob';
3
-
4
- describe(`conversion of dataurl to blob`, () => {
5
- it(`converts base64 without charset dataurl`, () => {
6
- return expect(
7
- dataurlToBlob(`data:text/html;base64,PGgxPkhlbGxvLCBXb3JsZCE8L2gxPg==`).then(async (blob) => [
8
- blob.type,
9
- await blobToBinaryString(blob),
10
- ]),
11
- ).resolves.toEqual([`text/html`, `<h1>Hello, World!</h1>`]);
12
- });
13
-
14
- it(`converts base64 with charset dataurl`, () => {
15
- return expect(
16
- dataurlToBlob(`data:text/html;charset=utf-8;base64,PGgxPkhlbGxvLCBXb3JsZCE8L2gxPg==`).then(async (blob) => [
17
- blob.type,
18
- await blobToBinaryString(blob),
19
- ]),
20
- ).resolves.toEqual([`text/html`, `<h1>Hello, World!</h1>`]);
21
- });
22
-
23
- it(`converts non-encoded, escaped dataurl`, () => {
24
- return expect(
25
- dataurlToBlob(`data:text/html,%3Ch1%3EHello%2C%20World%21%3C%2Fh1%3E`).then(async (blob) => [
26
- blob.type,
27
- await blobToBinaryString(blob),
28
- ]),
29
- ).resolves.toEqual([`text/html`, `<h1>Hello, World!</h1>`]);
30
- });
31
-
32
- it(`converts non-encoded, non-escaped dataurl`, () => {
33
- return expect(
34
- dataurlToBlob(`data:text/html,<h1>Hello, World!</h1>`).then(async (blob) => [
35
- blob.type,
36
- await blobToBinaryString(blob),
37
- ]),
38
- ).resolves.toEqual([`text/html`, `<h1>Hello, World!</h1>`]);
39
- });
40
-
41
- it(`checks that it is not wrong type`, () => {
42
- return expect(
43
- dataurlToBlob(`data:text/html;base64,PGgxPkhlbGxvLCBXb3JsZCE8L2gxPg==`).then((blob) => [blob.type]),
44
- ).resolves.not.toEqual(`text/plain`);
45
- });
46
-
47
- it(`checks that it is not wrong content`, () => {
48
- return expect(
49
- dataurlToBlob(`data:text/html;base64,PGgxPkhlbGxvLCBXb3JsZCE8L2gxPg==`).then(blobToBinaryString),
50
- ).resolves.not.toEqual(`<h1>Hello, Fooo!</h1>`);
51
- });
52
- });
@@ -1,42 +0,0 @@
1
- import { isValidDataurl } from './isValidDataurl';
2
-
3
- describe(`validation of urls`, () => {
4
- it(`is valid base64 without charset dataurl`, () => {
5
- expect(isValidDataurl(`data:text/plain;base64,PGgxPkhlbGxvLCBXb3JsZCE8L2gxPg==`)).toBe(true);
6
- });
7
-
8
- it(`is valid base64 with charset dataurl`, () => {
9
- expect(isValidDataurl(`data:text/plain;charset=utf-8;base64,PGgxPkhlbGxvLCBXb3JsZCE8L2gxPg==`)).toBe(true);
10
- });
11
-
12
- it(`is valid non-encoded, escaped dataurl`, () => {
13
- expect(isValidDataurl(`data:text/html,%3Ch1%3EHello%2C%20World%21%3C%2Fh1%3E`)).toBe(true);
14
- });
15
-
16
- it(`is valid non-encoded, non-escaped dataurl`, () => {
17
- expect(isValidDataurl(`data:text/html,<h1>Hello, World!</h1>`)).toBe(true);
18
- });
19
-
20
- /*
21
- TODO: [🏞️]
22
- it(`is corrupted dataurl`, () => {
23
- expect(isValidDataurl(`data:text/plain;charset=utf-8;base12345678,PGgxPkhlbGxvLCBXb3JsZCE8L2gxPg==`)).toBe(
24
- false,
25
- );
26
- expect(isValidDataurl(`data:text/plain;base64 PGgxPkhlbGxvLCBXb3JsZCE8L2gxPg==`)).toBe(false);
27
- expect(isValidDataurl(`data:PGgxPkhlbGxvLCBXb3JsZCE8L2gxPg==`)).toBe(false);
28
- });
29
- */
30
-
31
- it(`is valid url BUT not dataurl`, () => {
32
- expect(isValidDataurl(`https://collboard.com/`)).toBe(false);
33
- expect(isValidDataurl(`http://localhost:9977/fooo/add`)).toBe(false);
34
- });
35
-
36
- it(`is NOT valid url`, () => {
37
- expect(isValidDataurl(``)).toBe(false);
38
- expect(isValidDataurl(`Invalid URL`)).toBe(false);
39
- expect(isValidDataurl(`aegfawsgsdasdg`)).toBe(false);
40
- expect(isValidDataurl(`wtf://collboard.com/`)).toBe(false);
41
- });
42
- });
@@ -1,13 +0,0 @@
1
- import { shorten } from './shorten';
2
-
3
- describe('how shorten works', () => {
4
- it('will shorten', () => {
5
- expect(shorten('Foo', 2)).toEqual('F…');
6
- expect(shorten('Foooooooooooooooooooooooooooooooo', 5)).toEqual('Fooo…');
7
- });
8
-
9
- it('will keep short', () => {
10
- expect(shorten('Foo', 3)).toEqual('Foo');
11
- expect(shorten('Foo', 4)).toEqual('Foo');
12
- });
13
- });
@@ -1,80 +0,0 @@
1
- #!/usr/bin/env ts-node
2
-
3
- import * as dotenv from 'dotenv';
4
-
5
- dotenv.config({ path: '.env' });
6
-
7
- import colors from 'colors'; // <- TODO: [🔶] Make system to put color and style to both node and browser
8
- import { writeFile } from 'fs/promises';
9
- import { join } from 'path';
10
- import { $provideExecutablesForNode } from '../../../executables/$provideExecutablesForNode';
11
- import { usageToHuman } from '../../../execution/utils/usageToHuman';
12
- import { $provideLlmToolsForTestingAndScriptsAndPlayground } from '../../../llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground';
13
- import { stringifyPipelineJson } from '../../../utils/editable/utils/stringifyPipelineJson';
14
- import { $provideFilesystemForNode } from '../../_common/register/$provideFilesystemForNode';
15
- import { makeKnowledgeSourceHandler } from '../../_common/utils/makeKnowledgeSourceHandler';
16
- import { DocumentScraper } from '../DocumentScraper';
17
-
18
- playground()
19
- .catch((error) => {
20
- console.error(colors.bgRed(error.name || 'NamelessError'));
21
- console.error(error);
22
- process.exit(1);
23
- })
24
- .then(() => {
25
- process.exit(0);
26
- });
27
-
28
- /**
29
- * Handles playground.
30
- */
31
- async function playground() {
32
- console.info(`🧸 Scrape knowledge from documents (playground)`);
33
-
34
- // Do here stuff you want to test
35
- //========================================>
36
-
37
- //const example = '10-simple.docx';
38
- const example = '10-simple.odt';
39
- // <- TODO: [👩🏿‍🤝‍👩🏼] Read here the examples directory and itterate through all of them
40
-
41
- const llmTools = await $provideLlmToolsForTestingAndScriptsAndPlayground({ isCacheReloaded: true });
42
- const rootDirname = join(__dirname, '..', 'examples');
43
-
44
- const documentScraper = new DocumentScraper(
45
- {
46
- fs: $provideFilesystemForNode(),
47
- llm: await $provideLlmToolsForTestingAndScriptsAndPlayground(),
48
- executables: await $provideExecutablesForNode(),
49
- },
50
- {
51
- rootDirname,
52
- },
53
- );
54
-
55
- const knowledge = await documentScraper.scrape(
56
- await makeKnowledgeSourceHandler(
57
- { knowledgeSourceContent: example },
58
- { fs: $provideFilesystemForNode() },
59
- { rootDirname },
60
- ),
61
- );
62
-
63
- console.info(colors.cyan(usageToHuman(llmTools.getTotalUsage())));
64
- console.info(colors.bgGreen(' Knowledge: '));
65
- console.info(knowledge);
66
-
67
- await writeFile(
68
- join(
69
- __dirname,
70
- `../examples/${example}.knowledge.json` /* <- TODO: [👩🏿‍🤝‍👩🏼] Read here the examples directory and itterate through all of them */,
71
- ),
72
- stringifyPipelineJson(knowledge),
73
- 'utf-8',
74
- );
75
- /**/
76
-
77
- //========================================/
78
- }
79
-
80
- // Note: [⚫] Code for playground [document-scraper-playground](src/scrapers/document/playground/document-scraper-playground.ts) should never be published in any package
@@ -1,19 +0,0 @@
1
- {
2
- "compilerOptions": {
3
- "target": "esnext",
4
- "module": "commonjs",
5
- "downlevelIteration": true,
6
- "allowJs": true,
7
- "moduleResolution": "node",
8
- "forceConsistentCasingInFileNames": true,
9
- "noImplicitReturns": true,
10
- "noImplicitThis": true,
11
- "noImplicitAny": true,
12
- "strictNullChecks": true,
13
- "experimentalDecorators": true,
14
- "noUnusedLocals": false,
15
- "resolveJsonModule": true,
16
- "esModuleInterop": true
17
- },
18
- "exclude": ["node_modules"]
19
- }
@@ -1,80 +0,0 @@
1
- #!/usr/bin/env ts-node
2
-
3
- import * as dotenv from 'dotenv';
4
-
5
- dotenv.config({ path: '.env' });
6
-
7
- import colors from 'colors'; // <- TODO: [🔶] Make system to put color and style to both node and browser
8
- import { writeFile } from 'fs/promises';
9
- import { join } from 'path';
10
- import { $provideExecutablesForNode } from '../../../executables/$provideExecutablesForNode';
11
- import { usageToHuman } from '../../../execution/utils/usageToHuman';
12
- import { $provideLlmToolsForTestingAndScriptsAndPlayground } from '../../../llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground';
13
- import { stringifyPipelineJson } from '../../../utils/editable/utils/stringifyPipelineJson';
14
- import { $provideFilesystemForNode } from '../../_common/register/$provideFilesystemForNode';
15
- import { makeKnowledgeSourceHandler } from '../../_common/utils/makeKnowledgeSourceHandler';
16
- import { LegacyDocumentScraper } from '../LegacyDocumentScraper';
17
-
18
- playground()
19
- .catch((error) => {
20
- console.error(colors.bgRed(error.name || 'NamelessError'));
21
- console.error(error);
22
- process.exit(1);
23
- })
24
- .then(() => {
25
- process.exit(0);
26
- });
27
-
28
- /**
29
- * Handles playground.
30
- */
31
- async function playground() {
32
- console.info(`🧸 Scrape knowledge from legacy documents (playground)`);
33
-
34
- // Do here stuff you want to test
35
- //========================================>
36
-
37
- //const example = '10-simple.doc';
38
- const example = '10-simple.rtf';
39
- // <- TODO: [👩🏿‍🤝‍👩🏼] Read here the examples directory and itterate through all of them
40
-
41
- const llmTools = await $provideLlmToolsForTestingAndScriptsAndPlayground({ isCacheReloaded: true });
42
- const rootDirname = join(__dirname, '..', 'examples');
43
-
44
- const legacyDocumentScraper = new LegacyDocumentScraper(
45
- {
46
- fs: $provideFilesystemForNode(),
47
- llm: await $provideLlmToolsForTestingAndScriptsAndPlayground(),
48
- executables: await $provideExecutablesForNode(),
49
- },
50
- {
51
- rootDirname,
52
- },
53
- );
54
-
55
- const knowledge = await legacyDocumentScraper.scrape(
56
- await makeKnowledgeSourceHandler(
57
- { knowledgeSourceContent: example },
58
- { fs: $provideFilesystemForNode() },
59
- { rootDirname },
60
- ),
61
- );
62
-
63
- console.info(colors.cyan(usageToHuman(llmTools.getTotalUsage())));
64
- console.info(colors.bgGreen(' Knowledge: '));
65
- console.info(knowledge);
66
-
67
- await writeFile(
68
- join(
69
- __dirname,
70
- `../examples/${example}.knowledge.json` /* <- TODO: [👩🏿‍🤝‍👩🏼] Read here the examples directory and itterate through all of them */,
71
- ),
72
- stringifyPipelineJson(knowledge),
73
- 'utf-8',
74
- );
75
- /**/
76
-
77
- //========================================/
78
- }
79
-
80
- // Note: [⚫] Code for playground [legacy-document-scraper-playground](src/scrapers/document-legacy/playground/legacy-document-scraper-playground.ts) should never be published in any package
@@ -1,19 +0,0 @@
1
- {
2
- "compilerOptions": {
3
- "target": "esnext",
4
- "module": "commonjs",
5
- "downlevelIteration": true,
6
- "allowJs": true,
7
- "moduleResolution": "node",
8
- "forceConsistentCasingInFileNames": true,
9
- "noImplicitReturns": true,
10
- "noImplicitThis": true,
11
- "noImplicitAny": true,
12
- "strictNullChecks": true,
13
- "experimentalDecorators": true,
14
- "noUnusedLocals": false,
15
- "resolveJsonModule": true,
16
- "esModuleInterop": true
17
- },
18
- "exclude": ["node_modules"]
19
- }