@promptbook/cli 0.112.0-104 → 0.112.0-105

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (174) hide show
  1. package/apps/agents-server/src/app/admin/image-generator-test/ImageAttachmentsEditor.tsx +19 -3
  2. package/apps/agents-server/src/app/admin/limits/LimitsClient.tsx +11 -12
  3. package/apps/agents-server/src/app/admin/metadata/MetadataClient.tsx +34 -2
  4. package/apps/agents-server/src/app/admin/servers/CreateServerDialog.tsx +6 -1
  5. package/apps/agents-server/src/app/admin/servers/useCreateServerWizard.ts +13 -1
  6. package/apps/agents-server/src/app/agents/[agentName]/AgentChatWrapper.tsx +11 -2
  7. package/apps/agents-server/src/app/agents/[agentName]/AgentProfileChat.tsx +11 -1
  8. package/apps/agents-server/src/app/agents/[agentName]/book/BookEditorWrapper.tsx +7 -1
  9. package/apps/agents-server/src/app/agents/[agentName]/chat/CanonicalAgentChatSurface.tsx +11 -1
  10. package/apps/agents-server/src/app/agents/[agentName]/images/default-avatar.png/route.ts +6 -2
  11. package/apps/agents-server/src/app/api/images/[filename]/route.ts +6 -2
  12. package/apps/agents-server/src/app/api/internal/agent-runner-limits/route.ts +51 -0
  13. package/apps/agents-server/src/app/api/upload/route.ts +48 -12
  14. package/apps/agents-server/src/app/layout.tsx +13 -0
  15. package/apps/agents-server/src/components/FileUploadAvailability/FileUploadAvailabilityContext.tsx +50 -0
  16. package/apps/agents-server/src/components/FileUploadAvailability/FileUploadUnavailableNotice.tsx +45 -0
  17. package/apps/agents-server/src/components/LayoutWrapper/LayoutWrapper.tsx +85 -76
  18. package/apps/agents-server/src/components/NewAgentDialog/NewAgentDialog.tsx +7 -3
  19. package/apps/agents-server/src/components/NewAgentDialog/NewAgentWizardKnowledgeStep.tsx +6 -0
  20. package/apps/agents-server/src/components/NewAgentDialog/useNewAgentWizardKnowledgeState.ts +8 -1
  21. package/apps/agents-server/src/constants/serverLimits.ts +22 -2
  22. package/apps/agents-server/src/database/seedDefaultAgents.ts +218 -0
  23. package/apps/agents-server/src/tools/$provideCdnForServer.ts +72 -8
  24. package/apps/agents-server/src/utils/defaultAgents/loadDefaultAgentBooks.ts +103 -0
  25. package/apps/agents-server/src/utils/knowledge/createInlineKnowledgeSourceUploader.ts +24 -5
  26. package/apps/agents-server/src/utils/serverLimits.ts +26 -1
  27. package/apps/agents-server/src/utils/serverManagement/createManagedServer/seedServerDefaultAgents.ts +1 -85
  28. package/apps/agents-server/src/utils/shareTargetPayloads.ts +20 -2
  29. package/apps/agents-server/src/utils/upload/fileUploadAvailability.ts +91 -0
  30. package/apps/agents-server/src/utils/upload/uploadFileToServer.ts +46 -2
  31. package/esm/apps/agents-server/src/constants/federatedAgentImport.d.ts +42 -0
  32. package/esm/apps/agents-server/src/constants/serverLimits.d.ts +207 -0
  33. package/esm/apps/agents-server/src/constants/toolUsageLimits.d.ts +55 -0
  34. package/esm/index.es.js +602 -39
  35. package/esm/index.es.js.map +1 -1
  36. package/esm/scripts/run-agent-messages/main/AgentMessageFailureTracker.d.ts +27 -0
  37. package/esm/scripts/run-agent-messages/main/handleAgentWatchError.d.ts +4 -0
  38. package/esm/scripts/run-agent-messages/main/runAgentMessages.d.ts +1 -0
  39. package/esm/scripts/run-agent-messages/messages/moveAgentMessageToFailed.d.ts +17 -0
  40. package/esm/src/book-components/BookEditor/BookEditor.d.ts +5 -4
  41. package/esm/src/book-components/BookEditor/BookEditorTheme.d.ts +24 -0
  42. package/esm/src/book-components/BookEditor/useBookEditorMonacoLanguage.d.ts +1 -6
  43. package/esm/src/book-components/BookEditor/useBookEditorMonacoLifecycle.d.ts +1 -4
  44. package/esm/src/book-components/BookEditor/useBookEditorMonacoStyles.d.ts +2 -1
  45. package/esm/src/cli/cli-commands/agent-folder/agentProjectPaths.d.ts +6 -0
  46. package/esm/src/version.d.ts +1 -1
  47. package/package.json +1 -1
  48. package/src/avatars/visuals/octopus3d3AvatarVisual.ts +28 -29
  49. package/src/book-components/BookEditor/BookEditor.tsx +10 -7
  50. package/src/book-components/BookEditor/BookEditorMonaco.tsx +3 -1
  51. package/src/book-components/BookEditor/BookEditorTheme.ts +32 -0
  52. package/src/book-components/BookEditor/useBookEditorMonacoLanguage.ts +12 -15
  53. package/src/book-components/BookEditor/useBookEditorMonacoLifecycle.ts +1 -5
  54. package/src/book-components/BookEditor/useBookEditorMonacoStyles.ts +2 -1
  55. package/src/cli/cli-commands/agent-folder/agentProjectPaths.ts +7 -0
  56. package/src/cli/cli-commands/agents-server/buildAgentsServer.ts +109 -9
  57. package/src/cli/cli-commands/agents-server/startAgentsServer.ts +132 -4
  58. package/src/other/templates/getTemplatesPipelineCollection.ts +729 -854
  59. package/src/version.ts +2 -2
  60. package/src/versions.txt +1 -0
  61. package/umd/apps/agents-server/src/constants/federatedAgentImport.d.ts +42 -0
  62. package/umd/apps/agents-server/src/constants/serverLimits.d.ts +207 -0
  63. package/umd/apps/agents-server/src/constants/toolUsageLimits.d.ts +55 -0
  64. package/umd/index.umd.js +602 -39
  65. package/umd/index.umd.js.map +1 -1
  66. package/umd/scripts/run-agent-messages/main/AgentMessageFailureTracker.d.ts +27 -0
  67. package/umd/scripts/run-agent-messages/main/handleAgentWatchError.d.ts +4 -0
  68. package/umd/scripts/run-agent-messages/main/runAgentMessages.d.ts +1 -0
  69. package/umd/scripts/run-agent-messages/messages/moveAgentMessageToFailed.d.ts +17 -0
  70. package/umd/src/book-components/BookEditor/BookEditor.d.ts +5 -4
  71. package/umd/src/book-components/BookEditor/BookEditorTheme.d.ts +24 -0
  72. package/umd/src/book-components/BookEditor/useBookEditorMonacoLanguage.d.ts +1 -6
  73. package/umd/src/book-components/BookEditor/useBookEditorMonacoLifecycle.d.ts +1 -4
  74. package/umd/src/book-components/BookEditor/useBookEditorMonacoStyles.d.ts +2 -1
  75. package/umd/src/cli/cli-commands/agent-folder/agentProjectPaths.d.ts +6 -0
  76. package/umd/src/version.d.ts +1 -1
  77. package/apps/agents-server/src/message-providers/email/_common/utils/parseEmailAddress.test.ts.todo +0 -108
  78. package/apps/agents-server/src/message-providers/email/_common/utils/parseEmailAddresses.test.ts.todo +0 -117
  79. package/apps/agents-server/src/message-providers/email/_common/utils/stringifyEmailAddress.test.ts.todo +0 -119
  80. package/apps/agents-server/src/message-providers/email/_common/utils/stringifyEmailAddresses.test.ts.todo +0 -74
  81. package/apps/agents-server/src/utils/defaultAgents/defaultAgents.ts +0 -168
  82. package/apps/agents-server/src/utils/defaultAgents/installDefaultAgents.ts +0 -139
  83. package/apps/agents-server/tests/e2e/authentication-and-navigation.spec.ts.todo +0 -178
  84. package/src/_packages/browser.index.ts +0 -31
  85. package/src/_packages/browser.readme.md +0 -43
  86. package/src/book-2.0/agent-source/parseAgentSourceWithCommitments.test.ts.todo +0 -265
  87. package/src/book-components/BookEditor/BookEditorMonaco.test.tsx.todo +0 -115
  88. package/src/book-components/Chat/utils/renderMarkdown.test.ts.tmp +0 -199
  89. package/src/collection/agent-collection/constructors/agent-collection-in-directory/AgentCollectionInDirectory.test.ts.todo +0 -131
  90. package/src/commands/_common/parseCommand.test.ts.todo +0 -48
  91. package/src/commitments/META_LINK/META_LINK.test.ts.todo +0 -75
  92. package/src/conversion/validation/pipelineStringToJson-errors.test.ts.todo +0 -33
  93. package/src/dialogs/simple-prompt/SimplePromptInterfaceTools.ts +0 -51
  94. package/src/executables/browsers/locateSafari.test.ts.tmp +0 -15
  95. package/src/execution/PromptbookFetch.test-type.ts +0 -14
  96. package/src/execution/createPipelineExecutor/00-createPipelineExecutor.test.ts.todo +0 -0
  97. package/src/execution/execution-report/executionReportJsonToString.test.ts.todo +0 -83
  98. package/src/execution/utils/usageToHuman.test.ts.todo +0 -80
  99. package/src/llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground.ts +0 -76
  100. package/src/llm-providers/_common/utils/assertUniqueModels.ts +0 -27
  101. package/src/llm-providers/_multiple/playground/playground.ts +0 -141
  102. package/src/llm-providers/_multiple/playground/tsconfig.json +0 -19
  103. package/src/llm-providers/agent/playground/playground.ts +0 -190
  104. package/src/llm-providers/agent/playground/tsconfig.json +0 -19
  105. package/src/llm-providers/anthropic-claude/playground/playground.ts +0 -99
  106. package/src/llm-providers/anthropic-claude/playground/tsconfig.json +0 -19
  107. package/src/llm-providers/azure-openai/playground/playground.ts +0 -101
  108. package/src/llm-providers/azure-openai/playground/tsconfig.json +0 -19
  109. package/src/llm-providers/ollama/playground/playground.ts +0 -120
  110. package/src/llm-providers/ollama/playground/tsconfig.json +0 -19
  111. package/src/llm-providers/openai/playground/playground.ts +0 -406
  112. package/src/llm-providers/openai/playground/tsconfig.json +0 -19
  113. package/src/llm-providers/remote/playground/playground.ts +0 -144
  114. package/src/llm-providers/remote/playground/tsconfig.json +0 -19
  115. package/src/llm-providers/vercel/playground/playground.ts +0 -133
  116. package/src/llm-providers/vercel/playground/tsconfig.json +0 -19
  117. package/src/personas/preparePersona.test.ts.todo +0 -126
  118. package/src/playground/backup/_playground-boilerplate.ts.txt +0 -37
  119. package/src/playground/backup/playground-agent-os.txt +0 -62
  120. package/src/playground/backup/playground-brj-app.ts.txt +0 -302
  121. package/src/playground/backup/playground-browser-playwright.txt +0 -110
  122. package/src/playground/backup/playground-claude-mcp.txt +0 -43
  123. package/src/playground/backup/playground-document-conversion.txt +0 -84
  124. package/src/playground/backup/playground-glob.ts.txt +0 -42
  125. package/src/playground/backup/playground-mcp-server.txt +0 -1
  126. package/src/playground/backup/playground-openai-agent-kit.txt +0 -73
  127. package/src/playground/backup/playground-openai-function-calling.txt +0 -131
  128. package/src/playground/backup/playground-openai-streaming.ts.txt +0 -68
  129. package/src/playground/backup/playground-scrape-knowledge.txt +0 -65
  130. package/src/playground/backup/playground-scraperFetch.ts.txt +0 -44
  131. package/src/playground/backup/playground-using-openai-compatible-route-on-agents-server.ts.txt +0 -49
  132. package/src/playground/backup/playground-write-pavolhejny-bio.txt +0 -120
  133. package/src/playground/permanent/_boilerplate.ts +0 -54
  134. package/src/playground/permanent/agent-with-browser-playground.ts +0 -92
  135. package/src/playground/permanent/error-handling-playground.ts +0 -103
  136. package/src/playground/playground.ts +0 -36
  137. package/src/playground/tsconfig.json +0 -19
  138. package/src/scrapers/_boilerplate/BoilerplateScraper.test.ts.todo +0 -73
  139. package/src/scrapers/_boilerplate/playground/boilerplate-scraper-playground.ts +0 -79
  140. package/src/scrapers/_boilerplate/playground/tsconfig.json +0 -19
  141. package/src/scrapers/_common/utils/files/blobToDataurl.test.ts.todo +0 -17
  142. package/src/scrapers/_common/utils/files/dataurlToBlob.test.ts.todo +0 -52
  143. package/src/scrapers/_common/utils/files/isValidDataurl.test.ts.todo +0 -42
  144. package/src/scrapers/_common/utils/files/shorten.test.ts.todo +0 -13
  145. package/src/scrapers/document/playground/document-scraper-playground.ts +0 -80
  146. package/src/scrapers/document/playground/tsconfig.json +0 -19
  147. package/src/scrapers/document-legacy/playground/legacy-document-scraper-playground.ts +0 -80
  148. package/src/scrapers/document-legacy/playground/tsconfig.json +0 -19
  149. package/src/scrapers/markdown/playground/markdown-scraper-playground.ts +0 -74
  150. package/src/scrapers/markdown/playground/tsconfig.json +0 -19
  151. package/src/scrapers/markitdown/MarkitdownScraper.test.ts.todo +0 -132
  152. package/src/scrapers/markitdown/playground/markitdown-scraper-playground.ts +0 -91
  153. package/src/scrapers/markitdown/playground/tsconfig.json +0 -19
  154. package/src/scrapers/pdf/PdfScraper.test.ts.todo +0 -52
  155. package/src/scrapers/pdf/playground/pdf-scraper-playground.ts +0 -75
  156. package/src/scrapers/pdf/playground/tsconfig.json +0 -19
  157. package/src/scrapers/website/playground/tsconfig.json +0 -19
  158. package/src/scrapers/website/playground/website-scraper-playground.ts +0 -82
  159. package/src/storage/_common/PromptbookStorage.test-type.ts +0 -14
  160. package/src/storage/local-storage/getIndexedDbStorage.ts +0 -36
  161. package/src/storage/local-storage/getLocalStorage.ts +0 -33
  162. package/src/storage/local-storage/getSessionStorage.ts +0 -33
  163. package/src/storage/local-storage/utils/IndexedDbStorageOptions.ts +0 -16
  164. package/src/storage/local-storage/utils/makePromptbookStorageFromIndexedDb.ts +0 -58
  165. package/src/storage/local-storage/utils/makePromptbookStorageFromWebStorage.ts +0 -45
  166. package/src/transpilers/formatted-book-in-markdown/FormattedBookInMarkdownTranspiler.test.ts.todo +0 -35
  167. package/src/transpilers/openai-sdk/playground/playground.ts +0 -85
  168. package/src/transpilers/openai-sdk/playground/tmp/chatbot-openaisdk-1.js +0 -194
  169. package/src/transpilers/openai-sdk/playground/tmp/package.json +0 -3
  170. package/src/transpilers/openai-sdk/playground/tsconfig.json +0 -18
  171. package/src/utils/editable/utils/findUsableParameters.test.ts.todo +0 -43
  172. package/src/utils/editable/utils/stringifyPipelineJson.test.ts.todo +0 -38
  173. package/src/utils/markdown/prettifyMarkdown.test.ts.tmp +0 -42
  174. package/src/utils/serialization/serializeToPromptbookJavascript.test.ts.todo +0 -116
@@ -1,103 +0,0 @@
1
- #!/usr/bin/env ts-node
2
-
3
- // Note: [❌] Turning off some global checks for playground file:
4
- // spell-checker: disable
5
- /* eslint-disable */
6
-
7
- import * as dotenv from 'dotenv';
8
-
9
- dotenv.config({ path: '.env' });
10
-
11
- import colors from 'colors';
12
- import { join } from 'path';
13
- import { spaceTrim } from 'spacetrim';
14
- import { compilePipeline } from '../../conversion/compilePipeline';
15
- import { CallbackInterfaceTools } from '../../dialogs/callback/CallbackInterfaceTools';
16
- import { createPipelineExecutor } from '../../execution/createPipelineExecutor/00-createPipelineExecutor';
17
- import { MockedEchoLlmExecutionTools } from '../../llm-providers/mocked/MockedEchoLlmExecutionTools';
18
- import type { PipelineString } from '../../pipeline/PipelineString';
19
-
20
- if (process.cwd() !== join(__dirname, '../../..')) {
21
- console.error(
22
- colors.red(
23
- spaceTrim(`
24
- CWD must be root of the project
25
-
26
- Script: error-handling-playground.ts
27
- Current CWD: ${process.cwd()}
28
- Expected CWD: ${join(__dirname, '../../..')}
29
- `),
30
- ),
31
- );
32
- process.exit(1);
33
- }
34
-
35
- playground()
36
- .catch((error) => {
37
- console.error(colors.bgRed(error.name || 'NamelessError'));
38
- console.error(error);
39
- process.exit(1);
40
- })
41
- .then(() => {
42
- process.exit(0);
43
- });
44
-
45
- /**
46
- * Handles playground.
47
- */
48
- async function playground() {
49
- console.info(`🧸 Error handling Playground`);
50
-
51
- // Do here stuff you want to test
52
- //========================================>
53
-
54
- const pipelineExecutor = await getPipelineExecutor();
55
- const result = await pipelineExecutor({}).asPromise({ isCrashedOnError: false });
56
-
57
- console.info('result', result);
58
-
59
- async function getPipelineExecutor() {
60
- const pipeline = await compilePipeline(
61
- spaceTrim(`
62
- # Example prompt
63
-
64
- Show how to use a simple chat prompt
65
-
66
- - PROMPTBOOK VERSION 1.0.0
67
- - PIPELINE URL https://promptbook.studio/examples/pipeline.book
68
- - MODEL VARIANT Chat
69
- - MODEL NAME gpt-3.5-turbo
70
- - INPUT PARAMETER {thing} Any thing to buy
71
- - OUTPUT PARAMETER {response}
72
-
73
- ## Prompt
74
-
75
- \`\`\`
76
- One day I went to the shop and bought {thing}.
77
- Now I have {thing}.
78
- \`\`\`
79
-
80
- -> {response}
81
- `) as PipelineString,
82
- // <- TODO: [📼] Use`book\`` string literal notation
83
- );
84
- const pipelineExecutor = createPipelineExecutor({
85
- pipeline,
86
- tools: {
87
- llm: new MockedEchoLlmExecutionTools({ isVerbose: true }),
88
- script: [],
89
- userInterface: new CallbackInterfaceTools({
90
- isVerbose: true,
91
- async callback() {
92
- return 'Hello';
93
- },
94
- }),
95
- },
96
- });
97
- return pipelineExecutor;
98
- }
99
-
100
- //========================================/
101
- }
102
-
103
- // Note: [⚫] Code for permanent playground [error-handling-playground](src/playground/permanent/error-handling-playground.ts) should never be published in any package
@@ -1,36 +0,0 @@
1
- #!/usr/bin/env ts-node
2
-
3
- import * as dotenv from 'dotenv';
4
-
5
- dotenv.config({ path: '.env' });
6
-
7
- import colors from 'colors';
8
- import { join } from 'path';
9
-
10
- if (process.cwd() !== join(__dirname, '../..')) {
11
- console.error(colors.red(`CWD must be root of the project`));
12
- process.exit(1);
13
- }
14
-
15
- playground()
16
- .catch((error) => {
17
- console.error(colors.bgRed(error.name || 'NamelessError'));
18
- console.error(error);
19
- process.exit(1);
20
- })
21
- .then(() => {
22
- process.exit(0);
23
- });
24
-
25
- async function playground() {
26
- console.info(`🧸 Playground`);
27
-
28
- // Do here stuff you want to test
29
- //========================================>
30
-
31
- //========================================/
32
-
33
- console.info(`[ Done 🧸 Playground ]`);
34
- }
35
-
36
- /** Note: [⚫] Code for archived playground [_playground-boilerplate.ts](src/playground/backup/_playground-boilerplate.ts.txt) should never be published in any package */
@@ -1,19 +0,0 @@
1
- {
2
- "compilerOptions": {
3
- "target": "esnext",
4
- "module": "commonjs",
5
- "downlevelIteration": true,
6
- "allowJs": true,
7
- "moduleResolution": "node",
8
- "forceConsistentCasingInFileNames": true,
9
- "noImplicitReturns": true,
10
- "noImplicitThis": true,
11
- "noImplicitAny": true,
12
- "strictNullChecks": true,
13
- "experimentalDecorators": true,
14
- "noUnusedLocals": false,
15
- "resolveJsonModule": true,
16
- "esModuleInterop": true
17
- },
18
- "exclude": ["node_modules"]
19
- }
@@ -1,73 +0,0 @@
1
- // TODO: @@ Uncomment this test when making new scraper from boilerplate
2
-
3
- import { describe, expect, it } from '@jest/globals';
4
- import { join } from 'path';
5
- import { $provideExecutablesForNode } from '../../executables/$provideExecutablesForNode';
6
- import { $provideLlmToolsForTestingAndScriptsAndPlayground } from '../../llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground';
7
- import { $provideFilesystemForNode } from '../_common/register/$provideFilesystemForNode';
8
- import { makeKnowledgeSourceHandler } from '../_common/utils/makeKnowledgeSourceHandler';
9
- import { BoilerplateScraper } from './BoilerplateScraper';
10
-
11
- describe('how creating knowledge from @@ works', () => {
12
- const rootDirname = join(__dirname, 'examples');
13
-
14
- const boilerplateScraperPromise = (async () =>
15
- new BoilerplateScraper(
16
- {
17
- fs: $provideFilesystemForNode(),
18
- llm: await $provideLlmToolsForTestingAndScriptsAndPlayground(),
19
- executables: await $provideExecutablesForNode(),
20
- },
21
- {
22
- rootDirname,
23
- },
24
- ))();
25
-
26
- it('should scrape simple information from a @@ file', () =>
27
- expect(
28
- Promise.all([
29
- boilerplateScraperPromise,
30
- makeKnowledgeSourceHandler(
31
- {
32
- knowledgeSourceContent: '10-simple.@@',
33
- },
34
- { fs: $provideFilesystemForNode() },
35
- { rootDirname },
36
- ),
37
- ])
38
- .then(([boilerplateScraper, sourceHandler]) => boilerplateScraper.scrape(sourceHandler))
39
- .then((knowledge) => knowledge?.map(({ content }) => ({ content })))
40
- .then((knowledge) => knowledge?.slice(0, 1)),
41
- ).resolves.toMatchObject([
42
- {
43
- content: expect.stringMatching(/Springfield (is )?.*/i),
44
- },
45
- ]));
46
-
47
- // TODO: @@ Add all samples
48
-
49
- it('should NOT scrape irrelevant information', () =>
50
- expect(
51
- Promise.all([
52
- boilerplateScraperPromise,
53
- makeKnowledgeSourceHandler(
54
- {
55
- knowledgeSourceContent: '10-simple.@@',
56
- },
57
- { fs: $provideFilesystemForNode() },
58
- { rootDirname },
59
- ),
60
- ])
61
- .then(([boilerplateScraper, sourceHandler]) => boilerplateScraper.scrape(sourceHandler))
62
- .then((knowledge) => knowledge?.map(({ content }) => ({ content })))
63
- .then((knowledge) => knowledge?.slice(0, 1)),
64
- ).resolves.toMatchObject([
65
- {
66
- content: expect.not.stringMatching(/London (is )?.*/i),
67
- },
68
- ]));
69
- });
70
-
71
- /**
72
- * TODO: [📓] Maybe test all file in examples (not just one by one)
73
- */
@@ -1,79 +0,0 @@
1
- #!/usr/bin/env ts-node
2
-
3
- import * as dotenv from 'dotenv';
4
-
5
- dotenv.config({ path: '.env' });
6
-
7
- import colors from 'colors'; // <- TODO: [🔶] Make system to put color and style to both node and browser
8
- import { writeFile } from 'fs/promises';
9
- import { join } from 'path';
10
- import { $provideExecutablesForNode } from '../../../executables/$provideExecutablesForNode';
11
- import { usageToHuman } from '../../../execution/utils/usageToHuman';
12
- import { $provideLlmToolsForTestingAndScriptsAndPlayground } from '../../../llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground';
13
- import { stringifyPipelineJson } from '../../../utils/editable/utils/stringifyPipelineJson';
14
- import { $provideFilesystemForNode } from '../../_common/register/$provideFilesystemForNode';
15
- import { makeKnowledgeSourceHandler } from '../../_common/utils/makeKnowledgeSourceHandler';
16
- import { BoilerplateScraper } from '../BoilerplateScraper';
17
-
18
- playground()
19
- .catch((error) => {
20
- console.error(colors.bgRed(error.name || 'NamelessError'));
21
- console.error(error);
22
- process.exit(1);
23
- })
24
- .then(() => {
25
- process.exit(0);
26
- });
27
-
28
- /**
29
- * Handles playground.
30
- */
31
- async function playground() {
32
- console.info(`🧸 Scrape knowledge from @@ (playground)`);
33
-
34
- // Do here stuff you want to test
35
- //========================================>
36
-
37
- const example = '10-simple.@@';
38
- // <- TODO: [👩🏿‍🤝‍👩🏼] Read here the examples directory and itterate through all of them
39
-
40
- const llmTools = await $provideLlmToolsForTestingAndScriptsAndPlayground({ isCacheReloaded: true });
41
- const rootDirname = join(__dirname, '..', 'examples');
42
-
43
- const boilerplateScraper = new BoilerplateScraper(
44
- {
45
- fs: $provideFilesystemForNode(),
46
- llm: await $provideLlmToolsForTestingAndScriptsAndPlayground(),
47
- executables: await $provideExecutablesForNode(),
48
- },
49
- {
50
- rootDirname,
51
- },
52
- );
53
-
54
- const knowledge = await boilerplateScraper.scrape(
55
- await makeKnowledgeSourceHandler(
56
- { knowledgeSourceContent: example },
57
- { fs: $provideFilesystemForNode() },
58
- { rootDirname },
59
- ),
60
- );
61
-
62
- console.info(colors.cyan(usageToHuman(llmTools.getTotalUsage())));
63
- console.info(colors.bgGreen(' Knowledge: '));
64
- console.info(knowledge);
65
-
66
- await writeFile(
67
- join(
68
- __dirname,
69
- `../examples/${example}.knowledge.json` /* <- TODO: [👩🏿‍🤝‍👩🏼] Read here the examples directory and itterate through all of them */,
70
- ),
71
- stringifyPipelineJson(knowledge),
72
- 'utf-8',
73
- );
74
- /**/
75
-
76
- //========================================/
77
- }
78
-
79
- // Note: [⚫] Code for playground [boilerplate-scraper-playground](src/scrapers/_boilerplate/playground/boilerplate-scraper-playground.ts) should never be published in any package
@@ -1,19 +0,0 @@
1
- {
2
- "compilerOptions": {
3
- "target": "esnext",
4
- "module": "commonjs",
5
- "downlevelIteration": true,
6
- "allowJs": true,
7
- "moduleResolution": "node",
8
- "forceConsistentCasingInFileNames": true,
9
- "noImplicitReturns": true,
10
- "noImplicitThis": true,
11
- "noImplicitAny": true,
12
- "strictNullChecks": true,
13
- "experimentalDecorators": true,
14
- "noUnusedLocals": false,
15
- "resolveJsonModule": true,
16
- "esModuleInterop": true
17
- },
18
- "exclude": ["node_modules"]
19
- }
@@ -1,17 +0,0 @@
1
- import { describe, expect, it } from '@jest/globals';
2
- import { blobToDataurl } from './blobToDataurl';
3
-
4
- describe(`conversion of blob to dataurl`, () => {
5
- it(`converts blob to base64 encoded dataurl`, () =>
6
- expect(blobToDataurl(new Blob(
7
- [`<h1>Hello, World!</h1>`], { type: 'text/html' }))).resolves.toBe(
8
- `data:text/html;base64,PGgxPkhlbGxvLCBXb3JsZCE8L2gxPg==`,
9
- ));
10
- });
11
-
12
-
13
-
14
- /**
15
- * TODO: [🧺] In future uncomment
16
- * TODO: [🧺] Export via `@promptbook/pdf` and `@promptbook/docx`
17
- */
@@ -1,52 +0,0 @@
1
- import { blobToBinaryString } from './blobToBinaryString';
2
- import { dataurlToBlob } from './dataurlToBlob';
3
-
4
- describe(`conversion of dataurl to blob`, () => {
5
- it(`converts base64 without charset dataurl`, () => {
6
- return expect(
7
- dataurlToBlob(`data:text/html;base64,PGgxPkhlbGxvLCBXb3JsZCE8L2gxPg==`).then(async (blob) => [
8
- blob.type,
9
- await blobToBinaryString(blob),
10
- ]),
11
- ).resolves.toEqual([`text/html`, `<h1>Hello, World!</h1>`]);
12
- });
13
-
14
- it(`converts base64 with charset dataurl`, () => {
15
- return expect(
16
- dataurlToBlob(`data:text/html;charset=utf-8;base64,PGgxPkhlbGxvLCBXb3JsZCE8L2gxPg==`).then(async (blob) => [
17
- blob.type,
18
- await blobToBinaryString(blob),
19
- ]),
20
- ).resolves.toEqual([`text/html`, `<h1>Hello, World!</h1>`]);
21
- });
22
-
23
- it(`converts non-encoded, escaped dataurl`, () => {
24
- return expect(
25
- dataurlToBlob(`data:text/html,%3Ch1%3EHello%2C%20World%21%3C%2Fh1%3E`).then(async (blob) => [
26
- blob.type,
27
- await blobToBinaryString(blob),
28
- ]),
29
- ).resolves.toEqual([`text/html`, `<h1>Hello, World!</h1>`]);
30
- });
31
-
32
- it(`converts non-encoded, non-escaped dataurl`, () => {
33
- return expect(
34
- dataurlToBlob(`data:text/html,<h1>Hello, World!</h1>`).then(async (blob) => [
35
- blob.type,
36
- await blobToBinaryString(blob),
37
- ]),
38
- ).resolves.toEqual([`text/html`, `<h1>Hello, World!</h1>`]);
39
- });
40
-
41
- it(`checks that it is not wrong type`, () => {
42
- return expect(
43
- dataurlToBlob(`data:text/html;base64,PGgxPkhlbGxvLCBXb3JsZCE8L2gxPg==`).then((blob) => [blob.type]),
44
- ).resolves.not.toEqual(`text/plain`);
45
- });
46
-
47
- it(`checks that it is not wrong content`, () => {
48
- return expect(
49
- dataurlToBlob(`data:text/html;base64,PGgxPkhlbGxvLCBXb3JsZCE8L2gxPg==`).then(blobToBinaryString),
50
- ).resolves.not.toEqual(`<h1>Hello, Fooo!</h1>`);
51
- });
52
- });
@@ -1,42 +0,0 @@
1
- import { isValidDataurl } from './isValidDataurl';
2
-
3
- describe(`validation of urls`, () => {
4
- it(`is valid base64 without charset dataurl`, () => {
5
- expect(isValidDataurl(`data:text/plain;base64,PGgxPkhlbGxvLCBXb3JsZCE8L2gxPg==`)).toBe(true);
6
- });
7
-
8
- it(`is valid base64 with charset dataurl`, () => {
9
- expect(isValidDataurl(`data:text/plain;charset=utf-8;base64,PGgxPkhlbGxvLCBXb3JsZCE8L2gxPg==`)).toBe(true);
10
- });
11
-
12
- it(`is valid non-encoded, escaped dataurl`, () => {
13
- expect(isValidDataurl(`data:text/html,%3Ch1%3EHello%2C%20World%21%3C%2Fh1%3E`)).toBe(true);
14
- });
15
-
16
- it(`is valid non-encoded, non-escaped dataurl`, () => {
17
- expect(isValidDataurl(`data:text/html,<h1>Hello, World!</h1>`)).toBe(true);
18
- });
19
-
20
- /*
21
- TODO: [🏞️]
22
- it(`is corrupted dataurl`, () => {
23
- expect(isValidDataurl(`data:text/plain;charset=utf-8;base12345678,PGgxPkhlbGxvLCBXb3JsZCE8L2gxPg==`)).toBe(
24
- false,
25
- );
26
- expect(isValidDataurl(`data:text/plain;base64 PGgxPkhlbGxvLCBXb3JsZCE8L2gxPg==`)).toBe(false);
27
- expect(isValidDataurl(`data:PGgxPkhlbGxvLCBXb3JsZCE8L2gxPg==`)).toBe(false);
28
- });
29
- */
30
-
31
- it(`is valid url BUT not dataurl`, () => {
32
- expect(isValidDataurl(`https://collboard.com/`)).toBe(false);
33
- expect(isValidDataurl(`http://localhost:9977/fooo/add`)).toBe(false);
34
- });
35
-
36
- it(`is NOT valid url`, () => {
37
- expect(isValidDataurl(``)).toBe(false);
38
- expect(isValidDataurl(`Invalid URL`)).toBe(false);
39
- expect(isValidDataurl(`aegfawsgsdasdg`)).toBe(false);
40
- expect(isValidDataurl(`wtf://collboard.com/`)).toBe(false);
41
- });
42
- });
@@ -1,13 +0,0 @@
1
- import { shorten } from './shorten';
2
-
3
- describe('how shorten works', () => {
4
- it('will shorten', () => {
5
- expect(shorten('Foo', 2)).toEqual('F…');
6
- expect(shorten('Foooooooooooooooooooooooooooooooo', 5)).toEqual('Fooo…');
7
- });
8
-
9
- it('will keep short', () => {
10
- expect(shorten('Foo', 3)).toEqual('Foo');
11
- expect(shorten('Foo', 4)).toEqual('Foo');
12
- });
13
- });
@@ -1,80 +0,0 @@
1
- #!/usr/bin/env ts-node
2
-
3
- import * as dotenv from 'dotenv';
4
-
5
- dotenv.config({ path: '.env' });
6
-
7
- import colors from 'colors'; // <- TODO: [🔶] Make system to put color and style to both node and browser
8
- import { writeFile } from 'fs/promises';
9
- import { join } from 'path';
10
- import { $provideExecutablesForNode } from '../../../executables/$provideExecutablesForNode';
11
- import { usageToHuman } from '../../../execution/utils/usageToHuman';
12
- import { $provideLlmToolsForTestingAndScriptsAndPlayground } from '../../../llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground';
13
- import { stringifyPipelineJson } from '../../../utils/editable/utils/stringifyPipelineJson';
14
- import { $provideFilesystemForNode } from '../../_common/register/$provideFilesystemForNode';
15
- import { makeKnowledgeSourceHandler } from '../../_common/utils/makeKnowledgeSourceHandler';
16
- import { DocumentScraper } from '../DocumentScraper';
17
-
18
- playground()
19
- .catch((error) => {
20
- console.error(colors.bgRed(error.name || 'NamelessError'));
21
- console.error(error);
22
- process.exit(1);
23
- })
24
- .then(() => {
25
- process.exit(0);
26
- });
27
-
28
- /**
29
- * Handles playground.
30
- */
31
- async function playground() {
32
- console.info(`🧸 Scrape knowledge from documents (playground)`);
33
-
34
- // Do here stuff you want to test
35
- //========================================>
36
-
37
- //const example = '10-simple.docx';
38
- const example = '10-simple.odt';
39
- // <- TODO: [👩🏿‍🤝‍👩🏼] Read here the examples directory and itterate through all of them
40
-
41
- const llmTools = await $provideLlmToolsForTestingAndScriptsAndPlayground({ isCacheReloaded: true });
42
- const rootDirname = join(__dirname, '..', 'examples');
43
-
44
- const documentScraper = new DocumentScraper(
45
- {
46
- fs: $provideFilesystemForNode(),
47
- llm: await $provideLlmToolsForTestingAndScriptsAndPlayground(),
48
- executables: await $provideExecutablesForNode(),
49
- },
50
- {
51
- rootDirname,
52
- },
53
- );
54
-
55
- const knowledge = await documentScraper.scrape(
56
- await makeKnowledgeSourceHandler(
57
- { knowledgeSourceContent: example },
58
- { fs: $provideFilesystemForNode() },
59
- { rootDirname },
60
- ),
61
- );
62
-
63
- console.info(colors.cyan(usageToHuman(llmTools.getTotalUsage())));
64
- console.info(colors.bgGreen(' Knowledge: '));
65
- console.info(knowledge);
66
-
67
- await writeFile(
68
- join(
69
- __dirname,
70
- `../examples/${example}.knowledge.json` /* <- TODO: [👩🏿‍🤝‍👩🏼] Read here the examples directory and itterate through all of them */,
71
- ),
72
- stringifyPipelineJson(knowledge),
73
- 'utf-8',
74
- );
75
- /**/
76
-
77
- //========================================/
78
- }
79
-
80
- // Note: [⚫] Code for playground [document-scraper-playground](src/scrapers/document/playground/document-scraper-playground.ts) should never be published in any package
@@ -1,19 +0,0 @@
1
- {
2
- "compilerOptions": {
3
- "target": "esnext",
4
- "module": "commonjs",
5
- "downlevelIteration": true,
6
- "allowJs": true,
7
- "moduleResolution": "node",
8
- "forceConsistentCasingInFileNames": true,
9
- "noImplicitReturns": true,
10
- "noImplicitThis": true,
11
- "noImplicitAny": true,
12
- "strictNullChecks": true,
13
- "experimentalDecorators": true,
14
- "noUnusedLocals": false,
15
- "resolveJsonModule": true,
16
- "esModuleInterop": true
17
- },
18
- "exclude": ["node_modules"]
19
- }
@@ -1,80 +0,0 @@
1
- #!/usr/bin/env ts-node
2
-
3
- import * as dotenv from 'dotenv';
4
-
5
- dotenv.config({ path: '.env' });
6
-
7
- import colors from 'colors'; // <- TODO: [🔶] Make system to put color and style to both node and browser
8
- import { writeFile } from 'fs/promises';
9
- import { join } from 'path';
10
- import { $provideExecutablesForNode } from '../../../executables/$provideExecutablesForNode';
11
- import { usageToHuman } from '../../../execution/utils/usageToHuman';
12
- import { $provideLlmToolsForTestingAndScriptsAndPlayground } from '../../../llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground';
13
- import { stringifyPipelineJson } from '../../../utils/editable/utils/stringifyPipelineJson';
14
- import { $provideFilesystemForNode } from '../../_common/register/$provideFilesystemForNode';
15
- import { makeKnowledgeSourceHandler } from '../../_common/utils/makeKnowledgeSourceHandler';
16
- import { LegacyDocumentScraper } from '../LegacyDocumentScraper';
17
-
18
- playground()
19
- .catch((error) => {
20
- console.error(colors.bgRed(error.name || 'NamelessError'));
21
- console.error(error);
22
- process.exit(1);
23
- })
24
- .then(() => {
25
- process.exit(0);
26
- });
27
-
28
- /**
29
- * Handles playground.
30
- */
31
- async function playground() {
32
- console.info(`🧸 Scrape knowledge from legacy documents (playground)`);
33
-
34
- // Do here stuff you want to test
35
- //========================================>
36
-
37
- //const example = '10-simple.doc';
38
- const example = '10-simple.rtf';
39
- // <- TODO: [👩🏿‍🤝‍👩🏼] Read here the examples directory and itterate through all of them
40
-
41
- const llmTools = await $provideLlmToolsForTestingAndScriptsAndPlayground({ isCacheReloaded: true });
42
- const rootDirname = join(__dirname, '..', 'examples');
43
-
44
- const legacyDocumentScraper = new LegacyDocumentScraper(
45
- {
46
- fs: $provideFilesystemForNode(),
47
- llm: await $provideLlmToolsForTestingAndScriptsAndPlayground(),
48
- executables: await $provideExecutablesForNode(),
49
- },
50
- {
51
- rootDirname,
52
- },
53
- );
54
-
55
- const knowledge = await legacyDocumentScraper.scrape(
56
- await makeKnowledgeSourceHandler(
57
- { knowledgeSourceContent: example },
58
- { fs: $provideFilesystemForNode() },
59
- { rootDirname },
60
- ),
61
- );
62
-
63
- console.info(colors.cyan(usageToHuman(llmTools.getTotalUsage())));
64
- console.info(colors.bgGreen(' Knowledge: '));
65
- console.info(knowledge);
66
-
67
- await writeFile(
68
- join(
69
- __dirname,
70
- `../examples/${example}.knowledge.json` /* <- TODO: [👩🏿‍🤝‍👩🏼] Read here the examples directory and itterate through all of them */,
71
- ),
72
- stringifyPipelineJson(knowledge),
73
- 'utf-8',
74
- );
75
- /**/
76
-
77
- //========================================/
78
- }
79
-
80
- // Note: [⚫] Code for playground [legacy-document-scraper-playground](src/scrapers/document-legacy/playground/legacy-document-scraper-playground.ts) should never be published in any package
@@ -1,19 +0,0 @@
1
- {
2
- "compilerOptions": {
3
- "target": "esnext",
4
- "module": "commonjs",
5
- "downlevelIteration": true,
6
- "allowJs": true,
7
- "moduleResolution": "node",
8
- "forceConsistentCasingInFileNames": true,
9
- "noImplicitReturns": true,
10
- "noImplicitThis": true,
11
- "noImplicitAny": true,
12
- "strictNullChecks": true,
13
- "experimentalDecorators": true,
14
- "noUnusedLocals": false,
15
- "resolveJsonModule": true,
16
- "esModuleInterop": true
17
- },
18
- "exclude": ["node_modules"]
19
- }