@promptbook/cli 0.112.0-104 → 0.112.0-105
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/apps/agents-server/src/app/admin/image-generator-test/ImageAttachmentsEditor.tsx +19 -3
- package/apps/agents-server/src/app/admin/limits/LimitsClient.tsx +11 -12
- package/apps/agents-server/src/app/admin/metadata/MetadataClient.tsx +34 -2
- package/apps/agents-server/src/app/admin/servers/CreateServerDialog.tsx +6 -1
- package/apps/agents-server/src/app/admin/servers/useCreateServerWizard.ts +13 -1
- package/apps/agents-server/src/app/agents/[agentName]/AgentChatWrapper.tsx +11 -2
- package/apps/agents-server/src/app/agents/[agentName]/AgentProfileChat.tsx +11 -1
- package/apps/agents-server/src/app/agents/[agentName]/book/BookEditorWrapper.tsx +7 -1
- package/apps/agents-server/src/app/agents/[agentName]/chat/CanonicalAgentChatSurface.tsx +11 -1
- package/apps/agents-server/src/app/agents/[agentName]/images/default-avatar.png/route.ts +6 -2
- package/apps/agents-server/src/app/api/images/[filename]/route.ts +6 -2
- package/apps/agents-server/src/app/api/internal/agent-runner-limits/route.ts +51 -0
- package/apps/agents-server/src/app/api/upload/route.ts +48 -12
- package/apps/agents-server/src/app/layout.tsx +13 -0
- package/apps/agents-server/src/components/FileUploadAvailability/FileUploadAvailabilityContext.tsx +50 -0
- package/apps/agents-server/src/components/FileUploadAvailability/FileUploadUnavailableNotice.tsx +45 -0
- package/apps/agents-server/src/components/LayoutWrapper/LayoutWrapper.tsx +85 -76
- package/apps/agents-server/src/components/NewAgentDialog/NewAgentDialog.tsx +7 -3
- package/apps/agents-server/src/components/NewAgentDialog/NewAgentWizardKnowledgeStep.tsx +6 -0
- package/apps/agents-server/src/components/NewAgentDialog/useNewAgentWizardKnowledgeState.ts +8 -1
- package/apps/agents-server/src/constants/serverLimits.ts +22 -2
- package/apps/agents-server/src/database/seedDefaultAgents.ts +218 -0
- package/apps/agents-server/src/tools/$provideCdnForServer.ts +72 -8
- package/apps/agents-server/src/utils/defaultAgents/loadDefaultAgentBooks.ts +103 -0
- package/apps/agents-server/src/utils/knowledge/createInlineKnowledgeSourceUploader.ts +24 -5
- package/apps/agents-server/src/utils/serverLimits.ts +26 -1
- package/apps/agents-server/src/utils/serverManagement/createManagedServer/seedServerDefaultAgents.ts +1 -85
- package/apps/agents-server/src/utils/shareTargetPayloads.ts +20 -2
- package/apps/agents-server/src/utils/upload/fileUploadAvailability.ts +91 -0
- package/apps/agents-server/src/utils/upload/uploadFileToServer.ts +46 -2
- package/esm/apps/agents-server/src/constants/federatedAgentImport.d.ts +42 -0
- package/esm/apps/agents-server/src/constants/serverLimits.d.ts +207 -0
- package/esm/apps/agents-server/src/constants/toolUsageLimits.d.ts +55 -0
- package/esm/index.es.js +602 -39
- package/esm/index.es.js.map +1 -1
- package/esm/scripts/run-agent-messages/main/AgentMessageFailureTracker.d.ts +27 -0
- package/esm/scripts/run-agent-messages/main/handleAgentWatchError.d.ts +4 -0
- package/esm/scripts/run-agent-messages/main/runAgentMessages.d.ts +1 -0
- package/esm/scripts/run-agent-messages/messages/moveAgentMessageToFailed.d.ts +17 -0
- package/esm/src/book-components/BookEditor/BookEditor.d.ts +5 -4
- package/esm/src/book-components/BookEditor/BookEditorTheme.d.ts +24 -0
- package/esm/src/book-components/BookEditor/useBookEditorMonacoLanguage.d.ts +1 -6
- package/esm/src/book-components/BookEditor/useBookEditorMonacoLifecycle.d.ts +1 -4
- package/esm/src/book-components/BookEditor/useBookEditorMonacoStyles.d.ts +2 -1
- package/esm/src/cli/cli-commands/agent-folder/agentProjectPaths.d.ts +6 -0
- package/esm/src/version.d.ts +1 -1
- package/package.json +1 -1
- package/src/avatars/visuals/octopus3d3AvatarVisual.ts +28 -29
- package/src/book-components/BookEditor/BookEditor.tsx +10 -7
- package/src/book-components/BookEditor/BookEditorMonaco.tsx +3 -1
- package/src/book-components/BookEditor/BookEditorTheme.ts +32 -0
- package/src/book-components/BookEditor/useBookEditorMonacoLanguage.ts +12 -15
- package/src/book-components/BookEditor/useBookEditorMonacoLifecycle.ts +1 -5
- package/src/book-components/BookEditor/useBookEditorMonacoStyles.ts +2 -1
- package/src/cli/cli-commands/agent-folder/agentProjectPaths.ts +7 -0
- package/src/cli/cli-commands/agents-server/buildAgentsServer.ts +109 -9
- package/src/cli/cli-commands/agents-server/startAgentsServer.ts +132 -4
- package/src/other/templates/getTemplatesPipelineCollection.ts +729 -854
- package/src/version.ts +2 -2
- package/src/versions.txt +1 -0
- package/umd/apps/agents-server/src/constants/federatedAgentImport.d.ts +42 -0
- package/umd/apps/agents-server/src/constants/serverLimits.d.ts +207 -0
- package/umd/apps/agents-server/src/constants/toolUsageLimits.d.ts +55 -0
- package/umd/index.umd.js +602 -39
- package/umd/index.umd.js.map +1 -1
- package/umd/scripts/run-agent-messages/main/AgentMessageFailureTracker.d.ts +27 -0
- package/umd/scripts/run-agent-messages/main/handleAgentWatchError.d.ts +4 -0
- package/umd/scripts/run-agent-messages/main/runAgentMessages.d.ts +1 -0
- package/umd/scripts/run-agent-messages/messages/moveAgentMessageToFailed.d.ts +17 -0
- package/umd/src/book-components/BookEditor/BookEditor.d.ts +5 -4
- package/umd/src/book-components/BookEditor/BookEditorTheme.d.ts +24 -0
- package/umd/src/book-components/BookEditor/useBookEditorMonacoLanguage.d.ts +1 -6
- package/umd/src/book-components/BookEditor/useBookEditorMonacoLifecycle.d.ts +1 -4
- package/umd/src/book-components/BookEditor/useBookEditorMonacoStyles.d.ts +2 -1
- package/umd/src/cli/cli-commands/agent-folder/agentProjectPaths.d.ts +6 -0
- package/umd/src/version.d.ts +1 -1
- package/apps/agents-server/src/message-providers/email/_common/utils/parseEmailAddress.test.ts.todo +0 -108
- package/apps/agents-server/src/message-providers/email/_common/utils/parseEmailAddresses.test.ts.todo +0 -117
- package/apps/agents-server/src/message-providers/email/_common/utils/stringifyEmailAddress.test.ts.todo +0 -119
- package/apps/agents-server/src/message-providers/email/_common/utils/stringifyEmailAddresses.test.ts.todo +0 -74
- package/apps/agents-server/src/utils/defaultAgents/defaultAgents.ts +0 -168
- package/apps/agents-server/src/utils/defaultAgents/installDefaultAgents.ts +0 -139
- package/apps/agents-server/tests/e2e/authentication-and-navigation.spec.ts.todo +0 -178
- package/src/_packages/browser.index.ts +0 -31
- package/src/_packages/browser.readme.md +0 -43
- package/src/book-2.0/agent-source/parseAgentSourceWithCommitments.test.ts.todo +0 -265
- package/src/book-components/BookEditor/BookEditorMonaco.test.tsx.todo +0 -115
- package/src/book-components/Chat/utils/renderMarkdown.test.ts.tmp +0 -199
- package/src/collection/agent-collection/constructors/agent-collection-in-directory/AgentCollectionInDirectory.test.ts.todo +0 -131
- package/src/commands/_common/parseCommand.test.ts.todo +0 -48
- package/src/commitments/META_LINK/META_LINK.test.ts.todo +0 -75
- package/src/conversion/validation/pipelineStringToJson-errors.test.ts.todo +0 -33
- package/src/dialogs/simple-prompt/SimplePromptInterfaceTools.ts +0 -51
- package/src/executables/browsers/locateSafari.test.ts.tmp +0 -15
- package/src/execution/PromptbookFetch.test-type.ts +0 -14
- package/src/execution/createPipelineExecutor/00-createPipelineExecutor.test.ts.todo +0 -0
- package/src/execution/execution-report/executionReportJsonToString.test.ts.todo +0 -83
- package/src/execution/utils/usageToHuman.test.ts.todo +0 -80
- package/src/llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground.ts +0 -76
- package/src/llm-providers/_common/utils/assertUniqueModels.ts +0 -27
- package/src/llm-providers/_multiple/playground/playground.ts +0 -141
- package/src/llm-providers/_multiple/playground/tsconfig.json +0 -19
- package/src/llm-providers/agent/playground/playground.ts +0 -190
- package/src/llm-providers/agent/playground/tsconfig.json +0 -19
- package/src/llm-providers/anthropic-claude/playground/playground.ts +0 -99
- package/src/llm-providers/anthropic-claude/playground/tsconfig.json +0 -19
- package/src/llm-providers/azure-openai/playground/playground.ts +0 -101
- package/src/llm-providers/azure-openai/playground/tsconfig.json +0 -19
- package/src/llm-providers/ollama/playground/playground.ts +0 -120
- package/src/llm-providers/ollama/playground/tsconfig.json +0 -19
- package/src/llm-providers/openai/playground/playground.ts +0 -406
- package/src/llm-providers/openai/playground/tsconfig.json +0 -19
- package/src/llm-providers/remote/playground/playground.ts +0 -144
- package/src/llm-providers/remote/playground/tsconfig.json +0 -19
- package/src/llm-providers/vercel/playground/playground.ts +0 -133
- package/src/llm-providers/vercel/playground/tsconfig.json +0 -19
- package/src/personas/preparePersona.test.ts.todo +0 -126
- package/src/playground/backup/_playground-boilerplate.ts.txt +0 -37
- package/src/playground/backup/playground-agent-os.txt +0 -62
- package/src/playground/backup/playground-brj-app.ts.txt +0 -302
- package/src/playground/backup/playground-browser-playwright.txt +0 -110
- package/src/playground/backup/playground-claude-mcp.txt +0 -43
- package/src/playground/backup/playground-document-conversion.txt +0 -84
- package/src/playground/backup/playground-glob.ts.txt +0 -42
- package/src/playground/backup/playground-mcp-server.txt +0 -1
- package/src/playground/backup/playground-openai-agent-kit.txt +0 -73
- package/src/playground/backup/playground-openai-function-calling.txt +0 -131
- package/src/playground/backup/playground-openai-streaming.ts.txt +0 -68
- package/src/playground/backup/playground-scrape-knowledge.txt +0 -65
- package/src/playground/backup/playground-scraperFetch.ts.txt +0 -44
- package/src/playground/backup/playground-using-openai-compatible-route-on-agents-server.ts.txt +0 -49
- package/src/playground/backup/playground-write-pavolhejny-bio.txt +0 -120
- package/src/playground/permanent/_boilerplate.ts +0 -54
- package/src/playground/permanent/agent-with-browser-playground.ts +0 -92
- package/src/playground/permanent/error-handling-playground.ts +0 -103
- package/src/playground/playground.ts +0 -36
- package/src/playground/tsconfig.json +0 -19
- package/src/scrapers/_boilerplate/BoilerplateScraper.test.ts.todo +0 -73
- package/src/scrapers/_boilerplate/playground/boilerplate-scraper-playground.ts +0 -79
- package/src/scrapers/_boilerplate/playground/tsconfig.json +0 -19
- package/src/scrapers/_common/utils/files/blobToDataurl.test.ts.todo +0 -17
- package/src/scrapers/_common/utils/files/dataurlToBlob.test.ts.todo +0 -52
- package/src/scrapers/_common/utils/files/isValidDataurl.test.ts.todo +0 -42
- package/src/scrapers/_common/utils/files/shorten.test.ts.todo +0 -13
- package/src/scrapers/document/playground/document-scraper-playground.ts +0 -80
- package/src/scrapers/document/playground/tsconfig.json +0 -19
- package/src/scrapers/document-legacy/playground/legacy-document-scraper-playground.ts +0 -80
- package/src/scrapers/document-legacy/playground/tsconfig.json +0 -19
- package/src/scrapers/markdown/playground/markdown-scraper-playground.ts +0 -74
- package/src/scrapers/markdown/playground/tsconfig.json +0 -19
- package/src/scrapers/markitdown/MarkitdownScraper.test.ts.todo +0 -132
- package/src/scrapers/markitdown/playground/markitdown-scraper-playground.ts +0 -91
- package/src/scrapers/markitdown/playground/tsconfig.json +0 -19
- package/src/scrapers/pdf/PdfScraper.test.ts.todo +0 -52
- package/src/scrapers/pdf/playground/pdf-scraper-playground.ts +0 -75
- package/src/scrapers/pdf/playground/tsconfig.json +0 -19
- package/src/scrapers/website/playground/tsconfig.json +0 -19
- package/src/scrapers/website/playground/website-scraper-playground.ts +0 -82
- package/src/storage/_common/PromptbookStorage.test-type.ts +0 -14
- package/src/storage/local-storage/getIndexedDbStorage.ts +0 -36
- package/src/storage/local-storage/getLocalStorage.ts +0 -33
- package/src/storage/local-storage/getSessionStorage.ts +0 -33
- package/src/storage/local-storage/utils/IndexedDbStorageOptions.ts +0 -16
- package/src/storage/local-storage/utils/makePromptbookStorageFromIndexedDb.ts +0 -58
- package/src/storage/local-storage/utils/makePromptbookStorageFromWebStorage.ts +0 -45
- package/src/transpilers/formatted-book-in-markdown/FormattedBookInMarkdownTranspiler.test.ts.todo +0 -35
- package/src/transpilers/openai-sdk/playground/playground.ts +0 -85
- package/src/transpilers/openai-sdk/playground/tmp/chatbot-openaisdk-1.js +0 -194
- package/src/transpilers/openai-sdk/playground/tmp/package.json +0 -3
- package/src/transpilers/openai-sdk/playground/tsconfig.json +0 -18
- package/src/utils/editable/utils/findUsableParameters.test.ts.todo +0 -43
- package/src/utils/editable/utils/stringifyPipelineJson.test.ts.todo +0 -38
- package/src/utils/markdown/prettifyMarkdown.test.ts.tmp +0 -42
- package/src/utils/serialization/serializeToPromptbookJavascript.test.ts.todo +0 -116
|
@@ -1,103 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env ts-node
|
|
2
|
-
|
|
3
|
-
// Note: [❌] Turning off some global checks for playground file:
|
|
4
|
-
// spell-checker: disable
|
|
5
|
-
/* eslint-disable */
|
|
6
|
-
|
|
7
|
-
import * as dotenv from 'dotenv';
|
|
8
|
-
|
|
9
|
-
dotenv.config({ path: '.env' });
|
|
10
|
-
|
|
11
|
-
import colors from 'colors';
|
|
12
|
-
import { join } from 'path';
|
|
13
|
-
import { spaceTrim } from 'spacetrim';
|
|
14
|
-
import { compilePipeline } from '../../conversion/compilePipeline';
|
|
15
|
-
import { CallbackInterfaceTools } from '../../dialogs/callback/CallbackInterfaceTools';
|
|
16
|
-
import { createPipelineExecutor } from '../../execution/createPipelineExecutor/00-createPipelineExecutor';
|
|
17
|
-
import { MockedEchoLlmExecutionTools } from '../../llm-providers/mocked/MockedEchoLlmExecutionTools';
|
|
18
|
-
import type { PipelineString } from '../../pipeline/PipelineString';
|
|
19
|
-
|
|
20
|
-
if (process.cwd() !== join(__dirname, '../../..')) {
|
|
21
|
-
console.error(
|
|
22
|
-
colors.red(
|
|
23
|
-
spaceTrim(`
|
|
24
|
-
CWD must be root of the project
|
|
25
|
-
|
|
26
|
-
Script: error-handling-playground.ts
|
|
27
|
-
Current CWD: ${process.cwd()}
|
|
28
|
-
Expected CWD: ${join(__dirname, '../../..')}
|
|
29
|
-
`),
|
|
30
|
-
),
|
|
31
|
-
);
|
|
32
|
-
process.exit(1);
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
playground()
|
|
36
|
-
.catch((error) => {
|
|
37
|
-
console.error(colors.bgRed(error.name || 'NamelessError'));
|
|
38
|
-
console.error(error);
|
|
39
|
-
process.exit(1);
|
|
40
|
-
})
|
|
41
|
-
.then(() => {
|
|
42
|
-
process.exit(0);
|
|
43
|
-
});
|
|
44
|
-
|
|
45
|
-
/**
|
|
46
|
-
* Handles playground.
|
|
47
|
-
*/
|
|
48
|
-
async function playground() {
|
|
49
|
-
console.info(`🧸 Error handling Playground`);
|
|
50
|
-
|
|
51
|
-
// Do here stuff you want to test
|
|
52
|
-
//========================================>
|
|
53
|
-
|
|
54
|
-
const pipelineExecutor = await getPipelineExecutor();
|
|
55
|
-
const result = await pipelineExecutor({}).asPromise({ isCrashedOnError: false });
|
|
56
|
-
|
|
57
|
-
console.info('result', result);
|
|
58
|
-
|
|
59
|
-
async function getPipelineExecutor() {
|
|
60
|
-
const pipeline = await compilePipeline(
|
|
61
|
-
spaceTrim(`
|
|
62
|
-
# Example prompt
|
|
63
|
-
|
|
64
|
-
Show how to use a simple chat prompt
|
|
65
|
-
|
|
66
|
-
- PROMPTBOOK VERSION 1.0.0
|
|
67
|
-
- PIPELINE URL https://promptbook.studio/examples/pipeline.book
|
|
68
|
-
- MODEL VARIANT Chat
|
|
69
|
-
- MODEL NAME gpt-3.5-turbo
|
|
70
|
-
- INPUT PARAMETER {thing} Any thing to buy
|
|
71
|
-
- OUTPUT PARAMETER {response}
|
|
72
|
-
|
|
73
|
-
## Prompt
|
|
74
|
-
|
|
75
|
-
\`\`\`
|
|
76
|
-
One day I went to the shop and bought {thing}.
|
|
77
|
-
Now I have {thing}.
|
|
78
|
-
\`\`\`
|
|
79
|
-
|
|
80
|
-
-> {response}
|
|
81
|
-
`) as PipelineString,
|
|
82
|
-
// <- TODO: [📼] Use`book\`` string literal notation
|
|
83
|
-
);
|
|
84
|
-
const pipelineExecutor = createPipelineExecutor({
|
|
85
|
-
pipeline,
|
|
86
|
-
tools: {
|
|
87
|
-
llm: new MockedEchoLlmExecutionTools({ isVerbose: true }),
|
|
88
|
-
script: [],
|
|
89
|
-
userInterface: new CallbackInterfaceTools({
|
|
90
|
-
isVerbose: true,
|
|
91
|
-
async callback() {
|
|
92
|
-
return 'Hello';
|
|
93
|
-
},
|
|
94
|
-
}),
|
|
95
|
-
},
|
|
96
|
-
});
|
|
97
|
-
return pipelineExecutor;
|
|
98
|
-
}
|
|
99
|
-
|
|
100
|
-
//========================================/
|
|
101
|
-
}
|
|
102
|
-
|
|
103
|
-
// Note: [⚫] Code for permanent playground [error-handling-playground](src/playground/permanent/error-handling-playground.ts) should never be published in any package
|
|
@@ -1,36 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env ts-node
|
|
2
|
-
|
|
3
|
-
import * as dotenv from 'dotenv';
|
|
4
|
-
|
|
5
|
-
dotenv.config({ path: '.env' });
|
|
6
|
-
|
|
7
|
-
import colors from 'colors';
|
|
8
|
-
import { join } from 'path';
|
|
9
|
-
|
|
10
|
-
if (process.cwd() !== join(__dirname, '../..')) {
|
|
11
|
-
console.error(colors.red(`CWD must be root of the project`));
|
|
12
|
-
process.exit(1);
|
|
13
|
-
}
|
|
14
|
-
|
|
15
|
-
playground()
|
|
16
|
-
.catch((error) => {
|
|
17
|
-
console.error(colors.bgRed(error.name || 'NamelessError'));
|
|
18
|
-
console.error(error);
|
|
19
|
-
process.exit(1);
|
|
20
|
-
})
|
|
21
|
-
.then(() => {
|
|
22
|
-
process.exit(0);
|
|
23
|
-
});
|
|
24
|
-
|
|
25
|
-
async function playground() {
|
|
26
|
-
console.info(`🧸 Playground`);
|
|
27
|
-
|
|
28
|
-
// Do here stuff you want to test
|
|
29
|
-
//========================================>
|
|
30
|
-
|
|
31
|
-
//========================================/
|
|
32
|
-
|
|
33
|
-
console.info(`[ Done 🧸 Playground ]`);
|
|
34
|
-
}
|
|
35
|
-
|
|
36
|
-
/** Note: [⚫] Code for archived playground [_playground-boilerplate.ts](src/playground/backup/_playground-boilerplate.ts.txt) should never be published in any package */
|
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"compilerOptions": {
|
|
3
|
-
"target": "esnext",
|
|
4
|
-
"module": "commonjs",
|
|
5
|
-
"downlevelIteration": true,
|
|
6
|
-
"allowJs": true,
|
|
7
|
-
"moduleResolution": "node",
|
|
8
|
-
"forceConsistentCasingInFileNames": true,
|
|
9
|
-
"noImplicitReturns": true,
|
|
10
|
-
"noImplicitThis": true,
|
|
11
|
-
"noImplicitAny": true,
|
|
12
|
-
"strictNullChecks": true,
|
|
13
|
-
"experimentalDecorators": true,
|
|
14
|
-
"noUnusedLocals": false,
|
|
15
|
-
"resolveJsonModule": true,
|
|
16
|
-
"esModuleInterop": true
|
|
17
|
-
},
|
|
18
|
-
"exclude": ["node_modules"]
|
|
19
|
-
}
|
|
@@ -1,73 +0,0 @@
|
|
|
1
|
-
// TODO: @@ Uncomment this test when making new scraper from boilerplate
|
|
2
|
-
|
|
3
|
-
import { describe, expect, it } from '@jest/globals';
|
|
4
|
-
import { join } from 'path';
|
|
5
|
-
import { $provideExecutablesForNode } from '../../executables/$provideExecutablesForNode';
|
|
6
|
-
import { $provideLlmToolsForTestingAndScriptsAndPlayground } from '../../llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground';
|
|
7
|
-
import { $provideFilesystemForNode } from '../_common/register/$provideFilesystemForNode';
|
|
8
|
-
import { makeKnowledgeSourceHandler } from '../_common/utils/makeKnowledgeSourceHandler';
|
|
9
|
-
import { BoilerplateScraper } from './BoilerplateScraper';
|
|
10
|
-
|
|
11
|
-
describe('how creating knowledge from @@ works', () => {
|
|
12
|
-
const rootDirname = join(__dirname, 'examples');
|
|
13
|
-
|
|
14
|
-
const boilerplateScraperPromise = (async () =>
|
|
15
|
-
new BoilerplateScraper(
|
|
16
|
-
{
|
|
17
|
-
fs: $provideFilesystemForNode(),
|
|
18
|
-
llm: await $provideLlmToolsForTestingAndScriptsAndPlayground(),
|
|
19
|
-
executables: await $provideExecutablesForNode(),
|
|
20
|
-
},
|
|
21
|
-
{
|
|
22
|
-
rootDirname,
|
|
23
|
-
},
|
|
24
|
-
))();
|
|
25
|
-
|
|
26
|
-
it('should scrape simple information from a @@ file', () =>
|
|
27
|
-
expect(
|
|
28
|
-
Promise.all([
|
|
29
|
-
boilerplateScraperPromise,
|
|
30
|
-
makeKnowledgeSourceHandler(
|
|
31
|
-
{
|
|
32
|
-
knowledgeSourceContent: '10-simple.@@',
|
|
33
|
-
},
|
|
34
|
-
{ fs: $provideFilesystemForNode() },
|
|
35
|
-
{ rootDirname },
|
|
36
|
-
),
|
|
37
|
-
])
|
|
38
|
-
.then(([boilerplateScraper, sourceHandler]) => boilerplateScraper.scrape(sourceHandler))
|
|
39
|
-
.then((knowledge) => knowledge?.map(({ content }) => ({ content })))
|
|
40
|
-
.then((knowledge) => knowledge?.slice(0, 1)),
|
|
41
|
-
).resolves.toMatchObject([
|
|
42
|
-
{
|
|
43
|
-
content: expect.stringMatching(/Springfield (is )?.*/i),
|
|
44
|
-
},
|
|
45
|
-
]));
|
|
46
|
-
|
|
47
|
-
// TODO: @@ Add all samples
|
|
48
|
-
|
|
49
|
-
it('should NOT scrape irrelevant information', () =>
|
|
50
|
-
expect(
|
|
51
|
-
Promise.all([
|
|
52
|
-
boilerplateScraperPromise,
|
|
53
|
-
makeKnowledgeSourceHandler(
|
|
54
|
-
{
|
|
55
|
-
knowledgeSourceContent: '10-simple.@@',
|
|
56
|
-
},
|
|
57
|
-
{ fs: $provideFilesystemForNode() },
|
|
58
|
-
{ rootDirname },
|
|
59
|
-
),
|
|
60
|
-
])
|
|
61
|
-
.then(([boilerplateScraper, sourceHandler]) => boilerplateScraper.scrape(sourceHandler))
|
|
62
|
-
.then((knowledge) => knowledge?.map(({ content }) => ({ content })))
|
|
63
|
-
.then((knowledge) => knowledge?.slice(0, 1)),
|
|
64
|
-
).resolves.toMatchObject([
|
|
65
|
-
{
|
|
66
|
-
content: expect.not.stringMatching(/London (is )?.*/i),
|
|
67
|
-
},
|
|
68
|
-
]));
|
|
69
|
-
});
|
|
70
|
-
|
|
71
|
-
/**
|
|
72
|
-
* TODO: [📓] Maybe test all file in examples (not just one by one)
|
|
73
|
-
*/
|
|
@@ -1,79 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env ts-node
|
|
2
|
-
|
|
3
|
-
import * as dotenv from 'dotenv';
|
|
4
|
-
|
|
5
|
-
dotenv.config({ path: '.env' });
|
|
6
|
-
|
|
7
|
-
import colors from 'colors'; // <- TODO: [🔶] Make system to put color and style to both node and browser
|
|
8
|
-
import { writeFile } from 'fs/promises';
|
|
9
|
-
import { join } from 'path';
|
|
10
|
-
import { $provideExecutablesForNode } from '../../../executables/$provideExecutablesForNode';
|
|
11
|
-
import { usageToHuman } from '../../../execution/utils/usageToHuman';
|
|
12
|
-
import { $provideLlmToolsForTestingAndScriptsAndPlayground } from '../../../llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground';
|
|
13
|
-
import { stringifyPipelineJson } from '../../../utils/editable/utils/stringifyPipelineJson';
|
|
14
|
-
import { $provideFilesystemForNode } from '../../_common/register/$provideFilesystemForNode';
|
|
15
|
-
import { makeKnowledgeSourceHandler } from '../../_common/utils/makeKnowledgeSourceHandler';
|
|
16
|
-
import { BoilerplateScraper } from '../BoilerplateScraper';
|
|
17
|
-
|
|
18
|
-
playground()
|
|
19
|
-
.catch((error) => {
|
|
20
|
-
console.error(colors.bgRed(error.name || 'NamelessError'));
|
|
21
|
-
console.error(error);
|
|
22
|
-
process.exit(1);
|
|
23
|
-
})
|
|
24
|
-
.then(() => {
|
|
25
|
-
process.exit(0);
|
|
26
|
-
});
|
|
27
|
-
|
|
28
|
-
/**
|
|
29
|
-
* Handles playground.
|
|
30
|
-
*/
|
|
31
|
-
async function playground() {
|
|
32
|
-
console.info(`🧸 Scrape knowledge from @@ (playground)`);
|
|
33
|
-
|
|
34
|
-
// Do here stuff you want to test
|
|
35
|
-
//========================================>
|
|
36
|
-
|
|
37
|
-
const example = '10-simple.@@';
|
|
38
|
-
// <- TODO: [👩🏿🤝👩🏼] Read here the examples directory and itterate through all of them
|
|
39
|
-
|
|
40
|
-
const llmTools = await $provideLlmToolsForTestingAndScriptsAndPlayground({ isCacheReloaded: true });
|
|
41
|
-
const rootDirname = join(__dirname, '..', 'examples');
|
|
42
|
-
|
|
43
|
-
const boilerplateScraper = new BoilerplateScraper(
|
|
44
|
-
{
|
|
45
|
-
fs: $provideFilesystemForNode(),
|
|
46
|
-
llm: await $provideLlmToolsForTestingAndScriptsAndPlayground(),
|
|
47
|
-
executables: await $provideExecutablesForNode(),
|
|
48
|
-
},
|
|
49
|
-
{
|
|
50
|
-
rootDirname,
|
|
51
|
-
},
|
|
52
|
-
);
|
|
53
|
-
|
|
54
|
-
const knowledge = await boilerplateScraper.scrape(
|
|
55
|
-
await makeKnowledgeSourceHandler(
|
|
56
|
-
{ knowledgeSourceContent: example },
|
|
57
|
-
{ fs: $provideFilesystemForNode() },
|
|
58
|
-
{ rootDirname },
|
|
59
|
-
),
|
|
60
|
-
);
|
|
61
|
-
|
|
62
|
-
console.info(colors.cyan(usageToHuman(llmTools.getTotalUsage())));
|
|
63
|
-
console.info(colors.bgGreen(' Knowledge: '));
|
|
64
|
-
console.info(knowledge);
|
|
65
|
-
|
|
66
|
-
await writeFile(
|
|
67
|
-
join(
|
|
68
|
-
__dirname,
|
|
69
|
-
`../examples/${example}.knowledge.json` /* <- TODO: [👩🏿🤝👩🏼] Read here the examples directory and itterate through all of them */,
|
|
70
|
-
),
|
|
71
|
-
stringifyPipelineJson(knowledge),
|
|
72
|
-
'utf-8',
|
|
73
|
-
);
|
|
74
|
-
/**/
|
|
75
|
-
|
|
76
|
-
//========================================/
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
// Note: [⚫] Code for playground [boilerplate-scraper-playground](src/scrapers/_boilerplate/playground/boilerplate-scraper-playground.ts) should never be published in any package
|
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"compilerOptions": {
|
|
3
|
-
"target": "esnext",
|
|
4
|
-
"module": "commonjs",
|
|
5
|
-
"downlevelIteration": true,
|
|
6
|
-
"allowJs": true,
|
|
7
|
-
"moduleResolution": "node",
|
|
8
|
-
"forceConsistentCasingInFileNames": true,
|
|
9
|
-
"noImplicitReturns": true,
|
|
10
|
-
"noImplicitThis": true,
|
|
11
|
-
"noImplicitAny": true,
|
|
12
|
-
"strictNullChecks": true,
|
|
13
|
-
"experimentalDecorators": true,
|
|
14
|
-
"noUnusedLocals": false,
|
|
15
|
-
"resolveJsonModule": true,
|
|
16
|
-
"esModuleInterop": true
|
|
17
|
-
},
|
|
18
|
-
"exclude": ["node_modules"]
|
|
19
|
-
}
|
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
import { describe, expect, it } from '@jest/globals';
|
|
2
|
-
import { blobToDataurl } from './blobToDataurl';
|
|
3
|
-
|
|
4
|
-
describe(`conversion of blob to dataurl`, () => {
|
|
5
|
-
it(`converts blob to base64 encoded dataurl`, () =>
|
|
6
|
-
expect(blobToDataurl(new Blob(
|
|
7
|
-
[`<h1>Hello, World!</h1>`], { type: 'text/html' }))).resolves.toBe(
|
|
8
|
-
`data:text/html;base64,PGgxPkhlbGxvLCBXb3JsZCE8L2gxPg==`,
|
|
9
|
-
));
|
|
10
|
-
});
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
/**
|
|
15
|
-
* TODO: [🧺] In future uncomment
|
|
16
|
-
* TODO: [🧺] Export via `@promptbook/pdf` and `@promptbook/docx`
|
|
17
|
-
*/
|
|
@@ -1,52 +0,0 @@
|
|
|
1
|
-
import { blobToBinaryString } from './blobToBinaryString';
|
|
2
|
-
import { dataurlToBlob } from './dataurlToBlob';
|
|
3
|
-
|
|
4
|
-
describe(`conversion of dataurl to blob`, () => {
|
|
5
|
-
it(`converts base64 without charset dataurl`, () => {
|
|
6
|
-
return expect(
|
|
7
|
-
dataurlToBlob(`data:text/html;base64,PGgxPkhlbGxvLCBXb3JsZCE8L2gxPg==`).then(async (blob) => [
|
|
8
|
-
blob.type,
|
|
9
|
-
await blobToBinaryString(blob),
|
|
10
|
-
]),
|
|
11
|
-
).resolves.toEqual([`text/html`, `<h1>Hello, World!</h1>`]);
|
|
12
|
-
});
|
|
13
|
-
|
|
14
|
-
it(`converts base64 with charset dataurl`, () => {
|
|
15
|
-
return expect(
|
|
16
|
-
dataurlToBlob(`data:text/html;charset=utf-8;base64,PGgxPkhlbGxvLCBXb3JsZCE8L2gxPg==`).then(async (blob) => [
|
|
17
|
-
blob.type,
|
|
18
|
-
await blobToBinaryString(blob),
|
|
19
|
-
]),
|
|
20
|
-
).resolves.toEqual([`text/html`, `<h1>Hello, World!</h1>`]);
|
|
21
|
-
});
|
|
22
|
-
|
|
23
|
-
it(`converts non-encoded, escaped dataurl`, () => {
|
|
24
|
-
return expect(
|
|
25
|
-
dataurlToBlob(`data:text/html,%3Ch1%3EHello%2C%20World%21%3C%2Fh1%3E`).then(async (blob) => [
|
|
26
|
-
blob.type,
|
|
27
|
-
await blobToBinaryString(blob),
|
|
28
|
-
]),
|
|
29
|
-
).resolves.toEqual([`text/html`, `<h1>Hello, World!</h1>`]);
|
|
30
|
-
});
|
|
31
|
-
|
|
32
|
-
it(`converts non-encoded, non-escaped dataurl`, () => {
|
|
33
|
-
return expect(
|
|
34
|
-
dataurlToBlob(`data:text/html,<h1>Hello, World!</h1>`).then(async (blob) => [
|
|
35
|
-
blob.type,
|
|
36
|
-
await blobToBinaryString(blob),
|
|
37
|
-
]),
|
|
38
|
-
).resolves.toEqual([`text/html`, `<h1>Hello, World!</h1>`]);
|
|
39
|
-
});
|
|
40
|
-
|
|
41
|
-
it(`checks that it is not wrong type`, () => {
|
|
42
|
-
return expect(
|
|
43
|
-
dataurlToBlob(`data:text/html;base64,PGgxPkhlbGxvLCBXb3JsZCE8L2gxPg==`).then((blob) => [blob.type]),
|
|
44
|
-
).resolves.not.toEqual(`text/plain`);
|
|
45
|
-
});
|
|
46
|
-
|
|
47
|
-
it(`checks that it is not wrong content`, () => {
|
|
48
|
-
return expect(
|
|
49
|
-
dataurlToBlob(`data:text/html;base64,PGgxPkhlbGxvLCBXb3JsZCE8L2gxPg==`).then(blobToBinaryString),
|
|
50
|
-
).resolves.not.toEqual(`<h1>Hello, Fooo!</h1>`);
|
|
51
|
-
});
|
|
52
|
-
});
|
|
@@ -1,42 +0,0 @@
|
|
|
1
|
-
import { isValidDataurl } from './isValidDataurl';
|
|
2
|
-
|
|
3
|
-
describe(`validation of urls`, () => {
|
|
4
|
-
it(`is valid base64 without charset dataurl`, () => {
|
|
5
|
-
expect(isValidDataurl(`data:text/plain;base64,PGgxPkhlbGxvLCBXb3JsZCE8L2gxPg==`)).toBe(true);
|
|
6
|
-
});
|
|
7
|
-
|
|
8
|
-
it(`is valid base64 with charset dataurl`, () => {
|
|
9
|
-
expect(isValidDataurl(`data:text/plain;charset=utf-8;base64,PGgxPkhlbGxvLCBXb3JsZCE8L2gxPg==`)).toBe(true);
|
|
10
|
-
});
|
|
11
|
-
|
|
12
|
-
it(`is valid non-encoded, escaped dataurl`, () => {
|
|
13
|
-
expect(isValidDataurl(`data:text/html,%3Ch1%3EHello%2C%20World%21%3C%2Fh1%3E`)).toBe(true);
|
|
14
|
-
});
|
|
15
|
-
|
|
16
|
-
it(`is valid non-encoded, non-escaped dataurl`, () => {
|
|
17
|
-
expect(isValidDataurl(`data:text/html,<h1>Hello, World!</h1>`)).toBe(true);
|
|
18
|
-
});
|
|
19
|
-
|
|
20
|
-
/*
|
|
21
|
-
TODO: [🏞️]
|
|
22
|
-
it(`is corrupted dataurl`, () => {
|
|
23
|
-
expect(isValidDataurl(`data:text/plain;charset=utf-8;base12345678,PGgxPkhlbGxvLCBXb3JsZCE8L2gxPg==`)).toBe(
|
|
24
|
-
false,
|
|
25
|
-
);
|
|
26
|
-
expect(isValidDataurl(`data:text/plain;base64 PGgxPkhlbGxvLCBXb3JsZCE8L2gxPg==`)).toBe(false);
|
|
27
|
-
expect(isValidDataurl(`data:PGgxPkhlbGxvLCBXb3JsZCE8L2gxPg==`)).toBe(false);
|
|
28
|
-
});
|
|
29
|
-
*/
|
|
30
|
-
|
|
31
|
-
it(`is valid url BUT not dataurl`, () => {
|
|
32
|
-
expect(isValidDataurl(`https://collboard.com/`)).toBe(false);
|
|
33
|
-
expect(isValidDataurl(`http://localhost:9977/fooo/add`)).toBe(false);
|
|
34
|
-
});
|
|
35
|
-
|
|
36
|
-
it(`is NOT valid url`, () => {
|
|
37
|
-
expect(isValidDataurl(``)).toBe(false);
|
|
38
|
-
expect(isValidDataurl(`Invalid URL`)).toBe(false);
|
|
39
|
-
expect(isValidDataurl(`aegfawsgsdasdg`)).toBe(false);
|
|
40
|
-
expect(isValidDataurl(`wtf://collboard.com/`)).toBe(false);
|
|
41
|
-
});
|
|
42
|
-
});
|
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
import { shorten } from './shorten';
|
|
2
|
-
|
|
3
|
-
describe('how shorten works', () => {
|
|
4
|
-
it('will shorten', () => {
|
|
5
|
-
expect(shorten('Foo', 2)).toEqual('F…');
|
|
6
|
-
expect(shorten('Foooooooooooooooooooooooooooooooo', 5)).toEqual('Fooo…');
|
|
7
|
-
});
|
|
8
|
-
|
|
9
|
-
it('will keep short', () => {
|
|
10
|
-
expect(shorten('Foo', 3)).toEqual('Foo');
|
|
11
|
-
expect(shorten('Foo', 4)).toEqual('Foo');
|
|
12
|
-
});
|
|
13
|
-
});
|
|
@@ -1,80 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env ts-node
|
|
2
|
-
|
|
3
|
-
import * as dotenv from 'dotenv';
|
|
4
|
-
|
|
5
|
-
dotenv.config({ path: '.env' });
|
|
6
|
-
|
|
7
|
-
import colors from 'colors'; // <- TODO: [🔶] Make system to put color and style to both node and browser
|
|
8
|
-
import { writeFile } from 'fs/promises';
|
|
9
|
-
import { join } from 'path';
|
|
10
|
-
import { $provideExecutablesForNode } from '../../../executables/$provideExecutablesForNode';
|
|
11
|
-
import { usageToHuman } from '../../../execution/utils/usageToHuman';
|
|
12
|
-
import { $provideLlmToolsForTestingAndScriptsAndPlayground } from '../../../llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground';
|
|
13
|
-
import { stringifyPipelineJson } from '../../../utils/editable/utils/stringifyPipelineJson';
|
|
14
|
-
import { $provideFilesystemForNode } from '../../_common/register/$provideFilesystemForNode';
|
|
15
|
-
import { makeKnowledgeSourceHandler } from '../../_common/utils/makeKnowledgeSourceHandler';
|
|
16
|
-
import { DocumentScraper } from '../DocumentScraper';
|
|
17
|
-
|
|
18
|
-
playground()
|
|
19
|
-
.catch((error) => {
|
|
20
|
-
console.error(colors.bgRed(error.name || 'NamelessError'));
|
|
21
|
-
console.error(error);
|
|
22
|
-
process.exit(1);
|
|
23
|
-
})
|
|
24
|
-
.then(() => {
|
|
25
|
-
process.exit(0);
|
|
26
|
-
});
|
|
27
|
-
|
|
28
|
-
/**
|
|
29
|
-
* Handles playground.
|
|
30
|
-
*/
|
|
31
|
-
async function playground() {
|
|
32
|
-
console.info(`🧸 Scrape knowledge from documents (playground)`);
|
|
33
|
-
|
|
34
|
-
// Do here stuff you want to test
|
|
35
|
-
//========================================>
|
|
36
|
-
|
|
37
|
-
//const example = '10-simple.docx';
|
|
38
|
-
const example = '10-simple.odt';
|
|
39
|
-
// <- TODO: [👩🏿🤝👩🏼] Read here the examples directory and itterate through all of them
|
|
40
|
-
|
|
41
|
-
const llmTools = await $provideLlmToolsForTestingAndScriptsAndPlayground({ isCacheReloaded: true });
|
|
42
|
-
const rootDirname = join(__dirname, '..', 'examples');
|
|
43
|
-
|
|
44
|
-
const documentScraper = new DocumentScraper(
|
|
45
|
-
{
|
|
46
|
-
fs: $provideFilesystemForNode(),
|
|
47
|
-
llm: await $provideLlmToolsForTestingAndScriptsAndPlayground(),
|
|
48
|
-
executables: await $provideExecutablesForNode(),
|
|
49
|
-
},
|
|
50
|
-
{
|
|
51
|
-
rootDirname,
|
|
52
|
-
},
|
|
53
|
-
);
|
|
54
|
-
|
|
55
|
-
const knowledge = await documentScraper.scrape(
|
|
56
|
-
await makeKnowledgeSourceHandler(
|
|
57
|
-
{ knowledgeSourceContent: example },
|
|
58
|
-
{ fs: $provideFilesystemForNode() },
|
|
59
|
-
{ rootDirname },
|
|
60
|
-
),
|
|
61
|
-
);
|
|
62
|
-
|
|
63
|
-
console.info(colors.cyan(usageToHuman(llmTools.getTotalUsage())));
|
|
64
|
-
console.info(colors.bgGreen(' Knowledge: '));
|
|
65
|
-
console.info(knowledge);
|
|
66
|
-
|
|
67
|
-
await writeFile(
|
|
68
|
-
join(
|
|
69
|
-
__dirname,
|
|
70
|
-
`../examples/${example}.knowledge.json` /* <- TODO: [👩🏿🤝👩🏼] Read here the examples directory and itterate through all of them */,
|
|
71
|
-
),
|
|
72
|
-
stringifyPipelineJson(knowledge),
|
|
73
|
-
'utf-8',
|
|
74
|
-
);
|
|
75
|
-
/**/
|
|
76
|
-
|
|
77
|
-
//========================================/
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
// Note: [⚫] Code for playground [document-scraper-playground](src/scrapers/document/playground/document-scraper-playground.ts) should never be published in any package
|
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"compilerOptions": {
|
|
3
|
-
"target": "esnext",
|
|
4
|
-
"module": "commonjs",
|
|
5
|
-
"downlevelIteration": true,
|
|
6
|
-
"allowJs": true,
|
|
7
|
-
"moduleResolution": "node",
|
|
8
|
-
"forceConsistentCasingInFileNames": true,
|
|
9
|
-
"noImplicitReturns": true,
|
|
10
|
-
"noImplicitThis": true,
|
|
11
|
-
"noImplicitAny": true,
|
|
12
|
-
"strictNullChecks": true,
|
|
13
|
-
"experimentalDecorators": true,
|
|
14
|
-
"noUnusedLocals": false,
|
|
15
|
-
"resolveJsonModule": true,
|
|
16
|
-
"esModuleInterop": true
|
|
17
|
-
},
|
|
18
|
-
"exclude": ["node_modules"]
|
|
19
|
-
}
|
|
@@ -1,80 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env ts-node
|
|
2
|
-
|
|
3
|
-
import * as dotenv from 'dotenv';
|
|
4
|
-
|
|
5
|
-
dotenv.config({ path: '.env' });
|
|
6
|
-
|
|
7
|
-
import colors from 'colors'; // <- TODO: [🔶] Make system to put color and style to both node and browser
|
|
8
|
-
import { writeFile } from 'fs/promises';
|
|
9
|
-
import { join } from 'path';
|
|
10
|
-
import { $provideExecutablesForNode } from '../../../executables/$provideExecutablesForNode';
|
|
11
|
-
import { usageToHuman } from '../../../execution/utils/usageToHuman';
|
|
12
|
-
import { $provideLlmToolsForTestingAndScriptsAndPlayground } from '../../../llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground';
|
|
13
|
-
import { stringifyPipelineJson } from '../../../utils/editable/utils/stringifyPipelineJson';
|
|
14
|
-
import { $provideFilesystemForNode } from '../../_common/register/$provideFilesystemForNode';
|
|
15
|
-
import { makeKnowledgeSourceHandler } from '../../_common/utils/makeKnowledgeSourceHandler';
|
|
16
|
-
import { LegacyDocumentScraper } from '../LegacyDocumentScraper';
|
|
17
|
-
|
|
18
|
-
playground()
|
|
19
|
-
.catch((error) => {
|
|
20
|
-
console.error(colors.bgRed(error.name || 'NamelessError'));
|
|
21
|
-
console.error(error);
|
|
22
|
-
process.exit(1);
|
|
23
|
-
})
|
|
24
|
-
.then(() => {
|
|
25
|
-
process.exit(0);
|
|
26
|
-
});
|
|
27
|
-
|
|
28
|
-
/**
|
|
29
|
-
* Handles playground.
|
|
30
|
-
*/
|
|
31
|
-
async function playground() {
|
|
32
|
-
console.info(`🧸 Scrape knowledge from legacy documents (playground)`);
|
|
33
|
-
|
|
34
|
-
// Do here stuff you want to test
|
|
35
|
-
//========================================>
|
|
36
|
-
|
|
37
|
-
//const example = '10-simple.doc';
|
|
38
|
-
const example = '10-simple.rtf';
|
|
39
|
-
// <- TODO: [👩🏿🤝👩🏼] Read here the examples directory and itterate through all of them
|
|
40
|
-
|
|
41
|
-
const llmTools = await $provideLlmToolsForTestingAndScriptsAndPlayground({ isCacheReloaded: true });
|
|
42
|
-
const rootDirname = join(__dirname, '..', 'examples');
|
|
43
|
-
|
|
44
|
-
const legacyDocumentScraper = new LegacyDocumentScraper(
|
|
45
|
-
{
|
|
46
|
-
fs: $provideFilesystemForNode(),
|
|
47
|
-
llm: await $provideLlmToolsForTestingAndScriptsAndPlayground(),
|
|
48
|
-
executables: await $provideExecutablesForNode(),
|
|
49
|
-
},
|
|
50
|
-
{
|
|
51
|
-
rootDirname,
|
|
52
|
-
},
|
|
53
|
-
);
|
|
54
|
-
|
|
55
|
-
const knowledge = await legacyDocumentScraper.scrape(
|
|
56
|
-
await makeKnowledgeSourceHandler(
|
|
57
|
-
{ knowledgeSourceContent: example },
|
|
58
|
-
{ fs: $provideFilesystemForNode() },
|
|
59
|
-
{ rootDirname },
|
|
60
|
-
),
|
|
61
|
-
);
|
|
62
|
-
|
|
63
|
-
console.info(colors.cyan(usageToHuman(llmTools.getTotalUsage())));
|
|
64
|
-
console.info(colors.bgGreen(' Knowledge: '));
|
|
65
|
-
console.info(knowledge);
|
|
66
|
-
|
|
67
|
-
await writeFile(
|
|
68
|
-
join(
|
|
69
|
-
__dirname,
|
|
70
|
-
`../examples/${example}.knowledge.json` /* <- TODO: [👩🏿🤝👩🏼] Read here the examples directory and itterate through all of them */,
|
|
71
|
-
),
|
|
72
|
-
stringifyPipelineJson(knowledge),
|
|
73
|
-
'utf-8',
|
|
74
|
-
);
|
|
75
|
-
/**/
|
|
76
|
-
|
|
77
|
-
//========================================/
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
// Note: [⚫] Code for playground [legacy-document-scraper-playground](src/scrapers/document-legacy/playground/legacy-document-scraper-playground.ts) should never be published in any package
|
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"compilerOptions": {
|
|
3
|
-
"target": "esnext",
|
|
4
|
-
"module": "commonjs",
|
|
5
|
-
"downlevelIteration": true,
|
|
6
|
-
"allowJs": true,
|
|
7
|
-
"moduleResolution": "node",
|
|
8
|
-
"forceConsistentCasingInFileNames": true,
|
|
9
|
-
"noImplicitReturns": true,
|
|
10
|
-
"noImplicitThis": true,
|
|
11
|
-
"noImplicitAny": true,
|
|
12
|
-
"strictNullChecks": true,
|
|
13
|
-
"experimentalDecorators": true,
|
|
14
|
-
"noUnusedLocals": false,
|
|
15
|
-
"resolveJsonModule": true,
|
|
16
|
-
"esModuleInterop": true
|
|
17
|
-
},
|
|
18
|
-
"exclude": ["node_modules"]
|
|
19
|
-
}
|