@promptbook/cli 0.112.0-104 → 0.112.0-105
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/apps/agents-server/src/app/admin/image-generator-test/ImageAttachmentsEditor.tsx +19 -3
- package/apps/agents-server/src/app/admin/limits/LimitsClient.tsx +11 -12
- package/apps/agents-server/src/app/admin/metadata/MetadataClient.tsx +34 -2
- package/apps/agents-server/src/app/admin/servers/CreateServerDialog.tsx +6 -1
- package/apps/agents-server/src/app/admin/servers/useCreateServerWizard.ts +13 -1
- package/apps/agents-server/src/app/agents/[agentName]/AgentChatWrapper.tsx +11 -2
- package/apps/agents-server/src/app/agents/[agentName]/AgentProfileChat.tsx +11 -1
- package/apps/agents-server/src/app/agents/[agentName]/book/BookEditorWrapper.tsx +7 -1
- package/apps/agents-server/src/app/agents/[agentName]/chat/CanonicalAgentChatSurface.tsx +11 -1
- package/apps/agents-server/src/app/agents/[agentName]/images/default-avatar.png/route.ts +6 -2
- package/apps/agents-server/src/app/api/images/[filename]/route.ts +6 -2
- package/apps/agents-server/src/app/api/internal/agent-runner-limits/route.ts +51 -0
- package/apps/agents-server/src/app/api/upload/route.ts +48 -12
- package/apps/agents-server/src/app/layout.tsx +13 -0
- package/apps/agents-server/src/components/FileUploadAvailability/FileUploadAvailabilityContext.tsx +50 -0
- package/apps/agents-server/src/components/FileUploadAvailability/FileUploadUnavailableNotice.tsx +45 -0
- package/apps/agents-server/src/components/LayoutWrapper/LayoutWrapper.tsx +85 -76
- package/apps/agents-server/src/components/NewAgentDialog/NewAgentDialog.tsx +7 -3
- package/apps/agents-server/src/components/NewAgentDialog/NewAgentWizardKnowledgeStep.tsx +6 -0
- package/apps/agents-server/src/components/NewAgentDialog/useNewAgentWizardKnowledgeState.ts +8 -1
- package/apps/agents-server/src/constants/serverLimits.ts +22 -2
- package/apps/agents-server/src/database/seedDefaultAgents.ts +218 -0
- package/apps/agents-server/src/tools/$provideCdnForServer.ts +72 -8
- package/apps/agents-server/src/utils/defaultAgents/loadDefaultAgentBooks.ts +103 -0
- package/apps/agents-server/src/utils/knowledge/createInlineKnowledgeSourceUploader.ts +24 -5
- package/apps/agents-server/src/utils/serverLimits.ts +26 -1
- package/apps/agents-server/src/utils/serverManagement/createManagedServer/seedServerDefaultAgents.ts +1 -85
- package/apps/agents-server/src/utils/shareTargetPayloads.ts +20 -2
- package/apps/agents-server/src/utils/upload/fileUploadAvailability.ts +91 -0
- package/apps/agents-server/src/utils/upload/uploadFileToServer.ts +46 -2
- package/esm/apps/agents-server/src/constants/federatedAgentImport.d.ts +42 -0
- package/esm/apps/agents-server/src/constants/serverLimits.d.ts +207 -0
- package/esm/apps/agents-server/src/constants/toolUsageLimits.d.ts +55 -0
- package/esm/index.es.js +602 -39
- package/esm/index.es.js.map +1 -1
- package/esm/scripts/run-agent-messages/main/AgentMessageFailureTracker.d.ts +27 -0
- package/esm/scripts/run-agent-messages/main/handleAgentWatchError.d.ts +4 -0
- package/esm/scripts/run-agent-messages/main/runAgentMessages.d.ts +1 -0
- package/esm/scripts/run-agent-messages/messages/moveAgentMessageToFailed.d.ts +17 -0
- package/esm/src/book-components/BookEditor/BookEditor.d.ts +5 -4
- package/esm/src/book-components/BookEditor/BookEditorTheme.d.ts +24 -0
- package/esm/src/book-components/BookEditor/useBookEditorMonacoLanguage.d.ts +1 -6
- package/esm/src/book-components/BookEditor/useBookEditorMonacoLifecycle.d.ts +1 -4
- package/esm/src/book-components/BookEditor/useBookEditorMonacoStyles.d.ts +2 -1
- package/esm/src/cli/cli-commands/agent-folder/agentProjectPaths.d.ts +6 -0
- package/esm/src/version.d.ts +1 -1
- package/package.json +1 -1
- package/src/avatars/visuals/octopus3d3AvatarVisual.ts +28 -29
- package/src/book-components/BookEditor/BookEditor.tsx +10 -7
- package/src/book-components/BookEditor/BookEditorMonaco.tsx +3 -1
- package/src/book-components/BookEditor/BookEditorTheme.ts +32 -0
- package/src/book-components/BookEditor/useBookEditorMonacoLanguage.ts +12 -15
- package/src/book-components/BookEditor/useBookEditorMonacoLifecycle.ts +1 -5
- package/src/book-components/BookEditor/useBookEditorMonacoStyles.ts +2 -1
- package/src/cli/cli-commands/agent-folder/agentProjectPaths.ts +7 -0
- package/src/cli/cli-commands/agents-server/buildAgentsServer.ts +109 -9
- package/src/cli/cli-commands/agents-server/startAgentsServer.ts +132 -4
- package/src/other/templates/getTemplatesPipelineCollection.ts +729 -854
- package/src/version.ts +2 -2
- package/src/versions.txt +1 -0
- package/umd/apps/agents-server/src/constants/federatedAgentImport.d.ts +42 -0
- package/umd/apps/agents-server/src/constants/serverLimits.d.ts +207 -0
- package/umd/apps/agents-server/src/constants/toolUsageLimits.d.ts +55 -0
- package/umd/index.umd.js +602 -39
- package/umd/index.umd.js.map +1 -1
- package/umd/scripts/run-agent-messages/main/AgentMessageFailureTracker.d.ts +27 -0
- package/umd/scripts/run-agent-messages/main/handleAgentWatchError.d.ts +4 -0
- package/umd/scripts/run-agent-messages/main/runAgentMessages.d.ts +1 -0
- package/umd/scripts/run-agent-messages/messages/moveAgentMessageToFailed.d.ts +17 -0
- package/umd/src/book-components/BookEditor/BookEditor.d.ts +5 -4
- package/umd/src/book-components/BookEditor/BookEditorTheme.d.ts +24 -0
- package/umd/src/book-components/BookEditor/useBookEditorMonacoLanguage.d.ts +1 -6
- package/umd/src/book-components/BookEditor/useBookEditorMonacoLifecycle.d.ts +1 -4
- package/umd/src/book-components/BookEditor/useBookEditorMonacoStyles.d.ts +2 -1
- package/umd/src/cli/cli-commands/agent-folder/agentProjectPaths.d.ts +6 -0
- package/umd/src/version.d.ts +1 -1
- package/apps/agents-server/src/message-providers/email/_common/utils/parseEmailAddress.test.ts.todo +0 -108
- package/apps/agents-server/src/message-providers/email/_common/utils/parseEmailAddresses.test.ts.todo +0 -117
- package/apps/agents-server/src/message-providers/email/_common/utils/stringifyEmailAddress.test.ts.todo +0 -119
- package/apps/agents-server/src/message-providers/email/_common/utils/stringifyEmailAddresses.test.ts.todo +0 -74
- package/apps/agents-server/src/utils/defaultAgents/defaultAgents.ts +0 -168
- package/apps/agents-server/src/utils/defaultAgents/installDefaultAgents.ts +0 -139
- package/apps/agents-server/tests/e2e/authentication-and-navigation.spec.ts.todo +0 -178
- package/src/_packages/browser.index.ts +0 -31
- package/src/_packages/browser.readme.md +0 -43
- package/src/book-2.0/agent-source/parseAgentSourceWithCommitments.test.ts.todo +0 -265
- package/src/book-components/BookEditor/BookEditorMonaco.test.tsx.todo +0 -115
- package/src/book-components/Chat/utils/renderMarkdown.test.ts.tmp +0 -199
- package/src/collection/agent-collection/constructors/agent-collection-in-directory/AgentCollectionInDirectory.test.ts.todo +0 -131
- package/src/commands/_common/parseCommand.test.ts.todo +0 -48
- package/src/commitments/META_LINK/META_LINK.test.ts.todo +0 -75
- package/src/conversion/validation/pipelineStringToJson-errors.test.ts.todo +0 -33
- package/src/dialogs/simple-prompt/SimplePromptInterfaceTools.ts +0 -51
- package/src/executables/browsers/locateSafari.test.ts.tmp +0 -15
- package/src/execution/PromptbookFetch.test-type.ts +0 -14
- package/src/execution/createPipelineExecutor/00-createPipelineExecutor.test.ts.todo +0 -0
- package/src/execution/execution-report/executionReportJsonToString.test.ts.todo +0 -83
- package/src/execution/utils/usageToHuman.test.ts.todo +0 -80
- package/src/llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground.ts +0 -76
- package/src/llm-providers/_common/utils/assertUniqueModels.ts +0 -27
- package/src/llm-providers/_multiple/playground/playground.ts +0 -141
- package/src/llm-providers/_multiple/playground/tsconfig.json +0 -19
- package/src/llm-providers/agent/playground/playground.ts +0 -190
- package/src/llm-providers/agent/playground/tsconfig.json +0 -19
- package/src/llm-providers/anthropic-claude/playground/playground.ts +0 -99
- package/src/llm-providers/anthropic-claude/playground/tsconfig.json +0 -19
- package/src/llm-providers/azure-openai/playground/playground.ts +0 -101
- package/src/llm-providers/azure-openai/playground/tsconfig.json +0 -19
- package/src/llm-providers/ollama/playground/playground.ts +0 -120
- package/src/llm-providers/ollama/playground/tsconfig.json +0 -19
- package/src/llm-providers/openai/playground/playground.ts +0 -406
- package/src/llm-providers/openai/playground/tsconfig.json +0 -19
- package/src/llm-providers/remote/playground/playground.ts +0 -144
- package/src/llm-providers/remote/playground/tsconfig.json +0 -19
- package/src/llm-providers/vercel/playground/playground.ts +0 -133
- package/src/llm-providers/vercel/playground/tsconfig.json +0 -19
- package/src/personas/preparePersona.test.ts.todo +0 -126
- package/src/playground/backup/_playground-boilerplate.ts.txt +0 -37
- package/src/playground/backup/playground-agent-os.txt +0 -62
- package/src/playground/backup/playground-brj-app.ts.txt +0 -302
- package/src/playground/backup/playground-browser-playwright.txt +0 -110
- package/src/playground/backup/playground-claude-mcp.txt +0 -43
- package/src/playground/backup/playground-document-conversion.txt +0 -84
- package/src/playground/backup/playground-glob.ts.txt +0 -42
- package/src/playground/backup/playground-mcp-server.txt +0 -1
- package/src/playground/backup/playground-openai-agent-kit.txt +0 -73
- package/src/playground/backup/playground-openai-function-calling.txt +0 -131
- package/src/playground/backup/playground-openai-streaming.ts.txt +0 -68
- package/src/playground/backup/playground-scrape-knowledge.txt +0 -65
- package/src/playground/backup/playground-scraperFetch.ts.txt +0 -44
- package/src/playground/backup/playground-using-openai-compatible-route-on-agents-server.ts.txt +0 -49
- package/src/playground/backup/playground-write-pavolhejny-bio.txt +0 -120
- package/src/playground/permanent/_boilerplate.ts +0 -54
- package/src/playground/permanent/agent-with-browser-playground.ts +0 -92
- package/src/playground/permanent/error-handling-playground.ts +0 -103
- package/src/playground/playground.ts +0 -36
- package/src/playground/tsconfig.json +0 -19
- package/src/scrapers/_boilerplate/BoilerplateScraper.test.ts.todo +0 -73
- package/src/scrapers/_boilerplate/playground/boilerplate-scraper-playground.ts +0 -79
- package/src/scrapers/_boilerplate/playground/tsconfig.json +0 -19
- package/src/scrapers/_common/utils/files/blobToDataurl.test.ts.todo +0 -17
- package/src/scrapers/_common/utils/files/dataurlToBlob.test.ts.todo +0 -52
- package/src/scrapers/_common/utils/files/isValidDataurl.test.ts.todo +0 -42
- package/src/scrapers/_common/utils/files/shorten.test.ts.todo +0 -13
- package/src/scrapers/document/playground/document-scraper-playground.ts +0 -80
- package/src/scrapers/document/playground/tsconfig.json +0 -19
- package/src/scrapers/document-legacy/playground/legacy-document-scraper-playground.ts +0 -80
- package/src/scrapers/document-legacy/playground/tsconfig.json +0 -19
- package/src/scrapers/markdown/playground/markdown-scraper-playground.ts +0 -74
- package/src/scrapers/markdown/playground/tsconfig.json +0 -19
- package/src/scrapers/markitdown/MarkitdownScraper.test.ts.todo +0 -132
- package/src/scrapers/markitdown/playground/markitdown-scraper-playground.ts +0 -91
- package/src/scrapers/markitdown/playground/tsconfig.json +0 -19
- package/src/scrapers/pdf/PdfScraper.test.ts.todo +0 -52
- package/src/scrapers/pdf/playground/pdf-scraper-playground.ts +0 -75
- package/src/scrapers/pdf/playground/tsconfig.json +0 -19
- package/src/scrapers/website/playground/tsconfig.json +0 -19
- package/src/scrapers/website/playground/website-scraper-playground.ts +0 -82
- package/src/storage/_common/PromptbookStorage.test-type.ts +0 -14
- package/src/storage/local-storage/getIndexedDbStorage.ts +0 -36
- package/src/storage/local-storage/getLocalStorage.ts +0 -33
- package/src/storage/local-storage/getSessionStorage.ts +0 -33
- package/src/storage/local-storage/utils/IndexedDbStorageOptions.ts +0 -16
- package/src/storage/local-storage/utils/makePromptbookStorageFromIndexedDb.ts +0 -58
- package/src/storage/local-storage/utils/makePromptbookStorageFromWebStorage.ts +0 -45
- package/src/transpilers/formatted-book-in-markdown/FormattedBookInMarkdownTranspiler.test.ts.todo +0 -35
- package/src/transpilers/openai-sdk/playground/playground.ts +0 -85
- package/src/transpilers/openai-sdk/playground/tmp/chatbot-openaisdk-1.js +0 -194
- package/src/transpilers/openai-sdk/playground/tmp/package.json +0 -3
- package/src/transpilers/openai-sdk/playground/tsconfig.json +0 -18
- package/src/utils/editable/utils/findUsableParameters.test.ts.todo +0 -43
- package/src/utils/editable/utils/stringifyPipelineJson.test.ts.todo +0 -38
- package/src/utils/markdown/prettifyMarkdown.test.ts.tmp +0 -42
- package/src/utils/serialization/serializeToPromptbookJavascript.test.ts.todo +0 -116
|
@@ -1,74 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env ts-node
|
|
2
|
-
|
|
3
|
-
import * as dotenv from 'dotenv';
|
|
4
|
-
|
|
5
|
-
dotenv.config({ path: '.env' });
|
|
6
|
-
|
|
7
|
-
import colors from 'colors'; // <- TODO: [🔶] Make system to put color and style to both node and browser
|
|
8
|
-
import { writeFile } from 'fs/promises';
|
|
9
|
-
import { join } from 'path';
|
|
10
|
-
import { usageToHuman } from '../../../execution/utils/usageToHuman';
|
|
11
|
-
import { $provideLlmToolsForTestingAndScriptsAndPlayground } from '../../../llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground';
|
|
12
|
-
import { stringifyPipelineJson } from '../../../utils/editable/utils/stringifyPipelineJson';
|
|
13
|
-
import { $provideFilesystemForNode } from '../../_common/register/$provideFilesystemForNode';
|
|
14
|
-
import { makeKnowledgeSourceHandler } from '../../_common/utils/makeKnowledgeSourceHandler';
|
|
15
|
-
import { MarkdownScraper } from '../MarkdownScraper';
|
|
16
|
-
|
|
17
|
-
playground()
|
|
18
|
-
.catch((error) => {
|
|
19
|
-
console.error(colors.bgRed(error.name || 'NamelessError'));
|
|
20
|
-
console.error(error);
|
|
21
|
-
process.exit(1);
|
|
22
|
-
})
|
|
23
|
-
.then(() => {
|
|
24
|
-
process.exit(0);
|
|
25
|
-
});
|
|
26
|
-
|
|
27
|
-
/**
|
|
28
|
-
* Handles playground.
|
|
29
|
-
*/
|
|
30
|
-
async function playground() {
|
|
31
|
-
console.info(`🧸 Scrape knowledge from Markdown (playground)`);
|
|
32
|
-
|
|
33
|
-
// Do here stuff you want to test
|
|
34
|
-
//========================================>
|
|
35
|
-
|
|
36
|
-
const example = '10-simple.md';
|
|
37
|
-
// <- TODO: [👩🏿🤝👩🏼] Read here the examples directory and itterate through all of them
|
|
38
|
-
|
|
39
|
-
const llmTools = await $provideLlmToolsForTestingAndScriptsAndPlayground({ isCacheReloaded: true });
|
|
40
|
-
const rootDirname = join(__dirname, '..', 'examples');
|
|
41
|
-
|
|
42
|
-
const markdownScraper = new MarkdownScraper(
|
|
43
|
-
{ llm: await $provideLlmToolsForTestingAndScriptsAndPlayground() },
|
|
44
|
-
{
|
|
45
|
-
rootDirname,
|
|
46
|
-
},
|
|
47
|
-
);
|
|
48
|
-
|
|
49
|
-
const knowledge = await markdownScraper.scrape(
|
|
50
|
-
await makeKnowledgeSourceHandler(
|
|
51
|
-
{ knowledgeSourceContent: example },
|
|
52
|
-
{ fs: $provideFilesystemForNode() },
|
|
53
|
-
{ rootDirname },
|
|
54
|
-
),
|
|
55
|
-
);
|
|
56
|
-
|
|
57
|
-
console.info(colors.cyan(usageToHuman(llmTools.getTotalUsage())));
|
|
58
|
-
console.info(colors.bgGreen(' Knowledge: '));
|
|
59
|
-
console.info(knowledge);
|
|
60
|
-
|
|
61
|
-
await writeFile(
|
|
62
|
-
join(
|
|
63
|
-
__dirname,
|
|
64
|
-
`../examples/${example}.knowledge.json` /* <- TODO: [👩🏿🤝👩🏼] Read here the examples directory and itterate through all of them */,
|
|
65
|
-
),
|
|
66
|
-
stringifyPipelineJson(knowledge),
|
|
67
|
-
'utf-8',
|
|
68
|
-
);
|
|
69
|
-
/**/
|
|
70
|
-
|
|
71
|
-
//========================================/
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
// Note: [⚫] Code for playground [markdown-scraper-playground](src/scrapers/markdown/playground/markdown-scraper-playground.ts) should never be published in any package
|
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"compilerOptions": {
|
|
3
|
-
"target": "esnext",
|
|
4
|
-
"module": "commonjs",
|
|
5
|
-
"downlevelIteration": true,
|
|
6
|
-
"allowJs": true,
|
|
7
|
-
"moduleResolution": "node",
|
|
8
|
-
"forceConsistentCasingInFileNames": true,
|
|
9
|
-
"noImplicitReturns": true,
|
|
10
|
-
"noImplicitThis": true,
|
|
11
|
-
"noImplicitAny": true,
|
|
12
|
-
"strictNullChecks": true,
|
|
13
|
-
"experimentalDecorators": true,
|
|
14
|
-
"noUnusedLocals": false,
|
|
15
|
-
"resolveJsonModule": true,
|
|
16
|
-
"esModuleInterop": true
|
|
17
|
-
},
|
|
18
|
-
"exclude": ["node_modules"]
|
|
19
|
-
}
|
|
@@ -1,132 +0,0 @@
|
|
|
1
|
-
/*
|
|
2
|
-
TODO: !! This test is not wotking in Jest:
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
Jest (promptbook)
|
|
6
|
-
npm warn cli npm v10.9.1 does not support Node.js v18.4.0. This version of npm supports the following node versions: `^18.17.0 || >=20.5.0`. You can find the latest version at https://nodejs.org/.
|
|
7
|
-
|
|
8
|
-
> promptbook@0.84.0-0 test
|
|
9
|
-
> jest --testLocationInResults --json --useStderr --outputFile C:\Users\me\AppData\Local\Temp\jest_runner_promptbook_me_2.json --testNamePattern how creating knowledge by MarkitdownScraper works should scrape simple information from a pdf file$ --no-coverage --reporters default --reporters c:\Users\me\.vscode\extensions\orta.vscode-jest-6.4.0\out\reporter.js --colors --watchAll=false --testPathPattern c:\\Users\\me\\work\\ai\\promptbook\\src\\scrapers\\markitdown\\MarkitdownScraper\.test\.ts
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
> Test run "by-file-test-pattern:46" started at 1/22/2025, 4:30:15 PM <
|
|
13
|
-
|
|
14
|
-
FAIL src/scrapers/markitdown/MarkitdownScraper.test.ts
|
|
15
|
-
● Test suite failed to run
|
|
16
|
-
|
|
17
|
-
Jest encountered an unexpected token
|
|
18
|
-
|
|
19
|
-
Jest failed to parse a file. This happens e.g. when your code or its dependencies use non-standard JavaScript syntax, or when Jest is not configured to support such syntax.
|
|
20
|
-
|
|
21
|
-
Out of the box Jest supports Babel, which will be used to transform your files into valid JS based on your Babel configuration.
|
|
22
|
-
|
|
23
|
-
By default "node_modules" folder is ignored by transformers.
|
|
24
|
-
|
|
25
|
-
Here's what you can do:
|
|
26
|
-
• If you are trying to use ECMAScript Modules, see https://jestjs.io/docs/ecmascript-modules for how to enable it.
|
|
27
|
-
• If you are trying to use TypeScript, see https://jestjs.io/docs/getting-started#using-typescript
|
|
28
|
-
• To have some of your "node_modules" files transformed, you can specify a custom "transformIgnorePatterns" in your config.
|
|
29
|
-
• If you need a custom transformation specify a "transform" option in your config.
|
|
30
|
-
• If you simply want to mock your non-JS modules (e.g. binary assets) you can stub them out with the "moduleNameMapper" config option.
|
|
31
|
-
|
|
32
|
-
You'll find more details and examples of these config options in the docs:
|
|
33
|
-
https://jestjs.io/docs/configuration
|
|
34
|
-
For information about custom transformations, see:
|
|
35
|
-
https://jestjs.io/docs/code-transformation
|
|
36
|
-
|
|
37
|
-
Details:
|
|
38
|
-
|
|
39
|
-
C:\Users\me\work\ai\promptbook\node_modules\eventsource-parser\stream.js:2
|
|
40
|
-
export * from './dist/stream.esm'
|
|
41
|
-
^^^^^^
|
|
42
|
-
|
|
43
|
-
SyntaxError: Unexpected token 'export'
|
|
44
|
-
|
|
45
|
-
at Runtime.createScriptFromCode (node_modules/jest-runtime/build/index.js:1728:14)
|
|
46
|
-
at Object.<anonymous> (node_modules/ai/node_modules/@ai-sdk/provider-utils/src/response-handler.ts:5:8)
|
|
47
|
-
|
|
48
|
-
Test Suites: 1 failed, 1 total
|
|
49
|
-
Tests: 0 total
|
|
50
|
-
Snapshots: 0 total
|
|
51
|
-
Time: 36.491 s
|
|
52
|
-
Ran all test suites matching /c:\\Users\\me\\work\\ai\\promptbook\\src\\scrapers\\markitdown\\MarkitdownScraper\.test\.ts/i with tests matching "how creating knowledge by MarkitdownScraper works should scrape simple information from a pdf file$".
|
|
53
|
-
|
|
54
|
-
> Test run "by-file-test-pattern:46" finished at 1/22/2025, 4:30:51 PM <
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
> Test run "by-file-test-pattern:46" exited at 1/22/2025, 4:30:51 PM <
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
*/
|
|
61
|
-
|
|
62
|
-
import { describe, expect, it } from '@jest/globals';
|
|
63
|
-
import { join } from 'path';
|
|
64
|
-
import { $provideExecutablesForNode } from '../../executables/$provideExecutablesForNode';
|
|
65
|
-
import { $provideLlmToolsForTestingAndScriptsAndPlayground } from '../../llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground';
|
|
66
|
-
import { $provideFilesystemForNode } from '../_common/register/$provideFilesystemForNode';
|
|
67
|
-
import { makeKnowledgeSourceHandler } from '../_common/utils/makeKnowledgeSourceHandler';
|
|
68
|
-
import { MarkitdownScraper } from './MarkitdownScraper';
|
|
69
|
-
|
|
70
|
-
describe('how creating knowledge by MarkitdownScraper works', () => {
|
|
71
|
-
const rootDirname = join(__dirname, 'examples');
|
|
72
|
-
|
|
73
|
-
const markitdownScraperPromise = (async () =>
|
|
74
|
-
new MarkitdownScraper(
|
|
75
|
-
{
|
|
76
|
-
fs: $provideFilesystemForNode(),
|
|
77
|
-
llm: await $provideLlmToolsForTestingAndScriptsAndPlayground(),
|
|
78
|
-
executables: await $provideExecutablesForNode(),
|
|
79
|
-
},
|
|
80
|
-
{
|
|
81
|
-
rootDirname,
|
|
82
|
-
},
|
|
83
|
-
))();
|
|
84
|
-
|
|
85
|
-
it('should scrape simple information from a pdf file', () =>
|
|
86
|
-
expect(
|
|
87
|
-
Promise.all([
|
|
88
|
-
markitdownScraperPromise,
|
|
89
|
-
makeKnowledgeSourceHandler(
|
|
90
|
-
{
|
|
91
|
-
knowledgeSourceContent: '10-simple.pdf',
|
|
92
|
-
},
|
|
93
|
-
{ fs: $provideFilesystemForNode() },
|
|
94
|
-
{ rootDirname },
|
|
95
|
-
),
|
|
96
|
-
])
|
|
97
|
-
.then(([markitdownScraper, sourceHandler]) => markitdownScraper.scrape(sourceHandler))
|
|
98
|
-
.then((knowledge) => knowledge?.map(({ content }) => ({ content })))
|
|
99
|
-
.then((knowledge) => knowledge?.slice(0, 1)),
|
|
100
|
-
).resolves.toMatchObject([
|
|
101
|
-
{
|
|
102
|
-
content: expect.stringMatching(/Springfield (is )?.*/i),
|
|
103
|
-
},
|
|
104
|
-
]));
|
|
105
|
-
|
|
106
|
-
// TODO: !! Add all samples
|
|
107
|
-
|
|
108
|
-
it('should NOT scrape irrelevant information', () =>
|
|
109
|
-
expect(
|
|
110
|
-
Promise.all([
|
|
111
|
-
markitdownScraperPromise,
|
|
112
|
-
makeKnowledgeSourceHandler(
|
|
113
|
-
{
|
|
114
|
-
knowledgeSourceContent: '10-simple.pdf',
|
|
115
|
-
},
|
|
116
|
-
{ fs: $provideFilesystemForNode() },
|
|
117
|
-
{ rootDirname },
|
|
118
|
-
),
|
|
119
|
-
])
|
|
120
|
-
.then(([markitdownScraper, sourceHandler]) => markitdownScraper.scrape(sourceHandler))
|
|
121
|
-
.then((knowledge) => knowledge?.map(({ content }) => ({ content })))
|
|
122
|
-
.then((knowledge) => knowledge?.slice(0, 1)),
|
|
123
|
-
).resolves.toMatchObject([
|
|
124
|
-
{
|
|
125
|
-
content: expect.not.stringMatching(/London (is )?.*/i),
|
|
126
|
-
},
|
|
127
|
-
]));
|
|
128
|
-
});
|
|
129
|
-
|
|
130
|
-
/**
|
|
131
|
-
* TODO: [📓] Maybe test all file in examples (not just one by one)
|
|
132
|
-
*/
|
|
@@ -1,91 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env ts-node
|
|
2
|
-
|
|
3
|
-
import * as dotenv from 'dotenv';
|
|
4
|
-
|
|
5
|
-
dotenv.config({ path: '.env' });
|
|
6
|
-
|
|
7
|
-
import colors from 'colors'; // <- TODO: [🔶] Make system to put color and style to both node and browser
|
|
8
|
-
import { writeFile } from 'fs/promises';
|
|
9
|
-
import { join } from 'path';
|
|
10
|
-
import { $provideExecutablesForNode } from '../../../executables/$provideExecutablesForNode';
|
|
11
|
-
import { usageToHuman } from '../../../execution/utils/usageToHuman';
|
|
12
|
-
import { $provideLlmToolsForTestingAndScriptsAndPlayground } from '../../../llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground';
|
|
13
|
-
import { stringifyPipelineJson } from '../../../utils/editable/utils/stringifyPipelineJson';
|
|
14
|
-
import { titleToName } from '../../../utils/normalization/titleToName';
|
|
15
|
-
import { isValidUrl } from '../../../utils/validators/url/isValidUrl';
|
|
16
|
-
import { $provideFilesystemForNode } from '../../_common/register/$provideFilesystemForNode';
|
|
17
|
-
import { makeKnowledgeSourceHandler } from '../../_common/utils/makeKnowledgeSourceHandler';
|
|
18
|
-
import { MarkitdownScraper } from '../MarkitdownScraper';
|
|
19
|
-
|
|
20
|
-
playground()
|
|
21
|
-
.catch((error) => {
|
|
22
|
-
console.error(colors.bgRed(error.name || 'NamelessError'));
|
|
23
|
-
console.error(error);
|
|
24
|
-
process.exit(1);
|
|
25
|
-
})
|
|
26
|
-
.then(() => {
|
|
27
|
-
process.exit(0);
|
|
28
|
-
});
|
|
29
|
-
|
|
30
|
-
/**
|
|
31
|
-
* Handles playground.
|
|
32
|
-
*/
|
|
33
|
-
async function playground() {
|
|
34
|
-
console.info(`🧸 Scrape knowledge by Markitdown (playground)`);
|
|
35
|
-
|
|
36
|
-
// Do here stuff you want to test
|
|
37
|
-
//========================================>
|
|
38
|
-
|
|
39
|
-
// const example = '10-simple.pdf';
|
|
40
|
-
// const example = 'https://collboard.fra1.cdn.digitaloceanspaces.com/ptbk/user/knowledge-source/4/c/pravidla-pro-zadatele-a-prijemce-3-0.pdf';
|
|
41
|
-
const example = 'https://collboard.fra1.cdn.digitaloceanspaces.com/ptbk/user/knowledge-source/b/d/10-simple.pdf';
|
|
42
|
-
// const example = 'https://collboard.fra1.cdn.digitaloceanspaces.com/ptbk/user/knowledge-source/4/3/10-simple.odt';
|
|
43
|
-
// const example = 'https://collboard.fra1.cdn.digitaloceanspaces.com/ptbk/user/knowledge-source/4/3/10-simple.odt';
|
|
44
|
-
// const example = 'https://collboard.fra1.cdn.digitaloceanspaces.com/ptbk/user/knowledge-source/7/f/10-simple.docx';
|
|
45
|
-
// const example = 'https://collboard.fra1.cdn.digitaloceanspaces.com/ptbk/user/knowledge-source/2/9/10-simple.rtf';
|
|
46
|
-
// const example = 'https://collboard.fra1.cdn.digitaloceanspaces.com/ptbk/user/knowledge-source/f/3/10-simple.doc';
|
|
47
|
-
|
|
48
|
-
// <- TODO: [👩🏿🤝👩🏼] Read here the examples directory and itterate through all of them
|
|
49
|
-
|
|
50
|
-
const llmTools = await $provideLlmToolsForTestingAndScriptsAndPlayground({ isCacheReloaded: true });
|
|
51
|
-
const rootDirname = isValidUrl(example) ? null : join(__dirname, '..', 'examples');
|
|
52
|
-
|
|
53
|
-
const markitdownScraper = new MarkitdownScraper(
|
|
54
|
-
{
|
|
55
|
-
fs: $provideFilesystemForNode(),
|
|
56
|
-
llm: await $provideLlmToolsForTestingAndScriptsAndPlayground(),
|
|
57
|
-
executables: await $provideExecutablesForNode(),
|
|
58
|
-
},
|
|
59
|
-
{
|
|
60
|
-
rootDirname,
|
|
61
|
-
},
|
|
62
|
-
);
|
|
63
|
-
|
|
64
|
-
const knowledge = await markitdownScraper.scrape(
|
|
65
|
-
await makeKnowledgeSourceHandler(
|
|
66
|
-
{ knowledgeSourceContent: example },
|
|
67
|
-
{ fs: $provideFilesystemForNode() },
|
|
68
|
-
{ rootDirname },
|
|
69
|
-
),
|
|
70
|
-
);
|
|
71
|
-
|
|
72
|
-
console.info(colors.cyan(usageToHuman(llmTools.getTotalUsage())));
|
|
73
|
-
console.info(colors.bgGreen(' Knowledge: '));
|
|
74
|
-
console.info(knowledge);
|
|
75
|
-
|
|
76
|
-
await writeFile(
|
|
77
|
-
join(
|
|
78
|
-
__dirname,
|
|
79
|
-
`../examples/${
|
|
80
|
-
isValidUrl(example) ? titleToName(example) : example
|
|
81
|
-
}.knowledge.json` /* <- TODO: [👩🏿🤝👩🏼] Read here the examples directory and itterate through all of them */,
|
|
82
|
-
),
|
|
83
|
-
stringifyPipelineJson(knowledge),
|
|
84
|
-
'utf-8',
|
|
85
|
-
);
|
|
86
|
-
/**/
|
|
87
|
-
|
|
88
|
-
//========================================/
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
// Note: [⚫] Code for playground [markitdown-scraper-playground](src/scrapers/markitdown/playground/markitdown-scraper-playground.ts) should never be published in any package
|
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"compilerOptions": {
|
|
3
|
-
"target": "esnext",
|
|
4
|
-
"module": "commonjs",
|
|
5
|
-
"downlevelIteration": true,
|
|
6
|
-
"allowJs": true,
|
|
7
|
-
"moduleResolution": "node",
|
|
8
|
-
"forceConsistentCasingInFileNames": true,
|
|
9
|
-
"noImplicitReturns": true,
|
|
10
|
-
"noImplicitThis": true,
|
|
11
|
-
"noImplicitAny": true,
|
|
12
|
-
"strictNullChecks": true,
|
|
13
|
-
"experimentalDecorators": true,
|
|
14
|
-
"noUnusedLocals": false,
|
|
15
|
-
"resolveJsonModule": true,
|
|
16
|
-
"esModuleInterop": true
|
|
17
|
-
},
|
|
18
|
-
"exclude": ["node_modules"]
|
|
19
|
-
}
|
|
@@ -1,52 +0,0 @@
|
|
|
1
|
-
import { describe, expect, it } from '@jest/globals';
|
|
2
|
-
import { join } from 'path';
|
|
3
|
-
import { $provideLlmToolsForTestingAndScriptsAndPlayground } from '../../llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground';
|
|
4
|
-
import { makeKnowledgeSourceHandler } from '../_common/utils/makeKnowledgeSourceHandler';
|
|
5
|
-
import { pdfScraper } from './PdfScraper';
|
|
6
|
-
|
|
7
|
-
// TODO: [📇] Uncomment this test
|
|
8
|
-
|
|
9
|
-
describe('how creating knowledge from pdf works', () => {
|
|
10
|
-
|
|
11
|
-
const rootDirname = join(__dirname, 'examples');
|
|
12
|
-
const pdfScraper = new PdfScraper(
|
|
13
|
-
{ llm: await $provideLlmToolsForTestingAndScriptsAndPlayground() },
|
|
14
|
-
{
|
|
15
|
-
rootDirname,
|
|
16
|
-
},
|
|
17
|
-
);
|
|
18
|
-
|
|
19
|
-
it('should scrape simple information from a .pdf file', () =>
|
|
20
|
-
expect(
|
|
21
|
-
Promise.resolve()
|
|
22
|
-
.then(() => makeKnowledgeSourceHandler(join(__dirname, 'examples/10-simple.pdf'),{rootDirname}))
|
|
23
|
-
.then((sourceHandler) =>
|
|
24
|
-
pdfScraper.scrape(sourceHandler),
|
|
25
|
-
)
|
|
26
|
-
.then((knowledge) => knowledge?.map(({ content }) => ({ content })))
|
|
27
|
-
.then((knowledge) => knowledge?.slice(0, 1)),
|
|
28
|
-
).resolves.toMatchObject([
|
|
29
|
-
{
|
|
30
|
-
content: expect.stringMatching(/Springfield (is )?.*/i),
|
|
31
|
-
},
|
|
32
|
-
]));
|
|
33
|
-
|
|
34
|
-
it('should NOT scrape irrelevant information', () =>
|
|
35
|
-
expect(
|
|
36
|
-
Promise.resolve()
|
|
37
|
-
.then(() => makeKnowledgeSourceHandler(join(__dirname, 'examples/10-simple.pdf'),{rootDirname}))
|
|
38
|
-
.then((sourceHandler) =>
|
|
39
|
-
pdfScraper.scrape(sourceHandler),
|
|
40
|
-
)
|
|
41
|
-
.then((knowledge) => knowledge?.map(({ content }) => ({ content })))
|
|
42
|
-
.then((knowledge) => knowledge?.slice(0, 1)),
|
|
43
|
-
).resolves.toMatchObject([
|
|
44
|
-
{
|
|
45
|
-
content: expect.not.stringMatching(/London (is )?.*/i),
|
|
46
|
-
},
|
|
47
|
-
]));
|
|
48
|
-
});
|
|
49
|
-
|
|
50
|
-
/**
|
|
51
|
-
* TODO: [📓] Maybe test all file in examples (not just 10-simple.pdf)
|
|
52
|
-
*/
|
|
@@ -1,75 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env ts-node
|
|
2
|
-
|
|
3
|
-
import * as dotenv from 'dotenv';
|
|
4
|
-
|
|
5
|
-
dotenv.config({ path: '.env' });
|
|
6
|
-
|
|
7
|
-
import colors from 'colors'; // <- TODO: [🔶] Make system to put color and style to both node and browser
|
|
8
|
-
import { writeFile } from 'fs/promises';
|
|
9
|
-
import { join } from 'path';
|
|
10
|
-
import { usageToHuman } from '../../../execution/utils/usageToHuman';
|
|
11
|
-
import { $provideLlmToolsForTestingAndScriptsAndPlayground } from '../../../llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground';
|
|
12
|
-
import { stringifyPipelineJson } from '../../../utils/editable/utils/stringifyPipelineJson';
|
|
13
|
-
import { $provideFilesystemForNode } from '../../_common/register/$provideFilesystemForNode';
|
|
14
|
-
import { makeKnowledgeSourceHandler } from '../../_common/utils/makeKnowledgeSourceHandler';
|
|
15
|
-
import { PdfScraper } from '../PdfScraper';
|
|
16
|
-
|
|
17
|
-
playground()
|
|
18
|
-
.catch((error) => {
|
|
19
|
-
console.error(colors.bgRed(error.name || 'NamelessError'));
|
|
20
|
-
console.error(error);
|
|
21
|
-
process.exit(1);
|
|
22
|
-
})
|
|
23
|
-
.then(() => {
|
|
24
|
-
process.exit(0);
|
|
25
|
-
});
|
|
26
|
-
|
|
27
|
-
/**
|
|
28
|
-
* Handles playground.
|
|
29
|
-
*/
|
|
30
|
-
async function playground() {
|
|
31
|
-
console.info(`🧸 Scrape knowledge from PDFs (playground)`);
|
|
32
|
-
|
|
33
|
-
// Do here stuff you want to test
|
|
34
|
-
//========================================>
|
|
35
|
-
|
|
36
|
-
//const example = '10-simple.doc';
|
|
37
|
-
const example = '10-simple.pdf';
|
|
38
|
-
// <- TODO: [👩🏿🤝👩🏼] Read here the examples directory and itterate through all of them
|
|
39
|
-
|
|
40
|
-
const llmTools = await $provideLlmToolsForTestingAndScriptsAndPlayground({ isCacheReloaded: true });
|
|
41
|
-
const rootDirname = join(__dirname, '..', 'examples');
|
|
42
|
-
|
|
43
|
-
const pdfScraper = new PdfScraper(
|
|
44
|
-
{ llm: await $provideLlmToolsForTestingAndScriptsAndPlayground() },
|
|
45
|
-
{
|
|
46
|
-
rootDirname,
|
|
47
|
-
},
|
|
48
|
-
);
|
|
49
|
-
|
|
50
|
-
const knowledge = await pdfScraper.scrape(
|
|
51
|
-
await makeKnowledgeSourceHandler(
|
|
52
|
-
{ knowledgeSourceContent: example },
|
|
53
|
-
{ fs: $provideFilesystemForNode() },
|
|
54
|
-
{ rootDirname },
|
|
55
|
-
),
|
|
56
|
-
);
|
|
57
|
-
|
|
58
|
-
console.info(colors.cyan(usageToHuman(llmTools.getTotalUsage())));
|
|
59
|
-
console.info(colors.bgGreen(' Knowledge: '));
|
|
60
|
-
console.info(knowledge);
|
|
61
|
-
|
|
62
|
-
await writeFile(
|
|
63
|
-
join(
|
|
64
|
-
__dirname,
|
|
65
|
-
`../examples/${example}.knowledge.json` /* <- TODO: [👩🏿🤝👩🏼] Read here the examples directory and itterate through all of them */,
|
|
66
|
-
),
|
|
67
|
-
stringifyPipelineJson(knowledge),
|
|
68
|
-
'utf-8',
|
|
69
|
-
);
|
|
70
|
-
/**/
|
|
71
|
-
|
|
72
|
-
//========================================/
|
|
73
|
-
}
|
|
74
|
-
|
|
75
|
-
// Note: [⚫] Code for playground [pdf-scraper-playground](src/scrapers/pdf/playground/pdf-scraper-playground.ts) should never be published in any package
|
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"compilerOptions": {
|
|
3
|
-
"target": "esnext",
|
|
4
|
-
"module": "commonjs",
|
|
5
|
-
"downlevelIteration": true,
|
|
6
|
-
"allowJs": true,
|
|
7
|
-
"moduleResolution": "node",
|
|
8
|
-
"forceConsistentCasingInFileNames": true,
|
|
9
|
-
"noImplicitReturns": true,
|
|
10
|
-
"noImplicitThis": true,
|
|
11
|
-
"noImplicitAny": true,
|
|
12
|
-
"strictNullChecks": true,
|
|
13
|
-
"experimentalDecorators": true,
|
|
14
|
-
"noUnusedLocals": false,
|
|
15
|
-
"resolveJsonModule": true,
|
|
16
|
-
"esModuleInterop": true
|
|
17
|
-
},
|
|
18
|
-
"exclude": ["node_modules"]
|
|
19
|
-
}
|
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"compilerOptions": {
|
|
3
|
-
"target": "esnext",
|
|
4
|
-
"module": "commonjs",
|
|
5
|
-
"downlevelIteration": true,
|
|
6
|
-
"allowJs": true,
|
|
7
|
-
"moduleResolution": "node",
|
|
8
|
-
"forceConsistentCasingInFileNames": true,
|
|
9
|
-
"noImplicitReturns": true,
|
|
10
|
-
"noImplicitThis": true,
|
|
11
|
-
"noImplicitAny": true,
|
|
12
|
-
"strictNullChecks": true,
|
|
13
|
-
"experimentalDecorators": true,
|
|
14
|
-
"noUnusedLocals": false,
|
|
15
|
-
"resolveJsonModule": true,
|
|
16
|
-
"esModuleInterop": true
|
|
17
|
-
},
|
|
18
|
-
"exclude": ["node_modules"]
|
|
19
|
-
}
|
|
@@ -1,82 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env ts-node
|
|
2
|
-
|
|
3
|
-
import * as dotenv from 'dotenv';
|
|
4
|
-
|
|
5
|
-
dotenv.config({ path: '.env' });
|
|
6
|
-
|
|
7
|
-
import colors from 'colors'; // <- TODO: [🔶] Make system to put color and style to both node and browser
|
|
8
|
-
import { writeFile } from 'fs/promises';
|
|
9
|
-
import { join } from 'path';
|
|
10
|
-
import { usageToHuman } from '../../../execution/utils/usageToHuman';
|
|
11
|
-
import { $provideLlmToolsForTestingAndScriptsAndPlayground } from '../../../llm-providers/_common/register/$provideLlmToolsForTestingAndScriptsAndPlayground';
|
|
12
|
-
import { stringifyPipelineJson } from '../../../utils/editable/utils/stringifyPipelineJson';
|
|
13
|
-
import { titleToName } from '../../../utils/normalization/titleToName';
|
|
14
|
-
import { $provideFilesystemForNode } from '../../_common/register/$provideFilesystemForNode';
|
|
15
|
-
import { makeKnowledgeSourceHandler } from '../../_common/utils/makeKnowledgeSourceHandler';
|
|
16
|
-
import { WebsiteScraper } from '../WebsiteScraper';
|
|
17
|
-
|
|
18
|
-
playground()
|
|
19
|
-
.catch((error) => {
|
|
20
|
-
console.error(colors.bgRed(error.name || 'NamelessError'));
|
|
21
|
-
console.error(error);
|
|
22
|
-
process.exit(1);
|
|
23
|
-
})
|
|
24
|
-
.then(() => {
|
|
25
|
-
process.exit(0);
|
|
26
|
-
});
|
|
27
|
-
|
|
28
|
-
/**
|
|
29
|
-
* Handles playground.
|
|
30
|
-
*/
|
|
31
|
-
async function playground() {
|
|
32
|
-
console.info(`🧸 Scrape knowledge from websites (playground)`);
|
|
33
|
-
|
|
34
|
-
// Do here stuff you want to test
|
|
35
|
-
//========================================>
|
|
36
|
-
|
|
37
|
-
//const example = 'https://www.pavolhejny.com/'; // <- TODO: Not scraping really important information, just one-two paragraph
|
|
38
|
-
const example = 'https://koralkykatlas.cz/cs/blog/prispevek/-rijna-zhorseni-kvality-kovove-bizuterie.html';
|
|
39
|
-
// <- TODO: [👩🏿🤝👩🏼] Read here website-scraper-playground.ts and itterate
|
|
40
|
-
|
|
41
|
-
const llmTools = await $provideLlmToolsForTestingAndScriptsAndPlayground({ isCacheReloaded: true });
|
|
42
|
-
const rootDirname = join(__dirname, 'examples');
|
|
43
|
-
|
|
44
|
-
const websiteScraper = new WebsiteScraper(
|
|
45
|
-
{ llm: await $provideLlmToolsForTestingAndScriptsAndPlayground(), fs: $provideFilesystemForNode() },
|
|
46
|
-
{
|
|
47
|
-
rootDirname,
|
|
48
|
-
},
|
|
49
|
-
);
|
|
50
|
-
|
|
51
|
-
const source = await makeKnowledgeSourceHandler(
|
|
52
|
-
{ knowledgeSourceContent: example },
|
|
53
|
-
{ fs: $provideFilesystemForNode() },
|
|
54
|
-
{ rootDirname },
|
|
55
|
-
);
|
|
56
|
-
|
|
57
|
-
const converted = await websiteScraper.$convert(source);
|
|
58
|
-
|
|
59
|
-
console.info(colors.bgGreen(' Converted: '), converted);
|
|
60
|
-
|
|
61
|
-
const knowledge = await websiteScraper.scrape(source);
|
|
62
|
-
|
|
63
|
-
console.info(colors.cyan(usageToHuman(llmTools.getTotalUsage())));
|
|
64
|
-
console.info(colors.bgGreen(' Knowledge: '));
|
|
65
|
-
console.info(knowledge);
|
|
66
|
-
|
|
67
|
-
await writeFile(
|
|
68
|
-
join(
|
|
69
|
-
__dirname,
|
|
70
|
-
`../examples/${titleToName(
|
|
71
|
-
example,
|
|
72
|
-
)}.knowledge.json` /* <- TODO: [👩🏿🤝👩🏼] Read here the examples directory and itterate through all of them */,
|
|
73
|
-
),
|
|
74
|
-
stringifyPipelineJson(knowledge),
|
|
75
|
-
'utf-8',
|
|
76
|
-
);
|
|
77
|
-
/**/
|
|
78
|
-
|
|
79
|
-
//========================================/
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
// Note: [⚫] Code for playground [website-scraper-playground](src/scrapers/website/playground/website-scraper-playground.ts) should never be published in any package
|
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
import { keepUnused } from '../../utils/organization/keepUnused';
|
|
2
|
-
import type { PromptbookStorage } from './PromptbookStorage';
|
|
3
|
-
|
|
4
|
-
// Note: Checking that `localStorage` and `sessionStorage` are implementing `PromptbookStorage<string>`
|
|
5
|
-
/**
|
|
6
|
-
* Constant for storage.
|
|
7
|
-
*/
|
|
8
|
-
let storage: PromptbookStorage<string>;
|
|
9
|
-
storage = localStorage;
|
|
10
|
-
storage = sessionStorage;
|
|
11
|
-
keepUnused(storage);
|
|
12
|
-
|
|
13
|
-
// Note: [⚪] This should never be in any released package
|
|
14
|
-
// TODO: Is this a good pattern to do type testing?
|
|
@@ -1,36 +0,0 @@
|
|
|
1
|
-
import { EnvironmentMismatchError } from '../../errors/EnvironmentMismatchError';
|
|
2
|
-
import { $isRunningInBrowser } from '../../utils/environment/$isRunningInBrowser';
|
|
3
|
-
import type { chococake } from '../../utils/organization/really_any';
|
|
4
|
-
import type { PromptbookStorage } from '../_common/PromptbookStorage';
|
|
5
|
-
import type { IndexedDbStorageOptions } from './utils/IndexedDbStorageOptions';
|
|
6
|
-
import { makePromptbookStorageFromIndexedDb } from './utils/makePromptbookStorageFromIndexedDb';
|
|
7
|
-
|
|
8
|
-
/**
|
|
9
|
-
* Cache storage
|
|
10
|
-
*
|
|
11
|
-
* @private internal cache for `getIndexedDbStorage`
|
|
12
|
-
*/
|
|
13
|
-
const indexedDbStorageCache = new Map<string, PromptbookStorage<chococake>>();
|
|
14
|
-
|
|
15
|
-
/**
|
|
16
|
-
* Gets wrapper around IndexedDB which can be used as PromptbookStorage
|
|
17
|
-
*
|
|
18
|
-
* @public exported from `@promptbook/browser`
|
|
19
|
-
*/
|
|
20
|
-
export function getIndexedDbStorage<TItem>(options: IndexedDbStorageOptions): PromptbookStorage<TItem> {
|
|
21
|
-
if (!$isRunningInBrowser()) {
|
|
22
|
-
throw new EnvironmentMismatchError(`You can get IndexedDB storage only in browser environment`);
|
|
23
|
-
}
|
|
24
|
-
|
|
25
|
-
const { databaseName, storeName } = options;
|
|
26
|
-
const cacheKey = `${databaseName}/${storeName}`;
|
|
27
|
-
if (indexedDbStorageCache.has(cacheKey)) {
|
|
28
|
-
return indexedDbStorageCache.get(cacheKey) as PromptbookStorage<TItem>;
|
|
29
|
-
}
|
|
30
|
-
|
|
31
|
-
const storage = makePromptbookStorageFromIndexedDb<TItem>({ databaseName, storeName });
|
|
32
|
-
indexedDbStorageCache.set(cacheKey, storage as PromptbookStorage<TItem>);
|
|
33
|
-
return storage;
|
|
34
|
-
}
|
|
35
|
-
|
|
36
|
-
// Note: [🔵] Code for browser storage helper [getIndexedDbStorage](src/storage/local-storage/getIndexedDbStorage.ts) should never be published outside of `@promptbook/browser`
|