@raviolelabs/engram-mcp 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +232 -0
- package/LICENSE +21 -0
- package/README.md +222 -0
- package/SKILL.md +299 -0
- package/dist/cloud/auth.d.ts +29 -0
- package/dist/cloud/auth.d.ts.map +1 -0
- package/dist/cloud/auth.js +132 -0
- package/dist/cloud/auth.js.map +1 -0
- package/dist/cloud/bridge-client.d.ts +10 -0
- package/dist/cloud/bridge-client.d.ts.map +1 -0
- package/dist/cloud/bridge-client.js +167 -0
- package/dist/cloud/bridge-client.js.map +1 -0
- package/dist/cloud/crypto.d.ts +42 -0
- package/dist/cloud/crypto.d.ts.map +1 -0
- package/dist/cloud/crypto.js +146 -0
- package/dist/cloud/crypto.js.map +1 -0
- package/dist/cloud/endpoints.d.ts +26 -0
- package/dist/cloud/endpoints.d.ts.map +1 -0
- package/dist/cloud/endpoints.js +26 -0
- package/dist/cloud/endpoints.js.map +1 -0
- package/dist/cloud/pairing.d.ts +30 -0
- package/dist/cloud/pairing.d.ts.map +1 -0
- package/dist/cloud/pairing.js +157 -0
- package/dist/cloud/pairing.js.map +1 -0
- package/dist/cloud/transit-poller.d.ts +35 -0
- package/dist/cloud/transit-poller.d.ts.map +1 -0
- package/dist/cloud/transit-poller.js +281 -0
- package/dist/cloud/transit-poller.js.map +1 -0
- package/dist/config/index.d.ts +3 -0
- package/dist/config/index.d.ts.map +1 -0
- package/dist/config/index.js +24 -0
- package/dist/config/index.js.map +1 -0
- package/dist/config/schema.d.ts +466 -0
- package/dist/config/schema.d.ts.map +1 -0
- package/dist/config/schema.js +171 -0
- package/dist/config/schema.js.map +1 -0
- package/dist/core/db/index.d.ts +7 -0
- package/dist/core/db/index.d.ts.map +1 -0
- package/dist/core/db/index.js +273 -0
- package/dist/core/db/index.js.map +1 -0
- package/dist/core/logger.d.ts +19 -0
- package/dist/core/logger.d.ts.map +1 -0
- package/dist/core/logger.js +223 -0
- package/dist/core/logger.js.map +1 -0
- package/dist/core/server/http.d.ts +15 -0
- package/dist/core/server/http.d.ts.map +1 -0
- package/dist/core/server/http.js +76 -0
- package/dist/core/server/http.js.map +1 -0
- package/dist/core/server/instructions.d.ts +2 -0
- package/dist/core/server/instructions.d.ts.map +1 -0
- package/dist/core/server/instructions.js +36 -0
- package/dist/core/server/instructions.js.map +1 -0
- package/dist/core/server/mcp-handler.d.ts +39 -0
- package/dist/core/server/mcp-handler.d.ts.map +1 -0
- package/dist/core/server/mcp-handler.js +204 -0
- package/dist/core/server/mcp-handler.js.map +1 -0
- package/dist/core/server/mcp-http.d.ts +4 -0
- package/dist/core/server/mcp-http.d.ts.map +1 -0
- package/dist/core/server/mcp-http.js +56 -0
- package/dist/core/server/mcp-http.js.map +1 -0
- package/dist/core/server/tool-router.d.ts +9 -0
- package/dist/core/server/tool-router.d.ts.map +1 -0
- package/dist/core/server/tool-router.js +25 -0
- package/dist/core/server/tool-router.js.map +1 -0
- package/dist/core/server/websocket.d.ts +4 -0
- package/dist/core/server/websocket.d.ts.map +1 -0
- package/dist/core/server/websocket.js +25 -0
- package/dist/core/server/websocket.js.map +1 -0
- package/dist/db/index.d.ts +2 -0
- package/dist/db/index.d.ts.map +1 -0
- package/dist/db/index.js +3 -0
- package/dist/db/index.js.map +1 -0
- package/dist/embeddings/index.d.ts +24 -0
- package/dist/embeddings/index.d.ts.map +1 -0
- package/dist/embeddings/index.js +86 -0
- package/dist/embeddings/index.js.map +1 -0
- package/dist/embeddings/providers/engram.d.ts +7 -0
- package/dist/embeddings/providers/engram.d.ts.map +1 -0
- package/dist/embeddings/providers/engram.js +67 -0
- package/dist/embeddings/providers/engram.js.map +1 -0
- package/dist/embeddings/providers/ollama.d.ts +3 -0
- package/dist/embeddings/providers/ollama.d.ts.map +1 -0
- package/dist/embeddings/providers/ollama.js +9 -0
- package/dist/embeddings/providers/ollama.js.map +1 -0
- package/dist/embeddings/providers/openai-compat.d.ts +7 -0
- package/dist/embeddings/providers/openai-compat.d.ts.map +1 -0
- package/dist/embeddings/providers/openai-compat.js +27 -0
- package/dist/embeddings/providers/openai-compat.js.map +1 -0
- package/dist/embeddings/providers/openai.d.ts +3 -0
- package/dist/embeddings/providers/openai.d.ts.map +1 -0
- package/dist/embeddings/providers/openai.js +12 -0
- package/dist/embeddings/providers/openai.js.map +1 -0
- package/dist/embeddings/providers/voyage.d.ts +3 -0
- package/dist/embeddings/providers/voyage.d.ts.map +1 -0
- package/dist/embeddings/providers/voyage.js +12 -0
- package/dist/embeddings/providers/voyage.js.map +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +3 -0
- package/dist/index.js.map +1 -0
- package/dist/ingest/jobs.d.ts +29 -0
- package/dist/ingest/jobs.d.ts.map +1 -0
- package/dist/ingest/jobs.js +131 -0
- package/dist/ingest/jobs.js.map +1 -0
- package/dist/logger.d.ts +2 -0
- package/dist/logger.d.ts.map +1 -0
- package/dist/logger.js +3 -0
- package/dist/logger.js.map +1 -0
- package/dist/mcp-server/server.d.ts +2 -0
- package/dist/mcp-server/server.d.ts.map +1 -0
- package/dist/mcp-server/server.js +3 -0
- package/dist/mcp-server/server.js.map +1 -0
- package/dist/mcp-server/tests/mcp-e2e.test.d.ts +2 -0
- package/dist/mcp-server/tests/mcp-e2e.test.d.ts.map +1 -0
- package/dist/mcp-server/tests/mcp-e2e.test.js +157 -0
- package/dist/mcp-server/tests/mcp-e2e.test.js.map +1 -0
- package/dist/mcp-server/tool-router.d.ts +2 -0
- package/dist/mcp-server/tool-router.d.ts.map +1 -0
- package/dist/mcp-server/tool-router.js +3 -0
- package/dist/mcp-server/tool-router.js.map +1 -0
- package/dist/memory/admin/tools.d.ts +6 -0
- package/dist/memory/admin/tools.d.ts.map +1 -0
- package/dist/memory/admin/tools.js +134 -0
- package/dist/memory/admin/tools.js.map +1 -0
- package/dist/memory/core/chunker.d.ts +6 -0
- package/dist/memory/core/chunker.d.ts.map +1 -0
- package/dist/memory/core/chunker.js +49 -0
- package/dist/memory/core/chunker.js.map +1 -0
- package/dist/memory/core/module-interface.d.ts +23 -0
- package/dist/memory/core/module-interface.d.ts.map +1 -0
- package/dist/memory/core/module-interface.js +2 -0
- package/dist/memory/core/module-interface.js.map +1 -0
- package/dist/memory/core/module-registry.d.ts +14 -0
- package/dist/memory/core/module-registry.d.ts.map +1 -0
- package/dist/memory/core/module-registry.js +45 -0
- package/dist/memory/core/module-registry.js.map +1 -0
- package/dist/memory/core/property-extractor.d.ts +6 -0
- package/dist/memory/core/property-extractor.d.ts.map +1 -0
- package/dist/memory/core/property-extractor.js +90 -0
- package/dist/memory/core/property-extractor.js.map +1 -0
- package/dist/memory/core/reindex.d.ts +11 -0
- package/dist/memory/core/reindex.d.ts.map +1 -0
- package/dist/memory/core/reindex.js +55 -0
- package/dist/memory/core/reindex.js.map +1 -0
- package/dist/memory/core/source-registry.d.ts +42 -0
- package/dist/memory/core/source-registry.d.ts.map +1 -0
- package/dist/memory/core/source-registry.js +86 -0
- package/dist/memory/core/source-registry.js.map +1 -0
- package/dist/memory/core/store.d.ts +40 -0
- package/dist/memory/core/store.d.ts.map +1 -0
- package/dist/memory/core/store.js +257 -0
- package/dist/memory/core/store.js.map +1 -0
- package/dist/memory/core/wikilinks.d.ts +13 -0
- package/dist/memory/core/wikilinks.d.ts.map +1 -0
- package/dist/memory/core/wikilinks.js +25 -0
- package/dist/memory/core/wikilinks.js.map +1 -0
- package/dist/memory/modules/_custom/generic-module.d.ts +7 -0
- package/dist/memory/modules/_custom/generic-module.d.ts.map +1 -0
- package/dist/memory/modules/_custom/generic-module.js +108 -0
- package/dist/memory/modules/_custom/generic-module.js.map +1 -0
- package/dist/memory/modules/_custom/persistence.d.ts +15 -0
- package/dist/memory/modules/_custom/persistence.d.ts.map +1 -0
- package/dist/memory/modules/_custom/persistence.js +47 -0
- package/dist/memory/modules/_custom/persistence.js.map +1 -0
- package/dist/memory/modules/_custom/tests/custom-types.test.d.ts +2 -0
- package/dist/memory/modules/_custom/tests/custom-types.test.d.ts.map +1 -0
- package/dist/memory/modules/_custom/tests/custom-types.test.js +89 -0
- package/dist/memory/modules/_custom/tests/custom-types.test.js.map +1 -0
- package/dist/memory/modules/_custom/tools.d.ts +7 -0
- package/dist/memory/modules/_custom/tools.d.ts.map +1 -0
- package/dist/memory/modules/_custom/tools.js +72 -0
- package/dist/memory/modules/_custom/tools.js.map +1 -0
- package/dist/memory/modules/audio/ingest.d.ts +9 -0
- package/dist/memory/modules/audio/ingest.d.ts.map +1 -0
- package/dist/memory/modules/audio/ingest.js +32 -0
- package/dist/memory/modules/audio/ingest.js.map +1 -0
- package/dist/memory/modules/audio/module.d.ts +6 -0
- package/dist/memory/modules/audio/module.d.ts.map +1 -0
- package/dist/memory/modules/audio/module.js +18 -0
- package/dist/memory/modules/audio/module.js.map +1 -0
- package/dist/memory/modules/audio/tests/audio.test.d.ts +2 -0
- package/dist/memory/modules/audio/tests/audio.test.d.ts.map +1 -0
- package/dist/memory/modules/audio/tests/audio.test.js +57 -0
- package/dist/memory/modules/audio/tests/audio.test.js.map +1 -0
- package/dist/memory/modules/audio/tests/transcriber.test.d.ts +2 -0
- package/dist/memory/modules/audio/tests/transcriber.test.d.ts.map +1 -0
- package/dist/memory/modules/audio/tests/transcriber.test.js +27 -0
- package/dist/memory/modules/audio/tests/transcriber.test.js.map +1 -0
- package/dist/memory/modules/audio/tools.d.ts +5 -0
- package/dist/memory/modules/audio/tools.d.ts.map +1 -0
- package/dist/memory/modules/audio/tools.js +60 -0
- package/dist/memory/modules/audio/tools.js.map +1 -0
- package/dist/memory/modules/audio/transcriber.d.ts +15 -0
- package/dist/memory/modules/audio/transcriber.d.ts.map +1 -0
- package/dist/memory/modules/audio/transcriber.js +177 -0
- package/dist/memory/modules/audio/transcriber.js.map +1 -0
- package/dist/memory/modules/conversations/ingest.d.ts +10 -0
- package/dist/memory/modules/conversations/ingest.d.ts.map +1 -0
- package/dist/memory/modules/conversations/ingest.js +38 -0
- package/dist/memory/modules/conversations/ingest.js.map +1 -0
- package/dist/memory/modules/conversations/module.d.ts +6 -0
- package/dist/memory/modules/conversations/module.d.ts.map +1 -0
- package/dist/memory/modules/conversations/module.js +43 -0
- package/dist/memory/modules/conversations/module.js.map +1 -0
- package/dist/memory/modules/conversations/tests/conversations.test.d.ts +2 -0
- package/dist/memory/modules/conversations/tests/conversations.test.d.ts.map +1 -0
- package/dist/memory/modules/conversations/tests/conversations.test.js +70 -0
- package/dist/memory/modules/conversations/tests/conversations.test.js.map +1 -0
- package/dist/memory/modules/conversations/tools.d.ts +5 -0
- package/dist/memory/modules/conversations/tools.d.ts.map +1 -0
- package/dist/memory/modules/conversations/tools.js +75 -0
- package/dist/memory/modules/conversations/tools.js.map +1 -0
- package/dist/memory/modules/drive/connector.d.ts +19 -0
- package/dist/memory/modules/drive/connector.d.ts.map +1 -0
- package/dist/memory/modules/drive/connector.js +52 -0
- package/dist/memory/modules/drive/connector.js.map +1 -0
- package/dist/memory/modules/drive/ingest.d.ts +9 -0
- package/dist/memory/modules/drive/ingest.d.ts.map +1 -0
- package/dist/memory/modules/drive/ingest.js +27 -0
- package/dist/memory/modules/drive/ingest.js.map +1 -0
- package/dist/memory/modules/drive/module.d.ts +6 -0
- package/dist/memory/modules/drive/module.d.ts.map +1 -0
- package/dist/memory/modules/drive/module.js +31 -0
- package/dist/memory/modules/drive/module.js.map +1 -0
- package/dist/memory/modules/drive/oauth.d.ts +14 -0
- package/dist/memory/modules/drive/oauth.d.ts.map +1 -0
- package/dist/memory/modules/drive/oauth.js +130 -0
- package/dist/memory/modules/drive/oauth.js.map +1 -0
- package/dist/memory/modules/drive/tests/drive.test.d.ts +2 -0
- package/dist/memory/modules/drive/tests/drive.test.d.ts.map +1 -0
- package/dist/memory/modules/drive/tests/drive.test.js +66 -0
- package/dist/memory/modules/drive/tests/drive.test.js.map +1 -0
- package/dist/memory/modules/drive/tools.d.ts +5 -0
- package/dist/memory/modules/drive/tools.d.ts.map +1 -0
- package/dist/memory/modules/drive/tools.js +131 -0
- package/dist/memory/modules/drive/tools.js.map +1 -0
- package/dist/memory/modules/drive/watcher.d.ts +5 -0
- package/dist/memory/modules/drive/watcher.d.ts.map +1 -0
- package/dist/memory/modules/drive/watcher.js +46 -0
- package/dist/memory/modules/drive/watcher.js.map +1 -0
- package/dist/memory/modules/notes/ingest.d.ts +3 -0
- package/dist/memory/modules/notes/ingest.d.ts.map +1 -0
- package/dist/memory/modules/notes/ingest.js +30 -0
- package/dist/memory/modules/notes/ingest.js.map +1 -0
- package/dist/memory/modules/notes/module.d.ts +5 -0
- package/dist/memory/modules/notes/module.d.ts.map +1 -0
- package/dist/memory/modules/notes/module.js +28 -0
- package/dist/memory/modules/notes/module.js.map +1 -0
- package/dist/memory/modules/notes/tests/notes.test.d.ts +2 -0
- package/dist/memory/modules/notes/tests/notes.test.d.ts.map +1 -0
- package/dist/memory/modules/notes/tests/notes.test.js +59 -0
- package/dist/memory/modules/notes/tests/notes.test.js.map +1 -0
- package/dist/memory/modules/notes/tools.d.ts +5 -0
- package/dist/memory/modules/notes/tools.d.ts.map +1 -0
- package/dist/memory/modules/notes/tools.js +69 -0
- package/dist/memory/modules/notes/tools.js.map +1 -0
- package/dist/memory/modules/notion/connector.d.ts +10 -0
- package/dist/memory/modules/notion/connector.d.ts.map +1 -0
- package/dist/memory/modules/notion/connector.js +112 -0
- package/dist/memory/modules/notion/connector.js.map +1 -0
- package/dist/memory/modules/notion/ingest.d.ts +9 -0
- package/dist/memory/modules/notion/ingest.d.ts.map +1 -0
- package/dist/memory/modules/notion/ingest.js +24 -0
- package/dist/memory/modules/notion/ingest.js.map +1 -0
- package/dist/memory/modules/notion/module.d.ts +6 -0
- package/dist/memory/modules/notion/module.d.ts.map +1 -0
- package/dist/memory/modules/notion/module.js +31 -0
- package/dist/memory/modules/notion/module.js.map +1 -0
- package/dist/memory/modules/notion/oauth.d.ts +19 -0
- package/dist/memory/modules/notion/oauth.d.ts.map +1 -0
- package/dist/memory/modules/notion/oauth.js +117 -0
- package/dist/memory/modules/notion/oauth.js.map +1 -0
- package/dist/memory/modules/notion/tests/notion.test.d.ts +2 -0
- package/dist/memory/modules/notion/tests/notion.test.d.ts.map +1 -0
- package/dist/memory/modules/notion/tests/notion.test.js +53 -0
- package/dist/memory/modules/notion/tests/notion.test.js.map +1 -0
- package/dist/memory/modules/notion/tools.d.ts +5 -0
- package/dist/memory/modules/notion/tools.d.ts.map +1 -0
- package/dist/memory/modules/notion/tools.js +116 -0
- package/dist/memory/modules/notion/tools.js.map +1 -0
- package/dist/memory/modules/notion/watcher.d.ts +5 -0
- package/dist/memory/modules/notion/watcher.d.ts.map +1 -0
- package/dist/memory/modules/notion/watcher.js +41 -0
- package/dist/memory/modules/notion/watcher.js.map +1 -0
- package/dist/memory/modules/obsidian/ingest.d.ts +9 -0
- package/dist/memory/modules/obsidian/ingest.d.ts.map +1 -0
- package/dist/memory/modules/obsidian/ingest.js +80 -0
- package/dist/memory/modules/obsidian/ingest.js.map +1 -0
- package/dist/memory/modules/obsidian/module.d.ts +6 -0
- package/dist/memory/modules/obsidian/module.d.ts.map +1 -0
- package/dist/memory/modules/obsidian/module.js +31 -0
- package/dist/memory/modules/obsidian/module.js.map +1 -0
- package/dist/memory/modules/obsidian/tests/obsidian.test.d.ts +2 -0
- package/dist/memory/modules/obsidian/tests/obsidian.test.d.ts.map +1 -0
- package/dist/memory/modules/obsidian/tests/obsidian.test.js +65 -0
- package/dist/memory/modules/obsidian/tests/obsidian.test.js.map +1 -0
- package/dist/memory/modules/obsidian/tests/vault-reader.test.d.ts +2 -0
- package/dist/memory/modules/obsidian/tests/vault-reader.test.d.ts.map +1 -0
- package/dist/memory/modules/obsidian/tests/vault-reader.test.js +37 -0
- package/dist/memory/modules/obsidian/tests/vault-reader.test.js.map +1 -0
- package/dist/memory/modules/obsidian/tools.d.ts +5 -0
- package/dist/memory/modules/obsidian/tools.d.ts.map +1 -0
- package/dist/memory/modules/obsidian/tools.js +101 -0
- package/dist/memory/modules/obsidian/tools.js.map +1 -0
- package/dist/memory/modules/obsidian/vault-reader.d.ts +8 -0
- package/dist/memory/modules/obsidian/vault-reader.d.ts.map +1 -0
- package/dist/memory/modules/obsidian/vault-reader.js +82 -0
- package/dist/memory/modules/obsidian/vault-reader.js.map +1 -0
- package/dist/memory/modules/obsidian/watcher.d.ts +5 -0
- package/dist/memory/modules/obsidian/watcher.d.ts.map +1 -0
- package/dist/memory/modules/obsidian/watcher.js +83 -0
- package/dist/memory/modules/obsidian/watcher.js.map +1 -0
- package/dist/memory/modules/youtube/ingest.d.ts +20 -0
- package/dist/memory/modules/youtube/ingest.d.ts.map +1 -0
- package/dist/memory/modules/youtube/ingest.js +49 -0
- package/dist/memory/modules/youtube/ingest.js.map +1 -0
- package/dist/memory/modules/youtube/module.d.ts +11 -0
- package/dist/memory/modules/youtube/module.d.ts.map +1 -0
- package/dist/memory/modules/youtube/module.js +26 -0
- package/dist/memory/modules/youtube/module.js.map +1 -0
- package/dist/memory/modules/youtube/tests/channel.test.d.ts +2 -0
- package/dist/memory/modules/youtube/tests/channel.test.d.ts.map +1 -0
- package/dist/memory/modules/youtube/tests/channel.test.js +61 -0
- package/dist/memory/modules/youtube/tests/channel.test.js.map +1 -0
- package/dist/memory/modules/youtube/tests/transcript-fetcher.test.d.ts +2 -0
- package/dist/memory/modules/youtube/tests/transcript-fetcher.test.d.ts.map +1 -0
- package/dist/memory/modules/youtube/tests/transcript-fetcher.test.js +23 -0
- package/dist/memory/modules/youtube/tests/transcript-fetcher.test.js.map +1 -0
- package/dist/memory/modules/youtube/tests/youtube.test.d.ts +2 -0
- package/dist/memory/modules/youtube/tests/youtube.test.d.ts.map +1 -0
- package/dist/memory/modules/youtube/tests/youtube.test.js +52 -0
- package/dist/memory/modules/youtube/tests/youtube.test.js.map +1 -0
- package/dist/memory/modules/youtube/tools.d.ts +5 -0
- package/dist/memory/modules/youtube/tools.d.ts.map +1 -0
- package/dist/memory/modules/youtube/tools.js +182 -0
- package/dist/memory/modules/youtube/tools.js.map +1 -0
- package/dist/memory/modules/youtube/transcript-fetcher.d.ts +17 -0
- package/dist/memory/modules/youtube/transcript-fetcher.d.ts.map +1 -0
- package/dist/memory/modules/youtube/transcript-fetcher.js +178 -0
- package/dist/memory/modules/youtube/transcript-fetcher.js.map +1 -0
- package/dist/memory/modules/youtube/watcher.d.ts +30 -0
- package/dist/memory/modules/youtube/watcher.d.ts.map +1 -0
- package/dist/memory/modules/youtube/watcher.js +198 -0
- package/dist/memory/modules/youtube/watcher.js.map +1 -0
- package/dist/memory/public/tools.d.ts +5 -0
- package/dist/memory/public/tools.d.ts.map +1 -0
- package/dist/memory/public/tools.js +1761 -0
- package/dist/memory/public/tools.js.map +1 -0
- package/dist/private/algorithms/chunker-semantic.d.ts +3 -0
- package/dist/private/algorithms/chunker-semantic.d.ts.map +1 -0
- package/dist/private/algorithms/chunker-semantic.js +70 -0
- package/dist/private/algorithms/chunker-semantic.js.map +1 -0
- package/dist/private/algorithms/find-related-smart.d.ts +4 -0
- package/dist/private/algorithms/find-related-smart.d.ts.map +1 -0
- package/dist/private/algorithms/find-related-smart.js +52 -0
- package/dist/private/algorithms/find-related-smart.js.map +1 -0
- package/dist/private/algorithms/graph-semantic-edges.d.ts +4 -0
- package/dist/private/algorithms/graph-semantic-edges.d.ts.map +1 -0
- package/dist/private/algorithms/graph-semantic-edges.js +38 -0
- package/dist/private/algorithms/graph-semantic-edges.js.map +1 -0
- package/dist/private/algorithms/search-all-smart.d.ts +9 -0
- package/dist/private/algorithms/search-all-smart.d.ts.map +1 -0
- package/dist/private/algorithms/search-all-smart.js +62 -0
- package/dist/private/algorithms/search-all-smart.js.map +1 -0
- package/dist/private/index.d.ts +7 -0
- package/dist/private/index.d.ts.map +1 -0
- package/dist/private/index.js +39 -0
- package/dist/private/index.js.map +1 -0
- package/dist/private/prompts/extraction-system.d.ts +2 -0
- package/dist/private/prompts/extraction-system.d.ts.map +1 -0
- package/dist/private/prompts/extraction-system.js +15 -0
- package/dist/private/prompts/extraction-system.js.map +1 -0
- package/dist/private/prompts/suggest-properties.d.ts +2 -0
- package/dist/private/prompts/suggest-properties.d.ts.map +1 -0
- package/dist/private/prompts/suggest-properties.js +18 -0
- package/dist/private/prompts/suggest-properties.js.map +1 -0
- package/dist/private/tests/find-related-smart.test.d.ts +2 -0
- package/dist/private/tests/find-related-smart.test.d.ts.map +1 -0
- package/dist/private/tests/find-related-smart.test.js +86 -0
- package/dist/private/tests/find-related-smart.test.js.map +1 -0
- package/dist/private/tests/property-extractor-smart.test.d.ts +2 -0
- package/dist/private/tests/property-extractor-smart.test.d.ts.map +1 -0
- package/dist/private/tests/property-extractor-smart.test.js +26 -0
- package/dist/private/tests/property-extractor-smart.test.js.map +1 -0
- package/dist/scripts/install-ollama.d.ts +3 -0
- package/dist/scripts/install-ollama.d.ts.map +1 -0
- package/dist/scripts/install-ollama.js +78 -0
- package/dist/scripts/install-ollama.js.map +1 -0
- package/dist/scripts/install.d.ts +3 -0
- package/dist/scripts/install.d.ts.map +1 -0
- package/dist/scripts/install.js +191 -0
- package/dist/scripts/install.js.map +1 -0
- package/dist/scripts/pair.d.ts +3 -0
- package/dist/scripts/pair.d.ts.map +1 -0
- package/dist/scripts/pair.js +78 -0
- package/dist/scripts/pair.js.map +1 -0
- package/dist/scripts/rebuild.d.ts +20 -0
- package/dist/scripts/rebuild.d.ts.map +1 -0
- package/dist/scripts/rebuild.js +171 -0
- package/dist/scripts/rebuild.js.map +1 -0
- package/dist/scripts/reindex.d.ts +3 -0
- package/dist/scripts/reindex.d.ts.map +1 -0
- package/dist/scripts/reindex.js +23 -0
- package/dist/scripts/reindex.js.map +1 -0
- package/dist/scripts/serve.d.ts +3 -0
- package/dist/scripts/serve.d.ts.map +1 -0
- package/dist/scripts/serve.js +57 -0
- package/dist/scripts/serve.js.map +1 -0
- package/dist/scripts/service.d.ts +19 -0
- package/dist/scripts/service.d.ts.map +1 -0
- package/dist/scripts/service.js +257 -0
- package/dist/scripts/service.js.map +1 -0
- package/dist/server/api/daily.d.ts +3 -0
- package/dist/server/api/daily.d.ts.map +1 -0
- package/dist/server/api/daily.js +44 -0
- package/dist/server/api/daily.js.map +1 -0
- package/dist/server/api/graph.d.ts +26 -0
- package/dist/server/api/graph.d.ts.map +1 -0
- package/dist/server/api/graph.js +80 -0
- package/dist/server/api/graph.js.map +1 -0
- package/dist/server/api/integrations.d.ts +4 -0
- package/dist/server/api/integrations.d.ts.map +1 -0
- package/dist/server/api/integrations.js +228 -0
- package/dist/server/api/integrations.js.map +1 -0
- package/dist/server/api/memories.d.ts +4 -0
- package/dist/server/api/memories.d.ts.map +1 -0
- package/dist/server/api/memories.js +267 -0
- package/dist/server/api/memories.js.map +1 -0
- package/dist/server/api/reindex.d.ts +3 -0
- package/dist/server/api/reindex.d.ts.map +1 -0
- package/dist/server/api/reindex.js +18 -0
- package/dist/server/api/reindex.js.map +1 -0
- package/dist/server/api/settings.d.ts +3 -0
- package/dist/server/api/settings.d.ts.map +1 -0
- package/dist/server/api/settings.js +24 -0
- package/dist/server/api/settings.js.map +1 -0
- package/dist/server/api/sources.d.ts +4 -0
- package/dist/server/api/sources.d.ts.map +1 -0
- package/dist/server/api/sources.js +45 -0
- package/dist/server/api/sources.js.map +1 -0
- package/dist/server/api/sync-status.d.ts +3 -0
- package/dist/server/api/sync-status.d.ts.map +1 -0
- package/dist/server/api/sync-status.js +43 -0
- package/dist/server/api/sync-status.js.map +1 -0
- package/dist/server/api/types.d.ts +3 -0
- package/dist/server/api/types.d.ts.map +1 -0
- package/dist/server/api/types.js +20 -0
- package/dist/server/api/types.js.map +1 -0
- package/dist/server/api/views.d.ts +25 -0
- package/dist/server/api/views.d.ts.map +1 -0
- package/dist/server/api/views.js +54 -0
- package/dist/server/api/views.js.map +1 -0
- package/dist/server/index.d.ts +2 -0
- package/dist/server/index.d.ts.map +1 -0
- package/dist/server/index.js +3 -0
- package/dist/server/index.js.map +1 -0
- package/dist/sync/apply.d.ts +55 -0
- package/dist/sync/apply.d.ts.map +1 -0
- package/dist/sync/apply.js +277 -0
- package/dist/sync/apply.js.map +1 -0
- package/dist/sync/channel-client.d.ts +27 -0
- package/dist/sync/channel-client.d.ts.map +1 -0
- package/dist/sync/channel-client.js +154 -0
- package/dist/sync/channel-client.js.map +1 -0
- package/dist/sync/cloud-saves.d.ts +49 -0
- package/dist/sync/cloud-saves.d.ts.map +1 -0
- package/dist/sync/cloud-saves.js +182 -0
- package/dist/sync/cloud-saves.js.map +1 -0
- package/dist/sync/ed25519.d.ts +54 -0
- package/dist/sync/ed25519.d.ts.map +1 -0
- package/dist/sync/ed25519.js +136 -0
- package/dist/sync/ed25519.js.map +1 -0
- package/dist/sync/ops-log.d.ts +43 -0
- package/dist/sync/ops-log.d.ts.map +1 -0
- package/dist/sync/ops-log.js +153 -0
- package/dist/sync/ops-log.js.map +1 -0
- package/dist/sync/recovery-setup.d.ts +26 -0
- package/dist/sync/recovery-setup.d.ts.map +1 -0
- package/dist/sync/recovery-setup.js +113 -0
- package/dist/sync/recovery-setup.js.map +1 -0
- package/dist/sync/replay.d.ts +19 -0
- package/dist/sync/replay.d.ts.map +1 -0
- package/dist/sync/replay.js +59 -0
- package/dist/sync/replay.js.map +1 -0
- package/dist/sync/shamir.d.ts +22 -0
- package/dist/sync/shamir.d.ts.map +1 -0
- package/dist/sync/shamir.js +109 -0
- package/dist/sync/shamir.js.map +1 -0
- package/dist/sync/tests/apply.test.d.ts +4 -0
- package/dist/sync/tests/apply.test.d.ts.map +1 -0
- package/dist/sync/tests/apply.test.js +119 -0
- package/dist/sync/tests/apply.test.js.map +1 -0
- package/dist/sync/tests/ops-log.test.d.ts +2 -0
- package/dist/sync/tests/ops-log.test.d.ts.map +1 -0
- package/dist/sync/tests/ops-log.test.js +105 -0
- package/dist/sync/tests/ops-log.test.js.map +1 -0
- package/dist/sync/tests/two-device-sync.test.d.ts +2 -0
- package/dist/sync/tests/two-device-sync.test.d.ts.map +1 -0
- package/dist/sync/tests/two-device-sync.test.js +250 -0
- package/dist/sync/tests/two-device-sync.test.js.map +1 -0
- package/dist/sync/types.d.ts +87 -0
- package/dist/sync/types.d.ts.map +1 -0
- package/dist/sync/types.js +37 -0
- package/dist/sync/types.js.map +1 -0
- package/dist/tests/chunker.test.d.ts +2 -0
- package/dist/tests/chunker.test.d.ts.map +1 -0
- package/dist/tests/chunker.test.js +24 -0
- package/dist/tests/chunker.test.js.map +1 -0
- package/dist/tests/cloud-auth.test.d.ts +2 -0
- package/dist/tests/cloud-auth.test.d.ts.map +1 -0
- package/dist/tests/cloud-auth.test.js +75 -0
- package/dist/tests/cloud-auth.test.js.map +1 -0
- package/dist/tests/cloud-crypto.test.d.ts +2 -0
- package/dist/tests/cloud-crypto.test.d.ts.map +1 -0
- package/dist/tests/cloud-crypto.test.js +58 -0
- package/dist/tests/cloud-crypto.test.js.map +1 -0
- package/dist/tests/cloud-integration.test.d.ts +2 -0
- package/dist/tests/cloud-integration.test.d.ts.map +1 -0
- package/dist/tests/cloud-integration.test.js +193 -0
- package/dist/tests/cloud-integration.test.js.map +1 -0
- package/dist/tests/cloud-pairing.test.d.ts +2 -0
- package/dist/tests/cloud-pairing.test.d.ts.map +1 -0
- package/dist/tests/cloud-pairing.test.js +86 -0
- package/dist/tests/cloud-pairing.test.js.map +1 -0
- package/dist/tests/cloud-saves-integration.test.d.ts +2 -0
- package/dist/tests/cloud-saves-integration.test.d.ts.map +1 -0
- package/dist/tests/cloud-saves-integration.test.js +92 -0
- package/dist/tests/cloud-saves-integration.test.js.map +1 -0
- package/dist/tests/cloud-transit.test.d.ts +2 -0
- package/dist/tests/cloud-transit.test.d.ts.map +1 -0
- package/dist/tests/cloud-transit.test.js +263 -0
- package/dist/tests/cloud-transit.test.js.map +1 -0
- package/dist/tests/config.test.d.ts +2 -0
- package/dist/tests/config.test.d.ts.map +1 -0
- package/dist/tests/config.test.js +25 -0
- package/dist/tests/config.test.js.map +1 -0
- package/dist/tests/db.test.d.ts +2 -0
- package/dist/tests/db.test.d.ts.map +1 -0
- package/dist/tests/db.test.js +75 -0
- package/dist/tests/db.test.js.map +1 -0
- package/dist/tests/embeddings-providers.test.d.ts +2 -0
- package/dist/tests/embeddings-providers.test.d.ts.map +1 -0
- package/dist/tests/embeddings-providers.test.js +62 -0
- package/dist/tests/embeddings-providers.test.js.map +1 -0
- package/dist/tests/embeddings.test.d.ts +2 -0
- package/dist/tests/embeddings.test.d.ts.map +1 -0
- package/dist/tests/embeddings.test.js +22 -0
- package/dist/tests/embeddings.test.js.map +1 -0
- package/dist/tests/integrations-api.test.d.ts +2 -0
- package/dist/tests/integrations-api.test.d.ts.map +1 -0
- package/dist/tests/integrations-api.test.js +129 -0
- package/dist/tests/integrations-api.test.js.map +1 -0
- package/dist/tests/memory-store.test.d.ts +2 -0
- package/dist/tests/memory-store.test.d.ts.map +1 -0
- package/dist/tests/memory-store.test.js +129 -0
- package/dist/tests/memory-store.test.js.map +1 -0
- package/dist/tests/module-registry.test.d.ts +2 -0
- package/dist/tests/module-registry.test.d.ts.map +1 -0
- package/dist/tests/module-registry.test.js +44 -0
- package/dist/tests/module-registry.test.js.map +1 -0
- package/dist/tests/property-extractor.test.d.ts +2 -0
- package/dist/tests/property-extractor.test.d.ts.map +1 -0
- package/dist/tests/property-extractor.test.js +24 -0
- package/dist/tests/property-extractor.test.js.map +1 -0
- package/dist/tests/public-tools.test.d.ts +2 -0
- package/dist/tests/public-tools.test.d.ts.map +1 -0
- package/dist/tests/public-tools.test.js +270 -0
- package/dist/tests/public-tools.test.js.map +1 -0
- package/dist/tests/reindex.test.d.ts +2 -0
- package/dist/tests/reindex.test.d.ts.map +1 -0
- package/dist/tests/reindex.test.js +58 -0
- package/dist/tests/reindex.test.js.map +1 -0
- package/dist/tests/shamir.test.d.ts +2 -0
- package/dist/tests/shamir.test.d.ts.map +1 -0
- package/dist/tests/shamir.test.js +57 -0
- package/dist/tests/shamir.test.js.map +1 -0
- package/dist/tests/source-registry.test.d.ts +2 -0
- package/dist/tests/source-registry.test.d.ts.map +1 -0
- package/dist/tests/source-registry.test.js +58 -0
- package/dist/tests/source-registry.test.js.map +1 -0
- package/dist/tests/types.test.d.ts +2 -0
- package/dist/tests/types.test.d.ts.map +1 -0
- package/dist/tests/types.test.js +26 -0
- package/dist/tests/types.test.js.map +1 -0
- package/dist/tests/vector.test.d.ts +2 -0
- package/dist/tests/vector.test.d.ts.map +1 -0
- package/dist/tests/vector.test.js +61 -0
- package/dist/tests/vector.test.js.map +1 -0
- package/dist/tests/wikilinks.test.d.ts +2 -0
- package/dist/tests/wikilinks.test.d.ts.map +1 -0
- package/dist/tests/wikilinks.test.js +20 -0
- package/dist/tests/wikilinks.test.js.map +1 -0
- package/dist/tools/index.d.ts +22 -0
- package/dist/tools/index.d.ts.map +1 -0
- package/dist/tools/index.js +38 -0
- package/dist/tools/index.js.map +1 -0
- package/dist/types.d.ts +134 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +25 -0
- package/dist/types.js.map +1 -0
- package/dist/vector/store.d.ts +28 -0
- package/dist/vector/store.d.ts.map +1 -0
- package/dist/vector/store.js +132 -0
- package/dist/vector/store.js.map +1 -0
- package/dist/webapp/api/daily.d.ts +3 -0
- package/dist/webapp/api/daily.d.ts.map +1 -0
- package/dist/webapp/api/daily.js +44 -0
- package/dist/webapp/api/daily.js.map +1 -0
- package/dist/webapp/api/graph.d.ts +26 -0
- package/dist/webapp/api/graph.d.ts.map +1 -0
- package/dist/webapp/api/graph.js +80 -0
- package/dist/webapp/api/graph.js.map +1 -0
- package/dist/webapp/api/memories.d.ts +4 -0
- package/dist/webapp/api/memories.d.ts.map +1 -0
- package/dist/webapp/api/memories.js +70 -0
- package/dist/webapp/api/memories.js.map +1 -0
- package/dist/webapp/api/reindex.d.ts +3 -0
- package/dist/webapp/api/reindex.d.ts.map +1 -0
- package/dist/webapp/api/reindex.js +18 -0
- package/dist/webapp/api/reindex.js.map +1 -0
- package/dist/webapp/api/settings.d.ts +3 -0
- package/dist/webapp/api/settings.d.ts.map +1 -0
- package/dist/webapp/api/settings.js +24 -0
- package/dist/webapp/api/settings.js.map +1 -0
- package/dist/webapp/api/sources.d.ts +4 -0
- package/dist/webapp/api/sources.d.ts.map +1 -0
- package/dist/webapp/api/sources.js +45 -0
- package/dist/webapp/api/sources.js.map +1 -0
- package/dist/webapp/api/sync-status.d.ts +3 -0
- package/dist/webapp/api/sync-status.d.ts.map +1 -0
- package/dist/webapp/api/sync-status.js +43 -0
- package/dist/webapp/api/sync-status.js.map +1 -0
- package/dist/webapp/api/types.d.ts +3 -0
- package/dist/webapp/api/types.d.ts.map +1 -0
- package/dist/webapp/api/types.js +20 -0
- package/dist/webapp/api/types.js.map +1 -0
- package/dist/webapp/api/views.d.ts +25 -0
- package/dist/webapp/api/views.d.ts.map +1 -0
- package/dist/webapp/api/views.js +54 -0
- package/dist/webapp/api/views.js.map +1 -0
- package/dist/webapp/mcp-http.d.ts +2 -0
- package/dist/webapp/mcp-http.d.ts.map +1 -0
- package/dist/webapp/mcp-http.js +3 -0
- package/dist/webapp/mcp-http.js.map +1 -0
- package/dist/webapp/server.d.ts +2 -0
- package/dist/webapp/server.d.ts.map +1 -0
- package/dist/webapp/server.js +3 -0
- package/dist/webapp/server.js.map +1 -0
- package/dist/webapp/tests/api.test.d.ts +2 -0
- package/dist/webapp/tests/api.test.d.ts.map +1 -0
- package/dist/webapp/tests/api.test.js +125 -0
- package/dist/webapp/tests/api.test.js.map +1 -0
- package/dist/webapp/tests/mcp-http.test.d.ts +2 -0
- package/dist/webapp/tests/mcp-http.test.d.ts.map +1 -0
- package/dist/webapp/tests/mcp-http.test.js +47 -0
- package/dist/webapp/tests/mcp-http.test.js.map +1 -0
- package/dist/webapp/websocket.d.ts +2 -0
- package/dist/webapp/websocket.d.ts.map +1 -0
- package/dist/webapp/websocket.js +3 -0
- package/dist/webapp/websocket.js.map +1 -0
- package/package.json +128 -0
- package/src/private/README.md +49 -0
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import fs from 'fs/promises';
|
|
2
|
+
import path from 'path';
|
|
3
|
+
import { createLogger } from '../../../logger.js';
|
|
4
|
+
const log = createLogger('obsidian:vault-reader');
|
|
5
|
+
export async function readVault(vaultRoot) {
|
|
6
|
+
const root = path.resolve(vaultRoot);
|
|
7
|
+
const ignorePatterns = await loadIgnore(root);
|
|
8
|
+
const files = [];
|
|
9
|
+
await walk(root, root, ignorePatterns, files);
|
|
10
|
+
log.info(`Read ${files.length} markdown files from ${root}`);
|
|
11
|
+
return files;
|
|
12
|
+
}
|
|
13
|
+
async function loadIgnore(root) {
|
|
14
|
+
const ignoreFile = path.join(root, '.obsidianignore');
|
|
15
|
+
try {
|
|
16
|
+
const content = await fs.readFile(ignoreFile, 'utf-8');
|
|
17
|
+
return content
|
|
18
|
+
.split(/\r?\n/)
|
|
19
|
+
.map((l) => l.trim())
|
|
20
|
+
.filter((l) => l && !l.startsWith('#'))
|
|
21
|
+
.map(globToRegex);
|
|
22
|
+
}
|
|
23
|
+
catch {
|
|
24
|
+
return [];
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
function globToRegex(glob) {
|
|
28
|
+
let escaped = glob
|
|
29
|
+
.replace(/[.+^${}()|[\]\\]/g, '\\$&')
|
|
30
|
+
.replace(/\*/g, '.*')
|
|
31
|
+
.replace(/\?/g, '.');
|
|
32
|
+
if (!escaped.startsWith('/'))
|
|
33
|
+
escaped = `(^|/)${escaped}`;
|
|
34
|
+
return new RegExp(`${escaped}$`);
|
|
35
|
+
}
|
|
36
|
+
function isIgnored(rel, patterns) {
|
|
37
|
+
// Always ignore Obsidian config + system folders
|
|
38
|
+
if (rel.includes('/.obsidian/') || rel.startsWith('.obsidian/') || rel === '.obsidian')
|
|
39
|
+
return true;
|
|
40
|
+
if (rel.includes('/.trash/') || rel.startsWith('.trash/') || rel === '.trash')
|
|
41
|
+
return true;
|
|
42
|
+
for (const p of patterns) {
|
|
43
|
+
if (p.test(rel))
|
|
44
|
+
return true;
|
|
45
|
+
}
|
|
46
|
+
return false;
|
|
47
|
+
}
|
|
48
|
+
async function walk(root, current, ignore, out) {
|
|
49
|
+
let entries;
|
|
50
|
+
try {
|
|
51
|
+
entries = await fs.readdir(current, { withFileTypes: true });
|
|
52
|
+
}
|
|
53
|
+
catch (e) {
|
|
54
|
+
log.warn(`Failed to read ${current}: ${e instanceof Error ? e.message : String(e)}`);
|
|
55
|
+
return;
|
|
56
|
+
}
|
|
57
|
+
for (const e of entries) {
|
|
58
|
+
const abs = path.join(current, e.name);
|
|
59
|
+
const rel = path.relative(root, abs);
|
|
60
|
+
if (isIgnored(rel, ignore))
|
|
61
|
+
continue;
|
|
62
|
+
if (e.isDirectory()) {
|
|
63
|
+
await walk(root, abs, ignore, out);
|
|
64
|
+
}
|
|
65
|
+
else if (e.isFile() && abs.endsWith('.md')) {
|
|
66
|
+
try {
|
|
67
|
+
const content = await fs.readFile(abs, 'utf-8');
|
|
68
|
+
const stats = await fs.stat(abs);
|
|
69
|
+
out.push({
|
|
70
|
+
absolutePath: abs,
|
|
71
|
+
relativePath: rel,
|
|
72
|
+
content,
|
|
73
|
+
modifiedAt: stats.mtimeMs,
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
catch (e) {
|
|
77
|
+
log.warn(`Failed to read ${abs}: ${e instanceof Error ? e.message : String(e)}`);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
//# sourceMappingURL=vault-reader.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"vault-reader.js","sourceRoot":"","sources":["../../../../src/memory/modules/obsidian/vault-reader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,aAAa,CAAC;AAC7B,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAElD,MAAM,GAAG,GAAG,YAAY,CAAC,uBAAuB,CAAC,CAAC;AASlD,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,SAAiB;IAC/C,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IACrC,MAAM,cAAc,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,CAAC;IAC9C,MAAM,KAAK,GAAmB,EAAE,CAAC;IACjC,MAAM,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK,CAAC,CAAC;IAC9C,GAAG,CAAC,IAAI,CAAC,QAAQ,KAAK,CAAC,MAAM,wBAAwB,IAAI,EAAE,CAAC,CAAC;IAC7D,OAAO,KAAK,CAAC;AACf,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,IAAY;IACpC,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,iBAAiB,CAAC,CAAC;IACtD,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QACvD,OAAO,OAAO;aACX,KAAK,CAAC,OAAO,CAAC;aACd,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;aACpB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;aACtC,GAAG,CAAC,WAAW,CAAC,CAAC;IACtB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,SAAS,WAAW,CAAC,IAAY;IAC/B,IAAI,OAAO,GAAG,IAAI;SACf,OAAO,CAAC,mBAAmB,EAAE,MAAM,CAAC;SACpC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC;SACpB,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IACvB,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC;QAAE,OAAO,GAAG,QAAQ,OAAO,EAAE,CAAC;IAC1D,OAAO,IAAI,MAAM,CAAC,GAAG,OAAO,GAAG,CAAC,CAAC;AACnC,CAAC;AAED,SAAS,SAAS,CAAC,GAAW,EAAE,QAAkB;IAChD,iDAAiD;IACjD,IAAI,GAAG,CAAC,QAAQ,CAAC,aAAa,CAAC,IAAI,GAAG,CAAC,UAAU,CAAC,YAAY,CAAC,IAAI,GAAG,KAAK,WAAW;QAAE,OAAO,IAAI,CAAC;IACpG,IAAI,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,GAAG,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IAC3F,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QACzB,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;YAAE,OAAO,IAAI,CAAC;IAC/B,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,KAAK,UAAU,IAAI,CACjB,IAAY,EACZ,OAAe,EACf,MAAgB,EAChB,GAAmB;IAEnB,IAAI,OAA8B,CAAC;IACnC,IAAI,CAAC;QACH,OAAO,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IAC/D,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,GAAG,CAAC,IAAI,CAAC,kBAAkB,OAAO,KAAK,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACrF,OAAO;IACT,CAAC;IACD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;QACvC,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QACrC,IAAI,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC;YAAE,SAAS;QACrC,IAAI,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC;YACpB,MAAM,IAAI,CAAC,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,CAAC,CAAC;QACrC,CAAC;aAAM,IAAI,CAAC,CAAC,MAAM,EAAE,IAAI,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YAC7C,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;gBAChD,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACjC,GAAG,CAAC,IAAI,CAAC;oBACP,YAAY,EAAE,GAAG;oBACjB,YAAY,EAAE,GAAG;oBACjB,OAAO;oBACP,UAAU,EAAE,KAAK,CAAC,OAAO;iBAC1B,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,GAAG,CAAC,IAAI,CAAC,kBAAkB,GAAG,KAAK,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACnF,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import type { MemoryStore } from '../../core/store.js';
|
|
2
|
+
import type { EngramConfig } from '../../../config/schema.js';
|
|
3
|
+
export declare function startObsidianWatchers(store: MemoryStore, config: EngramConfig): void;
|
|
4
|
+
export declare function stopObsidianWatchers(): void;
|
|
5
|
+
//# sourceMappingURL=watcher.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"watcher.d.ts","sourceRoot":"","sources":["../../../../src/memory/modules/obsidian/watcher.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AAS9D,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,WAAW,EAAE,MAAM,EAAE,YAAY,GAAG,IAAI,CAkBpF;AAED,wBAAgB,oBAAoB,IAAI,IAAI,CAY3C"}
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import fs from 'fs';
|
|
2
|
+
import path from 'path';
|
|
3
|
+
import { createLogger } from '../../../logger.js';
|
|
4
|
+
import { sourceRegistry } from '../../core/source-registry.js';
|
|
5
|
+
import { buildObsidianItem } from './ingest.js';
|
|
6
|
+
const log = createLogger('obsidian:watcher');
|
|
7
|
+
const watchers = new Map();
|
|
8
|
+
const pendingByPath = new Map();
|
|
9
|
+
const DEBOUNCE_MS = 1500;
|
|
10
|
+
export function startObsidianWatchers(store, config) {
|
|
11
|
+
const embeddingModel = `${config.embeddings.provider}/${config.embeddings.model}`;
|
|
12
|
+
const sources = sourceRegistry.listEnabled('obsidian');
|
|
13
|
+
for (const s of sources) {
|
|
14
|
+
const vaultPath = s.external_id;
|
|
15
|
+
try {
|
|
16
|
+
const w = fs.watch(vaultPath, { recursive: true }, (eventType, filename) => {
|
|
17
|
+
if (!filename || !filename.endsWith('.md'))
|
|
18
|
+
return;
|
|
19
|
+
const abs = path.join(vaultPath, filename);
|
|
20
|
+
scheduleReingest(abs, vaultPath, store, embeddingModel, s.id);
|
|
21
|
+
});
|
|
22
|
+
watchers.set(vaultPath, w);
|
|
23
|
+
log.info(`Watching Obsidian vault ${vaultPath}`);
|
|
24
|
+
}
|
|
25
|
+
catch (e) {
|
|
26
|
+
const msg = e instanceof Error ? e.message : String(e);
|
|
27
|
+
log.warn(`Failed to watch ${vaultPath}: ${msg}`);
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
export function stopObsidianWatchers() {
|
|
32
|
+
for (const [p, w] of watchers) {
|
|
33
|
+
try {
|
|
34
|
+
w.close();
|
|
35
|
+
log.info(`Stopped watching ${p}`);
|
|
36
|
+
}
|
|
37
|
+
catch {
|
|
38
|
+
// ignore close errors
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
watchers.clear();
|
|
42
|
+
for (const t of pendingByPath.values())
|
|
43
|
+
clearTimeout(t);
|
|
44
|
+
pendingByPath.clear();
|
|
45
|
+
}
|
|
46
|
+
function scheduleReingest(abs, vaultPath, store, embeddingModel, sourceRegistryId) {
|
|
47
|
+
const existing = pendingByPath.get(abs);
|
|
48
|
+
if (existing)
|
|
49
|
+
clearTimeout(existing);
|
|
50
|
+
const timer = setTimeout(async () => {
|
|
51
|
+
pendingByPath.delete(abs);
|
|
52
|
+
try {
|
|
53
|
+
if (!fs.existsSync(abs)) {
|
|
54
|
+
await store.deleteBySourceId(`obsidian:${abs}`);
|
|
55
|
+
log.info(`Removed deleted file: ${abs}`);
|
|
56
|
+
sourceRegistry.recordSync(sourceRegistryId, new Date().toISOString());
|
|
57
|
+
return;
|
|
58
|
+
}
|
|
59
|
+
const stats = await fs.promises.stat(abs);
|
|
60
|
+
const content = await fs.promises.readFile(abs, 'utf-8');
|
|
61
|
+
const item = buildObsidianItem({
|
|
62
|
+
file: {
|
|
63
|
+
absolutePath: abs,
|
|
64
|
+
relativePath: path.relative(vaultPath, abs),
|
|
65
|
+
content,
|
|
66
|
+
modifiedAt: stats.mtimeMs,
|
|
67
|
+
},
|
|
68
|
+
vaultRoot: vaultPath,
|
|
69
|
+
embeddingModel,
|
|
70
|
+
});
|
|
71
|
+
await store.deleteBySourceId(item.source_id);
|
|
72
|
+
await store.insert(item);
|
|
73
|
+
sourceRegistry.recordSync(sourceRegistryId, new Date().toISOString());
|
|
74
|
+
log.info(`Re-ingested ${abs}`);
|
|
75
|
+
}
|
|
76
|
+
catch (e) {
|
|
77
|
+
const msg = e instanceof Error ? e.message : String(e);
|
|
78
|
+
log.warn(`Re-ingest failed for ${abs}: ${msg}`);
|
|
79
|
+
}
|
|
80
|
+
}, DEBOUNCE_MS);
|
|
81
|
+
pendingByPath.set(abs, timer);
|
|
82
|
+
}
|
|
83
|
+
//# sourceMappingURL=watcher.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"watcher.js","sourceRoot":"","sources":["../../../../src/memory/modules/obsidian/watcher.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,cAAc,EAAE,MAAM,+BAA+B,CAAC;AAG/D,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAEhD,MAAM,GAAG,GAAG,YAAY,CAAC,kBAAkB,CAAC,CAAC;AAE7C,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAwB,CAAC;AACjD,MAAM,aAAa,GAAG,IAAI,GAAG,EAA0B,CAAC;AACxD,MAAM,WAAW,GAAG,IAAI,CAAC;AAEzB,MAAM,UAAU,qBAAqB,CAAC,KAAkB,EAAE,MAAoB;IAC5E,MAAM,cAAc,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,IAAI,MAAM,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;IAClF,MAAM,OAAO,GAAG,cAAc,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;IACvD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,MAAM,SAAS,GAAG,CAAC,CAAC,WAAW,CAAC;QAChC,IAAI,CAAC;YACH,MAAM,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,EAAE,CAAC,SAAS,EAAE,QAAQ,EAAE,EAAE;gBACzE,IAAI,CAAC,QAAQ,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC;oBAAE,OAAO;gBACnD,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;gBAC3C,gBAAgB,CAAC,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;YAChE,CAAC,CAAC,CAAC;YACH,QAAQ,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;YAC3B,GAAG,CAAC,IAAI,CAAC,2BAA2B,SAAS,EAAE,CAAC,CAAC;QACnD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,GAAG,GAAG,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YACvD,GAAG,CAAC,IAAI,CAAC,mBAAmB,SAAS,KAAK,GAAG,EAAE,CAAC,CAAC;QACnD,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,oBAAoB;IAClC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,QAAQ,EAAE,CAAC;QAC9B,IAAI,CAAC;YACH,CAAC,CAAC,KAAK,EAAE,CAAC;YACV,GAAG,CAAC,IAAI,CAAC,oBAAoB,CAAC,EAAE,CAAC,CAAC;QACpC,CAAC;QAAC,MAAM,CAAC;YACP,sBAAsB;QACxB,CAAC;IACH,CAAC;IACD,QAAQ,CAAC,KAAK,EAAE,CAAC;IACjB,KAAK,MAAM,CAAC,IAAI,aAAa,CAAC,MAAM,EAAE;QAAE,YAAY,CAAC,CAAC,CAAC,CAAC;IACxD,aAAa,CAAC,KAAK,EAAE,CAAC;AACxB,CAAC;AAED,SAAS,gBAAgB,CACvB,GAAW,EACX,SAAiB,EACjB,KAAkB,EAClB,cAAsB,EACtB,gBAAwB;IAExB,MAAM,QAAQ,GAAG,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACxC,IAAI,QAAQ;QAAE,YAAY,CAAC,QAAQ,CAAC,CAAC;IAErC,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,IAAI,EAAE;QAClC,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC1B,IAAI,CAAC;YACH,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBACxB,MAAM,KAAK,CAAC,gBAAgB,CAAC,YAAY,GAAG,EAAE,CAAC,CAAC;gBAChD,GAAG,CAAC,IAAI,CAAC,yBAAyB,GAAG,EAAE,CAAC,CAAC;gBACzC,cAAc,CAAC,UAAU,CAAC,gBAAgB,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC;gBACtE,OAAO;YACT,CAAC;YACD,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC1C,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;YACzD,MAAM,IAAI,GAAG,iBAAiB,CAAC;gBAC7B,IAAI,EAAE;oBACJ,YAAY,EAAE,GAAG;oBACjB,YAAY,EAAE,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,GAAG,CAAC;oBAC3C,OAAO;oBACP,UAAU,EAAE,KAAK,CAAC,OAAO;iBAC1B;gBACD,SAAS,EAAE,SAAS;gBACpB,cAAc;aACf,CAAC,CAAC;YACH,MAAM,KAAK,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC7C,MAAM,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YACzB,cAAc,CAAC,UAAU,CAAC,gBAAgB,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC;YACtE,GAAG,CAAC,IAAI,CAAC,eAAe,GAAG,EAAE,CAAC,CAAC;QACjC,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,GAAG,GAAG,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YACvD,GAAG,CAAC,IAAI,CAAC,wBAAwB,GAAG,KAAK,GAAG,EAAE,CAAC,CAAC;QAClD,CAAC;IACH,CAAC,EAAE,WAAW,CAAC,CAAC;IAEhB,aAAa,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;AAChC,CAAC"}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { MemoryItem } from '../../../types.js';
|
|
2
|
+
import type { YoutubeTranscriptResult } from './transcript-fetcher.js';
|
|
3
|
+
import type { MemoryStore } from '../../core/store.js';
|
|
4
|
+
import type { EmbeddingsConfig } from '../../../config/schema.js';
|
|
5
|
+
import type { YoutubeConfig } from '../../../config/schema.js';
|
|
6
|
+
export interface IngestYouTubeOptions {
|
|
7
|
+
tags?: string[];
|
|
8
|
+
sourceContext?: string;
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* Full ingest pipeline for a YouTube URL: fetch transcript → build item → insert.
|
|
12
|
+
* Used by the channel watcher and playlist importer.
|
|
13
|
+
*/
|
|
14
|
+
export declare function ingestYouTube(url: string, store: MemoryStore, embeddingsConfig: EmbeddingsConfig, youtubeConfig: YoutubeConfig, options?: IngestYouTubeOptions): Promise<string>;
|
|
15
|
+
export interface YoutubeIngestInput {
|
|
16
|
+
transcript: YoutubeTranscriptResult;
|
|
17
|
+
embeddingModel: string;
|
|
18
|
+
}
|
|
19
|
+
export declare function buildYoutubeItem(input: YoutubeIngestInput): MemoryItem;
|
|
20
|
+
//# sourceMappingURL=ingest.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ingest.d.ts","sourceRoot":"","sources":["../../../../src/memory/modules/youtube/ingest.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,UAAU,EAAoB,MAAM,mBAAmB,CAAC;AACtE,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,yBAAyB,CAAC;AACvE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAC;AAClE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAE/D,MAAM,WAAW,oBAAoB;IACnC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED;;;GAGG;AACH,wBAAsB,aAAa,CACjC,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,WAAW,EAClB,gBAAgB,EAAE,gBAAgB,EAClC,aAAa,EAAE,aAAa,EAC5B,OAAO,CAAC,EAAE,oBAAoB,GAC7B,OAAO,CAAC,MAAM,CAAC,CAUjB;AAED,MAAM,WAAW,kBAAkB;IACjC,UAAU,EAAE,uBAAuB,CAAC;IACpC,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,kBAAkB,GAAG,UAAU,CA8BtE"}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
// src/memory/modules/youtube/ingest.ts
|
|
2
|
+
import { ulid } from 'ulid';
|
|
3
|
+
import { createHash } from 'crypto';
|
|
4
|
+
/**
|
|
5
|
+
* Full ingest pipeline for a YouTube URL: fetch transcript → build item → insert.
|
|
6
|
+
* Used by the channel watcher and playlist importer.
|
|
7
|
+
*/
|
|
8
|
+
export async function ingestYouTube(url, store, embeddingsConfig, youtubeConfig, options) {
|
|
9
|
+
const { fetchTranscript } = await import('./transcript-fetcher.js');
|
|
10
|
+
const transcript = await fetchTranscript(url, youtubeConfig);
|
|
11
|
+
const embeddingModel = `${embeddingsConfig.provider}/${embeddingsConfig.model}`;
|
|
12
|
+
const item = buildYoutubeItem({ transcript, embeddingModel });
|
|
13
|
+
if (options?.tags) {
|
|
14
|
+
item.properties.tags = options.tags;
|
|
15
|
+
}
|
|
16
|
+
await store.insert(item);
|
|
17
|
+
return item.id;
|
|
18
|
+
}
|
|
19
|
+
export function buildYoutubeItem(input) {
|
|
20
|
+
const now = new Date().toISOString();
|
|
21
|
+
const properties = {
|
|
22
|
+
title: input.transcript.title,
|
|
23
|
+
created_at: now,
|
|
24
|
+
ingested_at: now,
|
|
25
|
+
source_url: `https://www.youtube.com/watch?v=${input.transcript.video_id}`,
|
|
26
|
+
author: input.transcript.channel,
|
|
27
|
+
custom: {
|
|
28
|
+
video_id: input.transcript.video_id,
|
|
29
|
+
language: input.transcript.language,
|
|
30
|
+
segments: input.transcript.segments,
|
|
31
|
+
duration_seconds: input.transcript.segments.length > 0
|
|
32
|
+
? input.transcript.segments[input.transcript.segments.length - 1].start +
|
|
33
|
+
input.transcript.segments[input.transcript.segments.length - 1].duration
|
|
34
|
+
: 0,
|
|
35
|
+
},
|
|
36
|
+
};
|
|
37
|
+
return {
|
|
38
|
+
id: ulid(),
|
|
39
|
+
type: 'youtube',
|
|
40
|
+
source_id: `youtube:${input.transcript.video_id}`,
|
|
41
|
+
content: input.transcript.full_text,
|
|
42
|
+
content_hash: createHash('sha256').update(input.transcript.full_text).digest('hex'),
|
|
43
|
+
properties,
|
|
44
|
+
wikilinks: [],
|
|
45
|
+
related_ids: [],
|
|
46
|
+
embedding_model: input.embeddingModel,
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
//# sourceMappingURL=ingest.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ingest.js","sourceRoot":"","sources":["../../../../src/memory/modules/youtube/ingest.ts"],"names":[],"mappings":"AAAA,uCAAuC;AACvC,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAYpC;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,GAAW,EACX,KAAkB,EAClB,gBAAkC,EAClC,aAA4B,EAC5B,OAA8B;IAE9B,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,yBAAyB,CAAC,CAAC;IACpE,MAAM,UAAU,GAAG,MAAM,eAAe,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC;IAC7D,MAAM,cAAc,GAAG,GAAG,gBAAgB,CAAC,QAAQ,IAAI,gBAAgB,CAAC,KAAK,EAAE,CAAC;IAChF,MAAM,IAAI,GAAG,gBAAgB,CAAC,EAAE,UAAU,EAAE,cAAc,EAAE,CAAC,CAAC;IAC9D,IAAI,OAAO,EAAE,IAAI,EAAE,CAAC;QAClB,IAAI,CAAC,UAAU,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IACtC,CAAC;IACD,MAAM,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACzB,OAAO,IAAI,CAAC,EAAE,CAAC;AACjB,CAAC;AAOD,MAAM,UAAU,gBAAgB,CAAC,KAAyB;IACxD,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACrC,MAAM,UAAU,GAAqB;QACnC,KAAK,EAAE,KAAK,CAAC,UAAU,CAAC,KAAK;QAC7B,UAAU,EAAE,GAAG;QACf,WAAW,EAAE,GAAG;QAChB,UAAU,EAAE,mCAAmC,KAAK,CAAC,UAAU,CAAC,QAAQ,EAAE;QAC1E,MAAM,EAAE,KAAK,CAAC,UAAU,CAAC,OAAO;QAChC,MAAM,EAAE;YACN,QAAQ,EAAE,KAAK,CAAC,UAAU,CAAC,QAAQ;YACnC,QAAQ,EAAE,KAAK,CAAC,UAAU,CAAC,QAAQ;YACnC,QAAQ,EAAE,KAAK,CAAC,UAAU,CAAC,QAAQ;YACnC,gBAAgB,EACd,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC;gBAClC,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,KAAK;oBACrE,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,QAAQ;gBAC1E,CAAC,CAAC,CAAC;SACR;KACF,CAAC;IACF,OAAO;QACL,EAAE,EAAE,IAAI,EAAE;QACV,IAAI,EAAE,SAAS;QACf,SAAS,EAAE,WAAW,KAAK,CAAC,UAAU,CAAC,QAAQ,EAAE;QACjD,OAAO,EAAE,KAAK,CAAC,UAAU,CAAC,SAAS;QACnC,YAAY,EAAE,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;QACnF,UAAU;QACV,SAAS,EAAE,EAAE;QACb,WAAW,EAAE,EAAE;QACf,eAAe,EAAE,KAAK,CAAC,cAAc;KACtC,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { MemoryModule } from '../../core/module-interface.js';
|
|
2
|
+
import type { MemoryStore } from '../../core/store.js';
|
|
3
|
+
import type { EngramConfig } from '../../../config/schema.js';
|
|
4
|
+
export declare function createYoutubeModule(_config: EngramConfig): MemoryModule;
|
|
5
|
+
export declare function buildYoutubeModuleTools(store: MemoryStore, config: EngramConfig): import("../../core/module-interface.js").MCPToolDefinition[];
|
|
6
|
+
/**
|
|
7
|
+
* Start the YouTube channel cron — call this after the server is fully
|
|
8
|
+
* initialized and the store is available.
|
|
9
|
+
*/
|
|
10
|
+
export declare function startYoutubeChannelCron(store: MemoryStore, config: EngramConfig): void;
|
|
11
|
+
//# sourceMappingURL=module.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"module.d.ts","sourceRoot":"","sources":["../../../../src/memory/modules/youtube/module.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gCAAgC,CAAC;AACnE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AAI9D,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,YAAY,GAAG,YAAY,CAevE;AAED,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,WAAW,EAAE,MAAM,EAAE,YAAY,gEAE/E;AAED;;;GAGG;AACH,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,WAAW,EAAE,MAAM,EAAE,YAAY,GAAG,IAAI,CAEtF"}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { buildYoutubeTools } from './tools.js';
|
|
2
|
+
import { startChannelCron } from './watcher.js';
|
|
3
|
+
export function createYoutubeModule(_config) {
|
|
4
|
+
return {
|
|
5
|
+
id: 'youtube',
|
|
6
|
+
displayName: 'YouTube',
|
|
7
|
+
isCustom: false,
|
|
8
|
+
async onBoot() { },
|
|
9
|
+
async onShutdown() { },
|
|
10
|
+
async ingest() {
|
|
11
|
+
throw new Error('youtube.ingest is not used; use add_youtube_url MCP tool');
|
|
12
|
+
},
|
|
13
|
+
tools: [],
|
|
14
|
+
};
|
|
15
|
+
}
|
|
16
|
+
export function buildYoutubeModuleTools(store, config) {
|
|
17
|
+
return buildYoutubeTools(store, config);
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Start the YouTube channel cron — call this after the server is fully
|
|
21
|
+
* initialized and the store is available.
|
|
22
|
+
*/
|
|
23
|
+
export function startYoutubeChannelCron(store, config) {
|
|
24
|
+
startChannelCron(store, config.embeddings, config.youtube);
|
|
25
|
+
}
|
|
26
|
+
//# sourceMappingURL=module.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"module.js","sourceRoot":"","sources":["../../../../src/memory/modules/youtube/module.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAC/C,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAEhD,MAAM,UAAU,mBAAmB,CAAC,OAAqB;IACvD,OAAO;QACL,EAAE,EAAE,SAAS;QACb,WAAW,EAAE,SAAS;QACtB,QAAQ,EAAE,KAAK;QAEf,KAAK,CAAC,MAAM,KAAI,CAAC;QACjB,KAAK,CAAC,UAAU,KAAI,CAAC;QAErB,KAAK,CAAC,MAAM;YACV,MAAM,IAAI,KAAK,CAAC,0DAA0D,CAAC,CAAC;QAC9E,CAAC;QAED,KAAK,EAAE,EAAE;KACV,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,KAAkB,EAAE,MAAoB;IAC9E,OAAO,iBAAiB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;AAC1C,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,uBAAuB,CAAC,KAAkB,EAAE,MAAoB;IAC9E,gBAAgB,CAAC,KAAK,EAAE,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;AAC7D,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"channel.test.d.ts","sourceRoot":"","sources":["../../../../../src/memory/modules/youtube/tests/channel.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
// src/memory/modules/youtube/tests/channel.test.ts
|
|
2
|
+
import { describe, it, expect } from 'vitest';
|
|
3
|
+
import { resolveChannelId } from '../watcher.js';
|
|
4
|
+
const REAL_CHANNEL_ID = 'UCnUYZLuoy1rq1aVMwx4aTzw'; // Andrej Karpathy — stable, public
|
|
5
|
+
const REAL_CHANNEL_URL = 'https://www.youtube.com/@AndrejKarpathy';
|
|
6
|
+
describe('resolveChannelId', () => {
|
|
7
|
+
it('returns the ID unchanged when given a valid channel ID', async () => {
|
|
8
|
+
const id = await resolveChannelId(REAL_CHANNEL_ID);
|
|
9
|
+
expect(id).toBe(REAL_CHANNEL_ID);
|
|
10
|
+
});
|
|
11
|
+
it('resolves a channel URL to a channel ID', async () => {
|
|
12
|
+
let id;
|
|
13
|
+
try {
|
|
14
|
+
id = await resolveChannelId(REAL_CHANNEL_URL);
|
|
15
|
+
}
|
|
16
|
+
catch (e) {
|
|
17
|
+
const msg = e instanceof Error ? e.message : String(e);
|
|
18
|
+
// Network may be unavailable or YouTube may block — skip gracefully
|
|
19
|
+
if (msg.includes('HTTP') || msg.includes('fetch') || msg.includes('Cannot resolve')) {
|
|
20
|
+
console.warn(`Skipping resolveChannelId URL test — ${msg}`);
|
|
21
|
+
return;
|
|
22
|
+
}
|
|
23
|
+
throw e;
|
|
24
|
+
}
|
|
25
|
+
expect(id).toMatch(/^UC[A-Za-z0-9_-]{22}$/);
|
|
26
|
+
}, 20_000);
|
|
27
|
+
it('throws on unresolvable input', async () => {
|
|
28
|
+
await expect(resolveChannelId('not-a-channel')).rejects.toThrow(/Cannot resolve/);
|
|
29
|
+
});
|
|
30
|
+
});
|
|
31
|
+
describe('RSS feed fetch (real network)', () => {
|
|
32
|
+
it('fetches and parses the channel RSS feed', async () => {
|
|
33
|
+
const { XMLParser } = await import('fast-xml-parser');
|
|
34
|
+
const feedUrl = `https://www.youtube.com/feeds/videos.xml?channel_id=${REAL_CHANNEL_ID}`;
|
|
35
|
+
let res;
|
|
36
|
+
try {
|
|
37
|
+
res = await fetch(feedUrl, {
|
|
38
|
+
headers: {
|
|
39
|
+
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/124 Safari/537.36',
|
|
40
|
+
'Accept-Language': 'en-US,en;q=0.9',
|
|
41
|
+
},
|
|
42
|
+
signal: AbortSignal.timeout(15_000),
|
|
43
|
+
});
|
|
44
|
+
}
|
|
45
|
+
catch (e) {
|
|
46
|
+
// Network unavailable in CI — skip gracefully
|
|
47
|
+
console.warn(`Skipping RSS test — network error: ${e instanceof Error ? e.message : String(e)}`);
|
|
48
|
+
return;
|
|
49
|
+
}
|
|
50
|
+
if (!res.ok) {
|
|
51
|
+
// YouTube rate-limit or temporary error — skip gracefully instead of failing
|
|
52
|
+
console.warn(`Skipping RSS test — HTTP ${res.status} from YouTube`);
|
|
53
|
+
return;
|
|
54
|
+
}
|
|
55
|
+
const xml = await res.text();
|
|
56
|
+
const parser = new XMLParser({ ignoreAttributes: false });
|
|
57
|
+
const parsed = parser.parse(xml);
|
|
58
|
+
expect(parsed.feed).toBeDefined();
|
|
59
|
+
}, 20_000);
|
|
60
|
+
});
|
|
61
|
+
//# sourceMappingURL=channel.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"channel.test.js","sourceRoot":"","sources":["../../../../../src/memory/modules/youtube/tests/channel.test.ts"],"names":[],"mappings":"AAAA,mDAAmD;AACnD,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAEjD,MAAM,eAAe,GAAG,0BAA0B,CAAC,CAAC,mCAAmC;AACvF,MAAM,gBAAgB,GAAG,yCAAyC,CAAC;AAEnE,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;IAChC,EAAE,CAAC,wDAAwD,EAAE,KAAK,IAAI,EAAE;QACtE,MAAM,EAAE,GAAG,MAAM,gBAAgB,CAAC,eAAe,CAAC,CAAC;QACnD,MAAM,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IACnC,CAAC,CAAC,CAAC;IAEH,EAAE,CACA,wCAAwC,EACxC,KAAK,IAAI,EAAE;QACT,IAAI,EAAU,CAAC;QACf,IAAI,CAAC;YACH,EAAE,GAAG,MAAM,gBAAgB,CAAC,gBAAgB,CAAC,CAAC;QAChD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,GAAG,GAAG,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YACvD,oEAAoE;YACpE,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,CAAC;gBACpF,OAAO,CAAC,IAAI,CAAC,wCAAwC,GAAG,EAAE,CAAC,CAAC;gBAC5D,OAAO;YACT,CAAC;YACD,MAAM,CAAC,CAAC;QACV,CAAC;QACD,MAAM,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,uBAAuB,CAAC,CAAC;IAC9C,CAAC,EACD,MAAM,CACP,CAAC;IAEF,EAAE,CAAC,8BAA8B,EAAE,KAAK,IAAI,EAAE;QAC5C,MAAM,MAAM,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC;IACpF,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,+BAA+B,EAAE,GAAG,EAAE;IAC7C,EAAE,CACA,yCAAyC,EACzC,KAAK,IAAI,EAAE;QACT,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,MAAM,CAAC,iBAAiB,CAAC,CAAC;QACtD,MAAM,OAAO,GAAG,uDAAuD,eAAe,EAAE,CAAC;QACzF,IAAI,GAAa,CAAC;QAClB,IAAI,CAAC;YACH,GAAG,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE;gBACzB,OAAO,EAAE;oBACP,YAAY,EACV,6FAA6F;oBAC/F,iBAAiB,EAAE,gBAAgB;iBACpC;gBACD,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC;aACpC,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,8CAA8C;YAC9C,OAAO,CAAC,IAAI,CAAC,sCAAsC,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACjG,OAAO;QACT,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,6EAA6E;YAC7E,OAAO,CAAC,IAAI,CAAC,4BAA4B,GAAG,CAAC,MAAM,eAAe,CAAC,CAAC;YACpE,OAAO;QACT,CAAC;QACD,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAC7B,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC,EAAE,gBAAgB,EAAE,KAAK,EAAE,CAAC,CAAC;QAC1D,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAmC,CAAC;QACnE,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;IACpC,CAAC,EACD,MAAM,CACP,CAAC;AACJ,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"transcript-fetcher.test.d.ts","sourceRoot":"","sources":["../../../../../src/memory/modules/youtube/tests/transcript-fetcher.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
// src/memory/modules/youtube/tests/transcript-fetcher.test.ts
|
|
2
|
+
import { describe, it, expect } from 'vitest';
|
|
3
|
+
import { extractVideoId, fetchTranscript } from '../transcript-fetcher.js';
|
|
4
|
+
// Note: YouTube's timedtext API frequently returns empty XML for videos even when captions
|
|
5
|
+
// exist in captionTracks. We enable yt-dlp fallback for reliable transcript retrieval.
|
|
6
|
+
const config = { enabled: true, preferLanguage: 'en', fallbackToYtdlp: true };
|
|
7
|
+
// Stable target: TED Talk Sir Ken Robinson "Do schools kill creativity?"
|
|
8
|
+
// (long-standing video with reliable English captions)
|
|
9
|
+
const TEST_URL = 'https://www.youtube.com/watch?v=iG9CE55wbtY';
|
|
10
|
+
describe('youtube transcript fetcher', () => {
|
|
11
|
+
it('extracts video id from various URL shapes', () => {
|
|
12
|
+
expect(extractVideoId('https://www.youtube.com/watch?v=iG9CE55wbtY')).toBe('iG9CE55wbtY');
|
|
13
|
+
expect(extractVideoId('https://youtu.be/iG9CE55wbtY')).toBe('iG9CE55wbtY');
|
|
14
|
+
expect(extractVideoId('iG9CE55wbtY')).toBe('iG9CE55wbtY');
|
|
15
|
+
});
|
|
16
|
+
it('fetches an English transcript for a known video', async () => {
|
|
17
|
+
const result = await fetchTranscript(TEST_URL, config);
|
|
18
|
+
expect(result.video_id).toBe('iG9CE55wbtY');
|
|
19
|
+
expect(result.segments.length).toBeGreaterThan(50);
|
|
20
|
+
expect(result.full_text.toLowerCase()).toContain('creativity');
|
|
21
|
+
}, 60_000);
|
|
22
|
+
});
|
|
23
|
+
//# sourceMappingURL=transcript-fetcher.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"transcript-fetcher.test.js","sourceRoot":"","sources":["../../../../../src/memory/modules/youtube/tests/transcript-fetcher.test.ts"],"names":[],"mappings":"AAAA,8DAA8D;AAC9D,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAE3E,2FAA2F;AAC3F,uFAAuF;AACvF,MAAM,MAAM,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC;AAE9E,yEAAyE;AACzE,uDAAuD;AACvD,MAAM,QAAQ,GAAG,6CAA6C,CAAC;AAE/D,QAAQ,CAAC,4BAA4B,EAAE,GAAG,EAAE;IAC1C,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;QACnD,MAAM,CAAC,cAAc,CAAC,6CAA6C,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAC1F,MAAM,CAAC,cAAc,CAAC,8BAA8B,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAC3E,MAAM,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC5D,CAAC,CAAC,CAAC;IAEH,EAAE,CACA,iDAAiD,EACjD,KAAK,IAAI,EAAE;QACT,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QACvD,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAC5C,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC;QACnD,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;IACjE,CAAC,EACD,MAAM,CACP,CAAC;AACJ,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"youtube.test.d.ts","sourceRoot":"","sources":["../../../../../src/memory/modules/youtube/tests/youtube.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
// src/memory/modules/youtube/tests/youtube.test.ts
|
|
2
|
+
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
|
3
|
+
import fs from 'fs';
|
|
4
|
+
import os from 'os';
|
|
5
|
+
import path from 'path';
|
|
6
|
+
import { initDb, closeDb } from '../../../../db/index.js';
|
|
7
|
+
import { initVectorStore } from '../../../../vector/store.js';
|
|
8
|
+
import { MemoryStore } from '../../../core/store.js';
|
|
9
|
+
import { buildYoutubeModuleTools } from '../module.js';
|
|
10
|
+
const config = {
|
|
11
|
+
dataDir: '~/.engram',
|
|
12
|
+
embeddings: { provider: 'ollama', baseUrl: 'http://localhost:11434', model: 'nomic-embed-text', dimensions: 768 },
|
|
13
|
+
drive: undefined,
|
|
14
|
+
notion: undefined,
|
|
15
|
+
propertyExtraction: {
|
|
16
|
+
enabled: false,
|
|
17
|
+
baseUrl: 'http://localhost:11434',
|
|
18
|
+
model: 'llama3.2:3b',
|
|
19
|
+
maxTokens: 300,
|
|
20
|
+
},
|
|
21
|
+
whisper: { enabled: true, model: 'tiny.en', language: 'en' },
|
|
22
|
+
youtube: { enabled: true, preferLanguage: 'en', fallbackToYtdlp: true },
|
|
23
|
+
modules: { youtube: { enabled: true } },
|
|
24
|
+
mcp: { stdio: true, httpPort: 7777 },
|
|
25
|
+
};
|
|
26
|
+
describe('youtube module e2e (real video)', () => {
|
|
27
|
+
let tmpDir;
|
|
28
|
+
let store;
|
|
29
|
+
beforeEach(() => {
|
|
30
|
+
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'engram-yt-'));
|
|
31
|
+
initDb(tmpDir);
|
|
32
|
+
initVectorStore(tmpDir);
|
|
33
|
+
store = new MemoryStore({ embeddings: config.embeddings });
|
|
34
|
+
});
|
|
35
|
+
afterEach(() => {
|
|
36
|
+
closeDb();
|
|
37
|
+
fs.rmSync(tmpDir, { recursive: true, force: true });
|
|
38
|
+
});
|
|
39
|
+
it('add_youtube_url + search_youtube end-to-end', async () => {
|
|
40
|
+
const tools = buildYoutubeModuleTools(store, config);
|
|
41
|
+
const addRes = (await tools.find((t) => t.name === 'add_youtube_url').handler({
|
|
42
|
+
url: 'https://www.youtube.com/watch?v=iG9CE55wbtY',
|
|
43
|
+
}));
|
|
44
|
+
expect(addRes.id).toBeDefined();
|
|
45
|
+
expect(addRes.segments).toBeGreaterThan(50);
|
|
46
|
+
const hits = (await tools.find((t) => t.name === 'search_youtube').handler({
|
|
47
|
+
query: 'creativity in schools',
|
|
48
|
+
}));
|
|
49
|
+
expect(hits.some((h) => h.id === addRes.id)).toBe(true);
|
|
50
|
+
}, 120_000);
|
|
51
|
+
});
|
|
52
|
+
//# sourceMappingURL=youtube.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"youtube.test.js","sourceRoot":"","sources":["../../../../../src/memory/modules/youtube/tests/youtube.test.ts"],"names":[],"mappings":"AAAA,mDAAmD;AACnD,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACrE,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAC1D,OAAO,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;AAC9D,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AACrD,OAAO,EAAE,uBAAuB,EAAE,MAAM,cAAc,CAAC;AAGvD,MAAM,MAAM,GAAiB;IAC3B,OAAO,EAAE,WAAW;IACpB,UAAU,EAAE,EAAE,QAAQ,EAAE,QAAiB,EAAE,OAAO,EAAE,wBAAwB,EAAE,KAAK,EAAE,kBAAkB,EAAE,UAAU,EAAE,GAAG,EAAE;IAC1H,KAAK,EAAE,SAAS;IAChB,MAAM,EAAE,SAAS;IACjB,kBAAkB,EAAE;QAClB,OAAO,EAAE,KAAK;QACd,OAAO,EAAE,wBAAwB;QACjC,KAAK,EAAE,aAAa;QACpB,SAAS,EAAE,GAAG;KACf;IACD,OAAO,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,IAAI,EAAE;IAC5D,OAAO,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE;IACvE,OAAO,EAAE,EAAE,OAAO,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE;IACvC,GAAG,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE;CACrC,CAAC;AAEF,QAAQ,CAAC,iCAAiC,EAAE,GAAG,EAAE;IAC/C,IAAI,MAAc,CAAC;IACnB,IAAI,KAAkB,CAAC;IAEvB,UAAU,CAAC,GAAG,EAAE;QACd,MAAM,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,YAAY,CAAC,CAAC,CAAC;QAC9D,MAAM,CAAC,MAAM,CAAC,CAAC;QACf,eAAe,CAAC,MAAM,CAAC,CAAC;QACxB,KAAK,GAAG,IAAI,WAAW,CAAC,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC;IAC7D,CAAC,CAAC,CAAC;IAEH,SAAS,CAAC,GAAG,EAAE;QACb,OAAO,EAAE,CAAC;QACV,EAAE,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACtD,CAAC,CAAC,CAAC;IAEH,EAAE,CACA,6CAA6C,EAC7C,KAAK,IAAI,EAAE;QACT,MAAM,KAAK,GAAG,uBAAuB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QACrD,MAAM,MAAM,GAAG,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,iBAAiB,CAAE,CAAC,OAAO,CAAC;YAC7E,GAAG,EAAE,6CAA6C;SACnD,CAAC,CAAoD,CAAC;QACvD,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;QAChC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC;QAE5C,MAAM,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,gBAAgB,CAAE,CAAC,OAAO,CAAC;YAC1E,KAAK,EAAE,uBAAuB;SAC/B,CAAC,CAAyC,CAAC;QAC5C,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1D,CAAC,EACD,OAAO,CACR,CAAC;AACJ,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import type { MCPToolDefinition } from '../../core/module-interface.js';
|
|
2
|
+
import type { MemoryStore } from '../../core/store.js';
|
|
3
|
+
import type { EngramConfig } from '../../../config/schema.js';
|
|
4
|
+
export declare function buildYoutubeTools(store: MemoryStore, config: EngramConfig): MCPToolDefinition[];
|
|
5
|
+
//# sourceMappingURL=tools.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tools.d.ts","sourceRoot":"","sources":["../../../../src/memory/modules/youtube/tools.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,gCAAgC,CAAC;AACxE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AAO9D,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,WAAW,EAAE,MAAM,EAAE,YAAY,GAAG,iBAAiB,EAAE,CA+M/F"}
|