mulmoclaude 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +44 -0
- package/bin/mulmoclaude.js +202 -0
- package/bin/prepare-dist.js +93 -0
- package/client/assets/chunk-vKJrgz-R-C_I3GbVV.js +1 -0
- package/client/assets/html2canvas-Cx501zZr-BF5dYYkY.js +5 -0
- package/client/assets/index-D8rhwXLq.js +4906 -0
- package/client/assets/index-KNLBjwuh.css +1 -0
- package/client/assets/index.es-D4YyL_Dg-BfRHLTZV.js +5 -0
- package/client/assets/material-icons-Dr0goTwe.woff +0 -0
- package/client/assets/material-icons-kAwBdRge.woff2 +0 -0
- package/client/assets/material-icons-outlined-BpWbwl2n.woff +0 -0
- package/client/assets/material-icons-outlined-DZhiGvEA.woff2 +0 -0
- package/client/assets/material-icons-round-BDlwx-sv.woff +0 -0
- package/client/assets/material-icons-round-DrirKXBx.woff2 +0 -0
- package/client/assets/material-icons-sharp-CH1KkVu7.woff +0 -0
- package/client/assets/material-icons-sharp-gidztirS.woff2 +0 -0
- package/client/assets/material-icons-two-tone-B7wz7mED.woff +0 -0
- package/client/assets/material-icons-two-tone-DuNIpaEj.woff2 +0 -0
- package/client/assets/mulmo_bw-ERmkSv0a.png +0 -0
- package/client/assets/purify.es-Fx1Nqyry-PeS5RUhs.js +2 -0
- package/client/assets/typeof-DBp4T-Ny-BC0P-2DM.js +1 -0
- package/client/index.html +28 -0
- package/package.json +66 -0
- package/server/agent/attachmentConverter.ts +270 -0
- package/server/agent/config.ts +414 -0
- package/server/agent/index.ts +260 -0
- package/server/agent/mcp-server.ts +412 -0
- package/server/agent/mcp-tools/index.ts +63 -0
- package/server/agent/mcp-tools/x.ts +188 -0
- package/server/agent/plugin-names.ts +75 -0
- package/server/agent/prompt.ts +349 -0
- package/server/agent/resumeFailover.ts +129 -0
- package/server/agent/sandboxMounts.ts +329 -0
- package/server/agent/stream.ts +194 -0
- package/server/api/auth/bearerAuth.ts +61 -0
- package/server/api/auth/token.ts +98 -0
- package/server/api/csrfGuard.ts +85 -0
- package/server/api/routes/agent.ts +478 -0
- package/server/api/routes/chart.ts +98 -0
- package/server/api/routes/chat-index.ts +46 -0
- package/server/api/routes/config.ts +258 -0
- package/server/api/routes/dispatchResponse.ts +79 -0
- package/server/api/routes/files.ts +812 -0
- package/server/api/routes/html.ts +101 -0
- package/server/api/routes/image.ts +169 -0
- package/server/api/routes/mulmo-script.ts +712 -0
- package/server/api/routes/mulmoScriptValidate.ts +101 -0
- package/server/api/routes/notifications.ts +69 -0
- package/server/api/routes/pdf.ts +163 -0
- package/server/api/routes/plugins.ts +276 -0
- package/server/api/routes/presentHtml.ts +48 -0
- package/server/api/routes/roles.ts +125 -0
- package/server/api/routes/scheduler.ts +153 -0
- package/server/api/routes/schedulerHandlers.ts +151 -0
- package/server/api/routes/schedulerTasks.ts +163 -0
- package/server/api/routes/sessions.ts +294 -0
- package/server/api/routes/sessionsCursor.ts +59 -0
- package/server/api/routes/skills.ts +195 -0
- package/server/api/routes/sources.ts +540 -0
- package/server/api/routes/todos.ts +263 -0
- package/server/api/routes/todosColumnsHandlers.ts +347 -0
- package/server/api/routes/todosHandlers.ts +274 -0
- package/server/api/routes/todosItemsHandlers.ts +386 -0
- package/server/api/routes/wiki/pageIndex.ts +53 -0
- package/server/api/routes/wiki.ts +363 -0
- package/server/api/sandboxStatus.ts +64 -0
- package/server/events/notifications.ts +160 -0
- package/server/events/pub-sub/index.ts +45 -0
- package/server/events/relay-client.ts +288 -0
- package/server/events/scheduler-adapter.ts +302 -0
- package/server/events/session-store/index.ts +492 -0
- package/server/events/task-manager/index.ts +181 -0
- package/server/index.ts +572 -0
- package/server/system/config.ts +243 -0
- package/server/system/credentials.ts +220 -0
- package/server/system/docker.ts +97 -0
- package/server/system/env.ts +109 -0
- package/server/system/logger/config.ts +112 -0
- package/server/system/logger/formatters.ts +40 -0
- package/server/system/logger/index.ts +53 -0
- package/server/system/logger/rotation.ts +37 -0
- package/server/system/logger/sinks.ts +101 -0
- package/server/system/logger/types.ts +29 -0
- package/server/utils/date.ts +57 -0
- package/server/utils/errors.ts +7 -0
- package/server/utils/fetch.ts +27 -0
- package/server/utils/files/atomic.ts +125 -0
- package/server/utils/files/html-io.ts +20 -0
- package/server/utils/files/image-store.ts +66 -0
- package/server/utils/files/index.ts +45 -0
- package/server/utils/files/journal-io.ts +213 -0
- package/server/utils/files/json.ts +69 -0
- package/server/utils/files/markdown-store.ts +33 -0
- package/server/utils/files/naming.ts +50 -0
- package/server/utils/files/reference-dirs-io.ts +45 -0
- package/server/utils/files/roles-io.ts +45 -0
- package/server/utils/files/safe.ts +106 -0
- package/server/utils/files/scheduler-io.ts +20 -0
- package/server/utils/files/scheduler-overrides-io.ts +64 -0
- package/server/utils/files/session-io.ts +136 -0
- package/server/utils/files/spreadsheet-store.ts +63 -0
- package/server/utils/files/todos-io.ts +29 -0
- package/server/utils/files/user-tasks-io.ts +25 -0
- package/server/utils/files/workspace-io.ts +221 -0
- package/server/utils/gemini.ts +59 -0
- package/server/utils/gitignore.ts +69 -0
- package/server/utils/http.ts +15 -0
- package/server/utils/httpError.ts +61 -0
- package/server/utils/id.ts +16 -0
- package/server/utils/json.ts +83 -0
- package/server/utils/logBackgroundError.ts +22 -0
- package/server/utils/markdown.ts +82 -0
- package/server/utils/request.ts +29 -0
- package/server/utils/slug.ts +50 -0
- package/server/utils/spawn.ts +62 -0
- package/server/utils/time.ts +34 -0
- package/server/utils/types.ts +47 -0
- package/server/workspace/chat-index/index.ts +153 -0
- package/server/workspace/chat-index/indexer.ts +209 -0
- package/server/workspace/chat-index/paths.ts +34 -0
- package/server/workspace/chat-index/summarizer.ts +247 -0
- package/server/workspace/chat-index/types.ts +38 -0
- package/server/workspace/custom-dirs.ts +220 -0
- package/server/workspace/helps/business.md +104 -0
- package/server/workspace/helps/github.md +23 -0
- package/server/workspace/helps/index.md +60 -0
- package/server/workspace/helps/mulmoscript.md +249 -0
- package/server/workspace/helps/sandbox.md +90 -0
- package/server/workspace/helps/spreadsheet.md +43 -0
- package/server/workspace/helps/telegram.md +135 -0
- package/server/workspace/helps/wiki.md +131 -0
- package/server/workspace/journal/archivist.ts +386 -0
- package/server/workspace/journal/dailyPass.ts +743 -0
- package/server/workspace/journal/diff.ts +71 -0
- package/server/workspace/journal/index.ts +185 -0
- package/server/workspace/journal/indexFile.ts +136 -0
- package/server/workspace/journal/linkRewrite.ts +4 -0
- package/server/workspace/journal/memoryExtractor.ts +130 -0
- package/server/workspace/journal/optimizationPass.ts +160 -0
- package/server/workspace/journal/paths.ts +76 -0
- package/server/workspace/journal/state.ts +125 -0
- package/server/workspace/paths.ts +158 -0
- package/server/workspace/reference-dirs.ts +252 -0
- package/server/workspace/roles.ts +37 -0
- package/server/workspace/skills/discovery.ts +125 -0
- package/server/workspace/skills/index.ts +10 -0
- package/server/workspace/skills/parser.ts +144 -0
- package/server/workspace/skills/paths.ts +41 -0
- package/server/workspace/skills/scheduler.ts +149 -0
- package/server/workspace/skills/types.ts +30 -0
- package/server/workspace/skills/user-tasks.ts +257 -0
- package/server/workspace/skills/writer.ts +189 -0
- package/server/workspace/sources/arxivDiscovery.ts +182 -0
- package/server/workspace/sources/classifier.ts +268 -0
- package/server/workspace/sources/fetchers/arxiv.ts +170 -0
- package/server/workspace/sources/fetchers/github.ts +106 -0
- package/server/workspace/sources/fetchers/githubIssues.ts +208 -0
- package/server/workspace/sources/fetchers/githubReleases.ts +186 -0
- package/server/workspace/sources/fetchers/index.ts +71 -0
- package/server/workspace/sources/fetchers/registerAll.ts +15 -0
- package/server/workspace/sources/fetchers/rss.ts +141 -0
- package/server/workspace/sources/fetchers/rssParser.ts +295 -0
- package/server/workspace/sources/httpFetcher.ts +230 -0
- package/server/workspace/sources/interests.ts +120 -0
- package/server/workspace/sources/paths.ts +110 -0
- package/server/workspace/sources/pipeline/dedup.ts +60 -0
- package/server/workspace/sources/pipeline/fetch.ts +136 -0
- package/server/workspace/sources/pipeline/index.ts +249 -0
- package/server/workspace/sources/pipeline/notify.ts +72 -0
- package/server/workspace/sources/pipeline/plan.ts +66 -0
- package/server/workspace/sources/pipeline/summarize.ts +189 -0
- package/server/workspace/sources/pipeline/write.ts +185 -0
- package/server/workspace/sources/rateLimiter.ts +148 -0
- package/server/workspace/sources/registry.ts +326 -0
- package/server/workspace/sources/robots.ts +271 -0
- package/server/workspace/sources/sourceState.ts +135 -0
- package/server/workspace/sources/taxonomy.ts +74 -0
- package/server/workspace/sources/types.ts +144 -0
- package/server/workspace/sources/urls.ts +112 -0
- package/server/workspace/tool-trace/classify.ts +114 -0
- package/server/workspace/tool-trace/index.ts +250 -0
- package/server/workspace/tool-trace/writeSearch.ts +98 -0
- package/server/workspace/wiki-backlinks/index.ts +107 -0
- package/server/workspace/wiki-backlinks/sessionBacklinks.ts +144 -0
- package/server/workspace/workspace.ts +66 -0
- package/src/App.vue +720 -0
- package/src/assets/mulmo_bw.png +0 -0
- package/src/components/CanvasViewToggle.vue +27 -0
- package/src/components/ChatAttachmentPreview.vue +45 -0
- package/src/components/ChatImagePreview.vue +17 -0
- package/src/components/ChatInput.vue +208 -0
- package/src/components/FileContentHeader.vue +49 -0
- package/src/components/FileContentRenderer.vue +162 -0
- package/src/components/FileTree.vue +115 -0
- package/src/components/FileTreePane.vue +85 -0
- package/src/components/FilesView.vue +206 -0
- package/src/components/LockStatusPopup.vue +111 -0
- package/src/components/NotificationBell.vue +131 -0
- package/src/components/NotificationToast.vue +72 -0
- package/src/components/PluginLauncher.vue +138 -0
- package/src/components/RightSidebar.vue +113 -0
- package/src/components/RoleSelector.vue +64 -0
- package/src/components/SessionHistoryPanel.vue +176 -0
- package/src/components/SessionTabBar.vue +81 -0
- package/src/components/SettingsMcpTab.vue +350 -0
- package/src/components/SettingsModal.vue +275 -0
- package/src/components/SettingsReferenceDirsTab.vue +173 -0
- package/src/components/SettingsWorkspaceDirsTab.vue +174 -0
- package/src/components/SidebarHeader.vue +69 -0
- package/src/components/StackView.vue +360 -0
- package/src/components/SuggestionsPanel.vue +65 -0
- package/src/components/TodoExplorer.vue +358 -0
- package/src/components/ToolResultsPanel.vue +77 -0
- package/src/components/todo/TodoAddDialog.vue +131 -0
- package/src/components/todo/TodoEditDialog.vue +47 -0
- package/src/components/todo/TodoEditPanel.vue +113 -0
- package/src/components/todo/TodoKanbanView.vue +249 -0
- package/src/components/todo/TodoListView.vue +79 -0
- package/src/components/todo/TodoTableView.vue +177 -0
- package/src/composables/useActiveSession.ts +40 -0
- package/src/composables/useAppApi.ts +45 -0
- package/src/composables/useCanvasViewMode.ts +121 -0
- package/src/composables/useChatScroll.ts +47 -0
- package/src/composables/useClickOutside.ts +26 -0
- package/src/composables/useClipboardCopy.ts +44 -0
- package/src/composables/useContentDisplay.ts +52 -0
- package/src/composables/useDebugBeat.ts +23 -0
- package/src/composables/useDynamicFavicon.ts +115 -0
- package/src/composables/useEventListeners.ts +42 -0
- package/src/composables/useExpandedDirs.ts +64 -0
- package/src/composables/useFaviconState.ts +30 -0
- package/src/composables/useFileSelection.ts +115 -0
- package/src/composables/useFileSortMode.ts +24 -0
- package/src/composables/useFileTree.ts +85 -0
- package/src/composables/useFreshPluginData.ts +89 -0
- package/src/composables/useHealth.ts +38 -0
- package/src/composables/useImeAwareEnter.ts +57 -0
- package/src/composables/useKeyNavigation.ts +60 -0
- package/src/composables/useMarkdownLinkHandler.ts +46 -0
- package/src/composables/useMarkdownMode.ts +17 -0
- package/src/composables/useMcpTools.ts +71 -0
- package/src/composables/useMergedSessions.ts +27 -0
- package/src/composables/useNotifications.ts +90 -0
- package/src/composables/usePdfDownload.ts +60 -0
- package/src/composables/usePendingCalls.ts +77 -0
- package/src/composables/usePubSub.ts +85 -0
- package/src/composables/useRightSidebar.ts +23 -0
- package/src/composables/useRoles.ts +34 -0
- package/src/composables/useSandboxStatus.ts +67 -0
- package/src/composables/useSelectedResult.ts +49 -0
- package/src/composables/useSessionDerived.ts +51 -0
- package/src/composables/useSessionHistory.ts +81 -0
- package/src/composables/useSessionSync.ts +57 -0
- package/src/composables/useViewLayout.ts +55 -0
- package/src/config/apiRoutes.ts +173 -0
- package/src/config/pubsubChannels.ts +45 -0
- package/src/config/roles.ts +335 -0
- package/src/config/schedulerActions.ts +25 -0
- package/src/config/toolNames.ts +71 -0
- package/src/config/workspacePaths.ts +24 -0
- package/src/index.css +107 -0
- package/src/main.ts +25 -0
- package/src/plugins/canvas/Preview.vue +13 -0
- package/src/plugins/canvas/View.vue +333 -0
- package/src/plugins/canvas/definition.ts +38 -0
- package/src/plugins/canvas/index.ts +36 -0
- package/src/plugins/chart/Preview.vue +49 -0
- package/src/plugins/chart/View.vue +143 -0
- package/src/plugins/chart/definition.ts +58 -0
- package/src/plugins/chart/index.ts +52 -0
- package/src/plugins/editImage/Preview.vue +13 -0
- package/src/plugins/editImage/View.vue +13 -0
- package/src/plugins/editImage/definition.ts +27 -0
- package/src/plugins/editImage/index.ts +36 -0
- package/src/plugins/generateImage/Preview.vue +13 -0
- package/src/plugins/generateImage/View.vue +33 -0
- package/src/plugins/generateImage/definition.ts +32 -0
- package/src/plugins/generateImage/index.ts +56 -0
- package/src/plugins/manageRoles/Preview.vue +49 -0
- package/src/plugins/manageRoles/View.vue +525 -0
- package/src/plugins/manageRoles/definition.ts +43 -0
- package/src/plugins/manageRoles/index.ts +47 -0
- package/src/plugins/manageSkills/Preview.vue +21 -0
- package/src/plugins/manageSkills/View.vue +321 -0
- package/src/plugins/manageSkills/definition.ts +49 -0
- package/src/plugins/manageSkills/index.ts +49 -0
- package/src/plugins/manageSource/Preview.vue +33 -0
- package/src/plugins/manageSource/View.vue +697 -0
- package/src/plugins/manageSource/definition.ts +63 -0
- package/src/plugins/manageSource/index.ts +66 -0
- package/src/plugins/markdown/Preview.vue +77 -0
- package/src/plugins/markdown/View.vue +476 -0
- package/src/plugins/markdown/definition.ts +50 -0
- package/src/plugins/markdown/index.ts +36 -0
- package/src/plugins/presentHtml/Preview.vue +25 -0
- package/src/plugins/presentHtml/View.vue +52 -0
- package/src/plugins/presentHtml/definition.ts +27 -0
- package/src/plugins/presentHtml/helpers.ts +72 -0
- package/src/plugins/presentHtml/index.ts +41 -0
- package/src/plugins/presentMulmoScript/Preview.vue +23 -0
- package/src/plugins/presentMulmoScript/View.vue +1166 -0
- package/src/plugins/presentMulmoScript/definition.ts +95 -0
- package/src/plugins/presentMulmoScript/helpers.ts +162 -0
- package/src/plugins/presentMulmoScript/index.ts +40 -0
- package/src/plugins/scheduler/Preview.vue +67 -0
- package/src/plugins/scheduler/TasksTab.vue +205 -0
- package/src/plugins/scheduler/View.vue +565 -0
- package/src/plugins/scheduler/definition.ts +57 -0
- package/src/plugins/scheduler/index.ts +45 -0
- package/src/plugins/scheduler/viewModes.ts +26 -0
- package/src/plugins/spreadsheet/Preview.vue +29 -0
- package/src/plugins/spreadsheet/View.vue +997 -0
- package/src/plugins/spreadsheet/cellHighlights.ts +79 -0
- package/src/plugins/spreadsheet/definition.ts +121 -0
- package/src/plugins/spreadsheet/engine/calculator.ts +459 -0
- package/src/plugins/spreadsheet/engine/cellBuilder.ts +81 -0
- package/src/plugins/spreadsheet/engine/date-parser.ts +220 -0
- package/src/plugins/spreadsheet/engine/date-utils.ts +56 -0
- package/src/plugins/spreadsheet/engine/engine.ts +176 -0
- package/src/plugins/spreadsheet/engine/evaluator.ts +390 -0
- package/src/plugins/spreadsheet/engine/formatter.ts +172 -0
- package/src/plugins/spreadsheet/engine/formulaRefs.ts +101 -0
- package/src/plugins/spreadsheet/engine/functions/date.ts +299 -0
- package/src/plugins/spreadsheet/engine/functions/financial.ts +387 -0
- package/src/plugins/spreadsheet/engine/functions/index.ts +16 -0
- package/src/plugins/spreadsheet/engine/functions/logical.ts +262 -0
- package/src/plugins/spreadsheet/engine/functions/lookup.ts +400 -0
- package/src/plugins/spreadsheet/engine/functions/mathematical.ts +297 -0
- package/src/plugins/spreadsheet/engine/functions/statistical.ts +338 -0
- package/src/plugins/spreadsheet/engine/functions/text.ts +389 -0
- package/src/plugins/spreadsheet/engine/index.ts +27 -0
- package/src/plugins/spreadsheet/engine/jsonCellLocator.ts +111 -0
- package/src/plugins/spreadsheet/engine/parser.ts +143 -0
- package/src/plugins/spreadsheet/engine/registry.ts +150 -0
- package/src/plugins/spreadsheet/engine/responseDecoder.ts +67 -0
- package/src/plugins/spreadsheet/engine/types.ts +64 -0
- package/src/plugins/spreadsheet/index.ts +36 -0
- package/src/plugins/textResponse/Preview.vue +94 -0
- package/src/plugins/textResponse/View.vue +503 -0
- package/src/plugins/textResponse/definition.ts +34 -0
- package/src/plugins/textResponse/index.ts +27 -0
- package/src/plugins/textResponse/plugin.ts +29 -0
- package/src/plugins/textResponse/samples.ts +97 -0
- package/src/plugins/textResponse/types.ts +11 -0
- package/src/plugins/todo/Preview.vue +63 -0
- package/src/plugins/todo/View.vue +364 -0
- package/src/plugins/todo/composables/useTodos.ts +177 -0
- package/src/plugins/todo/definition.ts +45 -0
- package/src/plugins/todo/index.ts +61 -0
- package/src/plugins/todo/labels.ts +163 -0
- package/src/plugins/todo/priority.ts +98 -0
- package/src/plugins/todo/viewModes.ts +19 -0
- package/src/plugins/ui-image/ImagePreview.vue +23 -0
- package/src/plugins/ui-image/ImageView.vue +34 -0
- package/src/plugins/ui-image/index.ts +3 -0
- package/src/plugins/ui-image/types.ts +4 -0
- package/src/plugins/wiki/Preview.vue +65 -0
- package/src/plugins/wiki/View.vue +342 -0
- package/src/plugins/wiki/definition.ts +25 -0
- package/src/plugins/wiki/helpers.ts +59 -0
- package/src/plugins/wiki/index.ts +52 -0
- package/src/router/guards.ts +61 -0
- package/src/router/index.ts +50 -0
- package/src/tools/index.ts +52 -0
- package/src/tools/types.ts +27 -0
- package/src/types/events.ts +16 -0
- package/src/types/fileTree.ts +13 -0
- package/src/types/notification.ts +67 -0
- package/src/types/session.ts +116 -0
- package/src/types/sse.ts +90 -0
- package/src/types/toolCallHistory.ts +13 -0
- package/src/utils/agent/eventDispatch.ts +74 -0
- package/src/utils/agent/request.ts +55 -0
- package/src/utils/agent/toolCalls.ts +62 -0
- package/src/utils/api.ts +218 -0
- package/src/utils/canvas/viewMode.ts +46 -0
- package/src/utils/dom/authTokenMeta.ts +20 -0
- package/src/utils/dom/clickOutside.ts +11 -0
- package/src/utils/dom/externalLink.ts +57 -0
- package/src/utils/dom/scrollable.ts +24 -0
- package/src/utils/errors.ts +11 -0
- package/src/utils/files/expandedDirs.ts +25 -0
- package/src/utils/files/filename.ts +12 -0
- package/src/utils/files/sortChildren.ts +20 -0
- package/src/utils/filesPreview/schedulerPreview.ts +38 -0
- package/src/utils/filesPreview/todoPreview.ts +40 -0
- package/src/utils/format/date.ts +85 -0
- package/src/utils/format/frontmatter.ts +80 -0
- package/src/utils/format/jsonSyntax.ts +109 -0
- package/src/utils/html/previewCsp.ts +65 -0
- package/src/utils/image/resolve.ts +8 -0
- package/src/utils/image/rewriteMarkdownImageRefs.ts +182 -0
- package/src/utils/markdown/extractFirstH1.ts +39 -0
- package/src/utils/notification/dispatch.ts +22 -0
- package/src/utils/path/relativeLink.ts +130 -0
- package/src/utils/role/icon.ts +20 -0
- package/src/utils/role/merge.ts +10 -0
- package/src/utils/role/plugins.ts +12 -0
- package/src/utils/session/mergeSessions.ts +103 -0
- package/src/utils/session/seedRoleDefault.ts +35 -0
- package/src/utils/session/sessionEntries.ts +121 -0
- package/src/utils/session/sessionFactory.ts +22 -0
- package/src/utils/session/sessionHelpers.ts +99 -0
- package/src/utils/tools/dedup.ts +17 -0
- package/src/utils/tools/mcp.ts +33 -0
- package/src/utils/tools/pendingCalls.ts +16 -0
- package/src/utils/tools/result.ts +40 -0
- package/src/utils/types.ts +44 -0
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
// Path helpers for the source registry's on-disk layout.
|
|
2
|
+
//
|
|
3
|
+
// workspace/
|
|
4
|
+
// sources/
|
|
5
|
+
// <slug>.md ← source config
|
|
6
|
+
// _index.md ← auto-generated category index
|
|
7
|
+
// _state/
|
|
8
|
+
// <slug>.json ← runtime state per source
|
|
9
|
+
// robots/<host>.txt ← cached robots.txt
|
|
10
|
+
// news/
|
|
11
|
+
// daily/YYYY/MM/DD.md ← daily aggregated summary
|
|
12
|
+
// archive/<slug>/YYYY-MM.md ← per-source rolling archive
|
|
13
|
+
//
|
|
14
|
+
// Everything is derived from a single `workspaceRoot` argument so
|
|
15
|
+
// tests can target a `mkdtempSync` directory.
|
|
16
|
+
|
|
17
|
+
import path from "node:path";
|
|
18
|
+
import { isValidSlug } from "../../utils/slug.js";
|
|
19
|
+
|
|
20
|
+
export const SOURCES_DIR = "sources";
|
|
21
|
+
export const SOURCE_STATE_DIR = "_state";
|
|
22
|
+
export const ROBOTS_CACHE_DIR = "robots";
|
|
23
|
+
export const NEWS_DIR = "news";
|
|
24
|
+
export const DAILY_DIR = "daily";
|
|
25
|
+
export const ARCHIVE_DIR = "archive";
|
|
26
|
+
|
|
27
|
+
export function sourcesRoot(workspaceRoot: string): string {
|
|
28
|
+
return path.join(workspaceRoot, SOURCES_DIR);
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
// Enforced by every slug-accepting path builder so a caller can't
|
|
32
|
+
// accidentally pass `../other-source` (which path.join would
|
|
33
|
+
// happily resolve outside workspaceRoot).
|
|
34
|
+
function assertValidSlug(slug: string): void {
|
|
35
|
+
if (!isValidSlug(slug)) {
|
|
36
|
+
throw new Error(`[sources] invalid slug: "${slug}"`);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export function sourceFilePath(workspaceRoot: string, slug: string): string {
|
|
41
|
+
assertValidSlug(slug);
|
|
42
|
+
return path.join(sourcesRoot(workspaceRoot), `${slug}.md`);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export function sourceStateDir(workspaceRoot: string): string {
|
|
46
|
+
return path.join(sourcesRoot(workspaceRoot), SOURCE_STATE_DIR);
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export function sourceStatePath(workspaceRoot: string, slug: string): string {
|
|
50
|
+
assertValidSlug(slug);
|
|
51
|
+
return path.join(sourceStateDir(workspaceRoot), `${slug}.json`);
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export function robotsCacheDir(workspaceRoot: string): string {
|
|
55
|
+
return path.join(sourceStateDir(workspaceRoot), ROBOTS_CACHE_DIR);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export function robotsCachePath(workspaceRoot: string, host: string): string {
|
|
59
|
+
// Hosts can contain `:` (for explicit ports) which breaks on some
|
|
60
|
+
// filesystems. Colons → underscore. Other characters are ASCII
|
|
61
|
+
// letters, digits, dots, and hyphens per DNS rules so they're
|
|
62
|
+
// safe as-is.
|
|
63
|
+
const safe = host.replace(/:/g, "_");
|
|
64
|
+
return path.join(robotsCacheDir(workspaceRoot), `${safe}.txt`);
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
export function newsRoot(workspaceRoot: string): string {
|
|
68
|
+
return path.join(workspaceRoot, NEWS_DIR);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export function dailyNewsPath(workspaceRoot: string, isoDate: string): string {
|
|
72
|
+
// Validate shape at the boundary so an empty / bogus date can't
|
|
73
|
+
// produce "undefined/undefined/undefined.md" downstream.
|
|
74
|
+
const m = /^(\d{4})-(\d{2})-(\d{2})$/.exec(isoDate);
|
|
75
|
+
if (!m) {
|
|
76
|
+
throw new Error(`[sources] dailyNewsPath: expected YYYY-MM-DD, got "${isoDate}"`);
|
|
77
|
+
}
|
|
78
|
+
const [, year, month, day] = m;
|
|
79
|
+
return path.join(newsRoot(workspaceRoot), DAILY_DIR, year, month, `${day}.md`);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
export function archiveDir(workspaceRoot: string, slug: string): string {
|
|
83
|
+
assertValidSlug(slug);
|
|
84
|
+
return path.join(newsRoot(workspaceRoot), ARCHIVE_DIR, slug);
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
// Archive file path. Written as `<slug>/YYYY/MM.md` (year and
|
|
88
|
+
// month as nested directories) so long-running workspaces don't
|
|
89
|
+
// end up with 60+ files in a single source's archive dir —
|
|
90
|
+
// browsing a given year is one `cd YYYY/` away. Matches the
|
|
91
|
+
// daily-news layout (`daily/YYYY/MM/DD.md`).
|
|
92
|
+
//
|
|
93
|
+
// Input stays `YYYY-MM` so callers don't need to remember whether
|
|
94
|
+
// to split; we do the split here.
|
|
95
|
+
export function archivePath(workspaceRoot: string, slug: string, yearMonth: string): string {
|
|
96
|
+
assertValidSlug(slug);
|
|
97
|
+
const m = /^(\d{4})-(\d{2})$/.exec(yearMonth);
|
|
98
|
+
if (!m) {
|
|
99
|
+
throw new Error(`[sources] archivePath: expected YYYY-MM, got "${yearMonth}"`);
|
|
100
|
+
}
|
|
101
|
+
const [, year, month] = m;
|
|
102
|
+
return path.join(archiveDir(workspaceRoot, slug), year, `${month}.md`);
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
// Very conservative slug validator. The slug doubles as a filename
|
|
106
|
+
// and appears in URLs (via the manageSource plugin), so reject
|
|
107
|
+
// anything that could surprise the filesystem or the URL parser.
|
|
108
|
+
// Letters, digits, hyphens only. 1-64 chars. No leading / trailing
|
|
109
|
+
// hyphen. No consecutive hyphens.
|
|
110
|
+
// isValidSlug moved to server/utils/slug.ts — import from there.
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
// Cross-source dedup — pure.
|
|
2
|
+
//
|
|
3
|
+
// Per #188 Q3: per-source archives keep every item, but the daily
|
|
4
|
+
// summary step dedupes across sources so a single article that
|
|
5
|
+
// lands in three RSS feeds only appears once in the summary.
|
|
6
|
+
//
|
|
7
|
+
// Dedup key: `stableItemId` (SHA-256 prefix of the normalized URL,
|
|
8
|
+
// see server/sources/urls.ts — the item shape already carries this
|
|
9
|
+
// as `item.id`). Items retain the first-seen occurrence and drop
|
|
10
|
+
// subsequent ones. Order is preserved so the caller's sort (e.g.
|
|
11
|
+
// newest-first across all sources) survives dedup.
|
|
12
|
+
|
|
13
|
+
import type { SourceItem } from "../types.js";
|
|
14
|
+
|
|
15
|
+
export interface DedupStats {
|
|
16
|
+
uniqueCount: number;
|
|
17
|
+
duplicateCount: number;
|
|
18
|
+
// The winning sourceSlug per duplicate id — useful for the
|
|
19
|
+
// daily summary footer ("N duplicates across sources A, B"
|
|
20
|
+
// without naming item titles).
|
|
21
|
+
duplicateSlugsById: Map<string, string[]>;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export interface DedupResult {
|
|
25
|
+
items: SourceItem[];
|
|
26
|
+
stats: DedupStats;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
// Dedup an item list by `id` (stableItemId from urls.ts). Keeps
|
|
30
|
+
// the first occurrence; stats record which OTHER source slugs
|
|
31
|
+
// had duplicates of each kept item so the summary footer can
|
|
32
|
+
// credit them if needed.
|
|
33
|
+
export function dedupAcrossSources(items: readonly SourceItem[]): DedupResult {
|
|
34
|
+
const seen = new Set<string>();
|
|
35
|
+
const kept: SourceItem[] = [];
|
|
36
|
+
const duplicateSlugsById = new Map<string, string[]>();
|
|
37
|
+
let duplicateCount = 0;
|
|
38
|
+
for (const item of items) {
|
|
39
|
+
if (!seen.has(item.id)) {
|
|
40
|
+
seen.add(item.id);
|
|
41
|
+
kept.push(item);
|
|
42
|
+
continue;
|
|
43
|
+
}
|
|
44
|
+
duplicateCount++;
|
|
45
|
+
const dupSlugs = duplicateSlugsById.get(item.id) ?? [];
|
|
46
|
+
// Keep the slug list unique: a single source that emits the
|
|
47
|
+
// same item twice (e.g. feed pagination overlap) shouldn't
|
|
48
|
+
// inflate the "across sources" footer stat.
|
|
49
|
+
if (!dupSlugs.includes(item.sourceSlug)) dupSlugs.push(item.sourceSlug);
|
|
50
|
+
duplicateSlugsById.set(item.id, dupSlugs);
|
|
51
|
+
}
|
|
52
|
+
return {
|
|
53
|
+
items: kept,
|
|
54
|
+
stats: {
|
|
55
|
+
uniqueCount: kept.length,
|
|
56
|
+
duplicateCount,
|
|
57
|
+
duplicateSlugsById,
|
|
58
|
+
},
|
|
59
|
+
};
|
|
60
|
+
}
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
// Fetch-phase orchestrator.
|
|
2
|
+
//
|
|
3
|
+
// Given a planned list of eligible sources, runs each through
|
|
4
|
+
// its registered fetcher concurrently (Q7: parallel across
|
|
5
|
+
// hosts; same-host serialization happens inside HostRateLimiter
|
|
6
|
+
// at the HTTP layer). Failures are isolated per-source (Q8) —
|
|
7
|
+
// one bad fetch never aborts the pass.
|
|
8
|
+
//
|
|
9
|
+
// Each source produces a `FetchOutcome` summarizing success or
|
|
10
|
+
// failure. The next-state computation (backoff, failure counter,
|
|
11
|
+
// cursor persistence) is factored into `computeNextState` so the
|
|
12
|
+
// state-update policy is unit-testable without touching HTTP.
|
|
13
|
+
|
|
14
|
+
import type { FetcherDeps, FetchResult, SourceFetcher } from "../fetchers/index.js";
|
|
15
|
+
import type { FetcherKind, Source, SourceState } from "../types.js";
|
|
16
|
+
import { defaultSourceState } from "../types.js";
|
|
17
|
+
import { errorMessage } from "../../../utils/errors.js";
|
|
18
|
+
import { ONE_MINUTE_MS, ONE_DAY_MS } from "../../../utils/time.js";
|
|
19
|
+
|
|
20
|
+
// Outcome of one source's fetch attempt.
|
|
21
|
+
export type FetchOutcome =
|
|
22
|
+
| {
|
|
23
|
+
kind: "success";
|
|
24
|
+
sourceSlug: string;
|
|
25
|
+
items: FetchResult["items"];
|
|
26
|
+
cursor: FetchResult["cursor"];
|
|
27
|
+
}
|
|
28
|
+
| { kind: "no-fetcher"; sourceSlug: string; error: string }
|
|
29
|
+
| { kind: "error"; sourceSlug: string; error: string };
|
|
30
|
+
|
|
31
|
+
export interface FetchPhaseInput {
|
|
32
|
+
sources: readonly Source[];
|
|
33
|
+
statesBySlug: ReadonlyMap<string, SourceState>;
|
|
34
|
+
deps: FetcherDeps;
|
|
35
|
+
// Injected so tests don't depend on the module-level registry.
|
|
36
|
+
// Production passes `getFetcher` from fetchers/index.
|
|
37
|
+
getFetcher: (kind: FetcherKind) => SourceFetcher | null;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export interface FetchPhaseResult {
|
|
41
|
+
// In the original order of `input.sources` for caller
|
|
42
|
+
// ergonomics; the per-outcome `sourceSlug` is the authoritative
|
|
43
|
+
// key.
|
|
44
|
+
outcomes: FetchOutcome[];
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
// Run the fetch phase. All fetchers run in parallel
|
|
48
|
+
// (Promise.all) — same-host serialization is enforced deeper,
|
|
49
|
+
// inside `HostRateLimiter` via the fetchers' `fetchPolite`
|
|
50
|
+
// calls. A single-source error never throws out of here;
|
|
51
|
+
// failures are captured in `FetchOutcome.kind === "error"`.
|
|
52
|
+
export async function runFetchPhase(input: FetchPhaseInput): Promise<FetchPhaseResult> {
|
|
53
|
+
const outcomes = await Promise.all(
|
|
54
|
+
input.sources.map((source) => fetchOneSource(source, input.statesBySlug.get(source.slug) ?? defaultSourceState(source.slug), input.deps, input.getFetcher)),
|
|
55
|
+
);
|
|
56
|
+
return { outcomes };
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
async function fetchOneSource(
|
|
60
|
+
source: Source,
|
|
61
|
+
state: SourceState,
|
|
62
|
+
deps: FetcherDeps,
|
|
63
|
+
getFetcher: (kind: FetcherKind) => SourceFetcher | null,
|
|
64
|
+
): Promise<FetchOutcome> {
|
|
65
|
+
const fetcher = getFetcher(source.fetcherKind);
|
|
66
|
+
if (!fetcher) {
|
|
67
|
+
return {
|
|
68
|
+
kind: "no-fetcher",
|
|
69
|
+
sourceSlug: source.slug,
|
|
70
|
+
error: `no fetcher registered for kind "${source.fetcherKind}"`,
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
try {
|
|
74
|
+
const result = await fetcher.fetch(source, state, deps);
|
|
75
|
+
return {
|
|
76
|
+
kind: "success",
|
|
77
|
+
sourceSlug: source.slug,
|
|
78
|
+
items: result.items,
|
|
79
|
+
cursor: result.cursor,
|
|
80
|
+
};
|
|
81
|
+
} catch (err) {
|
|
82
|
+
return {
|
|
83
|
+
kind: "error",
|
|
84
|
+
sourceSlug: source.slug,
|
|
85
|
+
error: errorMessage(err),
|
|
86
|
+
};
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
// --- per-source state update --------------------------------------------
|
|
91
|
+
|
|
92
|
+
// Exponential backoff (in ms) for the Nth consecutive failure.
|
|
93
|
+
// Bounded at BACKOFF_MAX so even a permanently-broken source
|
|
94
|
+
// gets retried eventually.
|
|
95
|
+
export const BACKOFF_MAX_MS = ONE_DAY_MS;
|
|
96
|
+
|
|
97
|
+
export function backoffDelayMs(consecutiveFailures: number): number {
|
|
98
|
+
if (consecutiveFailures <= 0) return 0;
|
|
99
|
+
// 1m, 2m, 4m, 8m, 16m, ..., capped at 24h.
|
|
100
|
+
const base = ONE_MINUTE_MS;
|
|
101
|
+
const ms = base * 2 ** Math.min(consecutiveFailures - 1, 20);
|
|
102
|
+
return Math.min(ms, BACKOFF_MAX_MS);
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
// Compute the next per-source state given the outcome. Pure.
|
|
106
|
+
//
|
|
107
|
+
// On success:
|
|
108
|
+
// - lastFetchedAt = now
|
|
109
|
+
// - cursor = outcome.cursor (replace wholesale — fetchers
|
|
110
|
+
// return the merged cursor map)
|
|
111
|
+
// - consecutiveFailures = 0
|
|
112
|
+
// - nextAttemptAt = null
|
|
113
|
+
// On any non-success:
|
|
114
|
+
// - lastFetchedAt unchanged (we didn't successfully fetch)
|
|
115
|
+
// - cursor unchanged
|
|
116
|
+
// - consecutiveFailures += 1
|
|
117
|
+
// - nextAttemptAt = now + backoffDelayMs(newCount)
|
|
118
|
+
export function computeNextState(prev: SourceState, outcome: FetchOutcome, nowMs: number): SourceState {
|
|
119
|
+
if (outcome.kind === "success") {
|
|
120
|
+
return {
|
|
121
|
+
slug: prev.slug,
|
|
122
|
+
lastFetchedAt: new Date(nowMs).toISOString(),
|
|
123
|
+
cursor: outcome.cursor,
|
|
124
|
+
consecutiveFailures: 0,
|
|
125
|
+
nextAttemptAt: null,
|
|
126
|
+
};
|
|
127
|
+
}
|
|
128
|
+
const failures = prev.consecutiveFailures + 1;
|
|
129
|
+
return {
|
|
130
|
+
slug: prev.slug,
|
|
131
|
+
lastFetchedAt: prev.lastFetchedAt,
|
|
132
|
+
cursor: prev.cursor,
|
|
133
|
+
consecutiveFailures: failures,
|
|
134
|
+
nextAttemptAt: new Date(nowMs + backoffDelayMs(failures)).toISOString(),
|
|
135
|
+
};
|
|
136
|
+
}
|
|
@@ -0,0 +1,249 @@
|
|
|
1
|
+
// Top-level pipeline entry point.
|
|
2
|
+
//
|
|
3
|
+
// `runSourcesPipeline({ workspaceRoot, scheduleType, ... })`
|
|
4
|
+
// threads every phase in order:
|
|
5
|
+
//
|
|
6
|
+
// 1. Load sources from the registry
|
|
7
|
+
// 2. Read per-source state from `_state/<slug>.json`
|
|
8
|
+
// 3. Plan: filter by schedule + backoff
|
|
9
|
+
// 4. Fetch: per-source, parallel, failure-isolated
|
|
10
|
+
// 5. Dedup across sources (first occurrence wins)
|
|
11
|
+
// 6. Summarize via claude CLI (skipped for 0 items)
|
|
12
|
+
// 7. Write daily markdown + JSON block
|
|
13
|
+
// 8. Append every item to its per-source monthly archive
|
|
14
|
+
// 9. Persist updated per-source state back to disk
|
|
15
|
+
//
|
|
16
|
+
// Design follows #188 decisions: per-source try/catch (Q8),
|
|
17
|
+
// cross-source dedup only at summary step (Q3), local timezone
|
|
18
|
+
// (Q6), parallel across hosts (Q7 — enforced deeper by
|
|
19
|
+
// HostRateLimiter inside fetchPolite).
|
|
20
|
+
//
|
|
21
|
+
// Fully DI-threaded: `getFetcher`, `summarizeFn`, `now` are all
|
|
22
|
+
// parameters, and workspaceRoot is explicit. Tests can drive
|
|
23
|
+
// the whole pipeline end-to-end against a mkdtempSync workspace
|
|
24
|
+
// with stub fetchers and a fake summarize.
|
|
25
|
+
|
|
26
|
+
// Side-effect import: registers every production fetcher so
|
|
27
|
+
// `registryGetFetcher(kind)` below resolves. Without this the
|
|
28
|
+
// pipeline would run, report `no-fetcher` for every source, and
|
|
29
|
+
// write an empty daily file.
|
|
30
|
+
import "../fetchers/registerAll.js";
|
|
31
|
+
|
|
32
|
+
import { existsSync } from "fs";
|
|
33
|
+
import { listSources } from "../registry.js";
|
|
34
|
+
import { readManyStates, writeManyStates } from "../sourceState.js";
|
|
35
|
+
import { dailyNewsPath } from "../paths.js";
|
|
36
|
+
import { getFetcher as registryGetFetcher, type FetcherDeps, type SourceFetcher } from "../fetchers/index.js";
|
|
37
|
+
import type { FetcherKind, Source, SourceItem, SourceState, SourceSchedule } from "../types.js";
|
|
38
|
+
import { planEligibleSources } from "./plan.js";
|
|
39
|
+
import { runFetchPhase, computeNextState, type FetchOutcome } from "./fetch.js";
|
|
40
|
+
import { dedupAcrossSources, type DedupStats } from "./dedup.js";
|
|
41
|
+
import { makeDefaultSummarize, type SummarizeFn } from "./summarize.js";
|
|
42
|
+
import { writeDailyFile, appendItemsToArchives } from "./write.js";
|
|
43
|
+
import { runNotifyPhase } from "./notify.js";
|
|
44
|
+
import { discoverAndRegister } from "../arxivDiscovery.js";
|
|
45
|
+
import { log } from "../../../system/logger/index.js";
|
|
46
|
+
import { toLocalIsoDate } from "../../../utils/date.js";
|
|
47
|
+
|
|
48
|
+
export interface RunPipelineInput {
|
|
49
|
+
workspaceRoot: string;
|
|
50
|
+
scheduleType: SourceSchedule;
|
|
51
|
+
// Shared across all fetchers in the run (rate limiter, robots
|
|
52
|
+
// provider, fetch impl, timeout — assembled by the caller).
|
|
53
|
+
fetcherDeps: FetcherDeps;
|
|
54
|
+
// Pipeline-run clock. Production passes `() => Date.now()`.
|
|
55
|
+
// Tests pass a fixed millis so isoDate / backoff math is
|
|
56
|
+
// deterministic.
|
|
57
|
+
nowMs: () => number;
|
|
58
|
+
// Injection hooks.
|
|
59
|
+
getFetcher?: (kind: FetcherKind) => SourceFetcher | null;
|
|
60
|
+
summarizeFn?: SummarizeFn;
|
|
61
|
+
// For test instrumentation; ignored in production.
|
|
62
|
+
onProgress?: (phase: string) => void;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
export interface RunPipelineResult {
|
|
66
|
+
// Sources considered in this run.
|
|
67
|
+
plannedCount: number;
|
|
68
|
+
// Raw fetch outcomes (success / error / no-fetcher). In
|
|
69
|
+
// original plan order.
|
|
70
|
+
outcomes: FetchOutcome[];
|
|
71
|
+
// Items emitted after cross-source dedup, ready for
|
|
72
|
+
// summarization + archive append.
|
|
73
|
+
items: SourceItem[];
|
|
74
|
+
dedup: DedupStats;
|
|
75
|
+
// Absolute path of the daily markdown file written.
|
|
76
|
+
dailyPath: string;
|
|
77
|
+
archiveWrittenPaths: string[];
|
|
78
|
+
// Non-fatal errors from the archive append step.
|
|
79
|
+
archiveErrors: string[];
|
|
80
|
+
// Per-source post-run states, already persisted to disk.
|
|
81
|
+
nextStates: SourceState[];
|
|
82
|
+
// Local ISO date used for the daily header / filename.
|
|
83
|
+
isoDate: string;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
// Convert a wall-clock millis value to YYYY-MM-DD in LOCAL
|
|
87
|
+
// time, matching the #188 Q6 decision ("Local time, like the
|
|
88
|
+
// journal"). The journal's `toIsoDate` in paths.ts uses the
|
|
89
|
+
// Re-export for callers that imported from this module.
|
|
90
|
+
export { toLocalIsoDate } from "../../../utils/date.js";
|
|
91
|
+
|
|
92
|
+
// Convert a wall-clock millis value to the LOCAL year-month
|
|
93
|
+
// key (YYYY-MM) used as the archive fallback for items without
|
|
94
|
+
// a parseable publishedAt.
|
|
95
|
+
export function toLocalYearMonth(ms: number): string {
|
|
96
|
+
const d = new Date(ms);
|
|
97
|
+
const y = d.getFullYear();
|
|
98
|
+
const m = String(d.getMonth() + 1).padStart(2, "0");
|
|
99
|
+
return `${y}-${m}`;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
export async function runSourcesPipeline(input: RunPipelineInput): Promise<RunPipelineResult> {
|
|
103
|
+
const { workspaceRoot, scheduleType, fetcherDeps, nowMs, getFetcher = registryGetFetcher, onProgress = () => {} } = input;
|
|
104
|
+
|
|
105
|
+
const startMs = nowMs();
|
|
106
|
+
const isoDate = toLocalIsoDate(startMs);
|
|
107
|
+
const fallbackMonth = toLocalYearMonth(startMs);
|
|
108
|
+
const summarizeFn = input.summarizeFn ?? makeDefaultSummarize(isoDate);
|
|
109
|
+
|
|
110
|
+
// --- 0. Auto-discover arXiv sources from interests ------------------
|
|
111
|
+
// Best-effort: a bad interests.json or FS error must not abort the
|
|
112
|
+
// entire pipeline. The daily news fetch is more important than
|
|
113
|
+
// auto-registering arXiv sources.
|
|
114
|
+
onProgress("discover");
|
|
115
|
+
try {
|
|
116
|
+
await discoverAndRegister(workspaceRoot);
|
|
117
|
+
} catch (err) {
|
|
118
|
+
log.warn("pipeline", "arXiv auto-discovery failed (non-fatal)", {
|
|
119
|
+
error: String(err),
|
|
120
|
+
});
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
// --- 1. Load registry + state --------------------------------------
|
|
124
|
+
onProgress("load");
|
|
125
|
+
const allSources = await listSources(workspaceRoot);
|
|
126
|
+
const statesBySlug = await readManyStates(
|
|
127
|
+
workspaceRoot,
|
|
128
|
+
allSources.map((s) => s.slug),
|
|
129
|
+
);
|
|
130
|
+
|
|
131
|
+
// --- 2. Plan ------------------------------------------------------
|
|
132
|
+
onProgress("plan");
|
|
133
|
+
const eligible = planEligibleSources({
|
|
134
|
+
sources: allSources,
|
|
135
|
+
statesBySlug,
|
|
136
|
+
scheduleType,
|
|
137
|
+
nowMs: startMs,
|
|
138
|
+
});
|
|
139
|
+
if (eligible.length === 0) {
|
|
140
|
+
// Write an empty-day daily file so it's clear the pipeline
|
|
141
|
+
// ran. Archive append is a no-op. State untouched.
|
|
142
|
+
//
|
|
143
|
+
// But: if a previous pass today already produced a non-empty
|
|
144
|
+
// brief, don't clobber it. A same-day rerun with nothing due
|
|
145
|
+
// (all sources still in backoff / on "weekly" schedule) would
|
|
146
|
+
// otherwise wipe the morning's brief when re-triggered in the
|
|
147
|
+
// afternoon.
|
|
148
|
+
onProgress("write-empty");
|
|
149
|
+
const existingPath = dailyNewsPath(workspaceRoot, isoDate);
|
|
150
|
+
const dailyPath = existsSync(existingPath) ? existingPath : await writeDailyFile(workspaceRoot, isoDate, await summarizeFn([]), []);
|
|
151
|
+
return {
|
|
152
|
+
plannedCount: 0,
|
|
153
|
+
outcomes: [],
|
|
154
|
+
items: [],
|
|
155
|
+
dedup: {
|
|
156
|
+
uniqueCount: 0,
|
|
157
|
+
duplicateCount: 0,
|
|
158
|
+
duplicateSlugsById: new Map(),
|
|
159
|
+
},
|
|
160
|
+
dailyPath,
|
|
161
|
+
archiveWrittenPaths: [],
|
|
162
|
+
archiveErrors: [],
|
|
163
|
+
nextStates: [],
|
|
164
|
+
isoDate,
|
|
165
|
+
};
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
// --- 3. Fetch -----------------------------------------------------
|
|
169
|
+
onProgress("fetch");
|
|
170
|
+
const { outcomes } = await runFetchPhase({
|
|
171
|
+
sources: eligible,
|
|
172
|
+
statesBySlug,
|
|
173
|
+
deps: fetcherDeps,
|
|
174
|
+
getFetcher,
|
|
175
|
+
});
|
|
176
|
+
|
|
177
|
+
// --- 4. Dedup -----------------------------------------------------
|
|
178
|
+
onProgress("dedup");
|
|
179
|
+
const rawItems = flattenItems(outcomes);
|
|
180
|
+
const dedup = dedupAcrossSources(rawItems);
|
|
181
|
+
|
|
182
|
+
// --- 5. Notify (user interest matching) ----------------------------
|
|
183
|
+
onProgress("notify");
|
|
184
|
+
runNotifyPhase(dedup.items, workspaceRoot);
|
|
185
|
+
|
|
186
|
+
// --- 6. Summarize + write ----------------------------------------
|
|
187
|
+
onProgress("summarize");
|
|
188
|
+
const markdown = await summarizeFn(dedup.items);
|
|
189
|
+
|
|
190
|
+
onProgress("write"); // step 7
|
|
191
|
+
const dailyPath = await writeDailyFile(workspaceRoot, isoDate, markdown, dedup.items);
|
|
192
|
+
const archiveResult = await appendItemsToArchives(workspaceRoot, dedup.items, fallbackMonth);
|
|
193
|
+
|
|
194
|
+
// --- 8. Persist state ---------------------------------------------
|
|
195
|
+
onProgress("persist");
|
|
196
|
+
const nextStates = buildNextStates(eligible, statesBySlug, outcomes, nowMs());
|
|
197
|
+
await writeManyStates(workspaceRoot, nextStates);
|
|
198
|
+
|
|
199
|
+
onProgress("done");
|
|
200
|
+
return {
|
|
201
|
+
plannedCount: eligible.length,
|
|
202
|
+
outcomes,
|
|
203
|
+
items: dedup.items,
|
|
204
|
+
dedup: dedup.stats,
|
|
205
|
+
dailyPath,
|
|
206
|
+
archiveWrittenPaths: archiveResult.writtenPaths,
|
|
207
|
+
archiveErrors: archiveResult.errors,
|
|
208
|
+
nextStates,
|
|
209
|
+
isoDate,
|
|
210
|
+
};
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
// Flatten successful-outcome items into a single list for
|
|
214
|
+
// dedup. Keeps the original source ordering (planned sort
|
|
215
|
+
// order) so dedup preserves deterministic precedence.
|
|
216
|
+
function flattenItems(outcomes: readonly FetchOutcome[]): SourceItem[] {
|
|
217
|
+
const out: SourceItem[] = [];
|
|
218
|
+
for (const outcome of outcomes) {
|
|
219
|
+
if (outcome.kind !== "success") continue;
|
|
220
|
+
for (const item of outcome.items) out.push(item);
|
|
221
|
+
}
|
|
222
|
+
return out;
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
function buildNextStates(
|
|
226
|
+
eligible: readonly Source[],
|
|
227
|
+
statesBySlug: ReadonlyMap<string, SourceState>,
|
|
228
|
+
outcomes: readonly FetchOutcome[],
|
|
229
|
+
nowMs: number,
|
|
230
|
+
): SourceState[] {
|
|
231
|
+
const outcomeBySlug = new Map<string, FetchOutcome>();
|
|
232
|
+
for (const outcome of outcomes) {
|
|
233
|
+
outcomeBySlug.set(outcome.sourceSlug, outcome);
|
|
234
|
+
}
|
|
235
|
+
const nextStates: SourceState[] = [];
|
|
236
|
+
for (const source of eligible) {
|
|
237
|
+
const prev = statesBySlug.get(source.slug) ?? {
|
|
238
|
+
slug: source.slug,
|
|
239
|
+
lastFetchedAt: null,
|
|
240
|
+
cursor: {},
|
|
241
|
+
consecutiveFailures: 0,
|
|
242
|
+
nextAttemptAt: null,
|
|
243
|
+
};
|
|
244
|
+
const outcome = outcomeBySlug.get(source.slug);
|
|
245
|
+
if (!outcome) continue; // unreachable in practice; defensive
|
|
246
|
+
nextStates.push(computeNextState(prev, outcome, nowMs));
|
|
247
|
+
}
|
|
248
|
+
return nextStates;
|
|
249
|
+
}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
// Notify phase — score items by user interests and publish
|
|
2
|
+
// notifications for interesting findings (#466).
|
|
3
|
+
//
|
|
4
|
+
// Inserted between dedup and summarize in the pipeline.
|
|
5
|
+
// Skipped entirely when config/interests.json doesn't exist.
|
|
6
|
+
|
|
7
|
+
import { publishNotification } from "../../../events/notifications.js";
|
|
8
|
+
import { NOTIFICATION_KINDS, NOTIFICATION_PRIORITIES, NOTIFICATION_ACTION_TYPES, NOTIFICATION_VIEWS } from "../../../../src/types/notification.js";
|
|
9
|
+
import { loadInterests, scoreAndFilter, type ScoredItem } from "../interests.js";
|
|
10
|
+
import type { SourceItem } from "../types.js";
|
|
11
|
+
|
|
12
|
+
export interface NotifyPhaseResult {
|
|
13
|
+
notified: ScoredItem[];
|
|
14
|
+
skippedReason: string | null;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export function runNotifyPhase(items: readonly SourceItem[], workspaceRoot?: string): NotifyPhaseResult {
|
|
18
|
+
const profile = loadInterests(workspaceRoot);
|
|
19
|
+
if (!profile) {
|
|
20
|
+
return { notified: [], skippedReason: "no interests profile" };
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
const interesting = scoreAndFilter(items, profile);
|
|
24
|
+
if (interesting.length === 0) {
|
|
25
|
+
return { notified: [], skippedReason: "no items above threshold" };
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
publishBatchNotification(interesting);
|
|
29
|
+
return { notified: interesting, skippedReason: null };
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function formatSingleBody(item: SourceItem): string {
|
|
33
|
+
const suffix = item.summary ? " \u2014 " + item.summary : "";
|
|
34
|
+
return "From " + item.sourceSlug + suffix;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
function publishBatchNotification(scored: readonly ScoredItem[]): void {
|
|
38
|
+
if (scored.length === 1) {
|
|
39
|
+
const { item } = scored[0];
|
|
40
|
+
publishNotification({
|
|
41
|
+
kind: NOTIFICATION_KINDS.push,
|
|
42
|
+
title: item.title,
|
|
43
|
+
body: formatSingleBody(item),
|
|
44
|
+
priority: item.severity === "critical" ? NOTIFICATION_PRIORITIES.high : NOTIFICATION_PRIORITIES.normal,
|
|
45
|
+
action: {
|
|
46
|
+
type: NOTIFICATION_ACTION_TYPES.navigate,
|
|
47
|
+
view: NOTIFICATION_VIEWS.files,
|
|
48
|
+
},
|
|
49
|
+
});
|
|
50
|
+
return;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
const bullets = scored
|
|
54
|
+
.slice(0, 5)
|
|
55
|
+
.map((s) => `\u2022 ${s.item.title} (${s.item.sourceSlug})`)
|
|
56
|
+
.join("\n");
|
|
57
|
+
const extra = scored.length > 5 ? `\n+${scored.length - 5} more` : "";
|
|
58
|
+
|
|
59
|
+
// Preserve high priority if any item in the batch is critical
|
|
60
|
+
const hasCritical = scored.some((s) => s.item.severity === "critical");
|
|
61
|
+
|
|
62
|
+
publishNotification({
|
|
63
|
+
kind: NOTIFICATION_KINDS.push,
|
|
64
|
+
title: `${scored.length} interesting articles found`,
|
|
65
|
+
body: `${bullets}${extra}`,
|
|
66
|
+
priority: hasCritical ? NOTIFICATION_PRIORITIES.high : NOTIFICATION_PRIORITIES.normal,
|
|
67
|
+
action: {
|
|
68
|
+
type: NOTIFICATION_ACTION_TYPES.navigate,
|
|
69
|
+
view: NOTIFICATION_VIEWS.files,
|
|
70
|
+
},
|
|
71
|
+
});
|
|
72
|
+
}
|